Line data Source code
1 : import 'dart:async';
2 : import 'dart:convert';
3 : import 'dart:typed_data';
4 :
5 : import 'package:matrix/matrix.dart';
6 : import 'package:matrix/src/utils/crypto/crypto.dart';
7 : import 'package:matrix/src/voip/models/call_membership.dart';
8 :
9 : class LiveKitBackend extends CallBackend {
10 : final String livekitServiceUrl;
11 : final String livekitAlias;
12 :
13 : /// A delay after a member leaves before we create and publish a new key, because people
14 : /// tend to leave calls at the same time
15 : final Duration makeKeyDelay;
16 :
17 : /// The delay between creating and sending a new key and starting to encrypt with it. This gives others
18 : /// a chance to receive the new key to minimise the chance they don't get media they can't decrypt.
19 : /// The total time between a member leaving and the call switching to new keys is therefore
20 : /// makeKeyDelay + useKeyDelay
21 : final Duration useKeyDelay;
22 :
23 : @override
24 : final bool e2eeEnabled;
25 :
26 0 : LiveKitBackend({
27 : required this.livekitServiceUrl,
28 : required this.livekitAlias,
29 : super.type = 'livekit',
30 : this.e2eeEnabled = true,
31 : this.makeKeyDelay = CallTimeouts.makeKeyDelay,
32 : this.useKeyDelay = CallTimeouts.useKeyDelay,
33 : });
34 :
35 : Timer? _memberLeaveEncKeyRotateDebounceTimer;
36 :
37 : /// participant:keyIndex:keyBin
38 : final Map<CallParticipant, Map<int, Uint8List>> _encryptionKeysMap = {};
39 :
40 : final List<Future<void>> _setNewKeyTimeouts = [];
41 :
42 : int _indexCounter = 0;
43 :
44 : /// used to send the key again incase someone `onCallEncryptionKeyRequest` but don't just send
45 : /// the last one because you also cycle back in your window which means you
46 : /// could potentially end up sharing a past key
47 0 : int get latestLocalKeyIndex => _latestLocalKeyIndex;
48 : int _latestLocalKeyIndex = 0;
49 :
50 : /// the key currently being used by the local cryptor, can possibly not be the latest
51 : /// key, check `latestLocalKeyIndex` for latest key
52 0 : int get currentLocalKeyIndex => _currentLocalKeyIndex;
53 : int _currentLocalKeyIndex = 0;
54 :
55 0 : Map<int, Uint8List>? _getKeysForParticipant(CallParticipant participant) {
56 0 : return _encryptionKeysMap[participant];
57 : }
58 :
59 : /// always chooses the next possible index, we cycle after 16 because
60 : /// no real adv with infinite list
61 0 : int _getNewEncryptionKeyIndex() {
62 0 : final newIndex = _indexCounter % 16;
63 0 : _indexCounter++;
64 : return newIndex;
65 : }
66 :
67 0 : @override
68 : Future<void> preShareKey(GroupCallSession groupCall) async {
69 0 : await groupCall.onMemberStateChanged();
70 0 : await _changeEncryptionKey(groupCall, groupCall.participants, false);
71 : }
72 :
73 : /// makes a new e2ee key for local user and sets it with a delay if specified
74 : /// used on first join and when someone leaves
75 : ///
76 : /// also does the sending for you
77 0 : Future<void> _makeNewSenderKey(
78 : GroupCallSession groupCall, bool delayBeforeUsingKeyOurself) async {
79 0 : final key = secureRandomBytes(32);
80 0 : final keyIndex = _getNewEncryptionKeyIndex();
81 0 : Logs().i('[VOIP E2EE] Generated new key $key at index $keyIndex');
82 :
83 0 : await _setEncryptionKey(
84 : groupCall,
85 0 : groupCall.localParticipant!,
86 : keyIndex,
87 : key,
88 : delayBeforeUsingKeyOurself: delayBeforeUsingKeyOurself,
89 : send: true,
90 : );
91 : }
92 :
93 : DateTime lastRatchetAt = DateTime(1980);
94 :
95 : /// also does the sending for you
96 0 : Future<void> _ratchetLocalParticipantKey(
97 : GroupCallSession groupCall,
98 : List<CallParticipant> sendTo,
99 : ) async {
100 0 : final keyProvider = groupCall.voip.delegate.keyProvider;
101 :
102 : if (keyProvider == null) {
103 0 : throw MatrixSDKVoipException(
104 : '_ratchetKey called but KeyProvider was null');
105 : }
106 :
107 0 : final myKeys = _encryptionKeysMap[groupCall.localParticipant];
108 :
109 0 : if (myKeys == null || myKeys.isEmpty) {
110 0 : await _makeNewSenderKey(groupCall, false);
111 : return;
112 : }
113 :
114 0 : if (_currentLocalKeyIndex != _latestLocalKeyIndex) {
115 : /// Leave causes rotate, new user joins after making new key but
116 : /// before using new key, this then causes a ratchet of the latestLocalKey
117 : /// returns null until that key is set when useKeyDelay is done.
118 : ///
119 : /// You will see some onRatchetKey sending empty responses here
120 : /// therefore the below while loop.
121 0 : Logs().w(
122 0 : '[VOIP E2EE] Leave and join / rotate and ratchet scenario detected, expect ${useKeyDelay.inSeconds} seconds disruption. latest: $latestLocalKeyIndex, current: $currentLocalKeyIndex');
123 : }
124 :
125 0 : if (lastRatchetAt.isBefore(DateTime.now().subtract(makeKeyDelay))) {
126 : Uint8List? ratchedKey;
127 0 : while (ratchedKey == null || ratchedKey.isEmpty) {
128 0 : Logs().d(
129 0 : '[VOIP E2EE] Ignoring empty ratcheted key, probably waiting for useKeyDelay to finish, expect around ${useKeyDelay.inSeconds} seconds of disruption. latest: $latestLocalKeyIndex, current: $currentLocalKeyIndex');
130 0 : ratchedKey = await keyProvider.onRatchetKey(
131 0 : groupCall.localParticipant!,
132 0 : latestLocalKeyIndex,
133 : );
134 : }
135 0 : lastRatchetAt = DateTime.now();
136 0 : Logs().i(
137 0 : '[VOIP E2EE] Ratched latest key to $ratchedKey at idx $latestLocalKeyIndex');
138 0 : await _setEncryptionKey(
139 : groupCall,
140 0 : groupCall.localParticipant!,
141 0 : latestLocalKeyIndex,
142 : ratchedKey,
143 : delayBeforeUsingKeyOurself: false,
144 : send: true,
145 : sendTo: sendTo,
146 : );
147 : } else {
148 0 : Logs().d(
149 0 : '[VOIP E2EE] Skipped ratcheting because lastRatchet run was at ${lastRatchetAt.millisecondsSinceEpoch}');
150 : // send without setting because it is already set
151 0 : await _sendEncryptionKeysEvent(
152 : groupCall,
153 0 : latestLocalKeyIndex,
154 : sendTo: sendTo,
155 : );
156 : }
157 : }
158 :
159 0 : Future<void> _changeEncryptionKey(
160 : GroupCallSession groupCall,
161 : List<CallParticipant> anyJoined,
162 : bool delayBeforeUsingKeyOurself,
163 : ) async {
164 0 : if (!e2eeEnabled) return;
165 0 : if (groupCall.voip.enableSFUE2EEKeyRatcheting) {
166 0 : await _ratchetLocalParticipantKey(groupCall, anyJoined);
167 : } else {
168 0 : await _makeNewSenderKey(groupCall, delayBeforeUsingKeyOurself);
169 : }
170 : }
171 :
172 : /// sets incoming keys and also sends the key if it was for the local user
173 : /// if sendTo is null, its sent to all _participants, see `_sendEncryptionKeysEvent`
174 0 : Future<void> _setEncryptionKey(
175 : GroupCallSession groupCall,
176 : CallParticipant participant,
177 : int encryptionKeyIndex,
178 : Uint8List encryptionKeyBin, {
179 : bool delayBeforeUsingKeyOurself = false,
180 : bool send = false,
181 : List<CallParticipant>? sendTo,
182 : }) async {
183 : final encryptionKeys =
184 0 : _encryptionKeysMap[participant] ?? <int, Uint8List>{};
185 :
186 0 : encryptionKeys[encryptionKeyIndex] = encryptionKeyBin;
187 0 : _encryptionKeysMap[participant] = encryptionKeys;
188 0 : if (participant.isLocal) {
189 0 : _latestLocalKeyIndex = encryptionKeyIndex;
190 : }
191 :
192 : if (send) {
193 0 : await _sendEncryptionKeysEvent(
194 : groupCall,
195 : encryptionKeyIndex,
196 : sendTo: sendTo,
197 : );
198 : }
199 :
200 : if (delayBeforeUsingKeyOurself) {
201 : // now wait for the key to propogate and then set it, hopefully users can
202 : // stil decrypt everything
203 0 : final useKeyTimeout = Future<void>.delayed(useKeyDelay, () async {
204 0 : Logs().i(
205 0 : '[VOIP E2EE] setting key changed event for ${participant.id} idx $encryptionKeyIndex key $encryptionKeyBin');
206 0 : await groupCall.voip.delegate.keyProvider?.onSetEncryptionKey(
207 : participant, encryptionKeyBin, encryptionKeyIndex);
208 0 : if (participant.isLocal) {
209 0 : _currentLocalKeyIndex = encryptionKeyIndex;
210 : }
211 : });
212 0 : _setNewKeyTimeouts.add(useKeyTimeout);
213 : } else {
214 0 : Logs().i(
215 0 : '[VOIP E2EE] setting key changed event for ${participant.id} idx $encryptionKeyIndex key $encryptionKeyBin');
216 0 : await groupCall.voip.delegate.keyProvider?.onSetEncryptionKey(
217 : participant, encryptionKeyBin, encryptionKeyIndex);
218 0 : if (participant.isLocal) {
219 0 : _currentLocalKeyIndex = encryptionKeyIndex;
220 : }
221 : }
222 : }
223 :
224 : /// sends the enc key to the devices using todevice, passing a list of
225 : /// sendTo only sends events to them
226 : /// setting keyIndex to null will send the latestKey
227 0 : Future<void> _sendEncryptionKeysEvent(
228 : GroupCallSession groupCall,
229 : int keyIndex, {
230 : List<CallParticipant>? sendTo,
231 : }) async {
232 0 : final myKeys = _getKeysForParticipant(groupCall.localParticipant!);
233 0 : final myLatestKey = myKeys?[keyIndex];
234 :
235 : final sendKeysTo =
236 0 : sendTo ?? groupCall.participants.where((p) => !p.isLocal);
237 :
238 : if (myKeys == null || myLatestKey == null) {
239 0 : Logs().w(
240 : '[VOIP E2EE] _sendEncryptionKeysEvent Tried to send encryption keys event but no keys found!');
241 0 : await _makeNewSenderKey(groupCall, false);
242 0 : await _sendEncryptionKeysEvent(
243 : groupCall,
244 : keyIndex,
245 : sendTo: sendTo,
246 : );
247 : return;
248 : }
249 :
250 : try {
251 0 : final keyContent = EncryptionKeysEventContent(
252 0 : [EncryptionKeyEntry(keyIndex, base64Encode(myLatestKey))],
253 0 : groupCall.groupCallId,
254 : );
255 0 : final Map<String, Object> data = {
256 0 : ...keyContent.toJson(),
257 : // used to find group call in groupCalls when ToDeviceEvent happens,
258 : // plays nicely with backwards compatibility for mesh calls
259 0 : 'conf_id': groupCall.groupCallId,
260 0 : 'device_id': groupCall.client.deviceID!,
261 0 : 'room_id': groupCall.room.id,
262 : };
263 0 : await _sendToDeviceEvent(
264 : groupCall,
265 0 : sendTo ?? sendKeysTo.toList(),
266 : data,
267 : EventTypes.GroupCallMemberEncryptionKeys,
268 : );
269 : } catch (e, s) {
270 0 : Logs().e('[VOIP] Failed to send e2ee keys, retrying', e, s);
271 0 : await _sendEncryptionKeysEvent(
272 : groupCall,
273 : keyIndex,
274 : sendTo: sendTo,
275 : );
276 : }
277 : }
278 :
279 0 : Future<void> _sendToDeviceEvent(
280 : GroupCallSession groupCall,
281 : List<CallParticipant> remoteParticipants,
282 : Map<String, Object> data,
283 : String eventType,
284 : ) async {
285 0 : if (remoteParticipants.isEmpty) return;
286 0 : Logs().d(
287 0 : '[VOIP] _sendToDeviceEvent: sending ${data.toString()} to ${remoteParticipants.map((e) => e.id)} ');
288 : final txid =
289 0 : VoIP.customTxid ?? groupCall.client.generateUniqueTransactionId();
290 : final mustEncrypt =
291 0 : groupCall.room.encrypted && groupCall.client.encryptionEnabled;
292 :
293 : // could just combine the two but do not want to rewrite the enc thingy
294 : // wrappers here again.
295 0 : final List<DeviceKeys> mustEncryptkeysToSendTo = [];
296 : final Map<String, Map<String, Map<String, Object>>> unencryptedDataToSend =
297 0 : {};
298 :
299 0 : for (final participant in remoteParticipants) {
300 0 : if (participant.deviceId == null) continue;
301 : if (mustEncrypt) {
302 0 : await groupCall.client.userDeviceKeysLoading;
303 0 : final deviceKey = groupCall.client.userDeviceKeys[participant.userId]
304 0 : ?.deviceKeys[participant.deviceId];
305 : if (deviceKey != null) {
306 0 : mustEncryptkeysToSendTo.add(deviceKey);
307 : }
308 : } else {
309 0 : unencryptedDataToSend.addAll({
310 0 : participant.userId: {participant.deviceId!: data}
311 : });
312 : }
313 : }
314 :
315 : // prepped data, now we send
316 : if (mustEncrypt) {
317 0 : await groupCall.client.sendToDeviceEncrypted(
318 : mustEncryptkeysToSendTo,
319 : eventType,
320 : data,
321 : );
322 : } else {
323 0 : await groupCall.client.sendToDevice(
324 : eventType,
325 : txid,
326 : unencryptedDataToSend,
327 : );
328 : }
329 : }
330 :
331 0 : @override
332 : Map<String, Object?> toJson() {
333 0 : return {
334 0 : 'type': type,
335 0 : 'livekit_service_url': livekitServiceUrl,
336 0 : 'livekit_alias': livekitAlias,
337 : };
338 : }
339 :
340 0 : @override
341 : Future<void> requestEncrytionKey(
342 : GroupCallSession groupCall,
343 : List<CallParticipant> remoteParticipants,
344 : ) async {
345 0 : final Map<String, Object> data = {
346 0 : 'conf_id': groupCall.groupCallId,
347 0 : 'device_id': groupCall.client.deviceID!,
348 0 : 'room_id': groupCall.room.id,
349 : };
350 :
351 0 : await _sendToDeviceEvent(
352 : groupCall,
353 : remoteParticipants,
354 : data,
355 : EventTypes.GroupCallMemberEncryptionKeysRequest,
356 : );
357 : }
358 :
359 0 : @override
360 : Future<void> onCallEncryption(
361 : GroupCallSession groupCall,
362 : String userId,
363 : String deviceId,
364 : Map<String, dynamic> content,
365 : ) async {
366 0 : if (!e2eeEnabled) {
367 0 : Logs().w('[VOIP] got sframe key but we do not support e2ee');
368 : return;
369 : }
370 0 : final keyContent = EncryptionKeysEventContent.fromJson(content);
371 :
372 0 : final callId = keyContent.callId;
373 : final p =
374 0 : CallParticipant(groupCall.voip, userId: userId, deviceId: deviceId);
375 :
376 0 : if (keyContent.keys.isEmpty) {
377 0 : Logs().w(
378 0 : '[VOIP E2EE] Received m.call.encryption_keys where keys is empty: callId=$callId');
379 : return;
380 : } else {
381 0 : Logs().i(
382 0 : '[VOIP E2EE]: onCallEncryption, got keys from ${p.id} ${keyContent.toJson()}');
383 : }
384 :
385 0 : for (final key in keyContent.keys) {
386 0 : final encryptionKey = key.key;
387 0 : final encryptionKeyIndex = key.index;
388 0 : await _setEncryptionKey(
389 : groupCall,
390 : p,
391 : encryptionKeyIndex,
392 : // base64Decode here because we receive base64Encoded version
393 0 : base64Decode(encryptionKey),
394 : delayBeforeUsingKeyOurself: false,
395 : send: false,
396 : );
397 : }
398 : }
399 :
400 0 : @override
401 : Future<void> onCallEncryptionKeyRequest(
402 : GroupCallSession groupCall,
403 : String userId,
404 : String deviceId,
405 : Map<String, dynamic> content,
406 : ) async {
407 0 : if (!e2eeEnabled) {
408 0 : Logs().w('[VOIP] got sframe key request but we do not support e2ee');
409 : return;
410 : }
411 0 : final mems = groupCall.room.getCallMembershipsForUser(userId);
412 : if (mems
413 0 : .where(
414 0 : (mem) =>
415 0 : mem.callId == groupCall.groupCallId &&
416 0 : mem.userId == userId &&
417 0 : mem.deviceId == deviceId &&
418 0 : !mem.isExpired &&
419 : // sanity checks
420 0 : mem.backend.type == groupCall.backend.type &&
421 0 : mem.roomId == groupCall.room.id &&
422 0 : mem.application == groupCall.application,
423 : )
424 0 : .isNotEmpty) {
425 0 : Logs().d(
426 0 : '[VOIP] onCallEncryptionKeyRequest: request checks out, sending key on index: $latestLocalKeyIndex to $userId:$deviceId');
427 0 : await _sendEncryptionKeysEvent(
428 : groupCall,
429 0 : _latestLocalKeyIndex,
430 0 : sendTo: [
431 0 : CallParticipant(
432 0 : groupCall.voip,
433 : userId: userId,
434 : deviceId: deviceId,
435 : )
436 : ],
437 : );
438 : }
439 : }
440 :
441 0 : @override
442 : Future<void> onNewParticipant(
443 : GroupCallSession groupCall,
444 : List<CallParticipant> anyJoined,
445 : ) =>
446 0 : _changeEncryptionKey(groupCall, anyJoined, true);
447 :
448 0 : @override
449 : Future<void> onLeftParticipant(
450 : GroupCallSession groupCall,
451 : List<CallParticipant> anyLeft,
452 : ) async {
453 0 : _encryptionKeysMap.removeWhere((key, value) => anyLeft.contains(key));
454 :
455 : // debounce it because people leave at the same time
456 0 : if (_memberLeaveEncKeyRotateDebounceTimer != null) {
457 0 : _memberLeaveEncKeyRotateDebounceTimer!.cancel();
458 : }
459 0 : _memberLeaveEncKeyRotateDebounceTimer = Timer(makeKeyDelay, () async {
460 0 : await _makeNewSenderKey(groupCall, true);
461 : });
462 : }
463 :
464 0 : @override
465 : Future<void> dispose(GroupCallSession groupCall) async {
466 : // only remove our own, to save requesting if we join again, yes the other side
467 : // will send it anyway but welp
468 0 : _encryptionKeysMap.remove(groupCall.localParticipant!);
469 0 : _currentLocalKeyIndex = 0;
470 0 : _latestLocalKeyIndex = 0;
471 0 : _memberLeaveEncKeyRotateDebounceTimer?.cancel();
472 : }
473 :
474 0 : @override
475 : List<Map<String, String>>? getCurrentFeeds() {
476 : return null;
477 : }
478 :
479 0 : @override
480 : bool operator ==(Object other) =>
481 : identical(this, other) ||
482 0 : other is LiveKitBackend &&
483 0 : type == other.type &&
484 0 : livekitServiceUrl == other.livekitServiceUrl &&
485 0 : livekitAlias == other.livekitAlias;
486 :
487 0 : @override
488 : int get hashCode =>
489 0 : type.hashCode ^ livekitServiceUrl.hashCode ^ livekitAlias.hashCode;
490 :
491 : /// get everything else from your livekit sdk in your client
492 0 : @override
493 : Future<WrappedMediaStream?> initLocalStream(GroupCallSession groupCall,
494 : {WrappedMediaStream? stream}) async {
495 : return null;
496 : }
497 :
498 0 : @override
499 : CallParticipant? get activeSpeaker => null;
500 :
501 : /// these are unimplemented on purpose so that you know you have
502 : /// used the wrong method
503 0 : @override
504 : bool get isLocalVideoMuted =>
505 0 : throw UnimplementedError('Use livekit sdk for this');
506 :
507 0 : @override
508 : bool get isMicrophoneMuted =>
509 0 : throw UnimplementedError('Use livekit sdk for this');
510 :
511 0 : @override
512 : WrappedMediaStream? get localScreenshareStream =>
513 0 : throw UnimplementedError('Use livekit sdk for this');
514 :
515 0 : @override
516 : WrappedMediaStream? get localUserMediaStream =>
517 0 : throw UnimplementedError('Use livekit sdk for this');
518 :
519 0 : @override
520 : List<WrappedMediaStream> get screenShareStreams =>
521 0 : throw UnimplementedError('Use livekit sdk for this');
522 :
523 0 : @override
524 : List<WrappedMediaStream> get userMediaStreams =>
525 0 : throw UnimplementedError('Use livekit sdk for this');
526 :
527 0 : @override
528 : Future<void> setDeviceMuted(
529 : GroupCallSession groupCall, bool muted, MediaInputKind kind) async {
530 : return;
531 : }
532 :
533 0 : @override
534 : Future<void> setScreensharingEnabled(GroupCallSession groupCall, bool enabled,
535 : String desktopCapturerSourceId) async {
536 : return;
537 : }
538 :
539 0 : @override
540 : Future<void> setupP2PCallWithNewMember(GroupCallSession groupCall,
541 : CallParticipant rp, CallMembership mem) async {
542 : return;
543 : }
544 :
545 0 : @override
546 : Future<void> setupP2PCallsWithExistingMembers(
547 : GroupCallSession groupCall) async {
548 : return;
549 : }
550 :
551 0 : @override
552 : Future<void> updateMediaDeviceForCalls() async {
553 : return;
554 : }
555 : }
|
