diff --git a/package.json b/package.json index f34fab5f..6c74b84c 100644 --- a/package.json +++ b/package.json @@ -109,7 +109,7 @@ "livekit-client": "^2.13.0", "lodash-es": "^4.17.21", "loglevel": "^1.9.1", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=9779ac975df2f296958e3c4be254fa46ebd67ea4", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=2bb3b03a248e689f7460f4e70d5ffbf10353c725", "matrix-widget-api": "^1.14.0", "normalize.css": "^8.0.1", "observable-hooks": "^4.2.3", diff --git a/src/e2ee/matrixKeyProvider.ts b/src/e2ee/matrixKeyProvider.ts index a499f45c..166fd82c 100644 --- a/src/e2ee/matrixKeyProvider.ts +++ b/src/e2ee/matrixKeyProvider.ts @@ -6,18 +6,13 @@ Please see LICENSE in the repository root for full details. */ import { BaseKeyProvider } from "livekit-client"; -import { logger } from "matrix-js-sdk/lib/logger"; import { type MatrixRTCSession, MatrixRTCSessionEvent, } from "matrix-js-sdk/lib/matrixrtc"; import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; -import { firstValueFrom } from "rxjs"; - -import { - computeLivekitParticipantIdentity$, - livekitIdentityInput, -} from "../state/CallViewModel/remoteMembers/LivekitParticipantIdentity"; +import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; +const logger = rootLogger.getChild("[MatrixKeyProvider]"); export class MatrixKeyProvider extends BaseKeyProvider { private rtcSession?: MatrixRTCSession; @@ -32,6 +27,10 @@ export class MatrixKeyProvider extends BaseKeyProvider { MatrixRTCSessionEvent.EncryptionKeyChanged, this.onEncryptionKeyChanged, ); + this.rtcSession.off( + MatrixRTCSessionEvent.MembershipsChanged, + this.onMembershipsChanged, + ); } this.rtcSession = rtcSession; @@ -40,55 +39,86 @@ export class MatrixKeyProvider extends BaseKeyProvider { MatrixRTCSessionEvent.EncryptionKeyChanged, this.onEncryptionKeyChanged, ); + this.rtcSession.on( + MatrixRTCSessionEvent.MembershipsChanged, + this.onMembershipsChanged, + ); // The new session could be aware of keys of which the old session wasn't, // so emit key changed events this.rtcSession.reemitEncryptionKeys(); } + private keyCache = new Array<{ + membership: CallMembershipIdentityParts; + encryptionKey: Uint8Array; + encryptionKeyIndex: number; + }>(); + + private onMembershipsChanged = (): void => { + const duplicatedArray = this.keyCache; + // Reset key cache first. It will get repopulated when calling `onEncryptionKeyChanged` + this.keyCache = []; + let next = duplicatedArray.pop(); + while (next !== undefined) { + logger.debug( + "[KeyCache] remove key event from the cache and try adding it again. For membership: ", + next.membership, + ); + this.onEncryptionKeyChanged( + next.encryptionKey, + next.encryptionKeyIndex, + next.membership, + ); + next = duplicatedArray.pop(); + } + }; + private onEncryptionKeyChanged = ( encryptionKey: Uint8Array, encryptionKeyIndex: number, membership: CallMembershipIdentityParts, ): void => { - const unhashedIdentity = livekitIdentityInput(membership); - // This is the only way we can get the kind of the membership event we just received the key for. // best case we want to recompute this once the memberships change (you can receive the key before the participant...) - // - // TODO change this to `?? "rtc"` for newer versions. - const kind = - this.rtcSession?.memberships.find( - (m) => - m.userId === membership.userId && - m.deviceId === membership.deviceId && - m.memberId === membership.memberId, - )?.kind ?? "session"; + const membershipFull = this.rtcSession?.memberships.find( + (m) => + m.userId === membership.userId && + m.deviceId === membership.deviceId && + m.memberId === membership.memberId, + ); + if (!membershipFull) { + logger.debug( + "[KeyCache] Added key event to the cache because we do not have a membership for it (yet): ", + membership, + ); + this.keyCache.push({ membership, encryptionKey, encryptionKeyIndex }); + return; + } - Promise.all([ - crypto.subtle.importKey("raw", encryptionKey, "HKDF", false, [ + crypto.subtle + .importKey("raw", encryptionKey, "HKDF", false, [ "deriveBits", "deriveKey", - ]), - firstValueFrom(computeLivekitParticipantIdentity$(membership, kind)), - ]).then( - ([keyMaterial, livekitParticipantId]) => { - this.onSetEncryptionKey( - keyMaterial, - livekitParticipantId, - encryptionKeyIndex, - ); + ]) + .then( + (keyMaterial) => { + this.onSetEncryptionKey( + keyMaterial, + membershipFull.rtcBackendIdentity, + encryptionKeyIndex, + ); - logger.debug( - `Sent new key to livekit room=${this.rtcSession?.room.roomId} participantId=${livekitParticipantId} (before hash: ${unhashedIdentity}) encryptionKeyIndex=${encryptionKeyIndex}`, - ); - }, - (e) => { - logger.error( - `Failed to create key material from buffer for livekit room=${this.rtcSession?.room.roomId} participantId before hash=${unhashedIdentity} encryptionKeyIndex=${encryptionKeyIndex}`, - e, - ); - }, - ); + logger.debug( + `Sent new key to livekit room=${this.rtcSession?.room.roomId} participantId=${membershipFull.rtcBackendIdentity} (before hash: ${membershipFull.userId}) encryptionKeyIndex=${encryptionKeyIndex}`, + ); + }, + (e) => { + logger.error( + `Failed to create key material from buffer for livekit room=${this.rtcSession?.room.roomId} participantId before hash=${membershipFull.userId} encryptionKeyIndex=${encryptionKeyIndex}`, + e, + ); + }, + ); }; } diff --git a/src/state/CallViewModel/CallViewModel.test.ts b/src/state/CallViewModel/CallViewModel.test.ts index 505cb19f..6e3837c4 100644 --- a/src/state/CallViewModel/CallViewModel.test.ts +++ b/src/state/CallViewModel/CallViewModel.test.ts @@ -44,7 +44,6 @@ import { mockRtcMembership, testScope, exampleTransport, - mockComputeLivekitParticipantIdentity$, } from "../../utils/test.ts"; import { E2eeType } from "../../e2ee/e2eeType.ts"; import { @@ -88,14 +87,6 @@ vi.mock( }), ); -vi.mock( - import("./remoteMembers/LivekitParticipantIdentity.ts"), - async (importOriginal) => ({ - ...(await importOriginal()), - computeLivekitParticipantIdentity$: mockComputeLivekitParticipantIdentity$, - }), -); - const yesNo = { y: true, n: false, diff --git a/src/state/CallViewModel/remoteMembers/LivekitParticipantIdentity.ts b/src/state/CallViewModel/remoteMembers/LivekitParticipantIdentity.ts deleted file mode 100644 index e43d0bd1..00000000 --- a/src/state/CallViewModel/remoteMembers/LivekitParticipantIdentity.ts +++ /dev/null @@ -1,53 +0,0 @@ -/* -Copyright 2025 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { encodeUnpaddedBase64Url } from "matrix-js-sdk"; -import { sha256 } from "matrix-js-sdk/lib/digest"; -import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; -import { from, type Observable } from "rxjs"; - -const livekitParticipantIdentityCache = new Map(); - -/** - * The string that is computed based on the membership and used for the computing the hash. - * `${userId}:${deviceId}:${membershipID}` - * as the direct imput for: await sha256(input) - */ -export const livekitIdentityInput = ({ - userId, - deviceId, - memberId, -}: CallMembershipIdentityParts): string => `${userId}|${deviceId}|${memberId}`; - -export function computeLivekitParticipantIdentity$( - membership: CallMembershipIdentityParts, - kind: "rtc" | "session", -): Observable { - const compute = async (): Promise => { - switch (kind) { - case "rtc": { - const input = livekitIdentityInput(membership); - if (livekitParticipantIdentityCache.size > 400) - // prevent memory leaks in a stupid/simple way - livekitParticipantIdentityCache.clear(); - // TODO use non deprecated memberId - if (livekitParticipantIdentityCache.has(input)) - return livekitParticipantIdentityCache.get(input)!; - else { - const hashBuffer = await sha256(input); - const hashedString = encodeUnpaddedBase64Url(hashBuffer); - livekitParticipantIdentityCache.set(input, hashedString); - return hashedString; - } - } - case "session": - default: - return `${membership.userId}:${membership.deviceId}`; - } - }; - return from(compute()); -} diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts index 9178c347..a5d6b2f6 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts @@ -10,7 +10,7 @@ import { type LivekitTransport, type CallMembership, } from "matrix-js-sdk/lib/matrixrtc"; -import { combineLatest, filter, map, switchMap } from "rxjs"; +import { combineLatest, filter, map } from "rxjs"; import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { type Behavior } from "../../Behavior"; @@ -18,7 +18,6 @@ import { type IConnectionManager } from "./ConnectionManager"; import { Epoch, type ObservableScope } from "../../ObservableScope"; import { type Connection } from "./Connection"; import { generateItemsWithEpoch } from "../../../utils/observable"; -import { computeLivekitParticipantIdentity$ } from "./LivekitParticipantIdentity"; const logger = rootLogger.getChild("[MatrixLivekitMembers]"); @@ -82,45 +81,12 @@ export function createMatrixLivekitMembers$({ membershipsWithTransport$, connectionManager, }: Props): Behavior> { - /** - * This internal observable is used to compute the async sha256 hash of the user's identity. - * a promise is treated like an observable. So we can switchMap on the promise from the identity computation. - * The last update to `membershipsWithTransport$` will always be the last promise we pass to switchMap. - * So we will eventually always end up with the latest memberships and their identities. - */ - const membershipsWithTransportAndLivekitIdentity$ = - membershipsWithTransport$.pipe( - switchMap((membershipsWithTransport) => { - const { value, epoch } = membershipsWithTransport; - const membershipsWithTransportAndLkIdentityPromises = value.map( - (obj) => { - return computeLivekitParticipantIdentity$( - obj.membership, - obj.membership.kind, - ); - }, - ); - return combineLatest( - membershipsWithTransportAndLkIdentityPromises, - ).pipe( - map((identities) => { - const membershipsWithTransportAndLkIdentity = value.map( - ({ transport, membership }, index) => { - return { transport, membership, identity: identities[index] }; - }, - ); - return new Epoch(membershipsWithTransportAndLkIdentity, epoch); - }), - ); - }), - ); - /** * Stream of all the call members and their associated livekit data (if available). */ return scope.behavior( combineLatest([ - membershipsWithTransportAndLivekitIdentity$, + membershipsWithTransport$, connectionManager.connectionManagerData$, ]).pipe( filter((values) => @@ -131,37 +97,34 @@ export function createMatrixLivekitMembers$({ // Generator function. // creates an array of `{key, data}[]` // Each change in the keys (new key, missing key) will result in a call to the factory function. - function* ([membershipsWithTransportAndLivekitIdentity, managerData]) { - for (const { - membership, - transport, - identity, - } of membershipsWithTransportAndLivekitIdentity) { + function* ([membershipsWithTransport, managerData]) { + for (const { membership, transport } of membershipsWithTransport) { const participants = transport ? managerData.getParticipantForTransport(transport) : []; const participant = - participants.find((p) => p.identity == identity) ?? null; + participants.find( + (p) => p.identity == membership.rtcBackendIdentity, + ) ?? null; const connection = transport ? managerData.getConnectionForTransport(transport) : null; yield { - keys: [identity, membership.userId, membership.deviceId], + keys: [membership.userId, membership.deviceId], data: { membership, participant, connection }, }; } }, // Each update where the key of the generator array do not change will result in updates to the `data$` observable in the factory. - (scope, data$, identity, userId, deviceId) => { + (scope, data$, userId, deviceId) => { logger.debug( - `Generating member for livekitIdentity: ${identity}, userId:deviceId: ${userId}${deviceId}`, + `Generating member for livekitIdentity: ${data$.value.membership.rtcBackendIdentity}, userId:deviceId: ${userId}${deviceId}`, ); const { participant$, ...rest } = scope.splitBehavior(data$); // will only get called once per `participantId, userId` pair. // updates to data$ and as a result to displayName$ and mxcAvatarUrl$ are more frequent. return { - identity, userId, participant: { type: "remote" as const, value$: participant$ }, ...rest, diff --git a/src/state/CallViewModel/remoteMembers/integration.test.ts b/src/state/CallViewModel/remoteMembers/integration.test.ts index c9e02a7c..00062c60 100644 --- a/src/state/CallViewModel/remoteMembers/integration.test.ts +++ b/src/state/CallViewModel/remoteMembers/integration.test.ts @@ -22,7 +22,6 @@ import { ECConnectionFactory } from "./ConnectionFactory.ts"; import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts"; import { mockCallMembership, - mockComputeLivekitParticipantIdentity$, mockMediaDevices, ownMemberMock, withTestScheduler, @@ -45,11 +44,6 @@ let lkRoomFactory: () => LivekitRoom; const createdMockLivekitRooms: Map = new Map(); -vi.mock(import("./LivekitParticipantIdentity.ts"), async (importOriginal) => ({ - ...(await importOriginal()), - computeLivekitParticipantIdentity$: mockComputeLivekitParticipantIdentity$, -})); - beforeEach(() => { testScope = new ObservableScope(); mockClient = { diff --git a/src/utils/test.ts b/src/utils/test.ts index 968b7160..56148b32 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -252,7 +252,8 @@ export function mockRtcMembership( content: data, }); - const cms = new CallMembership(event, data); + const membershipData = CallMembership.membershipDataFromMatrixEvent(event); + const cms = new CallMembership(event, membershipData, "xx"); vi.mocked(cms).getTransport = vi.fn().mockReturnValue(fociPreferred[0]); return cms; } diff --git a/yarn.lock b/yarn.lock index b83976a2..c4b2e957 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7533,7 +7533,7 @@ __metadata: livekit-client: "npm:^2.13.0" lodash-es: "npm:^4.17.21" loglevel: "npm:^1.9.1" - matrix-js-sdk: "github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=9779ac975df2f296958e3c4be254fa46ebd67ea4" + matrix-js-sdk: "github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=2bb3b03a248e689f7460f4e70d5ffbf10353c725" matrix-widget-api: "npm:^1.14.0" normalize.css: "npm:^8.0.1" observable-hooks: "npm:^4.2.3" @@ -10338,9 +10338,9 @@ __metadata: languageName: node linkType: hard -"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=9779ac975df2f296958e3c4be254fa46ebd67ea4": +"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=2bb3b03a248e689f7460f4e70d5ffbf10353c725": version: 39.3.0 - resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=9779ac975df2f296958e3c4be254fa46ebd67ea4" + resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=2bb3b03a248e689f7460f4e70d5ffbf10353c725" dependencies: "@babel/runtime": "npm:^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm": "npm:^16.0.0" @@ -10356,7 +10356,7 @@ __metadata: sdp-transform: "npm:^3.0.0" unhomoglyph: "npm:^1.0.6" uuid: "npm:13" - checksum: 10c0/78c27847b58c229513bd28c4c4ad391d8af6722711d3d0f42e93a537d7a827a7233e920936dd8d7005c7893bad17a503c3f62b56ecfed3cf4ae81a5097b4ac21 + checksum: 10c0/2e7061f6e648c91aaeb30b3e01626d855e24efcb330bbe432fcba199bd46b0b0d998cbc545748e1c72a7b643d25581f988fcad9bbaa42912a6ec96a27c41d0de languageName: node linkType: hard