From 909d980dff83d96bd81db2d692b9c4f9f31745c9 Mon Sep 17 00:00:00 2001 From: Timo K Date: Mon, 15 Dec 2025 18:23:30 +0100 Subject: [PATCH 01/46] still with broken tests... --- package.json | 2 +- src/e2ee/matrixKeyProvider.ts | 65 ++- src/livekit/MatrixAudioRenderer.tsx | 3 +- src/room/InCallView.tsx | 1 + src/state/CallViewModel/CallViewModel.test.ts | 3 - src/state/CallViewModel/CallViewModel.ts | 25 +- .../remoteMembers/ConnectionManager.ts | 25 +- .../MatrixLivekitMembers.test.ts | 437 ++++++++---------- .../remoteMembers/MatrixLivekitMembers.ts | 107 ++++- yarn.lock | 42 +- 10 files changed, 353 insertions(+), 357 deletions(-) diff --git a/package.json b/package.json index 21c870ad..1efb504b 100644 --- a/package.json +++ b/package.json @@ -109,7 +109,7 @@ "livekit-client": "^2.13.0", "lodash-es": "^4.17.21", "loglevel": "^1.9.1", - "matrix-js-sdk": "^39.2.0", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=f5f1b8efb46b3d55a7eebfabb4a61496640b8b00", "matrix-widget-api": "^1.14.0", "normalize.css": "^8.0.1", "observable-hooks": "^4.2.3", diff --git a/src/e2ee/matrixKeyProvider.ts b/src/e2ee/matrixKeyProvider.ts index 95033f87..1d1c4588 100644 --- a/src/e2ee/matrixKeyProvider.ts +++ b/src/e2ee/matrixKeyProvider.ts @@ -11,6 +11,12 @@ import { type MatrixRTCSession, MatrixRTCSessionEvent, } from "matrix-js-sdk/lib/matrixrtc"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; + +import { + computeLivekitParticipantIdentity, + livekitIdentityInput, +} from "../state/CallViewModel/remoteMembers/MatrixLivekitMembers"; export class MatrixKeyProvider extends BaseKeyProvider { private rtcSession?: MatrixRTCSession; @@ -42,31 +48,46 @@ export class MatrixKeyProvider extends BaseKeyProvider { private onEncryptionKeyChanged = ( encryptionKey: Uint8Array, encryptionKeyIndex: number, - participantId: string, + membership: CallMembershipIdentityParts, ): void => { - crypto.subtle - .importKey("raw", encryptionKey, "HKDF", false, [ + const unhashedIdentity = livekitIdentityInput(membership); + + // This is the only way we can get the kind of the membership event we just received the key for. + // best case we want to recompute this once the memberships change (you can receive the key before the participant...) + // + // TODO change this to `?? "rtc"` for newer versions. + const kind = + this.rtcSession?.memberships.find( + (m) => + m.userId === membership.userId && + m.deviceId === membership.deviceId && + m.memberId === membership.memberId, + )?.kind ?? "session"; + + Promise.all([ + crypto.subtle.importKey("raw", encryptionKey, "HKDF", false, [ "deriveBits", "deriveKey", - ]) - .then( - (keyMaterial) => { - this.onSetEncryptionKey( - keyMaterial, - participantId, - encryptionKeyIndex, - ); + ]), + computeLivekitParticipantIdentity(membership, kind), + ]).then( + ([keyMaterial, livekitParticipantId]) => { + this.onSetEncryptionKey( + keyMaterial, + livekitParticipantId, + encryptionKeyIndex, + ); - logger.debug( - `Sent new key to livekit room=${this.rtcSession?.room.roomId} participantId=${participantId} encryptionKeyIndex=${encryptionKeyIndex}`, - ); - }, - (e) => { - logger.error( - `Failed to create key material from buffer for livekit room=${this.rtcSession?.room.roomId} participantId=${participantId} encryptionKeyIndex=${encryptionKeyIndex}`, - e, - ); - }, - ); + logger.debug( + `Sent new key to livekit room=${this.rtcSession?.room.roomId} participantId=${livekitParticipantId} (before hash: ${unhashedIdentity}) encryptionKeyIndex=${encryptionKeyIndex}`, + ); + }, + (e) => { + logger.error( + `Failed to create key material from buffer for livekit room=${this.rtcSession?.room.roomId} participantId before hash=${unhashedIdentity} encryptionKeyIndex=${encryptionKeyIndex}`, + e, + ); + }, + ); }; } diff --git a/src/livekit/MatrixAudioRenderer.tsx b/src/livekit/MatrixAudioRenderer.tsx index 5b1149e9..0fa5d000 100644 --- a/src/livekit/MatrixAudioRenderer.tsx +++ b/src/livekit/MatrixAudioRenderer.tsx @@ -15,7 +15,6 @@ import { type AudioTrackProps, } from "@livekit/components-react"; import { logger } from "matrix-js-sdk/lib/logger"; -import { type ParticipantId } from "matrix-js-sdk/lib/matrixrtc"; import { useEarpieceAudioConfig } from "../MediaDevicesContext"; import { useReactiveState } from "../useReactiveState"; @@ -32,7 +31,7 @@ export interface MatrixAudioRendererProps { * This list needs to be composed based on the matrixRTC members so that we do not play audio from users * that are not expected to be in the rtc session (local user is excluded). */ - validIdentities: ParticipantId[]; + validIdentities: string[]; /** * If set to `true`, mutes all audio tracks rendered by the component. * @remarks diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 6ae004d8..e9932fdc 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -785,6 +785,7 @@ export const InCallView: FC = ({ onTouchEnd={onControlsTouchEnd} /> )} + {!showControls &&
}
); diff --git a/src/state/CallViewModel/CallViewModel.test.ts b/src/state/CallViewModel/CallViewModel.test.ts index 2e5b5700..be598702 100644 --- a/src/state/CallViewModel/CallViewModel.test.ts +++ b/src/state/CallViewModel/CallViewModel.test.ts @@ -1248,9 +1248,6 @@ describe("CallViewModel", () => { y: () => { rtcSession.membershipStatus = Status.Connected; }, - n: () => { - rtcSession.membershipStatus = Status.Reconnecting; - }, }); schedule(probablyLeftMarbles, { y: () => { diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 5cc33f5d..289b642c 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -591,10 +591,9 @@ export function createCallViewModel$( const audioParticipants$ = scope.behavior( matrixLivekitMembers$.pipe( - switchMap((membersWithEpoch) => { - const members = membersWithEpoch.value; + switchMap((members) => { const a$ = combineLatest( - members.map((member) => + members.value.map((member) => combineLatest([member.connection$, member.participant$]).pipe( map(([connection, participant]) => { // do not render audio for local participant @@ -667,22 +666,22 @@ export function createCallViewModel$( generateItems( function* ([ localMatrixLivekitMember, - { value: matrixLivekitMembers }, + matrixLivekitMembers, duplicateTiles, ]) { - let localParticipantId: string | undefined = undefined; + let localUserMediaId: string | undefined = undefined; // add local member if available if (localMatrixLivekitMember) { const { userId, participant$, connection$, membership$ } = localMatrixLivekitMember; - localParticipantId = `${userId}:${membership$.value.deviceId}`; // should be membership$.value.membershipID which is not optional - // const participantId = membership$.value.membershipID; - if (localParticipantId) { + localUserMediaId = `${userId}:${membership$.value.deviceId}`; // should be membership$.value.membershipID which is not optional + + if (localUserMediaId) { for (let dup = 0; dup < 1 + duplicateTiles; dup++) { yield { keys: [ dup, - localParticipantId, + localUserMediaId, userId, participant$, connection$, @@ -698,13 +697,13 @@ export function createCallViewModel$( participant$, connection$, membership$, - } of matrixLivekitMembers) { - const participantId = `${userId}:${membership$.value.deviceId}`; - if (participantId === localParticipantId) continue; + } of matrixLivekitMembers.value) { + const userMediaId = `${userId}:${membership$.value.deviceId}`; + if (userMediaId === localUserMediaId) continue; // const participantId = membership$.value?.identity; for (let dup = 0; dup < 1 + duplicateTiles; dup++) { yield { - keys: [dup, participantId, userId, participant$, connection$], + keys: [dup, userMediaId, userId, participant$, connection$], data: undefined, }; } diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts index 755ba3dd..6660df62 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts @@ -6,10 +6,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { - type LivekitTransport, - type ParticipantId, -} from "matrix-js-sdk/lib/matrixrtc"; +import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { combineLatest, map, of, switchMap, tap } from "rxjs"; import { type Logger } from "matrix-js-sdk/lib/logger"; import { type LocalParticipant, type RemoteParticipant } from "livekit-client"; @@ -62,24 +59,8 @@ export class ConnectionManagerData { const key = transport.livekit_service_url + "|" + transport.livekit_alias; return this.store.get(key)?.[1] ?? []; } - /** - * Get all connections where the given participant is publishing. - * In theory, there could be several connections where the same participant is publishing but with - * only well behaving clients a participant should only be publishing on a single connection. - * @param participantId - */ - public getConnectionsForParticipant( - participantId: ParticipantId, - ): Connection[] { - const connections: Connection[] = []; - for (const [connection, participants] of this.store.values()) { - if (participants.some((p) => p.identity === participantId)) { - connections.push(connection); - } - } - return connections; - } } + interface Props { scope: ObservableScope; connectionFactory: ConnectionFactory; @@ -202,7 +183,7 @@ export function createConnectionManager$({ ); }), ), - new Epoch(new ConnectionManagerData()), + new Epoch(new ConnectionManagerData(), -1), ); return { connectionManagerData$ }; diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts index e675f723..c2e60798 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts @@ -10,8 +10,7 @@ import { type CallMembership, type LivekitTransport, } from "matrix-js-sdk/lib/matrixrtc"; -import { getParticipantId } from "matrix-js-sdk/lib/matrixrtc/utils"; -import { combineLatest, map, type Observable } from "rxjs"; +import { BehaviorSubject, combineLatest, map, type Observable } from "rxjs"; import { type IConnectionManager } from "./ConnectionManager.ts"; import { @@ -26,14 +25,19 @@ import { } from "../../ObservableScope.ts"; import { ConnectionManagerData } from "./ConnectionManager.ts"; import { + flushPromises, mockCallMembership, mockRemoteParticipant, withTestScheduler, } from "../../../utils/test.ts"; import { type Connection } from "./Connection.ts"; +import { constant } from "../../Behavior.ts"; let testScope: ObservableScope; +const fallbackMemberId = (userId: string, deviceId: string): string => + `${userId}:${deviceId}`; + const transportA: LivekitTransport = { type: "livekit", livekit_service_url: "https://lk.example.org", @@ -76,49 +80,41 @@ function epochMeWith$( ); } -test("should signal participant not yet connected to livekit", () => { - withTestScheduler(({ behavior, expectObservable }) => { - const { memberships$, membershipsWithTransport$ } = fromMemberships$( - behavior("a", { - a: [bobMembership], - }), - ); +test("should signal participant not yet connected to livekit", async () => { + const mockedMemberships$ = new BehaviorSubject([bobMembership]); + const mockConnectionManagerData$ = new BehaviorSubject( + new ConnectionManagerData(), + ); + const { memberships$, membershipsWithTransport$ } = + createEpochedMemberships$(mockedMemberships$); - const connectionManagerData$ = epochMeWith$( - memberships$, - behavior("a", { - a: new ConnectionManagerData(), - }), - ); + const connectionManagerData$ = epochMeWith$( + memberships$, + mockConnectionManagerData$, + ); - const matrixLivekitMember$ = createMatrixLivekitMembers$({ - scope: testScope, - membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), - connectionManager: { - connectionManagerData$: connectionManagerData$, - } as unknown as IConnectionManager, - }); - - expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", { - a: expect.toSatisfy((data: MatrixLivekitMember[]) => { - expect(data.length).toEqual(1); - expectObservable(data[0].membership$).toBe("a", { - a: bobMembership, - }); - expectObservable(data[0].participant$).toBe("a", { - a: null, - }); - expectObservable(data[0].connection$).toBe("a", { - a: null, - }); - return true; - }), - }); + const matrixLivekitMember$ = createMatrixLivekitMembers$({ + scope: testScope, + membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), + connectionManager: { + connectionManagerData$: connectionManagerData$, + } as unknown as IConnectionManager, }); + + await flushPromises(); + expect(matrixLivekitMember$.value.value).toSatisfy( + (data: MatrixLivekitMember[]) => { + expect(data.length).toEqual(1); + expect(data[0].membership$.value).toBe(bobMembership); + expect(data[0].participant$.value).toBe(null); + expect(data[0].connection$.value).toBe(null); + return true; + }, + ); }); // Helper to create epoch'ed memberships$ and membershipsWithTransport$ from memberships observable. -function fromMemberships$(m$: Observable): { +function createEpochedMemberships$(m$: Observable): { memberships$: Observable>; membershipsWithTransport$: Observable< Epoch<{ membership: CallMembership; transport?: LivekitTransport }[]> @@ -143,32 +139,115 @@ function fromMemberships$(m$: Observable): { }; } -test("should signal participant on a connection that is publishing", () => { - withTestScheduler(({ behavior, expectObservable }) => { - const bobParticipantId = getParticipantId( +test("should signal participant on a connection that is publishing", async () => { + const bobParticipantId = fallbackMemberId( + bobMembership.userId, + bobMembership.deviceId, + ); + + const { memberships$, membershipsWithTransport$ } = createEpochedMemberships$( + constant([bobMembership]), + ); + + const connection = { + transport: bobMembership.getTransport(bobMembership), + } as unknown as Connection; + const dataWithPublisher = new ConnectionManagerData(); + dataWithPublisher.add(connection, [ + mockRemoteParticipant({ identity: bobParticipantId }), + ]); + + const connectionManagerData$ = epochMeWith$( + memberships$, + constant(dataWithPublisher), + ); + + const matrixLivekitMember$ = createMatrixLivekitMembers$({ + scope: testScope, + membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), + connectionManager: { + connectionManagerData$: connectionManagerData$, + } as unknown as IConnectionManager, + }); + + await flushPromises(); + expect(matrixLivekitMember$.value.value).toSatisfy( + (data: MatrixLivekitMember[]) => { + expect(data.length).toEqual(1); + expect(data[0].membership$.value).toBe(bobMembership); + expect(data[0].participant$.value).toSatisfy((participant) => { + expect(participant).toBeDefined(); + expect(participant!.identity).toEqual(bobParticipantId); + return true; + }); + expect(data[0].connection$.value).toBe(connection); + return true; + }, + ); +}); + +test("should signal participant on a connection that is not publishing", async () => { + const { memberships$, membershipsWithTransport$ } = createEpochedMemberships$( + constant([bobMembership]), + ); + + const connection = { + transport: bobMembership.getTransport(bobMembership), + } as unknown as Connection; + const dataWithPublisher = new ConnectionManagerData(); + dataWithPublisher.add(connection, []); + + const connectionManagerData$ = epochMeWith$( + memberships$, + constant(dataWithPublisher), + ); + + const matrixLivekitMember$ = createMatrixLivekitMembers$({ + scope: testScope, + membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), + connectionManager: { + connectionManagerData$: connectionManagerData$, + } as unknown as IConnectionManager, + }); + await flushPromises(); + expect(matrixLivekitMember$.value.value).toSatisfy( + (data: MatrixLivekitMember[]) => { + expect(data.length).toEqual(1); + expect(data[0].membership$.value).toBe(bobMembership); + expect(data[0].participant$.value).toBe(null); + expect(data[0].connection$.value).toBe(connection); + return true; + }, + ); +}); + +describe("Publication edge case", () => { + test("bob is publishing in several connections", async () => { + const { memberships$, membershipsWithTransport$ } = + createEpochedMemberships$(constant([bobMembership, carlMembership])); + + const connectionWithPublisher = new ConnectionManagerData(); + const bobParticipantId = fallbackMemberId( bobMembership.userId, bobMembership.deviceId, ); - - const { memberships$, membershipsWithTransport$ } = fromMemberships$( - behavior("a", { - a: [bobMembership], - }), - ); - - const connection = { - transport: bobMembership.getTransport(bobMembership), + const connectionA = { + transport: transportA, } as unknown as Connection; - const dataWithPublisher = new ConnectionManagerData(); - dataWithPublisher.add(connection, [ + const connectionB = { + transport: transportB, + } as unknown as Connection; + + connectionWithPublisher.add(connectionA, [ + mockRemoteParticipant({ identity: bobParticipantId }), + ]); + connectionWithPublisher.add(connectionB, [ mockRemoteParticipant({ identity: bobParticipantId }), ]); const connectionManagerData$ = epochMeWith$( memberships$, - behavior("a", { - a: dataWithPublisher, - }), + constant(connectionWithPublisher), ); const matrixLivekitMember$ = createMatrixLivekitMembers$({ @@ -178,207 +257,73 @@ test("should signal participant on a connection that is publishing", () => { connectionManagerData$: connectionManagerData$, } as unknown as IConnectionManager, }); + await flushPromises(); + expect(matrixLivekitMember$.value.value).toSatisfy( + (data: MatrixLivekitMember[]) => { + expect(data.length).toEqual(2); + expect(data[0].membership$.value).toBe(bobMembership); + expect(data[0].connection$.value).toBe(connectionA); + expect(data[0].participant$.value).toSatisfy((participant) => { + expect(participant).toBeDefined(); + expect(participant!.identity).toEqual(bobParticipantId); + return true; + }); - expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", { - a: expect.toSatisfy((data: MatrixLivekitMember[]) => { - expect(data.length).toEqual(1); - expectObservable(data[0].membership$).toBe("a", { - a: bobMembership, - }); - expectObservable(data[0].participant$).toBe("a", { - a: expect.toSatisfy((participant) => { - expect(participant).toBeDefined(); - expect(participant!.identity).toEqual(bobParticipantId); - return true; - }), - }); - expectObservable(data[0].connection$).toBe("a", { - a: connection, - }); return true; - }), - }); - }); -}); - -test("should signal participant on a connection that is not publishing", () => { - withTestScheduler(({ behavior, expectObservable }) => { - const { memberships$, membershipsWithTransport$ } = fromMemberships$( - behavior("a", { - a: [bobMembership], - }), + }, ); - - const connection = { - transport: bobMembership.getTransport(bobMembership), - } as unknown as Connection; - const dataWithPublisher = new ConnectionManagerData(); - dataWithPublisher.add(connection, []); - - const connectionManagerData$ = epochMeWith$( - memberships$, - behavior("a", { - a: dataWithPublisher, - }), - ); - - const matrixLivekitMember$ = createMatrixLivekitMembers$({ - scope: testScope, - membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), - connectionManager: { - connectionManagerData$: connectionManagerData$, - } as unknown as IConnectionManager, - }); - - expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", { - a: expect.toSatisfy((data: MatrixLivekitMember[]) => { - expect(data.length).toEqual(1); - expectObservable(data[0].membership$).toBe("a", { - a: bobMembership, - }); - expectObservable(data[0].participant$).toBe("a", { - a: null, - }); - expectObservable(data[0].connection$).toBe("a", { - a: connection, - }); - return true; - }), - }); }); }); -describe("Publication edge case", () => { - test("bob is publishing in several connections", () => { - withTestScheduler(({ behavior, expectObservable }) => { - const { memberships$, membershipsWithTransport$ } = fromMemberships$( - behavior("a", { - a: [bobMembership, carlMembership], - }), - ); +test("bob is publishing in the wrong connection", async () => { + const mockedMemberships$ = new BehaviorSubject([ + bobMembership, + carlMembership, + ]); - const connectionWithPublisher = new ConnectionManagerData(); - const bobParticipantId = getParticipantId( - bobMembership.userId, - bobMembership.deviceId, - ); - const connectionA = { - transport: transportA, - } as unknown as Connection; - const connectionB = { - transport: transportB, - } as unknown as Connection; + const { memberships$, membershipsWithTransport$ } = + createEpochedMemberships$(mockedMemberships$); - connectionWithPublisher.add(connectionA, [ - mockRemoteParticipant({ identity: bobParticipantId }), - ]); - connectionWithPublisher.add(connectionB, [ - mockRemoteParticipant({ identity: bobParticipantId }), - ]); + const connectionWithPublisher = new ConnectionManagerData(); - const connectionManagerData$ = epochMeWith$( - memberships$, - behavior("a", { - a: connectionWithPublisher, - }), - ); + const bobParticipantId = fallbackMemberId( + bobMembership.userId, + bobMembership.deviceId, + ); + const connectionA = { transport: transportA } as unknown as Connection; + const connectionB = { transport: transportB } as unknown as Connection; - const matrixLivekitMember$ = createMatrixLivekitMembers$({ - scope: testScope, - membershipsWithTransport$: testScope.behavior( - membershipsWithTransport$, - ), - connectionManager: { - connectionManagerData$: connectionManagerData$, - } as unknown as IConnectionManager, - }); + // Bob is not publishing on A + connectionWithPublisher.add(connectionA, []); + // Bob is publishing on B but his membership says A + connectionWithPublisher.add(connectionB, [ + mockRemoteParticipant({ identity: bobParticipantId }), + ]); - expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe( - "a", - { - a: expect.toSatisfy((data: MatrixLivekitMember[]) => { - expect(data.length).toEqual(2); - expectObservable(data[0].membership$).toBe("a", { - a: bobMembership, - }); - expectObservable(data[0].connection$).toBe("a", { - // The real connection should be from transportA as per the membership - a: connectionA, - }); - expectObservable(data[0].participant$).toBe("a", { - a: expect.toSatisfy((participant) => { - expect(participant).toBeDefined(); - expect(participant!.identity).toEqual(bobParticipantId); - return true; - }), - }); - return true; - }), - }, - ); - }); + const connectionsWithPublisher$ = new BehaviorSubject( + connectionWithPublisher, + ); + const connectionManagerData$ = epochMeWith$( + memberships$, + connectionsWithPublisher$, + ); + + const matrixLivekitMember$ = createMatrixLivekitMembers$({ + scope: testScope, + membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), + connectionManager: { + connectionManagerData$: connectionManagerData$, + } as unknown as IConnectionManager, }); - test("bob is publishing in the wrong connection", () => { - withTestScheduler(({ behavior, expectObservable }) => { - const { memberships$, membershipsWithTransport$ } = fromMemberships$( - behavior("a", { - a: [bobMembership, carlMembership], - }), - ); - - const connectionWithPublisher = new ConnectionManagerData(); - const bobParticipantId = getParticipantId( - bobMembership.userId, - bobMembership.deviceId, - ); - const connectionA = { transport: transportA } as unknown as Connection; - const connectionB = { transport: transportB } as unknown as Connection; - - // Bob is not publishing on A - connectionWithPublisher.add(connectionA, []); - // Bob is publishing on B but his membership says A - connectionWithPublisher.add(connectionB, [ - mockRemoteParticipant({ identity: bobParticipantId }), - ]); - - const connectionManagerData$ = epochMeWith$( - memberships$, - behavior("a", { - a: connectionWithPublisher, - }), - ); - - const matrixLivekitMember$ = createMatrixLivekitMembers$({ - scope: testScope, - membershipsWithTransport$: testScope.behavior( - membershipsWithTransport$, - ), - connectionManager: { - connectionManagerData$: connectionManagerData$, - } as unknown as IConnectionManager, - }); - - expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe( - "a", - { - a: expect.toSatisfy((data: MatrixLivekitMember[]) => { - expect(data.length).toEqual(2); - expectObservable(data[0].membership$).toBe("a", { - a: bobMembership, - }); - expectObservable(data[0].connection$).toBe("a", { - // The real connection should be from transportA as per the membership - a: connectionA, - }); - expectObservable(data[0].participant$).toBe("a", { - // No participant as Bob is not publishing on his membership transport - a: null, - }); - return true; - }), - }, - ); - }); - }); + await flushPromises(); + expect(matrixLivekitMember$.value.value).toSatisfy( + (data: MatrixLivekitMember[]) => { + expect(data.length).toEqual(2); + expect(data[0].membership$.value).toBe(bobMembership); + expect(data[0].connection$.value).toBe(connectionA); + expect(data[0].participant$.value).toBe(null); + return true; + }, + ); }); diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts index 79ad933c..9c6a05c9 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts @@ -13,8 +13,11 @@ import { type LivekitTransport, type CallMembership, } from "matrix-js-sdk/lib/matrixrtc"; -import { combineLatest, filter, map } from "rxjs"; +import { combineLatest, filter, map, switchMap } from "rxjs"; import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; +import { sha256 } from "matrix-js-sdk/lib/digest"; +import { encodeUnpaddedBase64Url } from "matrix-js-sdk"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; import { type Behavior } from "../../Behavior"; import { type IConnectionManager } from "./ConnectionManager"; @@ -62,64 +65,89 @@ export function createMatrixLivekitMembers$({ membershipsWithTransport$, connectionManager, }: Props): Behavior> { + /** + * This internal observable is used to compute the async sha256 hash of the user's identity. + * a promise is treated like an observable. So we can switchMap on the promise from the identity computation. + * The last update to `membershipsWithTransport$` will always be the last promise we pass to switchMap. + * So we will eventually always end up with the latest memberships and their identities. + */ + const membershipsWithTransportAndLivekitIdentity$ = + membershipsWithTransport$.pipe( + switchMap(async (membershipsWithTransport) => { + const { value, epoch } = membershipsWithTransport; + const membershipsWithTransportAndLkIdentityPromises = value.map( + async (obj) => { + return computeLivekitParticipantIdentity( + obj.membership, + obj.membership.kind, + ); + }, + ); + const identities = await Promise.all( + membershipsWithTransportAndLkIdentityPromises, + ); + const membershipsWithTransportAndLkIdentity = value.map( + ({ transport, membership }, index) => { + return { transport, membership, identity: identities[index] }; + }, + ); + return new Epoch(membershipsWithTransportAndLkIdentity, epoch); + }), + ); + /** * Stream of all the call members and their associated livekit data (if available). */ - return scope.behavior( combineLatest([ - membershipsWithTransport$, + membershipsWithTransportAndLivekitIdentity$, connectionManager.connectionManagerData$, ]).pipe( filter((values) => values.every((value) => value.epoch === values[0].epoch), ), - map( - ([ - { value: membershipsWithTransports, epoch }, - { value: managerData }, - ]) => - new Epoch([membershipsWithTransports, managerData] as const, epoch), - ), + map(([x, y]) => new Epoch([x.value, y.value] as const, x.epoch)), generateItemsWithEpoch( // Generator function. // creates an array of `{key, data}[]` // Each change in the keys (new key, missing key) will result in a call to the factory function. - function* ([membershipsWithTransports, managerData]) { - for (const { membership, transport } of membershipsWithTransports) { - // TODO! cannot use membership.membershipID yet, Currently its hardcoded by the jwt service to - const participantId = /*membership.membershipID*/ `${membership.userId}:${membership.deviceId}`; - + function* ([membershipsWithTransportAndLivekitIdentity, managerData]) { + for (const { + membership, + transport, + identity, + } of membershipsWithTransportAndLivekitIdentity) { const participants = transport ? managerData.getParticipantForTransport(transport) : []; const participant = - participants.find((p) => p.identity == participantId) ?? null; + participants.find((p) => p.identity == identity) ?? null; const connection = transport ? managerData.getConnectionForTransport(transport) : null; yield { - keys: [participantId, membership.userId], + keys: [identity, membership.userId, membership.deviceId], data: { membership, participant, connection }, }; } }, // Each update where the key of the generator array do not change will result in updates to the `data$` observable in the factory. - (scope, data$, participantId, userId) => { + (scope, data$, identity, userId, deviceId) => { logger.debug( - `Generating member for participantId: ${participantId}, userId: ${userId}`, + `Generating member for livekitIdentity: ${identity}, userId:deviceId: ${userId}${deviceId}`, ); // will only get called once per `participantId, userId` pair. // updates to data$ and as a result to displayName$ and mxcAvatarUrl$ are more frequent. return { - participantId, + identity, userId, ...scope.splitBehavior(data$), }; }, ), ), + new Epoch([], -1), ); } @@ -136,3 +164,42 @@ export function areLivekitTransportsEqual( if (!t1 && !t2) return true; return false; } + +const livekitParticipantIdentityCache = new Map(); + +/** + * The string that is computed based on the membership and used for the computing the hash. + * `${userId}:${deviceId}:${membershipID}` + * as the direct imput for: await sha256(input) + */ +export const livekitIdentityInput = ({ + userId, + deviceId, + memberId, +}: CallMembershipIdentityParts): string => `${userId}|${deviceId}|${memberId}`; + +export async function computeLivekitParticipantIdentity( + membership: CallMembershipIdentityParts, + kind: "rtc" | "session", +): Promise { + switch (kind) { + case "rtc": { + const input = livekitIdentityInput(membership); + if (livekitParticipantIdentityCache.size > 400) + // prevent memory leaks in a stupid/simple way + livekitParticipantIdentityCache.clear(); + // TODO use non deprecated memberId + if (livekitParticipantIdentityCache.has(input)) + return livekitParticipantIdentityCache.get(input)!; + else { + const hashBuffer = await sha256(input); + const hashedString = encodeUnpaddedBase64Url(hashBuffer); + livekitParticipantIdentityCache.set(input, hashedString); + return hashedString; + } + } + case "session": + default: + return `${membership.userId}:${membership.deviceId}`; + } +} diff --git a/yarn.lock b/yarn.lock index 94b73130..707a6480 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2795,10 +2795,10 @@ __metadata: languageName: node linkType: hard -"@matrix-org/matrix-sdk-crypto-wasm@npm:^15.3.0": - version: 15.3.0 - resolution: "@matrix-org/matrix-sdk-crypto-wasm@npm:15.3.0" - checksum: 10c0/45628f36b7b0e54a8777ae67a7233dbdf3e3cf14e0d95d21f62f89a7ea7e3f907232f1eb7b1262193b1e227759fad47af829dcccc103ded89011f13c66f01d76 +"@matrix-org/matrix-sdk-crypto-wasm@npm:^16.0.0": + version: 16.0.0 + resolution: "@matrix-org/matrix-sdk-crypto-wasm@npm:16.0.0" + checksum: 10c0/13b4ede3e618da819957abff778afefcf3baf9a2faac04a36bb5a07a44fae2ea05fbfa072eb3408d48b2b7b9aaf27242ce52c594c8ce9bf1fb8b3aade2832be1 languageName: node linkType: hard @@ -6571,24 +6571,10 @@ __metadata: languageName: node linkType: hard -"caniuse-lite@npm:^1.0.30001688": - version: 1.0.30001701 - resolution: "caniuse-lite@npm:1.0.30001701" - checksum: 10c0/a814bd4dd8b49645ca51bc6ee42120660a36394bb54eb6084801d3f2bbb9471e5e1a9a8a25f44f83086a032d46e66b33031e2aa345f699b90a7e84a9836b819c - languageName: node - linkType: hard - -"caniuse-lite@npm:^1.0.30001702": - version: 1.0.30001720 - resolution: "caniuse-lite@npm:1.0.30001720" - checksum: 10c0/ba9f963364ec4bfc8359d15d7e2cf365185fa1fddc90b4f534c71befedae9b3dd0cd2583a25ffc168a02d7b61b6c18b59bda0a1828ea2a5250fd3e35c2c049e9 - languageName: node - linkType: hard - -"caniuse-lite@npm:^1.0.30001726": - version: 1.0.30001726 - resolution: "caniuse-lite@npm:1.0.30001726" - checksum: 10c0/2c5f91da7fd9ebf8c6b432818b1498ea28aca8de22b30dafabe2a2a6da1e014f10e67e14f8e68e872a0867b6b4cd6001558dde04e3ab9770c9252ca5c8849d0e +"caniuse-lite@npm:^1.0.30001688, caniuse-lite@npm:^1.0.30001702, caniuse-lite@npm:^1.0.30001726": + version: 1.0.30001760 + resolution: "caniuse-lite@npm:1.0.30001760" + checksum: 10c0/cee26dff5c5b15ba073ab230200e43c0d4e88dc3bac0afe0c9ab963df70aaa876c3e513dde42a027f317136bf6e274818d77b073708b74c5807dfad33c029d3c languageName: node linkType: hard @@ -7547,7 +7533,7 @@ __metadata: livekit-client: "npm:^2.13.0" lodash-es: "npm:^4.17.21" loglevel: "npm:^1.9.1" - matrix-js-sdk: "npm:^39.2.0" + matrix-js-sdk: "github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=f5f1b8efb46b3d55a7eebfabb4a61496640b8b00" matrix-widget-api: "npm:^1.14.0" normalize.css: "npm:^8.0.1" observable-hooks: "npm:^4.2.3" @@ -10352,12 +10338,12 @@ __metadata: languageName: node linkType: hard -"matrix-js-sdk@npm:^39.2.0": - version: 39.2.0 - resolution: "matrix-js-sdk@npm:39.2.0" +"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=f5f1b8efb46b3d55a7eebfabb4a61496640b8b00": + version: 39.3.0 + resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=f5f1b8efb46b3d55a7eebfabb4a61496640b8b00" dependencies: "@babel/runtime": "npm:^7.12.5" - "@matrix-org/matrix-sdk-crypto-wasm": "npm:^15.3.0" + "@matrix-org/matrix-sdk-crypto-wasm": "npm:^16.0.0" another-json: "npm:^0.2.0" bs58: "npm:^6.0.0" content-type: "npm:^1.0.4" @@ -10370,7 +10356,7 @@ __metadata: sdp-transform: "npm:^3.0.0" unhomoglyph: "npm:^1.0.6" uuid: "npm:13" - checksum: 10c0/f8b5261de2744305330ba3952821ca9303698170bfd3a0ff8a767b9286d4e8d4ed5aaf6fbaf8a1e8ff9dbd859102a2a47d882787e2da3b3078965bec00157959 + checksum: 10c0/9607b0c063c873a24c1a2d05cc7500d60c32556ec82b666ebaae5c5e829faf5bb7639780efddea7211e6b9873098bd53b97656f041e932e8b0de0c208ccabbff languageName: node linkType: hard From 5bc6ed5885c8de4220f7542fe4382c727c03629f Mon Sep 17 00:00:00 2001 From: Timo K Date: Mon, 15 Dec 2025 20:17:57 +0100 Subject: [PATCH 02/46] small refactor to make it testable. --- src/room/CallEventAudioRenderer.test.tsx | 8 +++ src/room/InCallView.test.tsx | 10 ++- src/state/CallViewModel/CallViewModel.test.ts | 22 +++++-- .../LivekitParticipantIdentity.ts | 53 +++++++++++++++ .../MatrixLivekitMembers.test.ts | 1 - .../remoteMembers/MatrixLivekitMembers.ts | 66 ++++--------------- .../remoteMembers/integration.test.ts | 6 ++ src/utils/test.ts | 28 +++++++- 8 files changed, 134 insertions(+), 60 deletions(-) create mode 100644 src/state/CallViewModel/remoteMembers/LivekitParticipantIdentity.ts diff --git a/src/room/CallEventAudioRenderer.test.tsx b/src/room/CallEventAudioRenderer.test.tsx index 733346eb..38f56b14 100644 --- a/src/room/CallEventAudioRenderer.test.tsx +++ b/src/room/CallEventAudioRenderer.test.tsx @@ -23,6 +23,7 @@ import { import { exampleTransport, + mockComputeLivekitParticipantIdentity$, mockMatrixRoomMember, mockRtcMembership, } from "../utils/test"; @@ -47,6 +48,13 @@ vitest.mock("../rtcSessionHelpers", async (importOriginal) => ({ ...(await importOriginal()), makeTransport: (): [LivekitTransport] => [exampleTransport], })); +vitest.mock( + import("../state/CallViewModel/remoteMembers/LivekitParticipantIdentity.ts"), + async (importOriginal) => ({ + ...(await importOriginal()), + computeLivekitParticipantIdentity$: mockComputeLivekitParticipantIdentity$, + }), +); afterEach(() => { vitest.clearAllMocks(); diff --git a/src/room/InCallView.test.tsx b/src/room/InCallView.test.tsx index a137074b..cd0af547 100644 --- a/src/room/InCallView.test.tsx +++ b/src/room/InCallView.test.tsx @@ -22,8 +22,8 @@ import { BrowserRouter } from "react-router-dom"; import { TooltipProvider } from "@vector-im/compound-web"; import { RoomContext, useLocalParticipant } from "@livekit/components-react"; -import { InCallView } from "./InCallView"; import { + mockComputeLivekitParticipantIdentity$, mockLivekitRoom, mockLocalParticipant, mockMatrixRoom, @@ -34,6 +34,7 @@ import { mockRtcMembership, type MockRTCSession, } from "../utils/test"; +import { InCallView } from "./InCallView"; import { E2eeType } from "../e2ee/e2eeType"; import { getBasicCallViewModelEnvironment } from "../utils/test-viewmodel"; import { alice, local } from "../utils/test-fixtures"; @@ -61,6 +62,13 @@ vi.mock("../livekit/MatrixAudioRenderer"); vi.mock("react-use-measure", () => ({ default: (): [() => void, object] => [(): void => {}, {}], })); +vi.mock( + import("../state/CallViewModel/remoteMembers/LivekitParticipantIdentity.ts"), + async (importOriginal) => ({ + ...(await importOriginal()), + computeLivekitParticipantIdentity$: mockComputeLivekitParticipantIdentity$, + }), +); const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC"); const localParticipant = mockLocalParticipant({ diff --git a/src/state/CallViewModel/CallViewModel.test.ts b/src/state/CallViewModel/CallViewModel.test.ts index be598702..58814bbe 100644 --- a/src/state/CallViewModel/CallViewModel.test.ts +++ b/src/state/CallViewModel/CallViewModel.test.ts @@ -44,6 +44,7 @@ import { mockRtcMembership, testScope, exampleTransport, + mockComputeLivekitParticipantIdentity$, } from "../../utils/test.ts"; import { E2eeType } from "../../e2ee/e2eeType.ts"; import { @@ -77,11 +78,22 @@ vi.mock("../e2ee/matrixKeyProvider"); const getUrlParams = vi.hoisted(() => vi.fn(() => ({}))); vi.mock("../UrlParams", () => ({ getUrlParams })); -vi.mock("../rtcSessionHelpers", async (importOriginal) => ({ - ...(await importOriginal()), - makeTransport: async (): Promise => - Promise.resolve(exampleTransport), -})); +vi.mock( + "../state/CallViewModel/localMember/localTransport", + async (importOriginal) => ({ + ...(await importOriginal()), + makeTransport: async (): Promise => + Promise.resolve(exampleTransport), + }), +); + +vi.mock( + import("./remoteMembers/LivekitParticipantIdentity.ts"), + async (importOriginal) => ({ + ...(await importOriginal()), + computeLivekitParticipantIdentity$: mockComputeLivekitParticipantIdentity$, + }), +); const yesNo = { y: true, diff --git a/src/state/CallViewModel/remoteMembers/LivekitParticipantIdentity.ts b/src/state/CallViewModel/remoteMembers/LivekitParticipantIdentity.ts new file mode 100644 index 00000000..e43d0bd1 --- /dev/null +++ b/src/state/CallViewModel/remoteMembers/LivekitParticipantIdentity.ts @@ -0,0 +1,53 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { encodeUnpaddedBase64Url } from "matrix-js-sdk"; +import { sha256 } from "matrix-js-sdk/lib/digest"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; +import { from, type Observable } from "rxjs"; + +const livekitParticipantIdentityCache = new Map(); + +/** + * The string that is computed based on the membership and used for the computing the hash. + * `${userId}:${deviceId}:${membershipID}` + * as the direct imput for: await sha256(input) + */ +export const livekitIdentityInput = ({ + userId, + deviceId, + memberId, +}: CallMembershipIdentityParts): string => `${userId}|${deviceId}|${memberId}`; + +export function computeLivekitParticipantIdentity$( + membership: CallMembershipIdentityParts, + kind: "rtc" | "session", +): Observable { + const compute = async (): Promise => { + switch (kind) { + case "rtc": { + const input = livekitIdentityInput(membership); + if (livekitParticipantIdentityCache.size > 400) + // prevent memory leaks in a stupid/simple way + livekitParticipantIdentityCache.clear(); + // TODO use non deprecated memberId + if (livekitParticipantIdentityCache.has(input)) + return livekitParticipantIdentityCache.get(input)!; + else { + const hashBuffer = await sha256(input); + const hashedString = encodeUnpaddedBase64Url(hashBuffer); + livekitParticipantIdentityCache.set(input, hashedString); + return hashedString; + } + } + case "session": + default: + return `${membership.userId}:${membership.deviceId}`; + } + }; + return from(compute()); +} diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts index c2e60798..68a67546 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts @@ -28,7 +28,6 @@ import { flushPromises, mockCallMembership, mockRemoteParticipant, - withTestScheduler, } from "../../../utils/test.ts"; import { type Connection } from "./Connection.ts"; import { constant } from "../../Behavior.ts"; diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts index 9c6a05c9..30abfc9b 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts @@ -15,15 +15,13 @@ import { } from "matrix-js-sdk/lib/matrixrtc"; import { combineLatest, filter, map, switchMap } from "rxjs"; import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; -import { sha256 } from "matrix-js-sdk/lib/digest"; -import { encodeUnpaddedBase64Url } from "matrix-js-sdk"; -import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; import { type Behavior } from "../../Behavior"; import { type IConnectionManager } from "./ConnectionManager"; import { Epoch, type ObservableScope } from "../../ObservableScope"; import { type Connection } from "./Connection"; import { generateItemsWithEpoch } from "../../../utils/observable"; +import { computeLivekitParticipantIdentity$ } from "./LivekitParticipantIdentity"; const logger = rootLogger.getChild("[MatrixLivekitMembers]"); @@ -73,25 +71,28 @@ export function createMatrixLivekitMembers$({ */ const membershipsWithTransportAndLivekitIdentity$ = membershipsWithTransport$.pipe( - switchMap(async (membershipsWithTransport) => { + switchMap((membershipsWithTransport) => { const { value, epoch } = membershipsWithTransport; const membershipsWithTransportAndLkIdentityPromises = value.map( - async (obj) => { - return computeLivekitParticipantIdentity( + (obj) => { + return computeLivekitParticipantIdentity$( obj.membership, obj.membership.kind, ); }, ); - const identities = await Promise.all( + return combineLatest( membershipsWithTransportAndLkIdentityPromises, + ).pipe( + map((identities) => { + const membershipsWithTransportAndLkIdentity = value.map( + ({ transport, membership }, index) => { + return { transport, membership, identity: identities[index] }; + }, + ); + return new Epoch(membershipsWithTransportAndLkIdentity, epoch); + }), ); - const membershipsWithTransportAndLkIdentity = value.map( - ({ transport, membership }, index) => { - return { transport, membership, identity: identities[index] }; - }, - ); - return new Epoch(membershipsWithTransportAndLkIdentity, epoch); }), ); @@ -164,42 +165,3 @@ export function areLivekitTransportsEqual( if (!t1 && !t2) return true; return false; } - -const livekitParticipantIdentityCache = new Map(); - -/** - * The string that is computed based on the membership and used for the computing the hash. - * `${userId}:${deviceId}:${membershipID}` - * as the direct imput for: await sha256(input) - */ -export const livekitIdentityInput = ({ - userId, - deviceId, - memberId, -}: CallMembershipIdentityParts): string => `${userId}|${deviceId}|${memberId}`; - -export async function computeLivekitParticipantIdentity( - membership: CallMembershipIdentityParts, - kind: "rtc" | "session", -): Promise { - switch (kind) { - case "rtc": { - const input = livekitIdentityInput(membership); - if (livekitParticipantIdentityCache.size > 400) - // prevent memory leaks in a stupid/simple way - livekitParticipantIdentityCache.clear(); - // TODO use non deprecated memberId - if (livekitParticipantIdentityCache.has(input)) - return livekitParticipantIdentityCache.get(input)!; - else { - const hashBuffer = await sha256(input); - const hashedString = encodeUnpaddedBase64Url(hashBuffer); - livekitParticipantIdentityCache.set(input, hashedString); - return hashedString; - } - } - case "session": - default: - return `${membership.userId}:${membership.deviceId}`; - } -} diff --git a/src/state/CallViewModel/remoteMembers/integration.test.ts b/src/state/CallViewModel/remoteMembers/integration.test.ts index e3aa6be8..3ad81fa8 100644 --- a/src/state/CallViewModel/remoteMembers/integration.test.ts +++ b/src/state/CallViewModel/remoteMembers/integration.test.ts @@ -22,6 +22,7 @@ import { ECConnectionFactory } from "./ConnectionFactory.ts"; import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts"; import { mockCallMembership, + mockComputeLivekitParticipantIdentity$, mockMediaDevices, withTestScheduler, } from "../../../utils/test.ts"; @@ -43,6 +44,11 @@ let lkRoomFactory: () => LivekitRoom; const createdMockLivekitRooms: Map = new Map(); +vi.mock(import("./LivekitParticipantIdentity.ts"), async (importOriginal) => ({ + ...(await importOriginal()), + computeLivekitParticipantIdentity$: mockComputeLivekitParticipantIdentity$, +})); + beforeEach(() => { testScope = new ObservableScope(); mockClient = { diff --git a/src/utils/test.ts b/src/utils/test.ts index bd7dcd6f..28fc8546 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -15,6 +15,7 @@ import { vitest, } from "vitest"; import { + encodeUnpaddedBase64, MatrixEvent, type Room as MatrixRoom, type Room, @@ -43,13 +44,14 @@ import { type Room as LivekitRoom, Track, } from "livekit-client"; -import { randomUUID } from "crypto"; +import { createHash, randomUUID } from "crypto"; import { type TrackReference } from "@livekit/components-core"; import EventEmitter from "events"; import { type KeyTransportEvents, type KeyTransportEventsHandlerMap, } from "matrix-js-sdk/lib/matrixrtc/IKeyTransport"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; import { LocalUserMediaViewModel, @@ -522,3 +524,27 @@ export function mockMuteStates( const observableScope = new ObservableScope(); return new MuteStates(observableScope, mockMediaDevices({}), joined$); } + +export const mockComputeLivekitParticipantIdentity$ = ( + membership: CallMembershipIdentityParts, + kind: "rtc" | "session", +): Observable => { + function sha256(commitmentStr: string): string { + return encodeUnpaddedBase64( + createHash("sha256").update(commitmentStr, "utf8").digest(), + ); + } + let hash; + switch (kind) { + case "rtc": { + hash = sha256( + `${membership.userId}|${membership.deviceId}|${membership.memberId}`, + ); + break; + } + case "session": + default: + hash = `${membership.userId}:${membership.deviceId}`; + } + return of(hash); +}; From 6fe6daba313b2cf55f582baf49d1e8f5091b5ed2 Mon Sep 17 00:00:00 2001 From: fkwp Date: Tue, 16 Dec 2025 11:20:38 +0100 Subject: [PATCH 03/46] switch synapse docker image to one with sticky event support --- dev-backend-docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-backend-docker-compose.yml b/dev-backend-docker-compose.yml index c7591847..5e955831 100644 --- a/dev-backend-docker-compose.yml +++ b/dev-backend-docker-compose.yml @@ -88,7 +88,7 @@ services: synapse: hostname: homeserver - image: docker.io/matrixdotorg/synapse:latest + image: ghcr.io/element-hq/synapse:pr-18968-dcb7678281bc02d4551043a6338fe5b7e6aa47ce pull_policy: always environment: - SYNAPSE_CONFIG_PATH=/data/cfg/homeserver.yaml @@ -106,7 +106,7 @@ services: synapse-1: hostname: homeserver-1 - image: docker.io/matrixdotorg/synapse:latest + image: ghcr.io/element-hq/synapse:pr-18968-dcb7678281bc02d4551043a6338fe5b7e6aa47ce pull_policy: always environment: - SYNAPSE_CONFIG_PATH=/data/cfg/homeserver.yaml From ff3d6bd088f5024207c2921830b374e0e9c0fa71 Mon Sep 17 00:00:00 2001 From: fkwp Date: Tue, 16 Dec 2025 11:22:25 +0100 Subject: [PATCH 04/46] enable sticky events --- backend/dev_homeserver-othersite.yaml | 2 ++ backend/dev_homeserver.yaml | 2 +- backend/playwright_homeserver-othersite.yaml | 2 ++ backend/playwright_homeserver.yaml | 2 ++ 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/backend/dev_homeserver-othersite.yaml b/backend/dev_homeserver-othersite.yaml index 947e33cd..81e775ca 100644 --- a/backend/dev_homeserver-othersite.yaml +++ b/backend/dev_homeserver-othersite.yaml @@ -38,6 +38,8 @@ experimental_features: # MSC4222 needed for syncv2 state_after. This allow clients to # correctly track the state of the room. msc4222_enabled: true + # sticky events for MatrixRTC user state + msc4354_enabled: true # The maximum allowed duration by which sent events can be delayed, as # per MSC4140. Must be a positive value if set. Defaults to no diff --git a/backend/dev_homeserver.yaml b/backend/dev_homeserver.yaml index fe89d95a..dc7b42c8 100644 --- a/backend/dev_homeserver.yaml +++ b/backend/dev_homeserver.yaml @@ -38,7 +38,7 @@ experimental_features: # MSC4222 needed for syncv2 state_after. This allow clients to # correctly track the state of the room. msc4222_enabled: true - # sticky events for matrixRTC user state + # sticky events for MatrixRTC user state msc4354_enabled: true # The maximum allowed duration by which sent events can be delayed, as diff --git a/backend/playwright_homeserver-othersite.yaml b/backend/playwright_homeserver-othersite.yaml index 5cb0dd65..35640ae9 100644 --- a/backend/playwright_homeserver-othersite.yaml +++ b/backend/playwright_homeserver-othersite.yaml @@ -38,6 +38,8 @@ experimental_features: # MSC4222 needed for syncv2 state_after. This allow clients to # correctly track the state of the room. msc4222_enabled: true + # sticky events for MatrixRTC user state + msc4354_enabled: true # The maximum allowed duration by which sent events can be delayed, as # per MSC4140. Must be a positive value if set. Defaults to no diff --git a/backend/playwright_homeserver.yaml b/backend/playwright_homeserver.yaml index 0d7b175c..a83247cd 100644 --- a/backend/playwright_homeserver.yaml +++ b/backend/playwright_homeserver.yaml @@ -38,6 +38,8 @@ experimental_features: # MSC4222 needed for syncv2 state_after. This allow clients to # correctly track the state of the room. msc4222_enabled: true + # sticky events for MatrixRTC user state + msc4354_enabled: true # The maximum allowed duration by which sent events can be delayed, as # per MSC4140. Must be a positive value if set. Defaults to no From ab7e3486b3ea1953f1435acbced309189476e78b Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 17 Dec 2025 09:53:49 +0100 Subject: [PATCH 05/46] Make use of the new jwt service endpoint (with delayed event delegation) This also does all the compatibility work. When to use which endpoint to authenticate agains a jwt service. --- src/livekit/openIDSFU.ts | 90 ++++++++++++++++--- src/state/CallViewModel/CallViewModel.ts | 32 ++++++- .../localMember/LocalMember.test.ts | 3 + .../CallViewModel/localMember/LocalMember.ts | 5 +- .../localMember/LocalTransport.test.ts | 18 +++- .../localMember/LocalTransport.ts | 63 +++++++++---- .../remoteMembers/Connection.test.ts | 17 ++-- .../CallViewModel/remoteMembers/Connection.ts | 20 ++++- .../remoteMembers/ConnectionFactory.ts | 18 +++- .../remoteMembers/ConnectionManager.test.ts | 13 ++- .../remoteMembers/ConnectionManager.ts | 27 ++++-- .../remoteMembers/ECConnectionFactory.test.ts | 20 ++++- .../remoteMembers/MatrixLivekitMembers.ts | 6 +- .../remoteMembers/integration.test.ts | 2 + src/state/SessionBehaviors.ts | 15 +++- src/utils/test.ts | 10 ++- yarn.lock | 9 +- 17 files changed, 294 insertions(+), 74 deletions(-) diff --git a/src/livekit/openIDSFU.ts b/src/livekit/openIDSFU.ts index 3ae003fb..f07bb035 100644 --- a/src/livekit/openIDSFU.ts +++ b/src/livekit/openIDSFU.ts @@ -7,9 +7,11 @@ Please see LICENSE in the repository root for full details. import { type IOpenIDToken, type MatrixClient } from "matrix-js-sdk"; import { logger } from "matrix-js-sdk/lib/logger"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; import { FailToGetOpenIdToken } from "../utils/errors"; import { doNetworkOperationWithRetry } from "../utils/matrix"; +import { Config } from "../config/Config"; export interface SFUConfig { url: string; @@ -33,8 +35,12 @@ export type OpenIDClientParts = Pick< */ export async function getSFUConfigWithOpenID( client: OpenIDClientParts, + membership: CallMembershipIdentityParts, serviceUrl: string, - matrixRoomId: string, + livekitRoomAlias: string, + matrix2jwt: boolean, + delayEndpointBaseUrl?: string, + delayId?: string, ): Promise { let openIdToken: IOpenIDToken; try { @@ -49,21 +55,31 @@ export async function getSFUConfigWithOpenID( logger.debug("Got openID token", openIdToken); logger.info(`Trying to get JWT for focus ${serviceUrl}...`); - const sfuConfig = await getLiveKitJWT( - client, + const args: [CallMembershipIdentityParts, string, string, IOpenIDToken] = [ + membership, serviceUrl, - matrixRoomId, + livekitRoomAlias, openIdToken, - ); - logger.info(`Got JWT from call's active focus URL.`); - - return sfuConfig; + ]; + if (matrix2jwt) { + const sfuConfig = await getLiveKitJWTWithDelayDelegation( + ...args, + delayEndpointBaseUrl, + delayId, + ); + logger.info(`Got JWT from call's active focus URL.`); + return sfuConfig; + } else { + const sfuConfig = await getLiveKitJWT(...args); + logger.info(`Got JWT from call's active focus URL.`); + return sfuConfig; + } } async function getLiveKitJWT( - client: OpenIDClientParts, + membership: CallMembershipIdentityParts, livekitServiceURL: string, - roomName: string, + livekitRoomAlias: string, openIDToken: IOpenIDToken, ): Promise { try { @@ -73,9 +89,9 @@ async function getLiveKitJWT( "Content-Type": "application/json", }, body: JSON.stringify({ - room: roomName, + room: livekitRoomAlias, openid_token: openIDToken, - device_id: client.getDeviceId(), + device_id: membership.deviceId, }), }); if (!res.ok) { @@ -86,3 +102,53 @@ async function getLiveKitJWT( throw new Error("SFU Config fetch failed with exception " + e); } } + +export async function getLiveKitJWTWithDelayDelegation( + membership: CallMembershipIdentityParts, + livekitServiceURL: string, + livekitRoomAlias: string, + openIDToken: IOpenIDToken, + delayEndpointBaseUrl?: string, + delayId?: string, +): Promise { + const { userId, deviceId, memberId } = membership; + + const body = { + room_id: livekitRoomAlias, + slot_id: "m.call#ROOM", + openid_token: openIDToken, + member: { + id: memberId, + claimed_user_id: userId, + claimed_device_id: deviceId, + }, + }; + + let bodyDalayParts = {}; + // Also check for empty string + if (delayId && delayEndpointBaseUrl) { + const delayTimeoutMs = + Config.get().matrix_rtc_session?.delayed_leave_event_delay_ms ?? 1000; + bodyDalayParts = { + delay_id: delayId, + delay_timeout: delayTimeoutMs, + delay_cs_api_url: delayEndpointBaseUrl, + }; + } + + try { + const res = await fetch(livekitServiceURL + "/get_token", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ ...body, ...bodyDalayParts }), + }); + if (!res.ok) { + throw new Error("SFU Config fetch failed with status code " + res.status); + } + return await res.json(); + } catch (e) { + throw new Error("SFU Config fetch failed with exception " + e); + } +} diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index e1869cf6..23c58268 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -41,10 +41,12 @@ import { } from "rxjs"; import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { + MembershipManagerEvent, type LivekitTransport, type MatrixRTCSession, } from "matrix-js-sdk/lib/matrixrtc"; import { type IWidgetApiRequest } from "matrix-widget-api"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; import { LocalUserMediaViewModel, @@ -98,7 +100,7 @@ import { type SpotlightLandscapeLayoutMedia, type SpotlightPortraitLayoutMedia, } from "../layout-types.ts"; -import { ElementCallError } from "../../utils/errors.ts"; +import { ElementCallError, UnknownCallError } from "../../utils/errors.ts"; import { type ObservableScope } from "../ObservableScope.ts"; import { createHomeserverConnected$ } from "./localMember/HomeserverConnected.ts"; import { @@ -375,8 +377,11 @@ export function createCallViewModel$( trackProcessorState$: Behavior, ): CallViewModel { const client = matrixRoom.client; - const userId = client.getUserId()!; - const deviceId = client.getDeviceId()!; + const userId = client.getUserId(); + const deviceId = client.getDeviceId(); + if (!(userId && deviceId)) + throw new UnknownCallError(new Error("userId and deviceId are required")); + const livekitKeyProvider = getE2eeKeyProvider( options.encryptionSystem, matrixRTCSession, @@ -407,10 +412,29 @@ export function createCallViewModel$( memberships$, ); + const ownMembershipIdentity: CallMembershipIdentityParts = { + userId, + deviceId, + memberId: `${userId}:${deviceId}`, + }; + const localTransport$ = createLocalTransport$({ scope: scope, memberships$: memberships$, + ownMembershipIdentity, client, + useMatrix2$: scope.behavior( + options.matrixRTCMode$.pipe(map((v) => v === MatrixRTCMode.Matrix_2_0)), + ), + delayId$: scope.behavior( + ( + fromEvent( + matrixRTCSession, + MembershipManagerEvent.DelayIdChanged, + ) as Observable + ).pipe(map((v) => v ?? null)), + matrixRTCSession.delayId ?? null, + ), roomId: matrixRoom.roomId, useOldestMember$: scope.behavior( options.matrixRTCMode$.pipe(map((v) => v === MatrixRTCMode.Legacy)), @@ -455,6 +479,7 @@ export function createCallViewModel$( ), ), logger: logger, + ownMembershipIdentity, }); const matrixLivekitMembers$ = createMatrixLivekitMembers$({ @@ -485,6 +510,7 @@ export function createCallViewModel$( joinMatrixRTC: (transport: LivekitTransport) => { return enterRTCSession( matrixRTCSession, + ownMembershipIdentity, transport, connectOptions$.value, ); diff --git a/src/state/CallViewModel/localMember/LocalMember.test.ts b/src/state/CallViewModel/localMember/LocalMember.test.ts index 6a9f196e..8de14039 100644 --- a/src/state/CallViewModel/localMember/LocalMember.test.ts +++ b/src/state/CallViewModel/localMember/LocalMember.test.ts @@ -24,6 +24,7 @@ import { mockLivekitRoom, mockMuteStates, withTestScheduler, + ownMemberMock, } from "../../../utils/test"; import { TransportState, @@ -108,6 +109,7 @@ describe("LocalMembership", () => { enterRTCSession( mockedSession, + ownMemberMock, { livekit_alias: "roomId", livekit_service_url: "http://my-well-known-service-url.com", @@ -166,6 +168,7 @@ describe("LocalMembership", () => { enterRTCSession( mockedSession, + ownMemberMock, { livekit_alias: "roomId", livekit_service_url: "http://my-well-known-service-url.com", diff --git a/src/state/CallViewModel/localMember/LocalMember.ts b/src/state/CallViewModel/localMember/LocalMember.ts index 40fb62d6..6f554423 100644 --- a/src/state/CallViewModel/localMember/LocalMember.ts +++ b/src/state/CallViewModel/localMember/LocalMember.ts @@ -34,6 +34,7 @@ import { } from "rxjs"; import { type Logger } from "matrix-js-sdk/lib/logger"; import { deepCompare } from "matrix-js-sdk/lib/utils"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; import { constant, type Behavior } from "../../Behavior.ts"; import { type IConnectionManager } from "../remoteMembers/ConnectionManager.ts"; @@ -657,6 +658,7 @@ interface EnterRTCSessionOptions { // Exported for unit testing export function enterRTCSession( rtcSession: MatrixRTCSession, + ownMembershipIdentity: CallMembershipIdentityParts, transport: LivekitTransport, { encryptMedia, matrixRTCMode }: EnterRTCSessionOptions, ): void { @@ -674,7 +676,8 @@ export function enterRTCSession( const multiSFU = matrixRTCMode !== MatrixRTCMode.Legacy; // Multi-sfu does not need a preferred foci list. just the focus that is actually used. // TODO where/how do we track errors originating from the ongoing rtcSession? - rtcSession.joinRoomSession( + rtcSession.joinRTCSession( + ownMembershipIdentity, multiSFU ? [] : [transport], multiSFU ? transport : undefined, { diff --git a/src/state/CallViewModel/localMember/LocalTransport.test.ts b/src/state/CallViewModel/localMember/LocalTransport.test.ts index c1c36fa5..ba030757 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.test.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.test.ts @@ -9,7 +9,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; import { BehaviorSubject } from "rxjs"; -import { mockConfig, flushPromises } from "../../../utils/test"; +import { mockConfig, flushPromises, ownMemberMock } from "../../../utils/test"; import { createLocalTransport$ } from "./LocalTransport"; import { constant } from "../../Behavior"; import { Epoch, ObservableScope } from "../../ObservableScope"; @@ -32,10 +32,14 @@ describe("LocalTransport", () => { memberships$: constant(new Epoch([])), client: { getDomain: () => "", + baseUrl: "example.org", // These won't be called in this error path but satisfy the type getOpenIdToken: vi.fn(), getDeviceId: vi.fn(), }, + ownMembershipIdentity: ownMemberMock, + useMatrix2$: constant(false), + delayId$: constant("delay_id_mock"), }); await flushPromises(); @@ -65,11 +69,15 @@ describe("LocalTransport", () => { useOldestMember$: constant(false), memberships$: constant(new Epoch([])), client: { + baseUrl: "https://lk.example.org", // Use empty domain to skip .well-known and use config directly getDomain: () => "", getOpenIdToken: vi.fn(), getDeviceId: vi.fn(), }, + ownMembershipIdentity: ownMemberMock, + useMatrix2$: constant(false), + delayId$: constant("delay_id_mock"), }); localTransport$.subscribe( (o) => observations.push(o), @@ -105,7 +113,11 @@ describe("LocalTransport", () => { getDomain: () => "", getOpenIdToken: vi.fn(), getDeviceId: vi.fn(), + baseUrl: "https://lk.example.org", }, + ownMembershipIdentity: ownMemberMock, + useMatrix2$: constant(false), + delayId$: constant("delay_id_mock"), }); openIdResolver.resolve?.({ url: "https://lk.example.org", jwt: "jwt" }); @@ -140,7 +152,11 @@ describe("LocalTransport", () => { getDomain: () => "", getOpenIdToken: vi.fn(), getDeviceId: vi.fn(), + baseUrl: "https://lk.example.org", }, + ownMembershipIdentity: ownMemberMock, + useMatrix2$: constant(false), + delayId$: constant("delay_id_mock"), }); openIdResolver.resolve?.({ url: "https://lk.example.org", jwt: "jwt" }); diff --git a/src/state/CallViewModel/localMember/LocalTransport.ts b/src/state/CallViewModel/localMember/LocalTransport.ts index 1320b8c4..6c3e1cd0 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.ts @@ -23,6 +23,7 @@ import { } from "rxjs"; import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; import { type Behavior } from "../../Behavior.ts"; import { type Epoch, type ObservableScope } from "../../ObservableScope.ts"; @@ -34,6 +35,7 @@ import { } from "../../../livekit/openIDSFU.ts"; import { areLivekitTransportsEqual } from "../remoteMembers/MatrixLivekitMembers.ts"; import { customLivekitUrl } from "../../../settings/settings.ts"; +import { type LivekitTransportWithVersion } from "../remoteMembers/ConnectionManager.ts"; const logger = rootLogger.getChild("[LocalTransport]"); @@ -44,10 +46,13 @@ const logger = rootLogger.getChild("[LocalTransport]"); */ interface Props { scope: ObservableScope; + ownMembershipIdentity: CallMembershipIdentityParts; memberships$: Behavior>; - client: Pick & OpenIDClientParts; + client: Pick & OpenIDClientParts; roomId: string; useOldestMember$: Behavior; + useMatrix2$: Behavior; + delayId$: Behavior; } /** @@ -62,20 +67,26 @@ interface Props { export const createLocalTransport$ = ({ scope, memberships$, + ownMembershipIdentity, client, roomId, useOldestMember$, -}: Props): Behavior => { + useMatrix2$, + delayId$, +}: Props): Behavior => { /** * The transport over which we should be actively publishing our media. * undefined when not joined. */ const oldestMemberTransport$ = scope.behavior( memberships$.pipe( - map( - (memberships) => - memberships.value[0]?.getTransport(memberships.value[0]) ?? null, - ), + map((memberships) => { + const oldestMember = memberships.value[0]; + const t = oldestMember?.getTransport(memberships.value[0]); + if (!t) return null; + // Here we will use the matrix2 information from the oldest member transport. + return { ...t, useMatrix2: oldestMember.kind === "rtc" }; + }), first((t) => t != null && isLivekitTransport(t)), ), null, @@ -87,12 +98,24 @@ export const createLocalTransport$ = ({ * * @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken */ - const preferredTransport$: Behavior = scope.behavior( - customLivekitUrl.value$.pipe( - switchMap((customUrl) => from(makeTransport(client, roomId, customUrl))), - ), - null, - ); + const preferredTransport$: Behavior = + scope.behavior( + combineLatest([customLivekitUrl.value$, useMatrix2$, delayId$]).pipe( + switchMap(([customUrl, useMatrix2, delayId]) => + from( + makeTransport( + client, + ownMembershipIdentity, + roomId, + customUrl, + useMatrix2, + delayId ?? undefined, + ), + ), + ), + ), + null, + ); /** * The chosen transport we should advertise in our MatrixRTC membership. @@ -123,10 +146,13 @@ const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci"; * @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken */ async function makeTransport( - client: Pick & OpenIDClientParts, + client: Pick & OpenIDClientParts, + membership: CallMembershipIdentityParts, roomId: string, urlFromDevSettings: string | null, -): Promise { + matrix2jwt = false, + delayId?: string, +): Promise { let transport: LivekitTransport | undefined; logger.trace("Searching for a preferred transport"); //TODO refactor this to use the jwt service returned alias. @@ -176,13 +202,18 @@ async function makeTransport( transport = transportFromConf; } - if (!transport) throw new MatrixRTCTransportMissingError(domain ?? ""); // this will call the jwt/sfu/get endpoint to pre create the livekit room. + if (!transport) throw new MatrixRTCTransportMissingError(domain ?? ""); + // this will call the jwt/sfu/get endpoint to pre create the livekit room. await getSFUConfigWithOpenID( client, + membership, transport.livekit_service_url, transport.livekit_alias, + matrix2jwt, + client.baseUrl, + delayId, ); - return transport; + return { ...transport, useMatrix2: matrix2jwt }; } diff --git a/src/state/CallViewModel/remoteMembers/Connection.test.ts b/src/state/CallViewModel/remoteMembers/Connection.test.ts index 30c934b9..533f451a 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.test.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.test.ts @@ -27,7 +27,6 @@ import EventEmitter from "events"; import { type IOpenIDToken } from "matrix-js-sdk"; import { logger } from "matrix-js-sdk/lib/logger"; -import type { LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { Connection, ConnectionState, @@ -39,7 +38,8 @@ import { ElementCallError, FailToGetOpenIdToken, } from "../../../utils/errors.ts"; -import { mockRemoteParticipant } from "../../../utils/test.ts"; +import { mockRemoteParticipant, ownMemberMock } from "../../../utils/test.ts"; +import { type LivekitTransportWithVersion } from "./ConnectionManager.ts"; let testScope: ObservableScope; @@ -50,10 +50,11 @@ let fakeLivekitRoom: MockedObject; let localParticipantEventEmiter: EventEmitter; let fakeLocalParticipant: MockedObject; -const livekitFocus: LivekitTransport = { +const livekitFocus: LivekitTransportWithVersion = { livekit_alias: "!roomID:example.org", livekit_service_url: "https://matrix-rtc.example.org/livekit/jwt", type: "livekit", + useMatrix2: false, }; function setupTest(): void { @@ -137,7 +138,7 @@ function setupRemoteConnection(): Connection { return Promise.resolve(); }); - return new Connection(opts, logger); + return new Connection(opts, logger, ownMemberMock); } afterEach(() => { @@ -156,7 +157,7 @@ describe("Start connection states", () => { scope: testScope, livekitRoomFactory: () => fakeLivekitRoom, }; - const connection = new Connection(opts, logger); + const connection = new Connection(opts, logger, ownMemberMock); expect(connection.state$.getValue()).toEqual("Initialized"); }); @@ -172,7 +173,7 @@ describe("Start connection states", () => { livekitRoomFactory: () => fakeLivekitRoom, }; - const connection = new Connection(opts, logger); + const connection = new Connection(opts, logger, ownMemberMock); const capturedStates: (ConnectionState | Error)[] = []; const s = connection.state$.subscribe((value) => { @@ -222,7 +223,7 @@ describe("Start connection states", () => { livekitRoomFactory: () => fakeLivekitRoom, }; - const connection = new Connection(opts, logger); + const connection = new Connection(opts, logger, ownMemberMock); const capturedStates: (ConnectionState | Error)[] = []; const s = connection.state$.subscribe((value) => { @@ -279,7 +280,7 @@ describe("Start connection states", () => { livekitRoomFactory: () => fakeLivekitRoom, }; - const connection = new Connection(opts, logger); + const connection = new Connection(opts, logger, ownMemberMock); const capturedStates: (ConnectionState | Error)[] = []; const s = connection.state$.subscribe((value) => { diff --git a/src/state/CallViewModel/remoteMembers/Connection.ts b/src/state/CallViewModel/remoteMembers/Connection.ts index 05d0ec9e..d32bbce6 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.ts @@ -18,6 +18,7 @@ import { import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { BehaviorSubject, map } from "rxjs"; import { type Logger } from "matrix-js-sdk/lib/logger"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; import { getSFUConfigWithOpenID, @@ -35,7 +36,7 @@ import { export interface ConnectionOpts { /** The media transport to connect to. */ - transport: LivekitTransport; + transport: LivekitTransport & { useMatrix2: boolean }; /** The Matrix client to use for OpenID and SFU config requests. */ client: OpenIDClientParts; /** The observable scope to use for this connection. */ @@ -88,7 +89,7 @@ export class Connection { /** * The media transport to connect to. */ - public readonly transport: LivekitTransport; + public readonly transport: LivekitTransport & { useMatrix2: boolean }; public readonly livekitRoom: LivekitRoom; @@ -189,9 +190,18 @@ export class Connection { protected async getSFUConfigWithOpenID(): Promise { return await getSFUConfigWithOpenID( this.client, + this.ownMembershipIdentity, this.transport.livekit_service_url, this.transport.livekit_alias, + this.transport.useMatrix2, ); + // client: OpenIDClientParts, + // membership: CallMembershipIdentityParts, + // serviceUrl: string, + // livekitRoomAlias: string, + // matrix2jwt: boolean, + // delayEndpointBaseUrl?: string, + // delayId?: string, } /** @@ -220,7 +230,11 @@ export class Connection { * * @param logger */ - public constructor(opts: ConnectionOpts, logger: Logger) { + public constructor( + opts: ConnectionOpts, + logger: Logger, + private ownMembershipIdentity: CallMembershipIdentityParts, + ) { this.logger = logger.getChild("[Connection]"); this.logger.info( `[Connection] Creating new connection to ${opts.transport.livekit_service_url} ${opts.transport.livekit_alias}`, diff --git a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts index 7c3a9eab..82a1a78a 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts @@ -5,7 +5,6 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { Room as LivekitRoom, type RoomOptions, @@ -15,6 +14,7 @@ import { } from "livekit-client"; import { type Logger } from "matrix-js-sdk/lib/logger"; import E2EEWorker from "livekit-client/e2ee-worker?worker"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; import { type ObservableScope } from "../../ObservableScope.ts"; import { Connection } from "./Connection.ts"; @@ -23,13 +23,15 @@ import type { MediaDevices } from "../../MediaDevices.ts"; import type { Behavior } from "../../Behavior.ts"; import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx"; import { defaultLiveKitOptions } from "../../../livekit/options.ts"; +import { type LivekitTransportWithVersion } from "./ConnectionManager.ts"; // TODO evaluate if this should be done like the Publisher Factory export interface ConnectionFactory { createConnection( - transport: LivekitTransport, + transport: LivekitTransportWithVersion, scope: ObservableScope, logger: Logger, + ownMembershipIdentity: CallMembershipIdentityParts, ): Connection; } @@ -77,10 +79,19 @@ export class ECConnectionFactory implements ConnectionFactory { this.livekitRoomFactory = livekitRoomFactory ?? defaultFactory; } + /** + * + * @param transport + * @param scope + * @param logger + * @param ownMembershipIdentity required to connect (using the jwt service) with the SFU. + * @returns + */ public createConnection( - transport: LivekitTransport, + transport: LivekitTransportWithVersion, scope: ObservableScope, logger: Logger, + ownMembershipIdentity: CallMembershipIdentityParts, ): Connection { return new Connection( { @@ -90,6 +101,7 @@ export class ECConnectionFactory implements ConnectionFactory { livekitRoomFactory: this.livekitRoomFactory, }, logger, + ownMembershipIdentity, ); } } diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts index 70bfb4de..4ab91646 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts @@ -14,26 +14,29 @@ import { logger } from "matrix-js-sdk/lib/logger"; import { Epoch, mapEpoch, ObservableScope } from "../../ObservableScope.ts"; import { createConnectionManager$, + type LivekitTransportWithVersion, type ConnectionManagerData, } from "./ConnectionManager.ts"; import { type ConnectionFactory } from "./ConnectionFactory.ts"; import { type Connection } from "./Connection.ts"; -import { withTestScheduler } from "../../../utils/test.ts"; +import { ownMemberMock, withTestScheduler } from "../../../utils/test.ts"; import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts"; import { type Behavior } from "../../Behavior.ts"; // Some test constants -const TRANSPORT_1: LivekitTransport = { +const TRANSPORT_1: LivekitTransportWithVersion = { type: "livekit", livekit_service_url: "https://lk.example.org", livekit_alias: "!alias:example.org", + useMatrix2: false, }; -const TRANSPORT_2: LivekitTransport = { +const TRANSPORT_2: LivekitTransportWithVersion = { type: "livekit", livekit_service_url: "https://lk.sample.com", livekit_alias: "!alias:sample.com", + useMatrix2: false, }; let fakeConnectionFactory: ConnectionFactory; @@ -80,6 +83,7 @@ describe("connections$ stream", () => { a: new Epoch([TRANSPORT_1, TRANSPORT_2], 0), }), logger: logger, + ownMembershipIdentity: ownMemberMock, }); expectObservable( @@ -124,6 +128,7 @@ describe("connections$ stream", () => { f: new Epoch([TRANSPORT_1, TRANSPORT_2], 5), }), logger: logger, + ownMembershipIdentity: ownMemberMock, }); expectObservable( @@ -166,6 +171,7 @@ describe("connections$ stream", () => { c: new Epoch([TRANSPORT_1], 2), }), logger: logger, + ownMembershipIdentity: ownMemberMock, }); expectObservable( @@ -279,6 +285,7 @@ describe("connectionManagerData$ stream", () => { a: new Epoch([TRANSPORT_1, TRANSPORT_2], 0), }), logger, + ownMembershipIdentity: ownMemberMock, }); expectObservable(connectionManagerData$).toBe("abcd", { diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts index 4303d50a..d5852d84 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts @@ -10,6 +10,7 @@ import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { combineLatest, map, of, switchMap, tap } from "rxjs"; import { type Logger } from "matrix-js-sdk/lib/logger"; import { type RemoteParticipant } from "livekit-client"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; import { type Behavior } from "../../Behavior.ts"; import { type Connection } from "./Connection.ts"; @@ -18,6 +19,10 @@ import { generateItemsWithEpoch } from "../../../utils/observable.ts"; import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts"; import { type ConnectionFactory } from "./ConnectionFactory.ts"; +export type LivekitTransportWithVersion = LivekitTransport & { + useMatrix2: boolean; +}; + export class ConnectionManagerData { private readonly store: Map = new Map(); @@ -59,8 +64,9 @@ export class ConnectionManagerData { interface Props { scope: ObservableScope; connectionFactory: ConnectionFactory; - inputTransports$: Behavior>; + inputTransports$: Behavior>; logger: Logger; + ownMembershipIdentity: CallMembershipIdentityParts; } // TODO - write test for scopes (do we really need to bind scope) @@ -87,6 +93,7 @@ export function createConnectionManager$({ connectionFactory, inputTransports$, logger: parentLogger, + ownMembershipIdentity, }: Props): IConnectionManager { const logger = parentLogger.getChild("[ConnectionManager]"); // TODO logger: only construct one logger from the client and make it compatible via a EC specific sing @@ -119,20 +126,26 @@ export function createConnectionManager$({ function* (transports) { for (const transport of transports) yield { - keys: [transport.livekit_service_url, transport.livekit_alias], + keys: [ + transport.livekit_service_url, + transport.livekit_alias, + transport.useMatrix2, + ], data: undefined, }; }, - (scope, _data$, serviceUrl, alias) => { + (scope, _data$, serviceUrl, alias, useMatrix2) => { logger.debug(`Creating connection to ${serviceUrl} (${alias})`); const connection = connectionFactory.createConnection( { type: "livekit", livekit_service_url: serviceUrl, livekit_alias: alias, + useMatrix2, }, scope, logger, + ownMembershipIdentity, ); // Start the connection immediately // Use connection state to track connection progress @@ -187,12 +200,12 @@ export function createConnectionManager$({ return { connectionManagerData$ }; } -function removeDuplicateTransports( - transports: LivekitTransport[], -): LivekitTransport[] { +function removeDuplicateTransports( + transports: T[], +): T[] { return transports.reduce((acc, transport) => { if (!acc.some((t) => areLivekitTransportsEqual(t, transport))) acc.push(transport); return acc; - }, [] as LivekitTransport[]); + }, [] as T[]); } diff --git a/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts b/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts index 0c439a6b..3c60e776 100644 --- a/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts +++ b/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts @@ -15,7 +15,11 @@ import EventEmitter from "events"; import { ObservableScope } from "../../ObservableScope.ts"; import { ECConnectionFactory } from "./ConnectionFactory.ts"; import type { OpenIDClientParts } from "../../../livekit/openIDSFU.ts"; -import { exampleTransport, mockMediaDevices } from "../../../utils/test.ts"; +import { + exampleTransport, + mockMediaDevices, + ownMemberMock, +} from "../../../utils/test.ts"; import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx"; import { constant } from "../../Behavior"; @@ -72,7 +76,12 @@ describe("ECConnectionFactory - Audio inputs options", () => { echo, noise, ); - ecConnectionFactory.createConnection(exampleTransport, testScope, logger); + ecConnectionFactory.createConnection( + exampleTransport, + testScope, + logger, + ownMemberMock, + ); // Check if Room was constructed with expected options expect(RoomConstructor).toHaveBeenCalledWith( @@ -113,7 +122,12 @@ describe("ECConnectionFactory - ControlledAudioDevice", () => { false, false, ); - ecConnectionFactory.createConnection(exampleTransport, testScope, logger); + ecConnectionFactory.createConnection( + exampleTransport, + testScope, + logger, + ownMemberMock, + ); // Check if Room was constructed with expected options expect(RoomConstructor).toHaveBeenCalledWith( diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts index 0c61ba06..9178c347 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts @@ -176,9 +176,9 @@ export function createMatrixLivekitMembers$({ // TODO add back in the callviewmodel pauseWhen(this.pretendToBeDisconnected$) // TODO add this to the JS-SDK -export function areLivekitTransportsEqual( - t1: LivekitTransport | null, - t2: LivekitTransport | null, +export function areLivekitTransportsEqual( + t1: T | null, + t2: T | null, ): boolean { if (t1 && t2) return t1.livekit_service_url === t2.livekit_service_url; // In case we have different lk rooms in the same SFU (depends on the livekit authorization service) diff --git a/src/state/CallViewModel/remoteMembers/integration.test.ts b/src/state/CallViewModel/remoteMembers/integration.test.ts index 202b3f56..c9e02a7c 100644 --- a/src/state/CallViewModel/remoteMembers/integration.test.ts +++ b/src/state/CallViewModel/remoteMembers/integration.test.ts @@ -24,6 +24,7 @@ import { mockCallMembership, mockComputeLivekitParticipantIdentity$, mockMediaDevices, + ownMemberMock, withTestScheduler, } from "../../../utils/test.ts"; import { type ProcessorState } from "../../../livekit/TrackProcessorContext.tsx"; @@ -128,6 +129,7 @@ test("bob, carl, then bob joining no tracks yet", () => { connectionFactory: ecConnectionFactory, inputTransports$: membershipsAndTransports.transports$, logger: logger, + ownMembershipIdentity: ownMemberMock, }); const matrixLivekitItems$ = createMatrixLivekitMembers$({ diff --git a/src/state/SessionBehaviors.ts b/src/state/SessionBehaviors.ts index e174a1cc..b61d2fe6 100644 --- a/src/state/SessionBehaviors.ts +++ b/src/state/SessionBehaviors.ts @@ -8,7 +8,6 @@ Please see LICENSE in the repository root for full details. import { type CallMembership, isLivekitTransport, - type LivekitTransport, type MatrixRTCSession, MatrixRTCSessionEvent, } from "matrix-js-sdk/lib/matrixrtc"; @@ -21,15 +20,18 @@ import { type ObservableScope, } from "./ObservableScope"; import { type Behavior } from "./Behavior"; +import { type LivekitTransportWithVersion } from "./CallViewModel/remoteMembers/ConnectionManager"; export const membershipsAndTransports$ = ( scope: ObservableScope, memberships$: Behavior>, ): { membershipsWithTransport$: Behavior< - Epoch<{ membership: CallMembership; transport?: LivekitTransport }[]> + Epoch< + { membership: CallMembership; transport?: LivekitTransportWithVersion }[] + > >; - transports$: Behavior>; + transports$: Behavior>; } => { /** * Lists the transports used by ourselves, plus all other MatrixRTC session @@ -47,7 +49,12 @@ export const membershipsAndTransports$ = ( const transport = membership.getTransport(oldestMembership); return { membership, - transport: isLivekitTransport(transport) ? transport : undefined, + transport: isLivekitTransport(transport) + ? { + ...transport, + useMatrix2: membership.kind === "rtc", + } + : undefined, }; }); }), diff --git a/src/utils/test.ts b/src/utils/test.ts index 7d251640..968b7160 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -25,7 +25,6 @@ import { import { CallMembership, type LivekitFocusSelection, - type LivekitTransport, type MatrixRTCSession, MatrixRTCSessionEvent, type MatrixRTCSessionEventHandlerMap, @@ -67,6 +66,7 @@ import { type MediaDevices } from "../state/MediaDevices"; import { type Behavior, constant } from "../state/Behavior"; import { ObservableScope } from "../state/ObservableScope"; import { MuteStates } from "../state/MuteStates"; +import { type LivekitTransportWithVersion } from "../state/CallViewModel/remoteMembers/ConnectionManager"; export function withFakeTimers(continuation: () => void): void { vi.useFakeTimers(); @@ -197,10 +197,11 @@ export function mockEmitter(): EmitterMock { }; } -export const exampleTransport: LivekitTransport = { +export const exampleTransport: LivekitTransportWithVersion = { type: "livekit", livekit_service_url: "https://lk.example.org", livekit_alias: "!alias:example.org", + useMatrix2: false, }; export function mockCallMembership( @@ -256,6 +257,11 @@ export function mockRtcMembership( return cms; } +export const ownMemberMock: CallMembershipIdentityParts = { + userId: "@alice:example.org", + deviceId: "DEVICE", + memberId: "@alice:example.org:DEVICE", +}; // Maybe it'd be good to move this to matrix-js-sdk? Our testing needs are // rather simple, but if one util to mock a member is good enough for us, maybe // it's useful for matrix-js-sdk consumers in general. diff --git a/yarn.lock b/yarn.lock index db1db491..a3211330 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10338,9 +10338,9 @@ __metadata: languageName: node linkType: hard -"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=f5f1b8efb46b3d55a7eebfabb4a61496640b8b00": - version: 39.3.0 - resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=f5f1b8efb46b3d55a7eebfabb4a61496640b8b00" +"matrix-js-sdk@portal:/Users/timo/Projects/matrix-js-sdk::locator=element-call%40workspace%3A.": + version: 0.0.0-use.local + resolution: "matrix-js-sdk@portal:/Users/timo/Projects/matrix-js-sdk::locator=element-call%40workspace%3A." dependencies: "@babel/runtime": "npm:^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm": "npm:^16.0.0" @@ -10356,9 +10356,8 @@ __metadata: sdp-transform: "npm:^3.0.0" unhomoglyph: "npm:^1.0.6" uuid: "npm:13" - checksum: 10c0/9607b0c063c873a24c1a2d05cc7500d60c32556ec82b666ebaae5c5e829faf5bb7639780efddea7211e6b9873098bd53b97656f041e932e8b0de0c208ccabbff languageName: node - linkType: hard + linkType: soft "matrix-widget-api@npm:^1.14.0": version: 1.15.0 From 50f3bf00aee60fde10012c8999599346819f87c0 Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 17 Dec 2025 10:22:46 +0100 Subject: [PATCH 06/46] use correct js-sdk --- package.json | 2 +- yarn.lock | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 35589771..f34fab5f 100644 --- a/package.json +++ b/package.json @@ -109,7 +109,7 @@ "livekit-client": "^2.13.0", "lodash-es": "^4.17.21", "loglevel": "^1.9.1", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=f5f1b8efb46b3d55a7eebfabb4a61496640b8b00", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=9779ac975df2f296958e3c4be254fa46ebd67ea4", "matrix-widget-api": "^1.14.0", "normalize.css": "^8.0.1", "observable-hooks": "^4.2.3", diff --git a/yarn.lock b/yarn.lock index a3211330..b83976a2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7533,7 +7533,7 @@ __metadata: livekit-client: "npm:^2.13.0" lodash-es: "npm:^4.17.21" loglevel: "npm:^1.9.1" - matrix-js-sdk: "github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=f5f1b8efb46b3d55a7eebfabb4a61496640b8b00" + matrix-js-sdk: "github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=9779ac975df2f296958e3c4be254fa46ebd67ea4" matrix-widget-api: "npm:^1.14.0" normalize.css: "npm:^8.0.1" observable-hooks: "npm:^4.2.3" @@ -10338,9 +10338,9 @@ __metadata: languageName: node linkType: hard -"matrix-js-sdk@portal:/Users/timo/Projects/matrix-js-sdk::locator=element-call%40workspace%3A.": - version: 0.0.0-use.local - resolution: "matrix-js-sdk@portal:/Users/timo/Projects/matrix-js-sdk::locator=element-call%40workspace%3A." +"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=9779ac975df2f296958e3c4be254fa46ebd67ea4": + version: 39.3.0 + resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=9779ac975df2f296958e3c4be254fa46ebd67ea4" dependencies: "@babel/runtime": "npm:^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm": "npm:^16.0.0" @@ -10356,8 +10356,9 @@ __metadata: sdp-transform: "npm:^3.0.0" unhomoglyph: "npm:^1.0.6" uuid: "npm:13" + checksum: 10c0/78c27847b58c229513bd28c4c4ad391d8af6722711d3d0f42e93a537d7a827a7233e920936dd8d7005c7893bad17a503c3f62b56ecfed3cf4ae81a5097b4ac21 languageName: node - linkType: soft + linkType: hard "matrix-widget-api@npm:^1.14.0": version: 1.15.0 From 55d18f10fe45cd78c528f46b96db8eecabc7c9ce Mon Sep 17 00:00:00 2001 From: Timo K Date: Fri, 19 Dec 2025 19:23:41 +0100 Subject: [PATCH 07/46] temp refactored membership rtcidentity --- package.json | 2 +- src/e2ee/matrixKeyProvider.ts | 110 +++++++++++------- src/state/CallViewModel/CallViewModel.test.ts | 9 -- .../LivekitParticipantIdentity.ts | 53 --------- .../remoteMembers/MatrixLivekitMembers.ts | 57 ++------- .../remoteMembers/integration.test.ts | 6 - src/utils/test.ts | 3 +- yarn.lock | 8 +- 8 files changed, 87 insertions(+), 161 deletions(-) delete mode 100644 src/state/CallViewModel/remoteMembers/LivekitParticipantIdentity.ts diff --git a/package.json b/package.json index f34fab5f..6c74b84c 100644 --- a/package.json +++ b/package.json @@ -109,7 +109,7 @@ "livekit-client": "^2.13.0", "lodash-es": "^4.17.21", "loglevel": "^1.9.1", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=9779ac975df2f296958e3c4be254fa46ebd67ea4", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=2bb3b03a248e689f7460f4e70d5ffbf10353c725", "matrix-widget-api": "^1.14.0", "normalize.css": "^8.0.1", "observable-hooks": "^4.2.3", diff --git a/src/e2ee/matrixKeyProvider.ts b/src/e2ee/matrixKeyProvider.ts index a499f45c..166fd82c 100644 --- a/src/e2ee/matrixKeyProvider.ts +++ b/src/e2ee/matrixKeyProvider.ts @@ -6,18 +6,13 @@ Please see LICENSE in the repository root for full details. */ import { BaseKeyProvider } from "livekit-client"; -import { logger } from "matrix-js-sdk/lib/logger"; import { type MatrixRTCSession, MatrixRTCSessionEvent, } from "matrix-js-sdk/lib/matrixrtc"; import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; -import { firstValueFrom } from "rxjs"; - -import { - computeLivekitParticipantIdentity$, - livekitIdentityInput, -} from "../state/CallViewModel/remoteMembers/LivekitParticipantIdentity"; +import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; +const logger = rootLogger.getChild("[MatrixKeyProvider]"); export class MatrixKeyProvider extends BaseKeyProvider { private rtcSession?: MatrixRTCSession; @@ -32,6 +27,10 @@ export class MatrixKeyProvider extends BaseKeyProvider { MatrixRTCSessionEvent.EncryptionKeyChanged, this.onEncryptionKeyChanged, ); + this.rtcSession.off( + MatrixRTCSessionEvent.MembershipsChanged, + this.onMembershipsChanged, + ); } this.rtcSession = rtcSession; @@ -40,55 +39,86 @@ export class MatrixKeyProvider extends BaseKeyProvider { MatrixRTCSessionEvent.EncryptionKeyChanged, this.onEncryptionKeyChanged, ); + this.rtcSession.on( + MatrixRTCSessionEvent.MembershipsChanged, + this.onMembershipsChanged, + ); // The new session could be aware of keys of which the old session wasn't, // so emit key changed events this.rtcSession.reemitEncryptionKeys(); } + private keyCache = new Array<{ + membership: CallMembershipIdentityParts; + encryptionKey: Uint8Array; + encryptionKeyIndex: number; + }>(); + + private onMembershipsChanged = (): void => { + const duplicatedArray = this.keyCache; + // Reset key cache first. It will get repopulated when calling `onEncryptionKeyChanged` + this.keyCache = []; + let next = duplicatedArray.pop(); + while (next !== undefined) { + logger.debug( + "[KeyCache] remove key event from the cache and try adding it again. For membership: ", + next.membership, + ); + this.onEncryptionKeyChanged( + next.encryptionKey, + next.encryptionKeyIndex, + next.membership, + ); + next = duplicatedArray.pop(); + } + }; + private onEncryptionKeyChanged = ( encryptionKey: Uint8Array, encryptionKeyIndex: number, membership: CallMembershipIdentityParts, ): void => { - const unhashedIdentity = livekitIdentityInput(membership); - // This is the only way we can get the kind of the membership event we just received the key for. // best case we want to recompute this once the memberships change (you can receive the key before the participant...) - // - // TODO change this to `?? "rtc"` for newer versions. - const kind = - this.rtcSession?.memberships.find( - (m) => - m.userId === membership.userId && - m.deviceId === membership.deviceId && - m.memberId === membership.memberId, - )?.kind ?? "session"; + const membershipFull = this.rtcSession?.memberships.find( + (m) => + m.userId === membership.userId && + m.deviceId === membership.deviceId && + m.memberId === membership.memberId, + ); + if (!membershipFull) { + logger.debug( + "[KeyCache] Added key event to the cache because we do not have a membership for it (yet): ", + membership, + ); + this.keyCache.push({ membership, encryptionKey, encryptionKeyIndex }); + return; + } - Promise.all([ - crypto.subtle.importKey("raw", encryptionKey, "HKDF", false, [ + crypto.subtle + .importKey("raw", encryptionKey, "HKDF", false, [ "deriveBits", "deriveKey", - ]), - firstValueFrom(computeLivekitParticipantIdentity$(membership, kind)), - ]).then( - ([keyMaterial, livekitParticipantId]) => { - this.onSetEncryptionKey( - keyMaterial, - livekitParticipantId, - encryptionKeyIndex, - ); + ]) + .then( + (keyMaterial) => { + this.onSetEncryptionKey( + keyMaterial, + membershipFull.rtcBackendIdentity, + encryptionKeyIndex, + ); - logger.debug( - `Sent new key to livekit room=${this.rtcSession?.room.roomId} participantId=${livekitParticipantId} (before hash: ${unhashedIdentity}) encryptionKeyIndex=${encryptionKeyIndex}`, - ); - }, - (e) => { - logger.error( - `Failed to create key material from buffer for livekit room=${this.rtcSession?.room.roomId} participantId before hash=${unhashedIdentity} encryptionKeyIndex=${encryptionKeyIndex}`, - e, - ); - }, - ); + logger.debug( + `Sent new key to livekit room=${this.rtcSession?.room.roomId} participantId=${membershipFull.rtcBackendIdentity} (before hash: ${membershipFull.userId}) encryptionKeyIndex=${encryptionKeyIndex}`, + ); + }, + (e) => { + logger.error( + `Failed to create key material from buffer for livekit room=${this.rtcSession?.room.roomId} participantId before hash=${membershipFull.userId} encryptionKeyIndex=${encryptionKeyIndex}`, + e, + ); + }, + ); }; } diff --git a/src/state/CallViewModel/CallViewModel.test.ts b/src/state/CallViewModel/CallViewModel.test.ts index 505cb19f..6e3837c4 100644 --- a/src/state/CallViewModel/CallViewModel.test.ts +++ b/src/state/CallViewModel/CallViewModel.test.ts @@ -44,7 +44,6 @@ import { mockRtcMembership, testScope, exampleTransport, - mockComputeLivekitParticipantIdentity$, } from "../../utils/test.ts"; import { E2eeType } from "../../e2ee/e2eeType.ts"; import { @@ -88,14 +87,6 @@ vi.mock( }), ); -vi.mock( - import("./remoteMembers/LivekitParticipantIdentity.ts"), - async (importOriginal) => ({ - ...(await importOriginal()), - computeLivekitParticipantIdentity$: mockComputeLivekitParticipantIdentity$, - }), -); - const yesNo = { y: true, n: false, diff --git a/src/state/CallViewModel/remoteMembers/LivekitParticipantIdentity.ts b/src/state/CallViewModel/remoteMembers/LivekitParticipantIdentity.ts deleted file mode 100644 index e43d0bd1..00000000 --- a/src/state/CallViewModel/remoteMembers/LivekitParticipantIdentity.ts +++ /dev/null @@ -1,53 +0,0 @@ -/* -Copyright 2025 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { encodeUnpaddedBase64Url } from "matrix-js-sdk"; -import { sha256 } from "matrix-js-sdk/lib/digest"; -import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; -import { from, type Observable } from "rxjs"; - -const livekitParticipantIdentityCache = new Map(); - -/** - * The string that is computed based on the membership and used for the computing the hash. - * `${userId}:${deviceId}:${membershipID}` - * as the direct imput for: await sha256(input) - */ -export const livekitIdentityInput = ({ - userId, - deviceId, - memberId, -}: CallMembershipIdentityParts): string => `${userId}|${deviceId}|${memberId}`; - -export function computeLivekitParticipantIdentity$( - membership: CallMembershipIdentityParts, - kind: "rtc" | "session", -): Observable { - const compute = async (): Promise => { - switch (kind) { - case "rtc": { - const input = livekitIdentityInput(membership); - if (livekitParticipantIdentityCache.size > 400) - // prevent memory leaks in a stupid/simple way - livekitParticipantIdentityCache.clear(); - // TODO use non deprecated memberId - if (livekitParticipantIdentityCache.has(input)) - return livekitParticipantIdentityCache.get(input)!; - else { - const hashBuffer = await sha256(input); - const hashedString = encodeUnpaddedBase64Url(hashBuffer); - livekitParticipantIdentityCache.set(input, hashedString); - return hashedString; - } - } - case "session": - default: - return `${membership.userId}:${membership.deviceId}`; - } - }; - return from(compute()); -} diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts index 9178c347..a5d6b2f6 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts @@ -10,7 +10,7 @@ import { type LivekitTransport, type CallMembership, } from "matrix-js-sdk/lib/matrixrtc"; -import { combineLatest, filter, map, switchMap } from "rxjs"; +import { combineLatest, filter, map } from "rxjs"; import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { type Behavior } from "../../Behavior"; @@ -18,7 +18,6 @@ import { type IConnectionManager } from "./ConnectionManager"; import { Epoch, type ObservableScope } from "../../ObservableScope"; import { type Connection } from "./Connection"; import { generateItemsWithEpoch } from "../../../utils/observable"; -import { computeLivekitParticipantIdentity$ } from "./LivekitParticipantIdentity"; const logger = rootLogger.getChild("[MatrixLivekitMembers]"); @@ -82,45 +81,12 @@ export function createMatrixLivekitMembers$({ membershipsWithTransport$, connectionManager, }: Props): Behavior> { - /** - * This internal observable is used to compute the async sha256 hash of the user's identity. - * a promise is treated like an observable. So we can switchMap on the promise from the identity computation. - * The last update to `membershipsWithTransport$` will always be the last promise we pass to switchMap. - * So we will eventually always end up with the latest memberships and their identities. - */ - const membershipsWithTransportAndLivekitIdentity$ = - membershipsWithTransport$.pipe( - switchMap((membershipsWithTransport) => { - const { value, epoch } = membershipsWithTransport; - const membershipsWithTransportAndLkIdentityPromises = value.map( - (obj) => { - return computeLivekitParticipantIdentity$( - obj.membership, - obj.membership.kind, - ); - }, - ); - return combineLatest( - membershipsWithTransportAndLkIdentityPromises, - ).pipe( - map((identities) => { - const membershipsWithTransportAndLkIdentity = value.map( - ({ transport, membership }, index) => { - return { transport, membership, identity: identities[index] }; - }, - ); - return new Epoch(membershipsWithTransportAndLkIdentity, epoch); - }), - ); - }), - ); - /** * Stream of all the call members and their associated livekit data (if available). */ return scope.behavior( combineLatest([ - membershipsWithTransportAndLivekitIdentity$, + membershipsWithTransport$, connectionManager.connectionManagerData$, ]).pipe( filter((values) => @@ -131,37 +97,34 @@ export function createMatrixLivekitMembers$({ // Generator function. // creates an array of `{key, data}[]` // Each change in the keys (new key, missing key) will result in a call to the factory function. - function* ([membershipsWithTransportAndLivekitIdentity, managerData]) { - for (const { - membership, - transport, - identity, - } of membershipsWithTransportAndLivekitIdentity) { + function* ([membershipsWithTransport, managerData]) { + for (const { membership, transport } of membershipsWithTransport) { const participants = transport ? managerData.getParticipantForTransport(transport) : []; const participant = - participants.find((p) => p.identity == identity) ?? null; + participants.find( + (p) => p.identity == membership.rtcBackendIdentity, + ) ?? null; const connection = transport ? managerData.getConnectionForTransport(transport) : null; yield { - keys: [identity, membership.userId, membership.deviceId], + keys: [membership.userId, membership.deviceId], data: { membership, participant, connection }, }; } }, // Each update where the key of the generator array do not change will result in updates to the `data$` observable in the factory. - (scope, data$, identity, userId, deviceId) => { + (scope, data$, userId, deviceId) => { logger.debug( - `Generating member for livekitIdentity: ${identity}, userId:deviceId: ${userId}${deviceId}`, + `Generating member for livekitIdentity: ${data$.value.membership.rtcBackendIdentity}, userId:deviceId: ${userId}${deviceId}`, ); const { participant$, ...rest } = scope.splitBehavior(data$); // will only get called once per `participantId, userId` pair. // updates to data$ and as a result to displayName$ and mxcAvatarUrl$ are more frequent. return { - identity, userId, participant: { type: "remote" as const, value$: participant$ }, ...rest, diff --git a/src/state/CallViewModel/remoteMembers/integration.test.ts b/src/state/CallViewModel/remoteMembers/integration.test.ts index c9e02a7c..00062c60 100644 --- a/src/state/CallViewModel/remoteMembers/integration.test.ts +++ b/src/state/CallViewModel/remoteMembers/integration.test.ts @@ -22,7 +22,6 @@ import { ECConnectionFactory } from "./ConnectionFactory.ts"; import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts"; import { mockCallMembership, - mockComputeLivekitParticipantIdentity$, mockMediaDevices, ownMemberMock, withTestScheduler, @@ -45,11 +44,6 @@ let lkRoomFactory: () => LivekitRoom; const createdMockLivekitRooms: Map = new Map(); -vi.mock(import("./LivekitParticipantIdentity.ts"), async (importOriginal) => ({ - ...(await importOriginal()), - computeLivekitParticipantIdentity$: mockComputeLivekitParticipantIdentity$, -})); - beforeEach(() => { testScope = new ObservableScope(); mockClient = { diff --git a/src/utils/test.ts b/src/utils/test.ts index 968b7160..56148b32 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -252,7 +252,8 @@ export function mockRtcMembership( content: data, }); - const cms = new CallMembership(event, data); + const membershipData = CallMembership.membershipDataFromMatrixEvent(event); + const cms = new CallMembership(event, membershipData, "xx"); vi.mocked(cms).getTransport = vi.fn().mockReturnValue(fociPreferred[0]); return cms; } diff --git a/yarn.lock b/yarn.lock index b83976a2..c4b2e957 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7533,7 +7533,7 @@ __metadata: livekit-client: "npm:^2.13.0" lodash-es: "npm:^4.17.21" loglevel: "npm:^1.9.1" - matrix-js-sdk: "github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=9779ac975df2f296958e3c4be254fa46ebd67ea4" + matrix-js-sdk: "github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=2bb3b03a248e689f7460f4e70d5ffbf10353c725" matrix-widget-api: "npm:^1.14.0" normalize.css: "npm:^8.0.1" observable-hooks: "npm:^4.2.3" @@ -10338,9 +10338,9 @@ __metadata: languageName: node linkType: hard -"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=9779ac975df2f296958e3c4be254fa46ebd67ea4": +"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=2bb3b03a248e689f7460f4e70d5ffbf10353c725": version: 39.3.0 - resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=9779ac975df2f296958e3c4be254fa46ebd67ea4" + resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=2bb3b03a248e689f7460f4e70d5ffbf10353c725" dependencies: "@babel/runtime": "npm:^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm": "npm:^16.0.0" @@ -10356,7 +10356,7 @@ __metadata: sdp-transform: "npm:^3.0.0" unhomoglyph: "npm:^1.0.6" uuid: "npm:13" - checksum: 10c0/78c27847b58c229513bd28c4c4ad391d8af6722711d3d0f42e93a537d7a827a7233e920936dd8d7005c7893bad17a503c3f62b56ecfed3cf4ae81a5097b4ac21 + checksum: 10c0/2e7061f6e648c91aaeb30b3e01626d855e24efcb330bbe432fcba199bd46b0b0d998cbc545748e1c72a7b643d25581f988fcad9bbaa42912a6ec96a27c41d0de languageName: node linkType: hard From 3ba2d13e2771dae0ca1e4687b690c6661f669f9a Mon Sep 17 00:00:00 2001 From: Timo K Date: Sun, 28 Dec 2025 21:04:46 +0100 Subject: [PATCH 08/46] use the js-sdk where the hashed rtcSessionIdeintity is already part of the event (no need to compute it in the encryption manager) --- package.json | 2 +- src/e2ee/matrixKeyProvider.ts | 55 ++--------------------------------- yarn.lock | 10 +++---- 3 files changed, 8 insertions(+), 59 deletions(-) diff --git a/package.json b/package.json index 6c74b84c..2611587a 100644 --- a/package.json +++ b/package.json @@ -109,7 +109,7 @@ "livekit-client": "^2.13.0", "lodash-es": "^4.17.21", "loglevel": "^1.9.1", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=2bb3b03a248e689f7460f4e70d5ffbf10353c725", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=8cea2c05839ebcaa90945504a453b9b1e1092fc4", "matrix-widget-api": "^1.14.0", "normalize.css": "^8.0.1", "observable-hooks": "^4.2.3", diff --git a/src/e2ee/matrixKeyProvider.ts b/src/e2ee/matrixKeyProvider.ts index 166fd82c..962c9bc6 100644 --- a/src/e2ee/matrixKeyProvider.ts +++ b/src/e2ee/matrixKeyProvider.ts @@ -10,7 +10,6 @@ import { type MatrixRTCSession, MatrixRTCSessionEvent, } from "matrix-js-sdk/lib/matrixrtc"; -import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; const logger = rootLogger.getChild("[MatrixKeyProvider]"); @@ -27,10 +26,6 @@ export class MatrixKeyProvider extends BaseKeyProvider { MatrixRTCSessionEvent.EncryptionKeyChanged, this.onEncryptionKeyChanged, ); - this.rtcSession.off( - MatrixRTCSessionEvent.MembershipsChanged, - this.onMembershipsChanged, - ); } this.rtcSession = rtcSession; @@ -39,63 +34,17 @@ export class MatrixKeyProvider extends BaseKeyProvider { MatrixRTCSessionEvent.EncryptionKeyChanged, this.onEncryptionKeyChanged, ); - this.rtcSession.on( - MatrixRTCSessionEvent.MembershipsChanged, - this.onMembershipsChanged, - ); // The new session could be aware of keys of which the old session wasn't, // so emit key changed events this.rtcSession.reemitEncryptionKeys(); } - private keyCache = new Array<{ - membership: CallMembershipIdentityParts; - encryptionKey: Uint8Array; - encryptionKeyIndex: number; - }>(); - - private onMembershipsChanged = (): void => { - const duplicatedArray = this.keyCache; - // Reset key cache first. It will get repopulated when calling `onEncryptionKeyChanged` - this.keyCache = []; - let next = duplicatedArray.pop(); - while (next !== undefined) { - logger.debug( - "[KeyCache] remove key event from the cache and try adding it again. For membership: ", - next.membership, - ); - this.onEncryptionKeyChanged( - next.encryptionKey, - next.encryptionKeyIndex, - next.membership, - ); - next = duplicatedArray.pop(); - } - }; - private onEncryptionKeyChanged = ( encryptionKey: Uint8Array, encryptionKeyIndex: number, - membership: CallMembershipIdentityParts, + rtcBackendIdentity: string, ): void => { - // This is the only way we can get the kind of the membership event we just received the key for. - // best case we want to recompute this once the memberships change (you can receive the key before the participant...) - const membershipFull = this.rtcSession?.memberships.find( - (m) => - m.userId === membership.userId && - m.deviceId === membership.deviceId && - m.memberId === membership.memberId, - ); - if (!membershipFull) { - logger.debug( - "[KeyCache] Added key event to the cache because we do not have a membership for it (yet): ", - membership, - ); - this.keyCache.push({ membership, encryptionKey, encryptionKeyIndex }); - return; - } - crypto.subtle .importKey("raw", encryptionKey, "HKDF", false, [ "deriveBits", @@ -105,7 +54,7 @@ export class MatrixKeyProvider extends BaseKeyProvider { (keyMaterial) => { this.onSetEncryptionKey( keyMaterial, - membershipFull.rtcBackendIdentity, + rtcBackendIdentity, encryptionKeyIndex, ); diff --git a/yarn.lock b/yarn.lock index c4b2e957..0ddaad61 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7533,7 +7533,7 @@ __metadata: livekit-client: "npm:^2.13.0" lodash-es: "npm:^4.17.21" loglevel: "npm:^1.9.1" - matrix-js-sdk: "github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=2bb3b03a248e689f7460f4e70d5ffbf10353c725" + matrix-js-sdk: "github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=8cea2c05839ebcaa90945504a453b9b1e1092fc4" matrix-widget-api: "npm:^1.14.0" normalize.css: "npm:^8.0.1" observable-hooks: "npm:^4.2.3" @@ -10338,9 +10338,9 @@ __metadata: languageName: node linkType: hard -"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=2bb3b03a248e689f7460f4e70d5ffbf10353c725": - version: 39.3.0 - resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=2bb3b03a248e689f7460f4e70d5ffbf10353c725" +"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=8cea2c05839ebcaa90945504a453b9b1e1092fc4": + version: 39.4.0 + resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=8cea2c05839ebcaa90945504a453b9b1e1092fc4" dependencies: "@babel/runtime": "npm:^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm": "npm:^16.0.0" @@ -10356,7 +10356,7 @@ __metadata: sdp-transform: "npm:^3.0.0" unhomoglyph: "npm:^1.0.6" uuid: "npm:13" - checksum: 10c0/2e7061f6e648c91aaeb30b3e01626d855e24efcb330bbe432fcba199bd46b0b0d998cbc545748e1c72a7b643d25581f988fcad9bbaa42912a6ec96a27c41d0de + checksum: 10c0/2375dd3d9191f78fe589b0d3170f3da7792ed469a81d3ba3cd12f4915fd33a859f8af3491edb9cf0cdaa1f881a3ea7c1bf7539e850ad0360ec9981271f462c81 languageName: node linkType: hard From 0f5c5d8be56d0f3bd522516cb7e693d4c879033a Mon Sep 17 00:00:00 2001 From: Timo K Date: Mon, 29 Dec 2025 17:38:54 +0100 Subject: [PATCH 09/46] cleanup based on new js-sdk impl --- package.json | 2 +- src/e2ee/matrixKeyProvider.ts | 6 +- src/livekit/openIDSFU.ts | 37 +++++++--- src/room/CallEventAudioRenderer.test.tsx | 8 --- src/room/InCallView.test.tsx | 8 --- src/settings/settings.ts | 6 ++ src/state/CallViewModel/CallViewModel.ts | 40 +++++------ .../localMember/LocalTransport.test.ts | 8 +-- .../localMember/LocalTransport.ts | 65 +++++++++-------- .../remoteMembers/Connection.test.ts | 4 +- .../CallViewModel/remoteMembers/Connection.ts | 23 +++--- .../remoteMembers/ConnectionFactory.ts | 9 ++- .../remoteMembers/ConnectionManager.test.ts | 21 +++--- .../remoteMembers/ConnectionManager.ts | 72 +++++++++++++------ .../remoteMembers/integration.test.ts | 10 ++- src/state/SessionBehaviors.ts | 15 ++-- src/utils/test.ts | 5 +- yarn.lock | 8 +-- 18 files changed, 191 insertions(+), 156 deletions(-) diff --git a/package.json b/package.json index 2611587a..346c12cf 100644 --- a/package.json +++ b/package.json @@ -109,7 +109,7 @@ "livekit-client": "^2.13.0", "lodash-es": "^4.17.21", "loglevel": "^1.9.1", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=8cea2c05839ebcaa90945504a453b9b1e1092fc4", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=6f0815930a008eff8f86e6e5748d447be0e7c25e", "matrix-widget-api": "^1.14.0", "normalize.css": "^8.0.1", "observable-hooks": "^4.2.3", diff --git a/src/e2ee/matrixKeyProvider.ts b/src/e2ee/matrixKeyProvider.ts index 962c9bc6..a9b0865f 100644 --- a/src/e2ee/matrixKeyProvider.ts +++ b/src/e2ee/matrixKeyProvider.ts @@ -11,6 +11,7 @@ import { MatrixRTCSessionEvent, } from "matrix-js-sdk/lib/matrixrtc"; import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; const logger = rootLogger.getChild("[MatrixKeyProvider]"); export class MatrixKeyProvider extends BaseKeyProvider { @@ -43,6 +44,7 @@ export class MatrixKeyProvider extends BaseKeyProvider { private onEncryptionKeyChanged = ( encryptionKey: Uint8Array, encryptionKeyIndex: number, + membershipParts: CallMembershipIdentityParts, rtcBackendIdentity: string, ): void => { crypto.subtle @@ -59,12 +61,12 @@ export class MatrixKeyProvider extends BaseKeyProvider { ); logger.debug( - `Sent new key to livekit room=${this.rtcSession?.room.roomId} participantId=${membershipFull.rtcBackendIdentity} (before hash: ${membershipFull.userId}) encryptionKeyIndex=${encryptionKeyIndex}`, + `Sent new key to livekit room=${this.rtcSession?.room.roomId} participantId=${rtcBackendIdentity} (before hash: ${membershipParts.userId}) encryptionKeyIndex=${encryptionKeyIndex}`, ); }, (e) => { logger.error( - `Failed to create key material from buffer for livekit room=${this.rtcSession?.room.roomId} participantId before hash=${membershipFull.userId} encryptionKeyIndex=${encryptionKeyIndex}`, + `Failed to create key material from buffer for livekit room=${this.rtcSession?.room.roomId} participantId before hash=${membershipParts.userId} encryptionKeyIndex=${encryptionKeyIndex}`, e, ); }, diff --git a/src/livekit/openIDSFU.ts b/src/livekit/openIDSFU.ts index f07bb035..aaf07615 100644 --- a/src/livekit/openIDSFU.ts +++ b/src/livekit/openIDSFU.ts @@ -6,8 +6,8 @@ Please see LICENSE in the repository root for full details. */ import { type IOpenIDToken, type MatrixClient } from "matrix-js-sdk"; -import { logger } from "matrix-js-sdk/lib/logger"; import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; +import { type Logger } from "matrix-js-sdk/lib/logger"; import { FailToGetOpenIdToken } from "../utils/errors"; import { doNetworkOperationWithRetry } from "../utils/matrix"; @@ -28,8 +28,17 @@ export type OpenIDClientParts = Pick< * to the matrix RTC backend in order to get acces to the SFU. * It has built-in retry for calls to the homeserver with a backoff policy. * @param client + * @param membership * @param serviceUrl - * @param matrixRoomId + * @param forceOldEndpoint This will use the old jwt endpoint which will create the rtc backend identity based on string concatination + * instead of a hash. + * This function by default uses whatever is possible with the current jwt service installed next to the SFU. + * For remote connections this does not matter, since we will not publish there we can rely on the newest option. + * For our own connection we can only use the hashed version if we also send the new matrix2.0 sticky events. + * @param livekitRoomAlias + * @param delayEndpointBaseUrl + * @param delayId + * @param logger * @returns Object containing the token information * @throws FailToGetOpenIdToken */ @@ -37,10 +46,11 @@ export async function getSFUConfigWithOpenID( client: OpenIDClientParts, membership: CallMembershipIdentityParts, serviceUrl: string, + forceOldJwtEndpoint: boolean, livekitRoomAlias: string, - matrix2jwt: boolean, delayEndpointBaseUrl?: string, delayId?: string, + logger?: Logger, ): Promise { let openIdToken: IOpenIDToken; try { @@ -52,26 +62,35 @@ export async function getSFUConfigWithOpenID( error instanceof Error ? error : new Error("Unknown error"), ); } - logger.debug("Got openID token", openIdToken); + logger?.debug("Got openID token", openIdToken); - logger.info(`Trying to get JWT for focus ${serviceUrl}...`); + logger?.info(`Trying to get JWT for focus ${serviceUrl}...`); const args: [CallMembershipIdentityParts, string, string, IOpenIDToken] = [ membership, serviceUrl, livekitRoomAlias, openIdToken, ]; - if (matrix2jwt) { + try { + // we do not want to try the old endpoint, since we are not sending the new matrix2.0 sticky events (no hashed identity in the event) + if (forceOldJwtEndpoint) throw new Error("Force old jwt endpoint"); + if (!delayId) + throw new Error("No delayId, Won't try matrix 2.0 jwt endpoint."); + const sfuConfig = await getLiveKitJWTWithDelayDelegation( ...args, delayEndpointBaseUrl, delayId, ); - logger.info(`Got JWT from call's active focus URL.`); + logger?.info(`Got JWT from call's active focus URL.`); return sfuConfig; - } else { + } catch (e) { + logger?.warn( + `Failed fetching jwt with matrix 2.0 endpoint (retry with legacy)`, + e, + ); const sfuConfig = await getLiveKitJWT(...args); - logger.info(`Got JWT from call's active focus URL.`); + logger?.info(`Got JWT from call's active focus URL.`); return sfuConfig; } } diff --git a/src/room/CallEventAudioRenderer.test.tsx b/src/room/CallEventAudioRenderer.test.tsx index 38f56b14..733346eb 100644 --- a/src/room/CallEventAudioRenderer.test.tsx +++ b/src/room/CallEventAudioRenderer.test.tsx @@ -23,7 +23,6 @@ import { import { exampleTransport, - mockComputeLivekitParticipantIdentity$, mockMatrixRoomMember, mockRtcMembership, } from "../utils/test"; @@ -48,13 +47,6 @@ vitest.mock("../rtcSessionHelpers", async (importOriginal) => ({ ...(await importOriginal()), makeTransport: (): [LivekitTransport] => [exampleTransport], })); -vitest.mock( - import("../state/CallViewModel/remoteMembers/LivekitParticipantIdentity.ts"), - async (importOriginal) => ({ - ...(await importOriginal()), - computeLivekitParticipantIdentity$: mockComputeLivekitParticipantIdentity$, - }), -); afterEach(() => { vitest.clearAllMocks(); diff --git a/src/room/InCallView.test.tsx b/src/room/InCallView.test.tsx index cd0af547..8ac4bccb 100644 --- a/src/room/InCallView.test.tsx +++ b/src/room/InCallView.test.tsx @@ -23,7 +23,6 @@ import { TooltipProvider } from "@vector-im/compound-web"; import { RoomContext, useLocalParticipant } from "@livekit/components-react"; import { - mockComputeLivekitParticipantIdentity$, mockLivekitRoom, mockLocalParticipant, mockMatrixRoom, @@ -62,13 +61,6 @@ vi.mock("../livekit/MatrixAudioRenderer"); vi.mock("react-use-measure", () => ({ default: (): [() => void, object] => [(): void => {}, {}], })); -vi.mock( - import("../state/CallViewModel/remoteMembers/LivekitParticipantIdentity.ts"), - async (importOriginal) => ({ - ...(await importOriginal()), - computeLivekitParticipantIdentity$: mockComputeLivekitParticipantIdentity$, - }), -); const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC"); const localParticipant = mockLocalParticipant({ diff --git a/src/settings/settings.ts b/src/settings/settings.ts index f85e1414..33408fd9 100644 --- a/src/settings/settings.ts +++ b/src/settings/settings.ts @@ -127,6 +127,12 @@ export const alwaysShowIphoneEarpiece = new Setting( export enum MatrixRTCMode { Legacy = "legacy", Compatibil = "compatibil", + /** This implies using + * - sticky events + * - hashed RTC backend identity + * - the new endpoint for the jwt token on the local membership (remote memberships will always try the new jwt endpoint first -> then the legacy one) + * - use the hashed identity for the local membership + */ Matrix_2_0 = "matrix_2_0", } diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 23c58268..922a390e 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -415,17 +415,18 @@ export function createCallViewModel$( const ownMembershipIdentity: CallMembershipIdentityParts = { userId, deviceId, + // TODO look into this!!! memberId: `${userId}:${deviceId}`, }; + const useOldJwtEndpoint$ = scope.behavior( + options.matrixRTCMode$.pipe(map((v) => v !== MatrixRTCMode.Matrix_2_0)), + ); const localTransport$ = createLocalTransport$({ scope: scope, memberships$: memberships$, ownMembershipIdentity, client, - useMatrix2$: scope.behavior( - options.matrixRTCMode$.pipe(map((v) => v === MatrixRTCMode.Matrix_2_0)), - ), delayId$: scope.behavior( ( fromEvent( @@ -436,6 +437,7 @@ export function createCallViewModel$( matrixRTCSession.delayId ?? null, ), roomId: matrixRoom.roomId, + useOldJwtEndpoint$, useOldestMember$: scope.behavior( options.matrixRTCMode$.pipe(map((v) => v === MatrixRTCMode.Legacy)), ), @@ -455,29 +457,19 @@ export function createCallViewModel$( const connectionManager = createConnectionManager$({ scope: scope, connectionFactory: connectionFactory, - inputTransports$: scope.behavior( - combineLatest( - [ - localTransport$.pipe( - catchError((e: unknown) => { - logger.info( - "dont pass local transport to createConnectionManager$. localTransport$ threw an error", - e, - ); - return of(null); - }), - ), - membershipsAndTransports.transports$, - ], - (localTransport, transports) => { - const localTransportAsArray = localTransport ? [localTransport] : []; - return transports.mapInner((transports) => [ - ...localTransportAsArray, - ...transports, - ]); - }, + localTransport$: scope.behavior( + localTransport$.pipe( + catchError((e: unknown) => { + logger.info( + "could not pass local transport to createConnectionManager$. localTransport$ threw an error", + e, + ); + return of(null); + }), ), ), + remoteTransports$: membershipsAndTransports.transports$, + forceOldJwtEndpointForLocalTransport$: useOldJwtEndpoint$, logger: logger, ownMembershipIdentity, }); diff --git a/src/state/CallViewModel/localMember/LocalTransport.test.ts b/src/state/CallViewModel/localMember/LocalTransport.test.ts index ba030757..e7df6e33 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.test.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.test.ts @@ -38,7 +38,7 @@ describe("LocalTransport", () => { getDeviceId: vi.fn(), }, ownMembershipIdentity: ownMemberMock, - useMatrix2$: constant(false), + useOldJwtEndpoint$: constant(false), delayId$: constant("delay_id_mock"), }); await flushPromises(); @@ -76,7 +76,7 @@ describe("LocalTransport", () => { getDeviceId: vi.fn(), }, ownMembershipIdentity: ownMemberMock, - useMatrix2$: constant(false), + useOldJwtEndpoint$: constant(false), delayId$: constant("delay_id_mock"), }); localTransport$.subscribe( @@ -116,7 +116,7 @@ describe("LocalTransport", () => { baseUrl: "https://lk.example.org", }, ownMembershipIdentity: ownMemberMock, - useMatrix2$: constant(false), + useOldJwtEndpoint$: constant(false), delayId$: constant("delay_id_mock"), }); @@ -155,7 +155,7 @@ describe("LocalTransport", () => { baseUrl: "https://lk.example.org", }, ownMembershipIdentity: ownMemberMock, - useMatrix2$: constant(false), + useOldJwtEndpoint$: constant(false), delayId$: constant("delay_id_mock"), }); diff --git a/src/state/CallViewModel/localMember/LocalTransport.ts b/src/state/CallViewModel/localMember/LocalTransport.ts index 6c3e1cd0..b44cf967 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.ts @@ -35,7 +35,6 @@ import { } from "../../../livekit/openIDSFU.ts"; import { areLivekitTransportsEqual } from "../remoteMembers/MatrixLivekitMembers.ts"; import { customLivekitUrl } from "../../../settings/settings.ts"; -import { type LivekitTransportWithVersion } from "../remoteMembers/ConnectionManager.ts"; const logger = rootLogger.getChild("[LocalTransport]"); @@ -51,7 +50,7 @@ interface Props { client: Pick & OpenIDClientParts; roomId: string; useOldestMember$: Behavior; - useMatrix2$: Behavior; + useOldJwtEndpoint$: Behavior; delayId$: Behavior; } @@ -62,6 +61,11 @@ interface Props { * @prop useOldestMember Whether to use the same transport as the oldest member. * This will only update once the first oldest member appears. Will not recompute if the oldest member leaves. * + * @prop useOldJwtEndpoint$ Whether to set forceOldJwtEndpoint the use the old JWT endpoint. + * This is used when the connection manager needs to know if it has to use the legacy endpoint which implies a string concatenated rtcBackendIdentity. + * (which is expected for non sticky event based rtc member events) + * @returns Behavior<(LivekitTransport & { forceOldJwtEndpoint: boolean }) | null> The `forceOldJwtEndpoint` field is added to let the connection EncryptionManager + * know that this transport is for the local member and it IS RELEVANT which jwt endpoint to use. (for the local member transport, we need to know which jwt endpoint to use) * @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken */ export const createLocalTransport$ = ({ @@ -71,21 +75,20 @@ export const createLocalTransport$ = ({ client, roomId, useOldestMember$, - useMatrix2$, + useOldJwtEndpoint$, delayId$, -}: Props): Behavior => { +}: Props): Behavior => { /** * The transport over which we should be actively publishing our media. * undefined when not joined. */ const oldestMemberTransport$ = scope.behavior( - memberships$.pipe( - map((memberships) => { + combineLatest([memberships$, useOldJwtEndpoint$]).pipe( + map(([memberships, forceOldJwtEndpoint]) => { const oldestMember = memberships.value[0]; - const t = oldestMember?.getTransport(memberships.value[0]); - if (!t) return null; - // Here we will use the matrix2 information from the oldest member transport. - return { ...t, useMatrix2: oldestMember.kind === "rtc" }; + const transport = oldestMember?.getTransport(memberships.value[0]); + if (!transport) return null; + return { ...transport, forceOldJwtEndpoint }; }), first((t) => t != null && isLivekitTransport(t)), ), @@ -98,24 +101,23 @@ export const createLocalTransport$ = ({ * * @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken */ - const preferredTransport$: Behavior = - scope.behavior( - combineLatest([customLivekitUrl.value$, useMatrix2$, delayId$]).pipe( - switchMap(([customUrl, useMatrix2, delayId]) => - from( - makeTransport( - client, - ownMembershipIdentity, - roomId, - customUrl, - useMatrix2, - delayId ?? undefined, - ), + const preferredTransport$: Behavior = scope.behavior( + combineLatest([customLivekitUrl.value$, delayId$, useOldJwtEndpoint$]).pipe( + switchMap(([customUrl, delayId, forceOldJwtEndpoint]) => + from( + makeTransport( + client, + ownMembershipIdentity, + roomId, + customUrl, + forceOldJwtEndpoint, + delayId ?? undefined, ), ), ), - null, - ); + ), + null, + ); /** * The chosen transport we should advertise in our MatrixRTC membership. @@ -131,7 +133,7 @@ export const createLocalTransport$ = ({ ? (oldestMemberTransport ?? preferredTransport) : preferredTransport, ), - distinctUntilChanged(areLivekitTransportsEqual), + distinctUntilChanged((t1, t2) => areLivekitTransportsEqual(t1, t2)), ), ); }; @@ -142,6 +144,8 @@ const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci"; * * @param client * @param roomId + * @param useMatrix2 This implies using the matrix2 jwt endpoint (including delayed event delegation of the jwt token) + * @param delayId * @returns * @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken */ @@ -150,9 +154,9 @@ async function makeTransport( membership: CallMembershipIdentityParts, roomId: string, urlFromDevSettings: string | null, - matrix2jwt = false, + forceOldJwtEndpoint: boolean, delayId?: string, -): Promise { +): Promise { let transport: LivekitTransport | undefined; logger.trace("Searching for a preferred transport"); //TODO refactor this to use the jwt service returned alias. @@ -209,11 +213,12 @@ async function makeTransport( client, membership, transport.livekit_service_url, + forceOldJwtEndpoint, transport.livekit_alias, - matrix2jwt, client.baseUrl, delayId, + logger, ); - return { ...transport, useMatrix2: matrix2jwt }; + return { ...transport, forceOldJwtEndpoint }; } diff --git a/src/state/CallViewModel/remoteMembers/Connection.test.ts b/src/state/CallViewModel/remoteMembers/Connection.test.ts index 533f451a..57578641 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.test.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.test.ts @@ -26,6 +26,7 @@ import fetchMock from "fetch-mock"; import EventEmitter from "events"; import { type IOpenIDToken } from "matrix-js-sdk"; import { logger } from "matrix-js-sdk/lib/logger"; +import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc/LivekitTransport"; import { Connection, @@ -39,7 +40,6 @@ import { FailToGetOpenIdToken, } from "../../../utils/errors.ts"; import { mockRemoteParticipant, ownMemberMock } from "../../../utils/test.ts"; -import { type LivekitTransportWithVersion } from "./ConnectionManager.ts"; let testScope: ObservableScope; @@ -50,7 +50,7 @@ let fakeLivekitRoom: MockedObject; let localParticipantEventEmiter: EventEmitter; let fakeLocalParticipant: MockedObject; -const livekitFocus: LivekitTransportWithVersion = { +const livekitFocus: LivekitTransport = { livekit_alias: "!roomID:example.org", livekit_service_url: "https://matrix-rtc.example.org/livekit/jwt", type: "livekit", diff --git a/src/state/CallViewModel/remoteMembers/Connection.ts b/src/state/CallViewModel/remoteMembers/Connection.ts index d32bbce6..e070b56b 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.ts @@ -35,8 +35,10 @@ import { } from "../../../utils/errors.ts"; export interface ConnectionOpts { + /** Whether we always try to connect to this connection via the legacy jwt endpoint. (no hash identity) */ + forceOldJwtEndpoint?: boolean; /** The media transport to connect to. */ - transport: LivekitTransport & { useMatrix2: boolean }; + transport: LivekitTransport; /** The Matrix client to use for OpenID and SFU config requests. */ client: OpenIDClientParts; /** The observable scope to use for this connection. */ @@ -89,7 +91,7 @@ export class Connection { /** * The media transport to connect to. */ - public readonly transport: LivekitTransport & { useMatrix2: boolean }; + public readonly transport: LivekitTransport; public readonly livekitRoom: LivekitRoom; @@ -192,16 +194,14 @@ export class Connection { this.client, this.ownMembershipIdentity, this.transport.livekit_service_url, + this.forceOldJwtEndpoint, this.transport.livekit_alias, - this.transport.useMatrix2, + // For the remote members we intentionally do not pass a delayEndpointBaseUrl. + undefined, + // and no delayId. + undefined, + this.logger, ); - // client: OpenIDClientParts, - // membership: CallMembershipIdentityParts, - // serviceUrl: string, - // livekitRoomAlias: string, - // matrix2jwt: boolean, - // delayEndpointBaseUrl?: string, - // delayId?: string, } /** @@ -222,7 +222,7 @@ export class Connection { private readonly client: OpenIDClientParts; private readonly logger: Logger; - + private readonly forceOldJwtEndpoint: boolean; /** * Creates a new connection to a matrix RTC LiveKit backend. * @@ -235,6 +235,7 @@ export class Connection { logger: Logger, private ownMembershipIdentity: CallMembershipIdentityParts, ) { + this.forceOldJwtEndpoint = opts.forceOldJwtEndpoint ?? false; this.logger = logger.getChild("[Connection]"); this.logger.info( `[Connection] Creating new connection to ${opts.transport.livekit_service_url} ${opts.transport.livekit_alias}`, diff --git a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts index 82a1a78a..94652d16 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts @@ -15,6 +15,7 @@ import { import { type Logger } from "matrix-js-sdk/lib/logger"; import E2EEWorker from "livekit-client/e2ee-worker?worker"; import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; +import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc/LivekitTransport"; import { type ObservableScope } from "../../ObservableScope.ts"; import { Connection } from "./Connection.ts"; @@ -23,15 +24,15 @@ import type { MediaDevices } from "../../MediaDevices.ts"; import type { Behavior } from "../../Behavior.ts"; import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx"; import { defaultLiveKitOptions } from "../../../livekit/options.ts"; -import { type LivekitTransportWithVersion } from "./ConnectionManager.ts"; // TODO evaluate if this should be done like the Publisher Factory export interface ConnectionFactory { createConnection( - transport: LivekitTransportWithVersion, + transport: LivekitTransport, scope: ObservableScope, logger: Logger, ownMembershipIdentity: CallMembershipIdentityParts, + forceOldJwtEndpoint?: boolean, ): Connection; } @@ -88,10 +89,11 @@ export class ECConnectionFactory implements ConnectionFactory { * @returns */ public createConnection( - transport: LivekitTransportWithVersion, + transport: LivekitTransport, scope: ObservableScope, logger: Logger, ownMembershipIdentity: CallMembershipIdentityParts, + forceOldJwtEndpoint?: boolean, ): Connection { return new Connection( { @@ -99,6 +101,7 @@ export class ECConnectionFactory implements ConnectionFactory { client: this.client, scope: scope, livekitRoomFactory: this.livekitRoomFactory, + forceOldJwtEndpoint, }, logger, ownMembershipIdentity, diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts index 4ab91646..088bf41b 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts @@ -14,29 +14,26 @@ import { logger } from "matrix-js-sdk/lib/logger"; import { Epoch, mapEpoch, ObservableScope } from "../../ObservableScope.ts"; import { createConnectionManager$, - type LivekitTransportWithVersion, type ConnectionManagerData, } from "./ConnectionManager.ts"; import { type ConnectionFactory } from "./ConnectionFactory.ts"; import { type Connection } from "./Connection.ts"; import { ownMemberMock, withTestScheduler } from "../../../utils/test.ts"; import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts"; -import { type Behavior } from "../../Behavior.ts"; +import { constant, type Behavior } from "../../Behavior.ts"; // Some test constants -const TRANSPORT_1: LivekitTransportWithVersion = { +const TRANSPORT_1: LivekitTransport = { type: "livekit", livekit_service_url: "https://lk.example.org", livekit_alias: "!alias:example.org", - useMatrix2: false, }; -const TRANSPORT_2: LivekitTransportWithVersion = { +const TRANSPORT_2: LivekitTransport = { type: "livekit", livekit_service_url: "https://lk.sample.com", livekit_alias: "!alias:sample.com", - useMatrix2: false, }; let fakeConnectionFactory: ConnectionFactory; @@ -79,7 +76,8 @@ describe("connections$ stream", () => { const { connectionManagerData$ } = createConnectionManager$({ scope: testScope, connectionFactory: fakeConnectionFactory, - inputTransports$: behavior("a", { + localTransport$: constant(null), + remoteTransports$: behavior("a", { a: new Epoch([TRANSPORT_1, TRANSPORT_2], 0), }), logger: logger, @@ -119,7 +117,8 @@ describe("connections$ stream", () => { const { connectionManagerData$ } = createConnectionManager$({ scope: testScope, connectionFactory: fakeConnectionFactory, - inputTransports$: behavior("abcdef", { + localTransport$: constant(null), + remoteTransports$: behavior("abcdef", { a: new Epoch([TRANSPORT_1], 0), b: new Epoch([TRANSPORT_1], 1), c: new Epoch([TRANSPORT_1], 2), @@ -165,7 +164,8 @@ describe("connections$ stream", () => { const { connectionManagerData$ } = createConnectionManager$({ scope: testScope, connectionFactory: fakeConnectionFactory, - inputTransports$: behavior("abc", { + localTransport$: constant(null), + remoteTransports$: behavior("abc", { a: new Epoch([TRANSPORT_1], 0), b: new Epoch([TRANSPORT_1, TRANSPORT_2], 1), c: new Epoch([TRANSPORT_1], 2), @@ -281,7 +281,8 @@ describe("connectionManagerData$ stream", () => { const { connectionManagerData$ } = createConnectionManager$({ scope: testScope, connectionFactory: fakeConnectionFactory, - inputTransports$: behavior("a", { + localTransport$: constant(null), + remoteTransports$: behavior("a", { a: new Epoch([TRANSPORT_1, TRANSPORT_2], 0), }), logger, diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts index d5852d84..6101f79b 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts @@ -7,22 +7,18 @@ Please see LICENSE in the repository root for full details. */ import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; -import { combineLatest, map, of, switchMap, tap } from "rxjs"; +import { combineLatest, map, of, switchMap } from "rxjs"; import { type Logger } from "matrix-js-sdk/lib/logger"; import { type RemoteParticipant } from "livekit-client"; import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; -import { type Behavior } from "../../Behavior.ts"; +import { constant, type Behavior } from "../../Behavior.ts"; import { type Connection } from "./Connection.ts"; import { Epoch, type ObservableScope } from "../../ObservableScope.ts"; import { generateItemsWithEpoch } from "../../../utils/observable.ts"; import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts"; import { type ConnectionFactory } from "./ConnectionFactory.ts"; -export type LivekitTransportWithVersion = LivekitTransport & { - useMatrix2: boolean; -}; - export class ConnectionManagerData { private readonly store: Map = new Map(); @@ -64,7 +60,9 @@ export class ConnectionManagerData { interface Props { scope: ObservableScope; connectionFactory: ConnectionFactory; - inputTransports$: Behavior>; + localTransport$: Behavior; + remoteTransports$: Behavior>; + forceOldJwtEndpointForLocalTransport$?: Behavior; logger: Logger; ownMembershipIdentity: CallMembershipIdentityParts; } @@ -91,13 +89,29 @@ export interface IConnectionManager { export function createConnectionManager$({ scope, connectionFactory, - inputTransports$, + localTransport$, + remoteTransports$, + forceOldJwtEndpointForLocalTransport$ = constant(false), logger: parentLogger, ownMembershipIdentity, }: Props): IConnectionManager { const logger = parentLogger.getChild("[ConnectionManager]"); // TODO logger: only construct one logger from the client and make it compatible via a EC specific sing + const allInputTransports$ = combineLatest([ + localTransport$, + remoteTransports$, + ]).pipe( + map(([localTransport, transports]) => { + const localTransportAsArray = localTransport ? [localTransport] : []; + return transports.mapInner((transports) => [ + ...localTransportAsArray, + ...transports, + ]); + }), + map((transports) => transports.mapInner(removeDuplicateTransports)), + ); + /** * All transports currently managed by the ConnectionManager. * @@ -106,14 +120,32 @@ export function createConnectionManager$({ * It is build based on the list of subscribed transports (`transportsSubscriptions$`). * externally this is modified via `registerTransports()`. */ - const transports$ = scope.behavior( - inputTransports$.pipe( - map((transports) => transports.mapInner(removeDuplicateTransports)), - tap(({ value: transports }) => { - logger.trace( - `Managing transports: ${transports.map((t) => t.livekit_service_url).join(", ")}`, - ); - }), + const transportsWithJwtTag$ = scope.behavior( + combineLatest([ + allInputTransports$, + localTransport$, + forceOldJwtEndpointForLocalTransport$, + ]).pipe( + map( + ([ + transports, + localTransport, + forceOldJwtEndpointForLocalTransport, + ]) => { + // nmodify only the local transport with forceOldJwtEndpointForLocalTransport + const index = transports.value.findIndex((t) => + areLivekitTransportsEqual(localTransport, t), + ); + transports.value[index].forceOldJwtEndpoint = + forceOldJwtEndpointForLocalTransport; + logger.trace( + `Managing transports: ${transports.value.map((t) => t.livekit_service_url).join(", ")}`, + ); + return transports as Epoch< + (LivekitTransport & { forceOldJwtEndpoint?: boolean })[] + >; + }, + ), ), ); @@ -121,7 +153,7 @@ export function createConnectionManager$({ * Connections for each transport in use by one or more session members. */ const connections$ = scope.behavior( - transports$.pipe( + transportsWithJwtTag$.pipe( generateItemsWithEpoch( function* (transports) { for (const transport of transports) @@ -129,23 +161,23 @@ export function createConnectionManager$({ keys: [ transport.livekit_service_url, transport.livekit_alias, - transport.useMatrix2, + transport.forceOldJwtEndpoint, ], data: undefined, }; }, - (scope, _data$, serviceUrl, alias, useMatrix2) => { + (scope, _data$, serviceUrl, alias, forceOldJwtEndpoint) => { logger.debug(`Creating connection to ${serviceUrl} (${alias})`); const connection = connectionFactory.createConnection( { type: "livekit", livekit_service_url: serviceUrl, livekit_alias: alias, - useMatrix2, }, scope, logger, ownMembershipIdentity, + forceOldJwtEndpoint, ); // Start the connection immediately // Use connection state to track connection progress diff --git a/src/state/CallViewModel/remoteMembers/integration.test.ts b/src/state/CallViewModel/remoteMembers/integration.test.ts index 00062c60..df10c861 100644 --- a/src/state/CallViewModel/remoteMembers/integration.test.ts +++ b/src/state/CallViewModel/remoteMembers/integration.test.ts @@ -13,11 +13,7 @@ import fetchMock from "fetch-mock"; import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { logger } from "matrix-js-sdk/lib/logger"; -import { - type Epoch, - ObservableScope, - trackEpoch, -} from "../../ObservableScope.ts"; +import { type Epoch, ObservableScope, trackEpoch } from "../../ObservableScope.ts"; import { ECConnectionFactory } from "./ConnectionFactory.ts"; import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts"; import { @@ -34,6 +30,7 @@ import { } from "./MatrixLivekitMembers.ts"; import { createConnectionManager$ } from "./ConnectionManager.ts"; import { membershipsAndTransports$ } from "../../SessionBehaviors.ts"; +import { constant } from "../../Behavior.ts"; // Test the integration of ConnectionManager and MatrixLivekitMerger @@ -121,7 +118,8 @@ test("bob, carl, then bob joining no tracks yet", () => { const connectionManager = createConnectionManager$({ scope: testScope, connectionFactory: ecConnectionFactory, - inputTransports$: membershipsAndTransports.transports$, + localTransport$: constant(null), + remoteTransports$: membershipsAndTransports.transports$, logger: logger, ownMembershipIdentity: ownMemberMock, }); diff --git a/src/state/SessionBehaviors.ts b/src/state/SessionBehaviors.ts index b61d2fe6..e174a1cc 100644 --- a/src/state/SessionBehaviors.ts +++ b/src/state/SessionBehaviors.ts @@ -8,6 +8,7 @@ Please see LICENSE in the repository root for full details. import { type CallMembership, isLivekitTransport, + type LivekitTransport, type MatrixRTCSession, MatrixRTCSessionEvent, } from "matrix-js-sdk/lib/matrixrtc"; @@ -20,18 +21,15 @@ import { type ObservableScope, } from "./ObservableScope"; import { type Behavior } from "./Behavior"; -import { type LivekitTransportWithVersion } from "./CallViewModel/remoteMembers/ConnectionManager"; export const membershipsAndTransports$ = ( scope: ObservableScope, memberships$: Behavior>, ): { membershipsWithTransport$: Behavior< - Epoch< - { membership: CallMembership; transport?: LivekitTransportWithVersion }[] - > + Epoch<{ membership: CallMembership; transport?: LivekitTransport }[]> >; - transports$: Behavior>; + transports$: Behavior>; } => { /** * Lists the transports used by ourselves, plus all other MatrixRTC session @@ -49,12 +47,7 @@ export const membershipsAndTransports$ = ( const transport = membership.getTransport(oldestMembership); return { membership, - transport: isLivekitTransport(transport) - ? { - ...transport, - useMatrix2: membership.kind === "rtc", - } - : undefined, + transport: isLivekitTransport(transport) ? transport : undefined, }; }); }), diff --git a/src/utils/test.ts b/src/utils/test.ts index a860bde0..02277af0 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -25,6 +25,7 @@ import { import { CallMembership, type LivekitFocusSelection, + type LivekitTransport, type MatrixRTCSession, MatrixRTCSessionEvent, type MatrixRTCSessionEventHandlerMap, @@ -66,7 +67,6 @@ import { type MediaDevices } from "../state/MediaDevices"; import { type Behavior, constant } from "../state/Behavior"; import { ObservableScope } from "../state/ObservableScope"; import { MuteStates } from "../state/MuteStates"; -import { type LivekitTransportWithVersion } from "../state/CallViewModel/remoteMembers/ConnectionManager"; export function withFakeTimers(continuation: () => void): void { vi.useFakeTimers(); @@ -197,11 +197,10 @@ export function mockEmitter(): EmitterMock { }; } -export const exampleTransport: LivekitTransportWithVersion = { +export const exampleTransport: LivekitTransport = { type: "livekit", livekit_service_url: "https://lk.example.org", livekit_alias: "!alias:example.org", - useMatrix2: false, }; export function mockCallMembership( diff --git a/yarn.lock b/yarn.lock index 0ddaad61..83555527 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7533,7 +7533,7 @@ __metadata: livekit-client: "npm:^2.13.0" lodash-es: "npm:^4.17.21" loglevel: "npm:^1.9.1" - matrix-js-sdk: "github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=8cea2c05839ebcaa90945504a453b9b1e1092fc4" + matrix-js-sdk: "github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=6f0815930a008eff8f86e6e5748d447be0e7c25e" matrix-widget-api: "npm:^1.14.0" normalize.css: "npm:^8.0.1" observable-hooks: "npm:^4.2.3" @@ -10338,9 +10338,9 @@ __metadata: languageName: node linkType: hard -"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=8cea2c05839ebcaa90945504a453b9b1e1092fc4": +"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=6f0815930a008eff8f86e6e5748d447be0e7c25e": version: 39.4.0 - resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=8cea2c05839ebcaa90945504a453b9b1e1092fc4" + resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=6f0815930a008eff8f86e6e5748d447be0e7c25e" dependencies: "@babel/runtime": "npm:^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm": "npm:^16.0.0" @@ -10356,7 +10356,7 @@ __metadata: sdp-transform: "npm:^3.0.0" unhomoglyph: "npm:^1.0.6" uuid: "npm:13" - checksum: 10c0/2375dd3d9191f78fe589b0d3170f3da7792ed469a81d3ba3cd12f4915fd33a859f8af3491edb9cf0cdaa1f881a3ea7c1bf7539e850ad0360ec9981271f462c81 + checksum: 10c0/a5a904a79f3660d1f6fe217195e662adf82af4a445681e47f292772d9d4d63ce60aaca209f40c41e2d659bee2b17cd5b3345bbad77795032057f2c0e3129cc77 languageName: node linkType: hard From 5d5d75ebdf3dab22f810eff4c64e8219c009bf74 Mon Sep 17 00:00:00 2001 From: Timo K Date: Mon, 5 Jan 2026 21:08:33 +0100 Subject: [PATCH 10/46] fixup merge --- sdk/main.ts | 7 +-- src/livekit/openIDSFU.test.ts | 7 +++ .../localMember/LocalTransport.test.ts | 19 +++++++- .../localMember/LocalTransport.ts | 47 +++++++++++-------- 4 files changed, 53 insertions(+), 27 deletions(-) diff --git a/sdk/main.ts b/sdk/main.ts index 376674a4..8d07ab07 100644 --- a/sdk/main.ts +++ b/sdk/main.ts @@ -101,12 +101,7 @@ export async function createMatrixRTCSdk( const mediaDevices = new MediaDevices(scope); const muteStates = new MuteStates(scope, mediaDevices, constant(true)); const slot = { application, id }; - const rtcSession = new MatrixRTCSession( - client, - room, - MatrixRTCSession.sessionMembershipsForSlot(room, slot), - slot, - ); + const rtcSession = new MatrixRTCSession(client, room, slot); const callViewModel = createCallViewModel$( scope, rtcSession, diff --git a/src/livekit/openIDSFU.test.ts b/src/livekit/openIDSFU.test.ts index 2a260b01..8b263662 100644 --- a/src/livekit/openIDSFU.test.ts +++ b/src/livekit/openIDSFU.test.ts @@ -18,6 +18,7 @@ import fetchMock from "fetch-mock"; import { getSFUConfigWithOpenID, type OpenIDClientParts } from "./openIDSFU"; import { testJWTToken } from "../utils/test-fixtures"; +import { ownMemberMock } from "../utils/test"; const sfuUrl = "https://sfu.example.org"; @@ -42,7 +43,9 @@ describe("getSFUConfigWithOpenID", () => { }); const config = await getSFUConfigWithOpenID( matrixClient, + ownMemberMock, "https://sfu.example.org", + false, "!example_room_id", ); expect(config).toEqual({ @@ -63,7 +66,9 @@ describe("getSFUConfigWithOpenID", () => { try { await getSFUConfigWithOpenID( matrixClient, + ownMemberMock, "https://sfu.example.org", + false, "!example_room_id", ); } catch (ex) { @@ -98,7 +103,9 @@ describe("getSFUConfigWithOpenID", () => { }); const config = await getSFUConfigWithOpenID( matrixClient, + ownMemberMock, "https://sfu.example.org", + false, "!example_room_id", ); expect(config).toEqual({ diff --git a/src/state/CallViewModel/localMember/LocalTransport.test.ts b/src/state/CallViewModel/localMember/LocalTransport.test.ts index c77297ef..2199ca94 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.test.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.test.ts @@ -49,6 +49,8 @@ describe("LocalTransport", () => { useOldestMember$: constant(false), memberships$: constant(new Epoch([])), client: { + // eslint-disable-next-line @typescript-eslint/naming-convention + _unstable_getRTCTransports: async () => Promise.resolve([]), getDomain: () => "", baseUrl: "example.org", // These won't be called in this error path but satisfy the type @@ -130,6 +132,8 @@ describe("LocalTransport", () => { useOldestMember$: constant(false), memberships$: constant(new Epoch([])), client: { + // eslint-disable-next-line @typescript-eslint/naming-convention + _unstable_getRTCTransports: async () => Promise.resolve([]), getDomain: () => "", getOpenIdToken: vi.fn(), getDeviceId: vi.fn(), @@ -140,7 +144,12 @@ describe("LocalTransport", () => { delayId$: constant("delay_id_mock"), }); - openIdResolver.resolve?.({ url: "https://lk.example.org", jwt: "jwt" }); + openIdResolver.resolve?.({ + url: "https://lk.example.org", + jwt: "jwt", + livekitAlias: "!room:example.org", + livekitIdentity: ownMemberMock.userId + ":" + ownMemberMock.deviceId, + }); expect(localTransport$.value).toBe(null); await flushPromises(); // final @@ -203,11 +212,15 @@ describe("LocalTransport", () => { mockConfig({}); customLivekitUrl.setValue(customLivekitUrl.defaultValue); localTransportOpts = { + ownMembershipIdentity: ownMemberMock, scope, roomId: "!example_room_id", useOldestMember$: constant(false), + useOldJwtEndpoint$: constant(false), + delayId$: constant(null), memberships$: constant(new Epoch([])), client: { + baseUrl: "https://example.org", getDomain: vi.fn().mockReturnValue(""), // eslint-disable-next-line @typescript-eslint/naming-convention _unstable_getRTCTransports: vi.fn().mockResolvedValue([]), @@ -317,11 +330,15 @@ describe("LocalTransport", () => { it("throws if no options are available", async () => { const localTransport$ = createLocalTransport$({ scope, + ownMembershipIdentity: ownMemberMock, roomId: "!example_room_id", useOldestMember$: constant(false), + useOldJwtEndpoint$: constant(false), + delayId$: constant(null), memberships$: constant(new Epoch([])), client: { getDomain: () => "", + baseUrl: "https://example.org", // eslint-disable-next-line @typescript-eslint/naming-convention _unstable_getRTCTransports: async () => Promise.resolve([]), // These won't be called in this error path but satisfy the type diff --git a/src/state/CallViewModel/localMember/LocalTransport.ts b/src/state/CallViewModel/localMember/LocalTransport.ts index d8b5dfce..1853ff6d 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.ts @@ -178,7 +178,6 @@ async function makeTransport( forceOldJwtEndpoint: boolean, delayId?: string, ): Promise { - let transport: LivekitTransport | undefined; logger.trace("Searching for a preferred transport"); // We will call `getSFUConfigWithOpenID` once per transport here as it's our @@ -194,32 +193,47 @@ async function makeTransport( logger.info("Using LiveKit transport from dev tools: ", urlFromDevSettings); // Validate that the SFU is up. Otherwise, we want to fail on this // as we don't permit other SFUs. - const config = await getSFUConfigWithOpenID( + // This will call the jwt/sfu/get endpoint to pre create the livekit room. + const { livekitAlias } = await getSFUConfigWithOpenID( client, + membership, urlFromDevSettings, + forceOldJwtEndpoint, roomId, + client.baseUrl, + delayId, + logger, ); return { type: "livekit", livekit_service_url: urlFromDevSettings, - livekit_alias: config.livekitAlias, + livekit_alias: livekitAlias, + forceOldJwtEndpoint, }; } async function getFirstUsableTransport( transports: Transport[], - ): Promise { + ): Promise<(LivekitTransport & { forceOldJwtEndpoint: boolean }) | null> { for (const potentialTransport of transports) { if (isLivekitTransportConfig(potentialTransport)) { try { + // This will call the jwt/sfu/get endpoint to pre create the livekit room. const { livekitAlias } = await getSFUConfigWithOpenID( client, + membership, potentialTransport.livekit_service_url, + forceOldJwtEndpoint, roomId, + client.baseUrl, + delayId, + logger, ); + return { ...potentialTransport, livekit_alias: livekitAlias, + forceOldJwtEndpoint, }; } catch (ex) { if (ex instanceof FailToGetOpenIdToken) { @@ -283,10 +297,16 @@ async function makeTransport( const urlFromConf = Config.get().livekit?.livekit_service_url; if (urlFromConf) { try { + // This will call the jwt/sfu/get endpoint to pre create the livekit room. const { livekitAlias } = await getSFUConfigWithOpenID( client, + membership, urlFromConf, + forceOldJwtEndpoint, roomId, + client.baseUrl, + delayId, + logger, ); const selectedTransport: LivekitTransport = { type: "livekit", @@ -294,7 +314,7 @@ async function makeTransport( livekit_alias: livekitAlias, }; logger.info("Using config SFU", selectedTransport); - return selectedTransport; + return { ...selectedTransport, forceOldJwtEndpoint }; } catch (ex) { if (ex instanceof FailToGetOpenIdToken) { throw ex; @@ -303,19 +323,6 @@ async function makeTransport( } } - if (!transport) throw new MatrixRTCTransportMissingError(domain ?? ""); - - // this will call the jwt/sfu/get endpoint to pre create the livekit room. - await getSFUConfigWithOpenID( - client, - membership, - transport.livekit_service_url, - forceOldJwtEndpoint, - transport.livekit_alias, - client.baseUrl, - delayId, - logger, - ); - - return { ...transport, forceOldJwtEndpoint }; + // If we do not have returned a transport by now we throw an error + throw new MatrixRTCTransportMissingError(domain ?? ""); } From 00fca6e3c731e24ed1ceffddde33e690412d3589 Mon Sep 17 00:00:00 2001 From: Timo K Date: Mon, 5 Jan 2026 21:17:37 +0100 Subject: [PATCH 11/46] simplify localTransport --- .../localMember/LocalTransport.ts | 73 +++++++------------ 1 file changed, 25 insertions(+), 48 deletions(-) diff --git a/src/state/CallViewModel/localMember/LocalTransport.ts b/src/state/CallViewModel/localMember/LocalTransport.ts index 1853ff6d..50d6cec6 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.ts @@ -180,6 +180,26 @@ async function makeTransport( ): Promise { logger.trace("Searching for a preferred transport"); + async function doOpenIdAndJWTFromUrl( + url: string, + ): Promise { + const { livekitAlias } = await getSFUConfigWithOpenID( + client, + membership, + url, + forceOldJwtEndpoint, + roomId, + client.baseUrl, + delayId, + logger, + ); + return { + type: "livekit", + livekit_service_url: url, + livekit_alias: livekitAlias, + forceOldJwtEndpoint, + }; + } // We will call `getSFUConfigWithOpenID` once per transport here as it's our // only mechanism of valiation. This means we will also ask the // homeserver for a OpenID token a few times. Since OpenID tokens are single @@ -190,26 +210,11 @@ async function makeTransport( // DEVTOOL: Highest priority: Load from devtool setting if (urlFromDevSettings !== null) { - logger.info("Using LiveKit transport from dev tools: ", urlFromDevSettings); // Validate that the SFU is up. Otherwise, we want to fail on this // as we don't permit other SFUs. // This will call the jwt/sfu/get endpoint to pre create the livekit room. - const { livekitAlias } = await getSFUConfigWithOpenID( - client, - membership, - urlFromDevSettings, - forceOldJwtEndpoint, - roomId, - client.baseUrl, - delayId, - logger, - ); - return { - type: "livekit", - livekit_service_url: urlFromDevSettings, - livekit_alias: livekitAlias, - forceOldJwtEndpoint, - }; + logger.info("Using LiveKit transport from dev tools: ", urlFromDevSettings); + return await doOpenIdAndJWTFromUrl(urlFromDevSettings); } async function getFirstUsableTransport( @@ -219,22 +224,9 @@ async function makeTransport( if (isLivekitTransportConfig(potentialTransport)) { try { // This will call the jwt/sfu/get endpoint to pre create the livekit room. - const { livekitAlias } = await getSFUConfigWithOpenID( - client, - membership, + return await doOpenIdAndJWTFromUrl( potentialTransport.livekit_service_url, - forceOldJwtEndpoint, - roomId, - client.baseUrl, - delayId, - logger, ); - - return { - ...potentialTransport, - livekit_alias: livekitAlias, - forceOldJwtEndpoint, - }; } catch (ex) { if (ex instanceof FailToGetOpenIdToken) { // Explictly throw these @@ -298,23 +290,8 @@ async function makeTransport( if (urlFromConf) { try { // This will call the jwt/sfu/get endpoint to pre create the livekit room. - const { livekitAlias } = await getSFUConfigWithOpenID( - client, - membership, - urlFromConf, - forceOldJwtEndpoint, - roomId, - client.baseUrl, - delayId, - logger, - ); - const selectedTransport: LivekitTransport = { - type: "livekit", - livekit_service_url: urlFromConf, - livekit_alias: livekitAlias, - }; - logger.info("Using config SFU", selectedTransport); - return { ...selectedTransport, forceOldJwtEndpoint }; + logger.info("Using config SFU", urlFromConf); + return await doOpenIdAndJWTFromUrl(urlFromConf); } catch (ex) { if (ex instanceof FailToGetOpenIdToken) { throw ex; From 15800872862fccd5839b4f64ad3cb8b27a570140 Mon Sep 17 00:00:00 2001 From: Timo K Date: Mon, 5 Jan 2026 21:24:52 +0100 Subject: [PATCH 12/46] use latest js-sdk --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 95c30ef6..1d4d6393 100644 --- a/package.json +++ b/package.json @@ -112,7 +112,7 @@ "livekit-client": "^2.13.0", "lodash-es": "^4.17.21", "loglevel": "^1.9.1", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=6f0815930a008eff8f86e6e5748d447be0e7c25e", + "matrix-js-sdk": "matrix-org/matrix-js-sdk#develop", "matrix-widget-api": "^1.14.0", "node-stdlib-browser": "^1.3.1", "normalize.css": "^8.0.1", diff --git a/yarn.lock b/yarn.lock index 1a200472..6c0e4948 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7951,7 +7951,7 @@ __metadata: livekit-client: "npm:^2.13.0" lodash-es: "npm:^4.17.21" loglevel: "npm:^1.9.1" - matrix-js-sdk: "github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=6f0815930a008eff8f86e6e5748d447be0e7c25e" + matrix-js-sdk: "matrix-org/matrix-js-sdk#develop" matrix-widget-api: "npm:^1.14.0" node-stdlib-browser: "npm:^1.3.1" normalize.css: "npm:^8.0.1" @@ -10939,9 +10939,9 @@ __metadata: languageName: node linkType: hard -"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=6f0815930a008eff8f86e6e5748d447be0e7c25e": +"matrix-js-sdk@matrix-org/matrix-js-sdk#develop": version: 39.4.0 - resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=6f0815930a008eff8f86e6e5748d447be0e7c25e" + resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=4b89fb23c54aaf7826bd127d8fa21cc7bb87688f" dependencies: "@babel/runtime": "npm:^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm": "npm:^16.0.0" @@ -10957,7 +10957,7 @@ __metadata: sdp-transform: "npm:^3.0.0" unhomoglyph: "npm:^1.0.6" uuid: "npm:13" - checksum: 10c0/a5a904a79f3660d1f6fe217195e662adf82af4a445681e47f292772d9d4d63ce60aaca209f40c41e2d659bee2b17cd5b3345bbad77795032057f2c0e3129cc77 + checksum: 10c0/bc7443bf67822e9bc7b8e531b4e61e6ebac41c2fd8047ac0567456c264ae0d1911fbef6e437d312a3adeead86cd5e7134944e3fd73d28002777618bc0ebaa1ca languageName: node linkType: hard From 69a4189517ca83016053b8d2f1722e2324076d45 Mon Sep 17 00:00:00 2001 From: Timo K Date: Mon, 5 Jan 2026 21:58:26 +0100 Subject: [PATCH 13/46] self review --- src/e2ee/matrixKeyProvider.ts | 4 ++-- src/livekit/openIDSFU.ts | 29 ++++++++++++++---------- src/room/InCallView.test.tsx | 2 +- src/room/InCallView.tsx | 1 - src/state/CallViewModel/CallViewModel.ts | 3 ++- 5 files changed, 22 insertions(+), 17 deletions(-) diff --git a/src/e2ee/matrixKeyProvider.ts b/src/e2ee/matrixKeyProvider.ts index d7aebc4b..63a96755 100644 --- a/src/e2ee/matrixKeyProvider.ts +++ b/src/e2ee/matrixKeyProvider.ts @@ -61,12 +61,12 @@ export class MatrixKeyProvider extends BaseKeyProvider { ); logger.debug( - `Sent new key to livekit room=${this.rtcSession?.room.roomId} participantId=${rtcBackendIdentity} (before hash: ${membershipParts.userId}) encryptionKeyIndex=${encryptionKeyIndex}`, + `Sent new key to livekit room=${this.rtcSession?.room.roomId} participantId=${rtcBackendIdentity} (before hash: ${membershipParts.userId}:${membershipParts.deviceId}) encryptionKeyIndex=${encryptionKeyIndex}`, ); }, (e) => { logger.error( - `Failed to create key material from buffer for livekit room=${this.rtcSession?.room.roomId} participantId before hash=${membershipParts.userId} encryptionKeyIndex=${encryptionKeyIndex}`, + `Failed to create key material from buffer for livekit room=${this.rtcSession?.room.roomId} participantId before hash=${membershipParts.userId}:${membershipParts.deviceId} encryptionKeyIndex=${encryptionKeyIndex}`, e, ); }, diff --git a/src/livekit/openIDSFU.ts b/src/livekit/openIDSFU.ts index 7d6dfc24..6728a243 100644 --- a/src/livekit/openIDSFU.ts +++ b/src/livekit/openIDSFU.ts @@ -20,6 +20,7 @@ export interface SFUConfig { url: string; jwt: string; livekitAlias: string; + // NOTE: Currently unused. livekitIdentity: string; } @@ -68,7 +69,7 @@ export type OpenIDClientParts = Pick< * @param client The Matrix client * @param membership * @param serviceUrl The URL of the livekit SFU service - * @param forceOldEndpoint This will use the old jwt endpoint which will create the rtc backend identity based on string concatination + * @param forceOldJwtEndpoint This will use the old jwt endpoint which will create the rtc backend identity based on string concatination * instead of a hash. * This function by default uses whatever is possible with the current jwt service installed next to the SFU. * For remote connections this does not matter, since we will not publish there we can rely on the newest option. @@ -103,12 +104,6 @@ export async function getSFUConfigWithOpenID( logger?.debug("Got openID token", openIdToken); logger?.info(`Trying to get JWT for focus ${serviceUrl}...`); - const args: [CallMembershipIdentityParts, string, string, IOpenIDToken] = [ - membership, - serviceUrl, - roomId, - openIdToken, - ]; let sfuConfig: { url: string; jwt: string }; try { @@ -118,7 +113,10 @@ export async function getSFUConfigWithOpenID( throw new Error("No delayId, Won't try matrix 2.0 jwt endpoint."); sfuConfig = await getLiveKitJWTWithDelayDelegation( - ...args, + membership, + serviceUrl, + roomId, + openIdToken, delayEndpointBaseUrl, delayId, ); @@ -128,23 +126,30 @@ export async function getSFUConfigWithOpenID( `Failed fetching jwt with matrix 2.0 endpoint (retry with legacy)`, e, ); - sfuConfig = await getLiveKitJWT(...args); + sfuConfig = await getLiveKitJWT( + membership.deviceId, + serviceUrl, + roomId, + openIdToken, + ); logger?.info(`Got JWT from call's active focus URL.`); } // Pull the details from the JWT const [, payloadStr] = sfuConfig.jwt.split("."); - + // TODO: Prefer Uint8Array.fromBase64 when widely available const payload = JSON.parse(global.atob(payloadStr)) as SFUJWTPayload; return { jwt: sfuConfig.jwt, url: sfuConfig.url, livekitAlias: payload.video.room, // NOTE: Currently unused. + // Probably also not helpful since we now compute the backendIdentity on joining the call so we can use it for the encryption manager. + // The only reason for us to know it locally is to connect the right users with the lk world. (and to set our own keys) livekitIdentity: payload.sub, }; } async function getLiveKitJWT( - membership: CallMembershipIdentityParts, + deviceId: string, livekitServiceURL: string, matrixRoomId: string, openIDToken: IOpenIDToken, @@ -159,7 +164,7 @@ async function getLiveKitJWT( // This is the actual livekit room alias. For the legacy jwt endpoint simply the room id was used. room: matrixRoomId, openid_token: openIDToken, - device_id: membership.deviceId, + device_id: deviceId, }), }); if (!res.ok) { diff --git a/src/room/InCallView.test.tsx b/src/room/InCallView.test.tsx index 8ac4bccb..a137074b 100644 --- a/src/room/InCallView.test.tsx +++ b/src/room/InCallView.test.tsx @@ -22,6 +22,7 @@ import { BrowserRouter } from "react-router-dom"; import { TooltipProvider } from "@vector-im/compound-web"; import { RoomContext, useLocalParticipant } from "@livekit/components-react"; +import { InCallView } from "./InCallView"; import { mockLivekitRoom, mockLocalParticipant, @@ -33,7 +34,6 @@ import { mockRtcMembership, type MockRTCSession, } from "../utils/test"; -import { InCallView } from "./InCallView"; import { E2eeType } from "../e2ee/e2eeType"; import { getBasicCallViewModelEnvironment } from "../utils/test-viewmodel"; import { alice, local } from "../utils/test-fixtures"; diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index a21c3302..fdbd4461 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -795,7 +795,6 @@ export const InCallView: FC = ({ onTouchEnd={onControlsTouchEnd} /> )} - {!showControls &&
}
); diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 9c2fbc3d..093abfad 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -423,7 +423,8 @@ export function createCallViewModel$( const ownMembershipIdentity: CallMembershipIdentityParts = { userId, deviceId, - // TODO look into this!!! + // This will eventually become the salt for the hash endpoint. + // For now we keep it as the user+device string since it is expected by non matrix matrixRTCMode === Legacy. memberId: `${userId}:${deviceId}`, }; From 556a87d1411d30e9b5aa600e33c97e2dfebce930 Mon Sep 17 00:00:00 2001 From: Timo K Date: Mon, 5 Jan 2026 22:20:19 +0100 Subject: [PATCH 14/46] fix js-doc --- src/livekit/openIDSFU.ts | 8 ++++---- src/state/CallViewModel/localMember/LocalMember.ts | 8 ++++---- src/state/CallViewModel/localMember/LocalTransport.ts | 6 ++++-- .../CallViewModel/remoteMembers/ConnectionFactory.ts | 11 ++++++----- .../CallViewModel/remoteMembers/ConnectionManager.ts | 10 +++++++--- .../remoteMembers/ECConnectionFactory.test.ts | 4 ++-- 6 files changed, 27 insertions(+), 20 deletions(-) diff --git a/src/livekit/openIDSFU.ts b/src/livekit/openIDSFU.ts index 6728a243..b89243c1 100644 --- a/src/livekit/openIDSFU.ts +++ b/src/livekit/openIDSFU.ts @@ -67,7 +67,7 @@ export type OpenIDClientParts = Pick< * to the matrix RTC backend in order to get acces to the SFU. * It has built-in retry for calls to the homeserver with a backoff policy. * @param client The Matrix client - * @param membership + * @param membership Our own membership identity parts used to send to jwt service. * @param serviceUrl The URL of the livekit SFU service * @param forceOldJwtEndpoint This will use the old jwt endpoint which will create the rtc backend identity based on string concatination * instead of a hash. @@ -75,9 +75,9 @@ export type OpenIDClientParts = Pick< * For remote connections this does not matter, since we will not publish there we can rely on the newest option. * For our own connection we can only use the hashed version if we also send the new matrix2.0 sticky events. * @param roomId The room id used in the jwt request. This is NOT the livekit_alias. The jwt service will provide the alias. It maps matrix room ids <-> Livekit aliases. - * @param delayEndpointBaseUrl - * @param delayId - * @param logger + * @param delayEndpointBaseUrl The URL of the matrix homeserver. + * @param delayId The delay id used for the jwt service to manage. + * @param logger optional logger. * @returns Object containing the token information * @throws FailToGetOpenIdToken */ diff --git a/src/state/CallViewModel/localMember/LocalMember.ts b/src/state/CallViewModel/localMember/LocalMember.ts index 890165dd..17f766ff 100644 --- a/src/state/CallViewModel/localMember/LocalMember.ts +++ b/src/state/CallViewModel/localMember/LocalMember.ts @@ -687,10 +687,9 @@ interface EnterRTCSessionOptions { * - Handles retries (fails only after several attempts) * * @param rtcSession - The MatrixRTCSession to join. + * @param ownMembershipIdentity - Options for entering the RTC session. * @param transport - The LivekitTransport to use for this session. - * @param options - Options for entering the RTC session. - * @param options.encryptMedia - Whether to encrypt media. - * @param options.matrixRTCMode - The Matrix RTC mode to use. + * @param options - `encryptMedia`: Whether to encrypt media `matrixRTCMode`: The Matrix RTC mode to use. * @throws If the widget could not send ElementWidgetActions.JoinCall action. */ // Exported for unit testing @@ -698,8 +697,9 @@ export function enterRTCSession( rtcSession: MatrixRTCSession, ownMembershipIdentity: CallMembershipIdentityParts, transport: LivekitTransport, - { encryptMedia, matrixRTCMode }: EnterRTCSessionOptions, + options: EnterRTCSessionOptions, ): void { + const { encryptMedia, matrixRTCMode } = options; PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date()); PosthogAnalytics.instance.eventCallStarted.track(rtcSession.room.roomId); diff --git a/src/state/CallViewModel/localMember/LocalTransport.ts b/src/state/CallViewModel/localMember/LocalTransport.ts index 50d6cec6..fa316805 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.ts @@ -159,10 +159,12 @@ const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci"; * 4. The transport configured in Element Call's config. * * @param client The authenticated Matrix client for the current user + * @param membership The membership identity of the user. * @param roomId The ID of the room to be connected to. * @param urlFromDevSettings Override URL provided by the user's local config. - * @param useMatrix2 This implies using the matrix2 jwt endpoint (including delayed event delegation of the jwt token) - * @param delayId + * @param forceOldJwtEndpoint Whether to force the old JWT endpoint (not hashing the backendIdentity). + * @param delayId the delay id passed to the jwt service. + * * @returns A fully validated transport config. * @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken */ diff --git a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts index 8e9c0dab..668538ac 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts @@ -31,8 +31,8 @@ export interface ConnectionFactory { createConnection( transport: LivekitTransport, scope: ObservableScope, - logger: Logger, ownMembershipIdentity: CallMembershipIdentityParts, + logger: Logger, forceOldJwtEndpoint?: boolean, ): Connection; } @@ -83,17 +83,18 @@ export class ECConnectionFactory implements ConnectionFactory { /** * - * @param transport - * @param scope - * @param logger + * @param transport The transport to use for this connection. + * @param scope The observable scope (used for clean-up) * @param ownMembershipIdentity required to connect (using the jwt service) with the SFU. + * @param logger The logger instance to use for this connection. + * @param forceOldJwtEndpoint Use the old JWT endpoint independent of what the sfu supports. * @returns */ public createConnection( transport: LivekitTransport, scope: ObservableScope, - logger: Logger, ownMembershipIdentity: CallMembershipIdentityParts, + logger: Logger, forceOldJwtEndpoint?: boolean, ): Connection { return new Connection( diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts index aa5a15ba..9d546d24 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts @@ -83,8 +83,12 @@ export interface IConnectionManager { * @param props - Configuration object * @param props.scope - The observable scope used by this object * @param props.connectionFactory - Used to create new connections - * @param props.inputTransports$ - A list of Behaviors each containing a LIST of LivekitTransport. - * @param props.logger - The logger to use + * @param props.localTransport$ - The local transport to use. (deduplicated with remoteTransports$) + * @param props.remoteTransports$ - All other transports. The connection manager will create connections for each transport. (deduplicated with localTransport$) + * @param props.ownMembershipIdentity - The own membership identity to use. + * @param props.logger - The logger to use. + * @param props.forceOldJwtEndpointForLocalTransport$ - Use the old JWT endpoint independent of what the sfu supports. Only applies for localTransport$. + * * Each of these behaviors can be interpreted as subscribed list of transports. * * Using `registerTransports` independent external modules can control what connections @@ -183,8 +187,8 @@ export function createConnectionManager$({ livekit_alias: alias, }, scope, - logger, ownMembershipIdentity, + logger, forceOldJwtEndpoint, ); // Start the connection immediately diff --git a/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts b/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts index 3c60e776..f28bd158 100644 --- a/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts +++ b/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts @@ -79,8 +79,8 @@ describe("ECConnectionFactory - Audio inputs options", () => { ecConnectionFactory.createConnection( exampleTransport, testScope, - logger, ownMemberMock, + logger, ); // Check if Room was constructed with expected options @@ -125,8 +125,8 @@ describe("ECConnectionFactory - ControlledAudioDevice", () => { ecConnectionFactory.createConnection( exampleTransport, testScope, - logger, ownMemberMock, + logger, ); // Check if Room was constructed with expected options From 83d04ac1222784de3af2997e2021ebf2a8766fdd Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 7 Jan 2026 13:26:37 +0100 Subject: [PATCH 15/46] fix tests and remove duplicated mocks. --- .../localMember/LocalMember.test.ts | 11 ++- .../localMember/LocalTransport.test.ts | 6 ++ .../localMember/LocalTransport.ts | 6 +- .../remoteMembers/ConnectionManager.ts | 8 +- .../MatrixLivekitMembers.test.ts | 6 +- .../MatrixMemberMetadata.test.ts | 72 ++++++++-------- .../remoteMembers/integration.test.ts | 8 +- src/utils/test-fixtures.ts | 14 ++-- src/utils/test.ts | 84 +++++++------------ 9 files changed, 103 insertions(+), 112 deletions(-) diff --git a/src/state/CallViewModel/localMember/LocalMember.test.ts b/src/state/CallViewModel/localMember/LocalMember.test.ts index edd5ea1e..8a7505eb 100644 --- a/src/state/CallViewModel/localMember/LocalMember.test.ts +++ b/src/state/CallViewModel/localMember/LocalMember.test.ts @@ -104,7 +104,7 @@ describe("LocalMembership", () => { getOldestMembership: vi.fn().mockReturnValue({ getPreferredFoci: vi.fn().mockReturnValue([focusFromOlderMembership]), }), - joinRoomSession: vi.fn(), + joinRTCSession: vi.fn(), }) as unknown as MatrixRTCSession; enterRTCSession( @@ -121,7 +121,12 @@ describe("LocalMembership", () => { }, ); - expect(mockedSession.joinRoomSession).toHaveBeenLastCalledWith( + expect(mockedSession.joinRTCSession).toHaveBeenLastCalledWith( + { + deviceId: "DEVICE", + memberId: "@alice:example.org:DEVICE", + userId: "@alice:example.org", + }, [ { livekit_alias: "roomId", @@ -163,7 +168,7 @@ describe("LocalMembership", () => { }, memberships: [], getFocusInUse: vi.fn(), - joinRoomSession: vi.fn(), + joinRTCSession: vi.fn(), }) as unknown as MatrixRTCSession; enterRTCSession( diff --git a/src/state/CallViewModel/localMember/LocalTransport.test.ts b/src/state/CallViewModel/localMember/LocalTransport.test.ts index 2199ca94..c37cab56 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.test.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.test.ts @@ -154,6 +154,7 @@ describe("LocalTransport", () => { await flushPromises(); // final expect(localTransport$.value).toStrictEqual({ + forceOldJwtEndpoint: false, livekit_alias: "!room:example.org", livekit_service_url: "https://lk.example.org", type: "livekit", @@ -195,6 +196,7 @@ describe("LocalTransport", () => { await flushPromises(); // final expect(localTransport$.value).toStrictEqual({ + forceOldJwtEndpoint: false, livekit_alias: "!example_room_id", livekit_service_url: "https://lk.example.org", type: "livekit", @@ -247,6 +249,7 @@ describe("LocalTransport", () => { expect(localTransport$.value).toBe(null); await flushPromises(); expect(localTransport$.value).toStrictEqual({ + forceOldJwtEndpoint: false, livekit_alias: "!example_room_id", livekit_service_url: "https://lk.example.org", type: "livekit", @@ -259,6 +262,7 @@ describe("LocalTransport", () => { expect(localTransport$.value).toBe(null); await flushPromises(); expect(localTransport$.value).toStrictEqual({ + forceOldJwtEndpoint: false, livekit_alias: "!example_room_id", livekit_service_url: "https://lk.example.org", type: "livekit", @@ -273,6 +277,7 @@ describe("LocalTransport", () => { expect(localTransport$.value).toBe(null); await flushPromises(); expect(localTransport$.value).toStrictEqual({ + forceOldJwtEndpoint: false, livekit_alias: "!example_room_id", livekit_service_url: "https://lk.example.org", type: "livekit", @@ -304,6 +309,7 @@ describe("LocalTransport", () => { expect(localTransport$.value).toBe(null); await flushPromises(); expect(localTransport$.value).toStrictEqual({ + forceOldJwtEndpoint: false, livekit_alias: "!example_room_id", livekit_service_url: "https://lk.example.org", type: "livekit", diff --git a/src/state/CallViewModel/localMember/LocalTransport.ts b/src/state/CallViewModel/localMember/LocalTransport.ts index fa316805..0dae3c99 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.ts @@ -84,7 +84,9 @@ export const createLocalTransport$ = ({ useOldestMember$, useOldJwtEndpoint$, delayId$, -}: Props): Behavior => { +}: Props): Behavior< + (LivekitTransport & { forceOldJwtEndpoint: boolean }) | null +> => { /** * The transport over which we should be actively publishing our media. * undefined when not joined. @@ -108,7 +110,7 @@ export const createLocalTransport$ = ({ * * @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken */ - const preferredTransport$: Behavior = scope.behavior( + const preferredTransport$ = scope.behavior( combineLatest([customLivekitUrl.value$, delayId$, useOldJwtEndpoint$]).pipe( switchMap(([customUrl, delayId, forceOldJwtEndpoint]) => from( diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts index 9d546d24..e5a542df 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts @@ -144,12 +144,14 @@ export function createConnectionManager$({ localTransport, forceOldJwtEndpointForLocalTransport, ]) => { - // nmodify only the local transport with forceOldJwtEndpointForLocalTransport + // modify only the local transport with forceOldJwtEndpointForLocalTransport const index = transports.value.findIndex((t) => areLivekitTransportsEqual(localTransport, t), ); - transports.value[index].forceOldJwtEndpoint = - forceOldJwtEndpointForLocalTransport; + if (index !== -1) { + transports.value[index].forceOldJwtEndpoint = + forceOldJwtEndpointForLocalTransport; + } logger.trace( `Managing transports: ${transports.value.map((t) => t.livekit_service_url).join(", ")}`, ); diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts index de0d7ecc..55549a10 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts @@ -26,7 +26,7 @@ import { import { ConnectionManagerData } from "./ConnectionManager.ts"; import { flushPromises, - mockCallMembership, + mockRtcMembership, mockRemoteParticipant, } from "../../../utils/test.ts"; import { type Connection } from "./Connection.ts"; @@ -49,12 +49,12 @@ const transportB: LivekitTransport = { livekit_alias: "!alias:sample.com", }; -const bobMembership = mockCallMembership( +const bobMembership = mockRtcMembership( "@bob:example.org", "DEV000", transportA, ); -const carlMembership = mockCallMembership( +const carlMembership = mockRtcMembership( "@carl:sample.com", "DEV111", transportB, diff --git a/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.test.ts b/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.test.ts index 6f392351..f7dd775c 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.test.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.test.ts @@ -18,7 +18,7 @@ import { it } from "vitest"; import { ObservableScope } from "../../ObservableScope.ts"; import type { Room as MatrixRoom } from "matrix-js-sdk/lib/models/room"; import { - mockCallMembership, + mockRtcMembership, mockMatrixRoomMember, withTestScheduler, } from "../../../utils/test.ts"; @@ -111,7 +111,7 @@ describe("MatrixMemberMetadata", () => { rawDisplayName: "it's a me", }); const memberships$ = behavior("a", { - a: [mockCallMembership("@local:example.com", "DEVICE1")], + a: [mockRtcMembership("@local:example.com", "DEVICE1")], }); const metadataStore = createMatrixMemberMetadata$( testScope, @@ -149,8 +149,8 @@ describe("MatrixMemberMetadata", () => { withTestScheduler(({ behavior, expectObservable }) => { const memberships$ = behavior("a", { a: [ - mockCallMembership("@alice:example.com", "DEVICE1"), - mockCallMembership("@bob:example.com", "DEVICE1"), + mockRtcMembership("@alice:example.com", "DEVICE1"), + mockRtcMembership("@bob:example.com", "DEVICE1"), ], }); const metadataStore = createMatrixMemberMetadata$( @@ -179,7 +179,7 @@ describe("MatrixMemberMetadata", () => { setUpBasicRoom(); const memberships$ = behavior("a", { - a: [mockCallMembership("@no-name:foo.bar", "D000")], + a: [mockRtcMembership("@no-name:foo.bar", "D000")], }); const metadataStore = createMatrixMemberMetadata$( testScope, @@ -201,11 +201,11 @@ describe("MatrixMemberMetadata", () => { const memberships$ = behavior("a", { a: [ - mockCallMembership("@bob:example.com", "DEVICE1"), - mockCallMembership("@bob:example.com", "DEVICE2"), - mockCallMembership("@bob:foo.bar", "BOB000"), - mockCallMembership("@carl:example.com", "C000"), - mockCallMembership("@evil:example.com", "E000"), + mockRtcMembership("@bob:example.com", "DEVICE1"), + mockRtcMembership("@bob:example.com", "DEVICE2"), + mockRtcMembership("@bob:foo.bar", "BOB000"), + mockRtcMembership("@carl:example.com", "C000"), + mockRtcMembership("@evil:example.com", "E000"), ], }); @@ -233,10 +233,10 @@ describe("MatrixMemberMetadata", () => { setUpBasicRoom(); const memberships$ = behavior("ab", { - a: [mockCallMembership("@bob:example.com", "DEVICE1")], + a: [mockRtcMembership("@bob:example.com", "DEVICE1")], b: [ - mockCallMembership("@bob:example.com", "DEVICE1"), - mockCallMembership("@bob:foo.bar", "BOB000"), + mockRtcMembership("@bob:example.com", "DEVICE1"), + mockRtcMembership("@bob:foo.bar", "BOB000"), ], }); @@ -262,10 +262,10 @@ describe("MatrixMemberMetadata", () => { const memberships$ = behavior("ab", { a: [ - mockCallMembership("@bob:example.com", "DEVICE1"), - mockCallMembership("@bob:foo.bar", "BOB000"), + mockRtcMembership("@bob:example.com", "DEVICE1"), + mockRtcMembership("@bob:foo.bar", "BOB000"), ], - b: [mockCallMembership("@bob:example.com", "DEVICE1")], + b: [mockRtcMembership("@bob:example.com", "DEVICE1")], }); const metadataStore = createMatrixMemberMetadata$( @@ -292,8 +292,8 @@ describe("MatrixMemberMetadata", () => { const memberships$ = behavior("a", { a: [ - mockCallMembership("@bob:example.com", "B000"), - mockCallMembership("@carl:example.com", "C000"), + mockRtcMembership("@bob:example.com", "B000"), + mockRtcMembership("@carl:example.com", "C000"), ], }); const metadataStore = createMatrixMemberMetadata$( @@ -331,16 +331,16 @@ describe("MatrixMemberMetadata", () => { // - room join/leave // - disambiguate const memberships$ = behavior("ab-d", { - a: [mockCallMembership(CARL, "C000")], + a: [mockRtcMembership(CARL, "C000")], b: [ - mockCallMembership(CARL, "C000"), + mockRtcMembership(CARL, "C000"), // bob joins - mockCallMembership(BOB, "B000"), + mockRtcMembership(BOB, "B000"), ], // c carl gets renamed to BOB d: [ // carl leaves - mockCallMembership(BOB, "B000"), + mockRtcMembership(BOB, "B000"), ], }); schedule("--a-", { @@ -379,8 +379,8 @@ describe("MatrixMemberMetadata", () => { it("should disambiguate users with invisible characters", () => { withTestScheduler(({ behavior, expectObservable }) => { - const bobRtcMember = mockCallMembership("@bob:example.org", "BBBB"); - const bobZeroWidthSpaceRtcMember = mockCallMembership( + const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB"); + const bobZeroWidthSpaceRtcMember = mockRtcMembership( "@bob2:example.org", "BBBB", ); @@ -397,9 +397,9 @@ describe("MatrixMemberMetadata", () => { fakeMemberWith(bobZeroWidthSpace); fakeMemberWith({ userId: "@carol:example.org" }); const memberships$ = behavior("ab", { - a: [mockCallMembership("@carol:example.org", "1111"), bobRtcMember], + a: [mockRtcMembership("@carol:example.org", "1111"), bobRtcMember], b: [ - mockCallMembership("@carol:example.org", "1111"), + mockRtcMembership("@carol:example.org", "1111"), bobRtcMember, bobZeroWidthSpaceRtcMember, ], @@ -450,8 +450,8 @@ describe("MatrixMemberMetadata", () => { it("should strip RTL characters from displayname", () => { withTestScheduler(({ behavior, expectObservable }) => { - const daveRtcMember = mockCallMembership("@dave:example.org", "DDDD"); - const daveRTLRtcMember = mockCallMembership( + const daveRtcMember = mockRtcMembership("@dave:example.org", "DDDD"); + const daveRTLRtcMember = mockRtcMembership( "@dave2:example.org", "DDDD", ); @@ -466,9 +466,9 @@ describe("MatrixMemberMetadata", () => { fakeMemberWith(daveRTL); fakeMemberWith(dave); const memberships$ = behavior("ab", { - a: [mockCallMembership("@carol:example.org", "DDDD")], + a: [mockRtcMembership("@carol:example.org", "DDDD")], b: [ - mockCallMembership("@carol:example.org", "DDDD"), + mockRtcMembership("@carol:example.org", "DDDD"), daveRtcMember, daveRTLRtcMember, ], @@ -527,8 +527,8 @@ describe("MatrixMemberMetadata", () => { }); const memberships$ = behavior("a", { a: [ - mockCallMembership("@local:example.com", "DEVICE1"), - mockCallMembership("@alice:example.com", "DEVICE1"), + mockRtcMembership("@local:example.com", "DEVICE1"), + mockRtcMembership("@alice:example.com", "DEVICE1"), ], }); const metadataStore = createMatrixMemberMetadata$( @@ -562,12 +562,12 @@ describe("MatrixMemberMetadata", () => { fakeMemberWith({ userId: "@carl:example.com" }); fakeMemberWith({ userId: "@bob:example.com" }); const memberships$ = behavior("ab-d", { - a: [mockCallMembership("@bob:example.com", "B000")], + a: [mockRtcMembership("@bob:example.com", "B000")], b: [ - mockCallMembership("@bob:example.com", "B000"), - mockCallMembership("@carl:example.com", "C000"), + mockRtcMembership("@bob:example.com", "B000"), + mockRtcMembership("@carl:example.com", "C000"), ], - d: [mockCallMembership("@carl:example.com", "C000")], + d: [mockRtcMembership("@carl:example.com", "C000")], }); const metadataStore = createMatrixMemberMetadata$( diff --git a/src/state/CallViewModel/remoteMembers/integration.test.ts b/src/state/CallViewModel/remoteMembers/integration.test.ts index 84e09487..c29f07c0 100644 --- a/src/state/CallViewModel/remoteMembers/integration.test.ts +++ b/src/state/CallViewModel/remoteMembers/integration.test.ts @@ -21,8 +21,8 @@ import { import { ECConnectionFactory } from "./ConnectionFactory.ts"; import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts"; import { - mockCallMembership, mockMediaDevices, + mockRtcMembership, ownMemberMock, withTestScheduler, } from "../../../utils/test.ts"; @@ -101,9 +101,9 @@ afterEach(() => { test("bob, carl, then bob joining no tracks yet", () => { withTestScheduler(({ expectObservable, behavior, scope }) => { - const bobMembership = mockCallMembership("@bob:example.com", "BDEV000"); - const carlMembership = mockCallMembership("@carl:example.com", "CDEV000"); - const daveMembership = mockCallMembership("@dave:foo.bar", "DDEV000"); + const bobMembership = mockRtcMembership("@bob:example.com", "BDEV000"); + const carlMembership = mockRtcMembership("@carl:example.com", "CDEV000"); + const daveMembership = mockRtcMembership("@dave:foo.bar", "DDEV000"); const eMarble = "abc"; const vMarble = "abc"; diff --git a/src/utils/test-fixtures.ts b/src/utils/test-fixtures.ts index f915bb19..dcdb9f9c 100644 --- a/src/utils/test-fixtures.ts +++ b/src/utils/test-fixtures.ts @@ -17,14 +17,16 @@ export const localRtcMemberDevice2 = mockRtcMembership( "2222", ); export const local = mockMatrixRoomMember(localRtcMember); -// export const localParticipant = mockLocalParticipant({ identity: "" }); + export const localId = `${local.userId}:${localRtcMember.deviceId}`; -export const aliceRtcMember = mockRtcMembership("@alice:example.org", "AAAA"); +export const aliceDeviceId = "AAAA"; +export const aliceUserId = "@alice:example.org"; +export const aliceId = `${aliceUserId}:${aliceDeviceId}`; +export const aliceRtcMember = mockRtcMembership(aliceUserId, aliceDeviceId); export const alice = mockMatrixRoomMember(aliceRtcMember, { rawDisplayName: "Alice", }); -export const aliceId = `${alice.userId}:${aliceRtcMember.deviceId}`; export const aliceParticipant = mockRemoteParticipant({ identity: aliceId }); export const aliceDoppelgangerRtcMember = mockRtcMembership( @@ -38,11 +40,13 @@ export const aliceDoppelganger = mockMatrixRoomMember( }, ); -export const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB"); +export const bobDeviceId = "BBBB"; +export const bobUserId = "@bob:example.org"; +export const bobId = `${bobUserId}:${bobDeviceId}`; +export const bobRtcMember = mockRtcMembership(bobUserId, bobDeviceId); export const bob = mockMatrixRoomMember(bobRtcMember, { rawDisplayName: "Bob", }); -export const bobId = `${bob.userId}:${bobRtcMember.deviceId}`; export const bobZeroWidthSpaceRtcMember = mockRtcMembership( "@bob2:example.org", diff --git a/src/utils/test.ts b/src/utils/test.ts index 02277af0..b19ea961 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -203,40 +203,30 @@ export const exampleTransport: LivekitTransport = { livekit_alias: "!alias:example.org", }; -export function mockCallMembership( - userId: string, - deviceId: string, - transport?: Transport, -): CallMembership { - const t = transport ?? transportForUser(userId); - return { - userId: userId, - deviceId: deviceId, - getTransport: vi.fn().mockReturnValue(t), - transports: [t], - } as unknown as CallMembership; -} - -function transportForUser(userId: string): Transport { - const domain = userId.split(":")[1]; - return { - type: "livekit", - livekit_service_url: `https://lk.${domain}`, - livekit_alias: `!alias:${domain}`, - }; -} - export function mockRtcMembership( user: string | RoomMember, deviceId: string, - callId = "", - fociPreferred: Transport[] = [exampleTransport], - focusActive: LivekitFocusSelection = { - type: "livekit", - focus_selection: "oldest_membership", + customOverwrites?: { + rtcBackendIdentity?: string; + callId?: string; + fociPreferred?: Transport[]; + focusActive?: LivekitFocusSelection; + membership?: Partial; }, - membership: Partial = {}, ): CallMembership { + // setup defaults based on overwrites and fallback values. + const { rtcBackendIdentity, callId, fociPreferred, focusActive, membership } = + { + fociPreferred: [exampleTransport], + focusActive: { + type: "livekit" as const, + focus_selection: "oldest_membership" as const, + }, + callId: "", + membership: {}, + ...customOverwrites, + }; + const data: SessionMembershipData = { application: "m.call", call_id: callId, @@ -245,15 +235,21 @@ export function mockRtcMembership( focus_active: focusActive, ...membership, }; + const userId = typeof user === "string" ? user : user.userId; const event = new MatrixEvent({ - sender: typeof user === "string" ? user : user.userId, + sender: userId, event_id: `$-ev-${randomUUID()}:example.org`, content: data, }); const membershipData = CallMembership.membershipDataFromMatrixEvent(event); - const cms = new CallMembership(event, membershipData, "xx"); + const cms = new CallMembership( + event, + membershipData, + rtcBackendIdentity ?? `${userId}:${deviceId}`, + ); vi.mocked(cms).getTransport = vi.fn().mockReturnValue(fociPreferred[0]); + return cms; } @@ -486,7 +482,7 @@ export class MockRTCSession extends TypedEventEmitter< if (value !== prev) this.emit(MembershipManagerEvent.ProbablyLeft, value); } - public async joinRoomSession(): Promise { + public async joinRTCSession(): Promise { return Promise.resolve(); } } @@ -535,27 +531,3 @@ export function mockMuteStates( const observableScope = new ObservableScope(); return new MuteStates(observableScope, mockMediaDevices({}), joined$); } - -export const mockComputeLivekitParticipantIdentity$ = ( - membership: CallMembershipIdentityParts, - kind: "rtc" | "session", -): Observable => { - function sha256(commitmentStr: string): string { - return encodeUnpaddedBase64( - createHash("sha256").update(commitmentStr, "utf8").digest(), - ); - } - let hash; - switch (kind) { - case "rtc": { - hash = sha256( - `${membership.userId}|${membership.deviceId}|${membership.memberId}`, - ); - break; - } - case "session": - default: - hash = `${membership.userId}:${membership.deviceId}`; - } - return of(hash); -}; From 6480df44e9c4706b6b49ba723b2e70233bb00eea Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 7 Jan 2026 15:10:24 +0100 Subject: [PATCH 16/46] add tests for open id delay fallback --- src/livekit/openIDSFU.test.ts | 99 +++++++++++++++++++ .../MatrixLivekitMembers.test.ts | 16 ++- src/utils/test.ts | 3 +- 3 files changed, 106 insertions(+), 12 deletions(-) diff --git a/src/livekit/openIDSFU.test.ts b/src/livekit/openIDSFU.test.ts index 8b263662..22e487f5 100644 --- a/src/livekit/openIDSFU.test.ts +++ b/src/livekit/openIDSFU.test.ts @@ -34,6 +34,7 @@ describe("getSFUConfigWithOpenID", () => { vitest.clearAllMocks(); fetchMock.reset(); }); + it("should handle fetching a token", async () => { fetchMock.post("https://sfu.example.org/sfu/get", () => { return { @@ -56,6 +57,7 @@ describe("getSFUConfigWithOpenID", () => { }); void (await fetchMock.flush()); }); + it("should fail if the SFU errors", async () => { fetchMock.post("https://sfu.example.org/sfu/get", () => { return { @@ -81,6 +83,103 @@ describe("getSFUConfigWithOpenID", () => { expect.fail("Expected test to throw;"); }); + it("should try legacy and then new endpoint with delay delegation", async () => { + fetchMock.post("https://sfu.example.org/get_token", () => { + return { + status: 500, + body: { error: "Test failure" }, + }; + }); + fetchMock.post("https://sfu.example.org/sfu/get", () => { + return { + status: 500, + body: { error: "Test failure" }, + }; + }); + try { + await getSFUConfigWithOpenID( + matrixClient, + ownMemberMock, + "https://sfu.example.org", + false, + "!example_room_id", + "https://matrix.homeserverserver.org", + "mock_delay_id", + ); + } catch (ex) { + logger.info(ex); + expect(((ex as Error).cause as Error).message).toEqual( + "SFU Config fetch failed with status code 500", + ); + void (await fetchMock.flush()); + } + const calls = fetchMock.calls(); + expect(calls.length).toBe(2); + + expect(calls[0][0]).toStrictEqual("https://sfu.example.org/get_token"); + expect(calls[0][1]).toStrictEqual({ + // check if it uses correct delayID! + body: '{"room_id":"!example_room_id","slot_id":"m.call#ROOM","member":{"id":"@alice:example.org:DEVICE","claimed_user_id":"@alice:example.org","claimed_device_id":"DEVICE"},"delay_id":"mock_delay_id","delay_timeout":1000,"delay_cs_api_url":"https://matrix.homeserverserver.org"}', + method: "POST", + headers: { + "Content-Type": "application/json", + }, + }); + + expect(calls[1][0]).toStrictEqual("https://sfu.example.org/sfu/get"); + + expect(calls[1][1]).toStrictEqual({ + body: '{"room":"!example_room_id","device_id":"DEVICE"}', + headers: { + "Content-Type": "application/json", + }, + method: "POST", + }); + }); + + it("dont try legacy if endpoint with delay delegation is sucessful", async () => { + fetchMock.post("https://sfu.example.org/get_token", () => { + return { + status: 200, + body: { url: sfuUrl, jwt: testJWTToken }, + }; + }); + fetchMock.post("https://sfu.example.org/sfu/get", () => { + return { + status: 500, + body: { error: "Test failure" }, + }; + }); + try { + await getSFUConfigWithOpenID( + matrixClient, + ownMemberMock, + "https://sfu.example.org", + false, + "!example_room_id", + "https://matrix.homeserverserver.org", + "mock_delay_id", + ); + } catch (ex) { + expect(((ex as Error).cause as Error).message).toEqual( + "SFU Config fetch failed with status code 500", + ); + void (await fetchMock.flush()); + } + const calls = fetchMock.calls(); + expect(calls.length).toBe(1); + + expect(calls[0][0]).toStrictEqual("https://sfu.example.org/get_token"); + expect(calls[0][1]).toStrictEqual({ + // check if it uses correct delayID! + body: '{"room_id":"!example_room_id","slot_id":"m.call#ROOM","member":{"id":"@alice:example.org:DEVICE","claimed_user_id":"@alice:example.org","claimed_device_id":"DEVICE"},"delay_id":"mock_delay_id","delay_timeout":1000,"delay_cs_api_url":"https://matrix.homeserverserver.org"}', + method: "POST", + headers: { + "Content-Type": "application/json", + }, + }); + }); + it("should retry fetching the openid token", async () => { let count = 0; matrixClient.getOpenIdToken.mockImplementation(async () => { diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts index 55549a10..5d34f7be 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts @@ -49,16 +49,12 @@ const transportB: LivekitTransport = { livekit_alias: "!alias:sample.com", }; -const bobMembership = mockRtcMembership( - "@bob:example.org", - "DEV000", - transportA, -); -const carlMembership = mockRtcMembership( - "@carl:sample.com", - "DEV111", - transportB, -); +const bobMembership = mockRtcMembership("@bob:example.org", "DEV000", { + fociPreferred: [transportA], +}); +const carlMembership = mockRtcMembership("@carl:sample.com", "DEV111", { + fociPreferred: [transportB], +}); beforeEach(() => { testScope = new ObservableScope(); diff --git a/src/utils/test.ts b/src/utils/test.ts index b19ea961..d24ad130 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -15,7 +15,6 @@ import { vitest, } from "vitest"; import { - encodeUnpaddedBase64, MatrixEvent, type Room as MatrixRoom, type Room, @@ -44,7 +43,7 @@ import { type Room as LivekitRoom, Track, } from "livekit-client"; -import { createHash, randomUUID } from "crypto"; +import { randomUUID } from "crypto"; import { type TrackReference } from "@livekit/components-core"; import EventEmitter from "events"; import { From d48042f5220fca828a471892f52ed23879261728 Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 7 Jan 2026 15:18:37 +0100 Subject: [PATCH 17/46] fix lint --- src/livekit/openIDSFU.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/livekit/openIDSFU.test.ts b/src/livekit/openIDSFU.test.ts index 22e487f5..aed66d33 100644 --- a/src/livekit/openIDSFU.test.ts +++ b/src/livekit/openIDSFU.test.ts @@ -107,7 +107,6 @@ describe("getSFUConfigWithOpenID", () => { "mock_delay_id", ); } catch (ex) { - logger.info(ex); expect(((ex as Error).cause as Error).message).toEqual( "SFU Config fetch failed with status code 500", ); From dd562bdaf51ec99c63c0a754c31bf2d9d90fe186 Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 7 Jan 2026 15:36:32 +0100 Subject: [PATCH 18/46] dont use throw for logic. --- src/livekit/openIDSFU.ts | 51 +++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/src/livekit/openIDSFU.ts b/src/livekit/openIDSFU.ts index b89243c1..6bade4ef 100644 --- a/src/livekit/openIDSFU.ts +++ b/src/livekit/openIDSFU.ts @@ -105,35 +105,42 @@ export async function getSFUConfigWithOpenID( logger?.info(`Trying to get JWT for focus ${serviceUrl}...`); - let sfuConfig: { url: string; jwt: string }; - try { - // we do not want to try the old endpoint, since we are not sending the new matrix2.0 sticky events (no hashed identity in the event) - if (forceOldJwtEndpoint) throw new Error("Force old jwt endpoint"); - if (!delayId) - throw new Error("No delayId, Won't try matrix 2.0 jwt endpoint."); + let sfuConfig: { url: string; jwt: string } | undefined; - sfuConfig = await getLiveKitJWTWithDelayDelegation( - membership, - serviceUrl, - roomId, - openIdToken, - delayEndpointBaseUrl, - delayId, - ); - logger?.info(`Got JWT from call's active focus URL.`); - } catch (e) { - logger?.warn( - `Failed fetching jwt with matrix 2.0 endpoint (retry with legacy)`, - e, - ); + // If forceOldJwtEndpoint is set we indicate that we do not want to try the new endpoint, + // since we are not sending the new matrix2.0 sticky events (no hashed identity in the event) + if (forceOldJwtEndpoint === false) { + try { + sfuConfig = await getLiveKitJWTWithDelayDelegation( + membership, + serviceUrl, + roomId, + openIdToken, + delayEndpointBaseUrl, + delayId, + ); + logger?.info(`Got JWT from call's active focus URL.`); + } catch (e) { + sfuConfig = undefined; + logger?.warn( + `Failed fetching jwt with matrix 2.0 endpoint (retry with legacy)`, + e, + ); + logger?.info(`Got JWT from call's active focus URL.`); + } + } + + // Either forceOldJwtEndpoint = true or getLiveKitJWTWithDelayDelegation throws -> reset sfuConfig = undefined + if (sfuConfig === undefined) { sfuConfig = await getLiveKitJWT( membership.deviceId, serviceUrl, roomId, openIdToken, ); - logger?.info(`Got JWT from call's active focus URL.`); - } // Pull the details from the JWT + } + + // Pull the details from the JWT const [, payloadStr] = sfuConfig.jwt.split("."); // TODO: Prefer Uint8Array.fromBase64 when widely available const payload = JSON.parse(global.atob(payloadStr)) as SFUJWTPayload; From d814f60f23b8d366bc7ef802ecfd87cbb24536ec Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 7 Jan 2026 15:47:44 +0100 Subject: [PATCH 19/46] review (docstrings) and remove unused artifacts. --- src/state/CallViewModel/CallViewModel.ts | 1 + .../localMember/LocalTransport.ts | 20 +++++++------------ .../remoteMembers/Connection.test.ts | 1 - 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 093abfad..9654920d 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -431,6 +431,7 @@ export function createCallViewModel$( const useOldJwtEndpoint$ = scope.behavior( matrixRTCMode$.pipe(map((v) => v !== MatrixRTCMode.Matrix_2_0)), ); + const localTransport$ = createLocalTransport$({ scope: scope, memberships$: memberships$, diff --git a/src/state/CallViewModel/localMember/LocalTransport.ts b/src/state/CallViewModel/localMember/LocalTransport.ts index 0dae3c99..8a6a750b 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.ts @@ -68,11 +68,10 @@ interface Props { * @prop useOldestMember Whether to use the same transport as the oldest member. * This will only update once the first oldest member appears. Will not recompute if the oldest member leaves. * - * @prop useOldJwtEndpoint$ Whether to set forceOldJwtEndpoint the use the old JWT endpoint. + * @prop useOldJwtEndpoint$ Whether to set forceOldJwtEndpoint on the returned transport and to use the old JWT endpoint. * This is used when the connection manager needs to know if it has to use the legacy endpoint which implies a string concatenated rtcBackendIdentity. * (which is expected for non sticky event based rtc member events) - * @returns Behavior<(LivekitTransport & { forceOldJwtEndpoint: boolean }) | null> The `forceOldJwtEndpoint` field is added to let the connection EncryptionManager - * know that this transport is for the local member and it IS RELEVANT which jwt endpoint to use. (for the local member transport, we need to know which jwt endpoint to use) + * @returns The local transport. It will be created using the correct sfu endpoint based on the useOldJwtEndpoint$ value. * @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken */ export const createLocalTransport$ = ({ @@ -84,9 +83,7 @@ export const createLocalTransport$ = ({ useOldestMember$, useOldJwtEndpoint$, delayId$, -}: Props): Behavior< - (LivekitTransport & { forceOldJwtEndpoint: boolean }) | null -> => { +}: Props): Behavior => { /** * The transport over which we should be actively publishing our media. * undefined when not joined. @@ -97,7 +94,7 @@ export const createLocalTransport$ = ({ const oldestMember = memberships.value[0]; const transport = oldestMember?.getTransport(memberships.value[0]); if (!transport) return null; - return { ...transport, forceOldJwtEndpoint }; + return transport; }), first((t) => t != null && isLivekitTransport(t)), ), @@ -181,12 +178,10 @@ async function makeTransport( urlFromDevSettings: string | null, forceOldJwtEndpoint: boolean, delayId?: string, -): Promise { +): Promise { logger.trace("Searching for a preferred transport"); - async function doOpenIdAndJWTFromUrl( - url: string, - ): Promise { + async function doOpenIdAndJWTFromUrl(url: string): Promise { const { livekitAlias } = await getSFUConfigWithOpenID( client, membership, @@ -201,7 +196,6 @@ async function makeTransport( type: "livekit", livekit_service_url: url, livekit_alias: livekitAlias, - forceOldJwtEndpoint, }; } // We will call `getSFUConfigWithOpenID` once per transport here as it's our @@ -223,7 +217,7 @@ async function makeTransport( async function getFirstUsableTransport( transports: Transport[], - ): Promise<(LivekitTransport & { forceOldJwtEndpoint: boolean }) | null> { + ): Promise { for (const potentialTransport of transports) { if (isLivekitTransportConfig(potentialTransport)) { try { diff --git a/src/state/CallViewModel/remoteMembers/Connection.test.ts b/src/state/CallViewModel/remoteMembers/Connection.test.ts index 916e7dd4..239a5c75 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.test.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.test.ts @@ -55,7 +55,6 @@ const livekitFocus: LivekitTransport = { livekit_alias: "!roomID:example.org", livekit_service_url: "https://matrix-rtc.example.org/livekit/jwt", type: "livekit", - useMatrix2: false, }; function setupTest(): void { From 75fca3108a17be45be925e67d86e48d4b5218674 Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 7 Jan 2026 16:08:20 +0100 Subject: [PATCH 20/46] cleanup an rename compatibility mode --- src/settings/DeveloperSettingsTab.tsx | 4 ++-- src/settings/settings.ts | 2 +- src/state/CallViewModel/CallViewModel.test.ts | 2 +- src/state/CallViewModel/CallViewModel.ts | 6 +++++- src/state/CallViewModel/localMember/LocalMember.ts | 4 +++- src/state/CallViewModelWidget.test.ts | 2 +- 6 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/settings/DeveloperSettingsTab.tsx b/src/settings/DeveloperSettingsTab.tsx index 254aaf0f..c88eadf0 100644 --- a/src/settings/DeveloperSettingsTab.tsx +++ b/src/settings/DeveloperSettingsTab.tsx @@ -275,8 +275,8 @@ export const DeveloperSettingsTab: FC = ({ name={matrixRTCModeRadioGroup} control={ } diff --git a/src/settings/settings.ts b/src/settings/settings.ts index 33408fd9..a674f1aa 100644 --- a/src/settings/settings.ts +++ b/src/settings/settings.ts @@ -126,7 +126,7 @@ export const alwaysShowIphoneEarpiece = new Setting( export enum MatrixRTCMode { Legacy = "legacy", - Compatibil = "compatibil", + Compatibility = "compatibility", /** This implies using * - sticky events * - hashed RTC backend identity diff --git a/src/state/CallViewModel/CallViewModel.test.ts b/src/state/CallViewModel/CallViewModel.test.ts index 6e3837c4..376d8986 100644 --- a/src/state/CallViewModel/CallViewModel.test.ts +++ b/src/state/CallViewModel/CallViewModel.test.ts @@ -235,7 +235,7 @@ const mockLegacyRingEvent = {} as { event_id: string } & ICallNotifyContent; describe.each([ [MatrixRTCMode.Legacy], - [MatrixRTCMode.Compatibil], + [MatrixRTCMode.Compatibility], [MatrixRTCMode.Matrix_2_0], ])("CallViewModel (%s mode)", (mode) => { const withCallViewModel = withCallViewModelInMode(mode); diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 9654920d..c75b3ec4 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -429,7 +429,11 @@ export function createCallViewModel$( }; const useOldJwtEndpoint$ = scope.behavior( - matrixRTCMode$.pipe(map((v) => v !== MatrixRTCMode.Matrix_2_0)), + matrixRTCMode$.pipe( + map( + (v) => v === MatrixRTCMode.Legacy || v === MatrixRTCMode.Compatibility, + ), + ), ); const localTransport$ = createLocalTransport$({ diff --git a/src/state/CallViewModel/localMember/LocalMember.ts b/src/state/CallViewModel/localMember/LocalMember.ts index 17f766ff..5b72266f 100644 --- a/src/state/CallViewModel/localMember/LocalMember.ts +++ b/src/state/CallViewModel/localMember/LocalMember.ts @@ -711,7 +711,9 @@ export function enterRTCSession( const useDeviceSessionMemberEvents = features?.feature_use_device_session_member_events; const { sendNotificationType: notificationType, callIntent } = getUrlParams(); - const multiSFU = matrixRTCMode !== MatrixRTCMode.Legacy; + const multiSFU = + matrixRTCMode === MatrixRTCMode.Compatibility || + matrixRTCMode === MatrixRTCMode.Matrix_2_0; // Multi-sfu does not need a preferred foci list. just the focus that is actually used. // TODO where/how do we track errors originating from the ongoing rtcSession? rtcSession.joinRTCSession( diff --git a/src/state/CallViewModelWidget.test.ts b/src/state/CallViewModelWidget.test.ts index 5d6442f1..76776720 100644 --- a/src/state/CallViewModelWidget.test.ts +++ b/src/state/CallViewModelWidget.test.ts @@ -37,7 +37,7 @@ vi.mock("../widget", () => ({ it.each([ [MatrixRTCMode.Legacy], - [MatrixRTCMode.Compatibil], + [MatrixRTCMode.Compatibility], [MatrixRTCMode.Matrix_2_0], ])( "expect leave when ElementWidgetActions.HangupCall is called (%s mode)", From d5ad2e38e2583670f4ec11e7a000691b478a536a Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 7 Jan 2026 16:09:08 +0100 Subject: [PATCH 21/46] fix up tests --- src/state/CallViewModel/localMember/LocalTransport.test.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/state/CallViewModel/localMember/LocalTransport.test.ts b/src/state/CallViewModel/localMember/LocalTransport.test.ts index c37cab56..2199ca94 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.test.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.test.ts @@ -154,7 +154,6 @@ describe("LocalTransport", () => { await flushPromises(); // final expect(localTransport$.value).toStrictEqual({ - forceOldJwtEndpoint: false, livekit_alias: "!room:example.org", livekit_service_url: "https://lk.example.org", type: "livekit", @@ -196,7 +195,6 @@ describe("LocalTransport", () => { await flushPromises(); // final expect(localTransport$.value).toStrictEqual({ - forceOldJwtEndpoint: false, livekit_alias: "!example_room_id", livekit_service_url: "https://lk.example.org", type: "livekit", @@ -249,7 +247,6 @@ describe("LocalTransport", () => { expect(localTransport$.value).toBe(null); await flushPromises(); expect(localTransport$.value).toStrictEqual({ - forceOldJwtEndpoint: false, livekit_alias: "!example_room_id", livekit_service_url: "https://lk.example.org", type: "livekit", @@ -262,7 +259,6 @@ describe("LocalTransport", () => { expect(localTransport$.value).toBe(null); await flushPromises(); expect(localTransport$.value).toStrictEqual({ - forceOldJwtEndpoint: false, livekit_alias: "!example_room_id", livekit_service_url: "https://lk.example.org", type: "livekit", @@ -277,7 +273,6 @@ describe("LocalTransport", () => { expect(localTransport$.value).toBe(null); await flushPromises(); expect(localTransport$.value).toStrictEqual({ - forceOldJwtEndpoint: false, livekit_alias: "!example_room_id", livekit_service_url: "https://lk.example.org", type: "livekit", @@ -309,7 +304,6 @@ describe("LocalTransport", () => { expect(localTransport$.value).toBe(null); await flushPromises(); expect(localTransport$.value).toStrictEqual({ - forceOldJwtEndpoint: false, livekit_alias: "!example_room_id", livekit_service_url: "https://lk.example.org", type: "livekit", From 0eeed4e18e58289b10b21b788281853b758a7938 Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 7 Jan 2026 17:02:06 +0100 Subject: [PATCH 22/46] fix test snapshot --- src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap b/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap index 4a2dada0..1c82d07b 100644 --- a/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap +++ b/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap @@ -284,7 +284,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = ` name="_r_0_" title="" type="radio" - value="compatibil" + value="compatibility" />
Date: Wed, 7 Jan 2026 17:21:08 +0100 Subject: [PATCH 23/46] add retries and be more specific when we fall back to legacy endpoint --- src/livekit/openIDSFU.ts | 133 +++++++++++++++++++++++---------------- 1 file changed, 79 insertions(+), 54 deletions(-) diff --git a/src/livekit/openIDSFU.ts b/src/livekit/openIDSFU.ts index 6bade4ef..8c4434bd 100644 --- a/src/livekit/openIDSFU.ts +++ b/src/livekit/openIDSFU.ts @@ -5,7 +5,12 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type IOpenIDToken, type MatrixClient } from "matrix-js-sdk"; +import { + HTTPError, + retryNetworkOperation, + type IOpenIDToken, + type MatrixClient, +} from "matrix-js-sdk"; import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; import { type Logger } from "matrix-js-sdk/lib/logger"; @@ -111,33 +116,49 @@ export async function getSFUConfigWithOpenID( // since we are not sending the new matrix2.0 sticky events (no hashed identity in the event) if (forceOldJwtEndpoint === false) { try { - sfuConfig = await getLiveKitJWTWithDelayDelegation( - membership, - serviceUrl, - roomId, - openIdToken, - delayEndpointBaseUrl, - delayId, - ); - logger?.info(`Got JWT from call's active focus URL.`); + await retryNetworkOperation(4, async () => { + sfuConfig = await getLiveKitJWTWithDelayDelegation( + membership, + serviceUrl, + roomId, + openIdToken, + delayEndpointBaseUrl, + delayId, + ); + logger?.info(`Got JWT from call's active focus URL.`); + }); } catch (e) { - sfuConfig = undefined; - logger?.warn( - `Failed fetching jwt with matrix 2.0 endpoint (retry with legacy)`, - e, - ); - logger?.info(`Got JWT from call's active focus URL.`); + if (e instanceof NotSupportedError) { + logger?.warn( + `Failed fetching jwt with matrix 2.0 endpoint (retry with legacy) Not supported`, + e, + ); + sfuConfig = undefined; + } else { + logger?.warn( + `Failed fetching jwt with matrix 2.0 endpoint other issues -> not going to try with legacy endpoint`, + e, + ); + } } } + // DEPRECATED // Either forceOldJwtEndpoint = true or getLiveKitJWTWithDelayDelegation throws -> reset sfuConfig = undefined if (sfuConfig === undefined) { - sfuConfig = await getLiveKitJWT( - membership.deviceId, - serviceUrl, - roomId, - openIdToken, - ); + await retryNetworkOperation(4, async () => { + sfuConfig = await getLiveKitJWT( + membership.deviceId, + serviceUrl, + roomId, + openIdToken, + ); + }); + logger?.info(`Got JWT from call's active focus URL.`); + } + + if (!sfuConfig) { + throw new Error("No `sfuConfig` after trying with old and new endpoints"); } // Pull the details from the JWT @@ -161,25 +182,28 @@ async function getLiveKitJWT( matrixRoomId: string, openIDToken: IOpenIDToken, ): Promise<{ url: string; jwt: string }> { - try { - const res = await fetch(livekitServiceURL + "/sfu/get", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ - // This is the actual livekit room alias. For the legacy jwt endpoint simply the room id was used. - room: matrixRoomId, - openid_token: openIDToken, - device_id: deviceId, - }), - }); - if (!res.ok) { - throw new Error("SFU Config fetch failed with status code " + res.status); - } - return await res.json(); - } catch (e) { - throw new Error("SFU Config fetch failed with exception", { cause: e }); + const res = await fetch(livekitServiceURL + "/sfu/get", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + // This is the actual livekit room alias. For the legacy jwt endpoint simply the room id was used. + room: matrixRoomId, + openid_token: openIDToken, + device_id: deviceId, + }), + }); + if (!res.ok) { + throw new Error("SFU Config fetch failed with status code " + res.status); + } + return await res.json(); +} + +class NotSupportedError extends Error { + public constructor(message: string) { + super(message); + this.name = "NotSupported"; } } @@ -216,19 +240,20 @@ export async function getLiveKitJWTWithDelayDelegation( }; } - try { - const res = await fetch(livekitServiceURL + "/get_token", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ ...body, ...bodyDalayParts }), - }); - if (!res.ok) { - throw new Error("SFU Config fetch failed with status code " + res.status); + const res = await fetch(livekitServiceURL + "/get_token", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ ...body, ...bodyDalayParts }), + }); + if (!res.ok) { + const msg = "SFU Config fetch failed with status code " + res.status; + if (res.status === 404) { + throw new NotSupportedError(msg); + } else { + throw new Error(msg); } - return await res.json(); - } catch (e) { - throw new Error("SFU Config fetch failed with exception " + e); } + return await res.json(); } From a5a4bb2b8233cebb01cd17aa025ed1b2e97247d3 Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 7 Jan 2026 17:38:29 +0100 Subject: [PATCH 24/46] add retries inside the `getLiveKitJWTWithDelayDelegation` and `getLiveKitJWT` functions. --- src/livekit/openIDSFU.ts | 90 +++++++++++++++++++++++----------------- 1 file changed, 52 insertions(+), 38 deletions(-) diff --git a/src/livekit/openIDSFU.ts b/src/livekit/openIDSFU.ts index 8c4434bd..cf9ad5bc 100644 --- a/src/livekit/openIDSFU.ts +++ b/src/livekit/openIDSFU.ts @@ -6,7 +6,6 @@ Please see LICENSE in the repository root for full details. */ import { - HTTPError, retryNetworkOperation, type IOpenIDToken, type MatrixClient, @@ -116,17 +115,15 @@ export async function getSFUConfigWithOpenID( // since we are not sending the new matrix2.0 sticky events (no hashed identity in the event) if (forceOldJwtEndpoint === false) { try { - await retryNetworkOperation(4, async () => { - sfuConfig = await getLiveKitJWTWithDelayDelegation( - membership, - serviceUrl, - roomId, - openIdToken, - delayEndpointBaseUrl, - delayId, - ); - logger?.info(`Got JWT from call's active focus URL.`); - }); + sfuConfig = await getLiveKitJWTWithDelayDelegation( + membership, + serviceUrl, + roomId, + openIdToken, + delayEndpointBaseUrl, + delayId, + ); + logger?.info(`Got JWT from call's active focus URL.`); } catch (e) { if (e instanceof NotSupportedError) { logger?.warn( @@ -146,14 +143,13 @@ export async function getSFUConfigWithOpenID( // DEPRECATED // Either forceOldJwtEndpoint = true or getLiveKitJWTWithDelayDelegation throws -> reset sfuConfig = undefined if (sfuConfig === undefined) { - await retryNetworkOperation(4, async () => { - sfuConfig = await getLiveKitJWT( - membership.deviceId, - serviceUrl, - roomId, - openIdToken, - ); - }); + sfuConfig = await getLiveKitJWT( + membership.deviceId, + serviceUrl, + roomId, + openIdToken, + ); + logger?.info(`Got JWT from call's active focus URL.`); } @@ -175,25 +171,33 @@ export async function getSFUConfigWithOpenID( livekitIdentity: payload.sub, }; } - +const RETRIES = 4; async function getLiveKitJWT( deviceId: string, livekitServiceURL: string, matrixRoomId: string, openIDToken: IOpenIDToken, ): Promise<{ url: string; jwt: string }> { - const res = await fetch(livekitServiceURL + "/sfu/get", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ - // This is the actual livekit room alias. For the legacy jwt endpoint simply the room id was used. - room: matrixRoomId, - openid_token: openIDToken, - device_id: deviceId, - }), + let res: Response | undefined; + await retryNetworkOperation(RETRIES, async () => { + res = await fetch(livekitServiceURL + "/sfu/get", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + // This is the actual livekit room alias. For the legacy jwt endpoint simply the room id was used. + room: matrixRoomId, + openid_token: openIDToken, + device_id: deviceId, + }), + }); }); + if (!res) { + throw new Error( + `Network error while connecting to jwt service after ${RETRIES} retries`, + ); + } if (!res.ok) { throw new Error("SFU Config fetch failed with status code " + res.status); } @@ -240,13 +244,23 @@ export async function getLiveKitJWTWithDelayDelegation( }; } - const res = await fetch(livekitServiceURL + "/get_token", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ ...body, ...bodyDalayParts }), + let res: Response | undefined; + + await retryNetworkOperation(RETRIES, async () => { + res = await fetch(livekitServiceURL + "/get_token", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ ...body, ...bodyDalayParts }), + }); }); + + if (!res) { + throw new Error( + `Network error while connecting to jwt service after ${RETRIES} retries`, + ); + } if (!res.ok) { const msg = "SFU Config fetch failed with status code " + res.status; if (res.status === 404) { From 385f63e83e5d0bec8184e62e06e1253d567f8c9a Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 7 Jan 2026 17:46:39 +0100 Subject: [PATCH 25/46] fix tests --- src/livekit/openIDSFU.test.ts | 6 +++--- src/state/CallViewModel/remoteMembers/Connection.test.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/livekit/openIDSFU.test.ts b/src/livekit/openIDSFU.test.ts index aed66d33..5f286958 100644 --- a/src/livekit/openIDSFU.test.ts +++ b/src/livekit/openIDSFU.test.ts @@ -74,7 +74,7 @@ describe("getSFUConfigWithOpenID", () => { "!example_room_id", ); } catch (ex) { - expect(((ex as Error).cause as Error).message).toEqual( + expect((ex as Error).message).toEqual( "SFU Config fetch failed with status code 500", ); void (await fetchMock.flush()); @@ -107,7 +107,7 @@ describe("getSFUConfigWithOpenID", () => { "mock_delay_id", ); } catch (ex) { - expect(((ex as Error).cause as Error).message).toEqual( + expect((ex as Error).message).toEqual( "SFU Config fetch failed with status code 500", ); void (await fetchMock.flush()); @@ -160,7 +160,7 @@ describe("getSFUConfigWithOpenID", () => { "mock_delay_id", ); } catch (ex) { - expect(((ex as Error).cause as Error).message).toEqual( + expect((ex as Error).message).toEqual( "SFU Config fetch failed with status code 500", ); void (await fetchMock.flush()); diff --git a/src/state/CallViewModel/remoteMembers/Connection.test.ts b/src/state/CallViewModel/remoteMembers/Connection.test.ts index 239a5c75..0130a5ce 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.test.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.test.ts @@ -259,7 +259,7 @@ describe("Start connection states", () => { capturedState.cause instanceof Error ) { expect(capturedState.cause.message).toContain( - "SFU Config fetch failed with exception", + "SFU Config fetch failed with status code 500", ); expect(connection.transport.livekit_alias).toEqual( livekitFocus.livekit_alias, From 1909aef1862c049482855483a172f15ba8ad0d74 Mon Sep 17 00:00:00 2001 From: Timo K Date: Thu, 8 Jan 2026 12:27:17 +0100 Subject: [PATCH 26/46] temp --- src/livekit/openIDSFU.ts | 5 +++-- src/state/CallViewModel/localMember/LocalMember.ts | 7 ++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/livekit/openIDSFU.ts b/src/livekit/openIDSFU.ts index cf9ad5bc..df1d02a5 100644 --- a/src/livekit/openIDSFU.ts +++ b/src/livekit/openIDSFU.ts @@ -133,7 +133,8 @@ export async function getSFUConfigWithOpenID( sfuConfig = undefined; } else { logger?.warn( - `Failed fetching jwt with matrix 2.0 endpoint other issues -> not going to try with legacy endpoint`, + `Failed fetching jwt with matrix 2.0 endpoint other issues ->`, + `(not going to try with legacy endpoint: forceOldJwtEndpoint is set to false, we did not get a not supported error from the sfu)`, e, ); } @@ -234,7 +235,7 @@ export async function getLiveKitJWTWithDelayDelegation( let bodyDalayParts = {}; // Also check for empty string - if (delayId && delayEndpointBaseUrl) { + if (delayId && delayEndpointBaseUrl && false) { const delayTimeoutMs = Config.get().matrix_rtc_session?.delayed_leave_event_delay_ms ?? 1000; bodyDalayParts = { diff --git a/src/state/CallViewModel/localMember/LocalMember.ts b/src/state/CallViewModel/localMember/LocalMember.ts index 5b72266f..eb506132 100644 --- a/src/state/CallViewModel/localMember/LocalMember.ts +++ b/src/state/CallViewModel/localMember/LocalMember.ts @@ -552,7 +552,12 @@ export const createLocalMembership$ = ({ ); const participant$ = scope.behavior( - localConnection$.pipe(map((c) => c?.livekitRoom?.localParticipant ?? null)), + localConnection$.pipe( + map((c) => c?.livekitRoom?.localParticipant ?? null), + tap((p) => { + logger.debug("participant$ updated:", p?.identity); + }), + ), ); // Pause upstream of all local media tracks when we're disconnected from From d4b06b0f9c505d33aaa560a2dad2c71deaa2b474 Mon Sep 17 00:00:00 2001 From: Timo K Date: Thu, 8 Jan 2026 14:27:47 +0100 Subject: [PATCH 27/46] fix connection recreation which breaks EC lk connection --- .../remoteMembers/ConnectionManager.ts | 54 +++++++++---------- 1 file changed, 25 insertions(+), 29 deletions(-) diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts index e5a542df..5c50f0cd 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts @@ -110,20 +110,6 @@ export function createConnectionManager$({ const logger = parentLogger.getChild("[ConnectionManager]"); // TODO logger: only construct one logger from the client and make it compatible via a EC specific sing - const allInputTransports$ = combineLatest([ - localTransport$, - remoteTransports$, - ]).pipe( - map(([localTransport, transports]) => { - const localTransportAsArray = localTransport ? [localTransport] : []; - return transports.mapInner((transports) => [ - ...localTransportAsArray, - ...transports, - ]); - }), - map((transports) => transports.mapInner(removeDuplicateTransports)), - ); - /** * All transports currently managed by the ConnectionManager. * @@ -134,30 +120,38 @@ export function createConnectionManager$({ */ const transportsWithJwtTag$ = scope.behavior( combineLatest([ - allInputTransports$, + remoteTransports$, localTransport$, forceOldJwtEndpointForLocalTransport$, ]).pipe( + // combine local and remote transports into one transport array + // and set the forceOldJwtEndpoint property on the local transport map( ([ - transports, + remoteTransports, localTransport, forceOldJwtEndpointForLocalTransport, ]) => { - // modify only the local transport with forceOldJwtEndpointForLocalTransport - const index = transports.value.findIndex((t) => - areLivekitTransportsEqual(localTransport, t), - ); - if (index !== -1) { - transports.value[index].forceOldJwtEndpoint = - forceOldJwtEndpointForLocalTransport; + let localTransportAsArray: (LivekitTransport & { + forceOldJwtEndpoint: boolean; + })[] = []; + if (localTransport) { + localTransportAsArray = [ + { + ...localTransport, + forceOldJwtEndpoint: forceOldJwtEndpointForLocalTransport, + }, + ]; } - logger.trace( - `Managing transports: ${transports.value.map((t) => t.livekit_service_url).join(", ")}`, + return new Epoch( + removeDuplicateTransports([ + ...localTransportAsArray, + ...remoteTransports.value, + ]) as (LivekitTransport & { + forceOldJwtEndpoint?: boolean; + })[], + remoteTransports.epoch, ); - return transports as Epoch< - (LivekitTransport & { forceOldJwtEndpoint?: boolean })[] - >; }, ), ), @@ -181,7 +175,9 @@ export function createConnectionManager$({ }; }, (scope, _data$, serviceUrl, alias, forceOldJwtEndpoint) => { - logger.debug(`Creating connection to ${serviceUrl} (${alias})`); + logger.debug( + `Creating connection to ${serviceUrl} (${alias}, forceOldJwtEndpoint: ${forceOldJwtEndpoint})`, + ); const connection = connectionFactory.createConnection( { type: "livekit", From 8fe49d681adeb2afa4ef24511a1523ee1f0b0be7 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 8 Jan 2026 19:49:47 +0000 Subject: [PATCH 28/46] Update Compound --- yarn.lock | 168 ++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 150 insertions(+), 18 deletions(-) diff --git a/yarn.lock b/yarn.lock index b350926d..4e7c6e37 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3838,6 +3838,25 @@ __metadata: languageName: node linkType: hard +"@radix-ui/react-arrow@npm:1.1.7": + version: 1.1.7 + resolution: "@radix-ui/react-arrow@npm:1.1.7" + dependencies: + "@radix-ui/react-primitive": "npm:2.1.3" + peerDependencies: + "@types/react": "*" + "@types/react-dom": "*" + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + "@types/react": + optional: true + "@types/react-dom": + optional: true + checksum: 10c0/c3b46766238b3ee2a394d8806a5141432361bf1425110c9f0dcf480bda4ebd304453a53f294b5399c6ee3ccfcae6fd544921fd01ddc379cf5942acdd7168664b + languageName: node + linkType: hard + "@radix-ui/react-collection@npm:1.1.1": version: 1.1.1 resolution: "@radix-ui/react-collection@npm:1.1.1" @@ -3908,16 +3927,16 @@ __metadata: languageName: node linkType: hard -"@radix-ui/react-context-menu@npm:^2.2.1": - version: 2.2.4 - resolution: "@radix-ui/react-context-menu@npm:2.2.4" +"@radix-ui/react-context-menu@npm:^2.2.16": + version: 2.2.16 + resolution: "@radix-ui/react-context-menu@npm:2.2.16" dependencies: - "@radix-ui/primitive": "npm:1.1.1" - "@radix-ui/react-context": "npm:1.1.1" - "@radix-ui/react-menu": "npm:2.1.4" - "@radix-ui/react-primitive": "npm:2.0.1" - "@radix-ui/react-use-callback-ref": "npm:1.1.0" - "@radix-ui/react-use-controllable-state": "npm:1.1.0" + "@radix-ui/primitive": "npm:1.1.3" + "@radix-ui/react-context": "npm:1.1.2" + "@radix-ui/react-menu": "npm:2.1.16" + "@radix-ui/react-primitive": "npm:2.1.3" + "@radix-ui/react-use-callback-ref": "npm:1.1.1" + "@radix-ui/react-use-controllable-state": "npm:1.2.2" peerDependencies: "@types/react": "*" "@types/react-dom": "*" @@ -3928,7 +3947,7 @@ __metadata: optional: true "@types/react-dom": optional: true - checksum: 10c0/f500590b1300dfcd8a2d0fb51fcada0e7d9a1a354ac239328ffdd32f3736bde888ebf0cd64d9039f7d894e3d13eb549a872359669de8c7ff128ee1afb9cf21a8 + checksum: 10c0/950f7559e65474a19145238cf44d744cb1e49be2221ff18436ba49b496b05ccf93bd3906aaa2c7ab76bc77daf694911a78442801e0053f57d2e57ebbfd281c49 languageName: node linkType: hard @@ -4228,6 +4247,42 @@ __metadata: languageName: node linkType: hard +"@radix-ui/react-menu@npm:2.1.16": + version: 2.1.16 + resolution: "@radix-ui/react-menu@npm:2.1.16" + dependencies: + "@radix-ui/primitive": "npm:1.1.3" + "@radix-ui/react-collection": "npm:1.1.7" + "@radix-ui/react-compose-refs": "npm:1.1.2" + "@radix-ui/react-context": "npm:1.1.2" + "@radix-ui/react-direction": "npm:1.1.1" + "@radix-ui/react-dismissable-layer": "npm:1.1.11" + "@radix-ui/react-focus-guards": "npm:1.1.3" + "@radix-ui/react-focus-scope": "npm:1.1.7" + "@radix-ui/react-id": "npm:1.1.1" + "@radix-ui/react-popper": "npm:1.2.8" + "@radix-ui/react-portal": "npm:1.1.9" + "@radix-ui/react-presence": "npm:1.1.5" + "@radix-ui/react-primitive": "npm:2.1.3" + "@radix-ui/react-roving-focus": "npm:1.1.11" + "@radix-ui/react-slot": "npm:1.2.3" + "@radix-ui/react-use-callback-ref": "npm:1.1.1" + aria-hidden: "npm:^1.2.4" + react-remove-scroll: "npm:^2.6.3" + peerDependencies: + "@types/react": "*" + "@types/react-dom": "*" + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + "@types/react": + optional: true + "@types/react-dom": + optional: true + checksum: 10c0/27516b2b987fa9181c4da8645000af8f60691866a349d7a46b9505fa7d2e9d92b9e364db4f7305d08e9e57d0e1afc8df8354f8ee3c12aa05c0100c16b0e76c27 + languageName: node + linkType: hard + "@radix-ui/react-menu@npm:2.1.4": version: 2.1.4 resolution: "@radix-ui/react-menu@npm:2.1.4" @@ -4292,6 +4347,34 @@ __metadata: languageName: node linkType: hard +"@radix-ui/react-popper@npm:1.2.8": + version: 1.2.8 + resolution: "@radix-ui/react-popper@npm:1.2.8" + dependencies: + "@floating-ui/react-dom": "npm:^2.0.0" + "@radix-ui/react-arrow": "npm:1.1.7" + "@radix-ui/react-compose-refs": "npm:1.1.2" + "@radix-ui/react-context": "npm:1.1.2" + "@radix-ui/react-primitive": "npm:2.1.3" + "@radix-ui/react-use-callback-ref": "npm:1.1.1" + "@radix-ui/react-use-layout-effect": "npm:1.1.1" + "@radix-ui/react-use-rect": "npm:1.1.1" + "@radix-ui/react-use-size": "npm:1.1.1" + "@radix-ui/rect": "npm:1.1.1" + peerDependencies: + "@types/react": "*" + "@types/react-dom": "*" + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + "@types/react": + optional: true + "@types/react-dom": + optional: true + checksum: 10c0/48e3f13eac3b8c13aca8ded37d74db17e1bb294da8d69f142ab6b8719a06c3f90051668bed64520bf9f3abdd77b382ce7ce209d056bb56137cecc949b69b421c + languageName: node + linkType: hard + "@radix-ui/react-portal@npm:1.1.3": version: 1.1.3 resolution: "@radix-ui/react-portal@npm:1.1.3" @@ -4476,6 +4559,33 @@ __metadata: languageName: node linkType: hard +"@radix-ui/react-roving-focus@npm:1.1.11": + version: 1.1.11 + resolution: "@radix-ui/react-roving-focus@npm:1.1.11" + dependencies: + "@radix-ui/primitive": "npm:1.1.3" + "@radix-ui/react-collection": "npm:1.1.7" + "@radix-ui/react-compose-refs": "npm:1.1.2" + "@radix-ui/react-context": "npm:1.1.2" + "@radix-ui/react-direction": "npm:1.1.1" + "@radix-ui/react-id": "npm:1.1.1" + "@radix-ui/react-primitive": "npm:2.1.3" + "@radix-ui/react-use-callback-ref": "npm:1.1.1" + "@radix-ui/react-use-controllable-state": "npm:1.2.2" + peerDependencies: + "@types/react": "*" + "@types/react-dom": "*" + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + "@types/react": + optional: true + "@types/react-dom": + optional: true + checksum: 10c0/2cd43339c36e89a3bf1db8aab34b939113dfbde56bf3a33df2d74757c78c9489b847b1962f1e2441c67e41817d120cb6177943e0f655f47bc1ff8e44fd55b1a2 + languageName: node + linkType: hard + "@radix-ui/react-separator@npm:^1.1.0": version: 1.1.1 resolution: "@radix-ui/react-separator@npm:1.1.1" @@ -4725,6 +4835,21 @@ __metadata: languageName: node linkType: hard +"@radix-ui/react-use-rect@npm:1.1.1": + version: 1.1.1 + resolution: "@radix-ui/react-use-rect@npm:1.1.1" + dependencies: + "@radix-ui/rect": "npm:1.1.1" + peerDependencies: + "@types/react": "*" + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + "@types/react": + optional: true + checksum: 10c0/271711404c05c589c8dbdaa748749e7daf44bcc6bffc9ecd910821c3ebca0ee245616cf5b39653ce690f53f875c3836fd3f36f51ab1c628273b6db599eee4864 + languageName: node + linkType: hard + "@radix-ui/react-use-size@npm:1.1.0": version: 1.1.0 resolution: "@radix-ui/react-use-size@npm:1.1.0" @@ -4781,6 +4906,13 @@ __metadata: languageName: node linkType: hard +"@radix-ui/rect@npm:1.1.1": + version: 1.1.1 + resolution: "@radix-ui/rect@npm:1.1.1" + checksum: 10c0/0dac4f0f15691199abe6a0e067821ddd9d0349c0c05f39834e4eafc8403caf724106884035ae91bbc826e10367e6a5672e7bec4d4243860fa7649de246b1f60b + languageName: node + linkType: hard + "@react-spring/animated@npm:~10.0.3": version: 10.0.3 resolution: "@react-spring/animated@npm:10.0.3" @@ -6026,8 +6158,8 @@ __metadata: linkType: hard "@vector-im/compound-design-tokens@npm:^6.0.0": - version: 6.0.0 - resolution: "@vector-im/compound-design-tokens@npm:6.0.0" + version: 6.6.0 + resolution: "@vector-im/compound-design-tokens@npm:6.6.0" peerDependencies: "@types/react": "*" react: ^17 || ^18 || ^19.0.0 @@ -6036,16 +6168,16 @@ __metadata: optional: true react: optional: true - checksum: 10c0/1af5b2b73a3a55149047cd0716f071b83a4df8a210c9ad432db4cc2f9b9e72e958f93ff850dbaddb88e37a01870c5eb810b03dfb0acc89cc147eaaf6cf1dada1 + checksum: 10c0/93b152dd1de96371f9b6b1f7dbcc381d7ab598031dbc900f52d610f015766c0d4426ae6e47d417e723bfb62d1a53099155b4d788848b78232916ba132c03c2fe languageName: node linkType: hard "@vector-im/compound-web@npm:^8.0.0": - version: 8.2.0 - resolution: "@vector-im/compound-web@npm:8.2.0" + version: 8.3.4 + resolution: "@vector-im/compound-web@npm:8.3.4" dependencies: "@floating-ui/react": "npm:^0.27.0" - "@radix-ui/react-context-menu": "npm:^2.2.1" + "@radix-ui/react-context-menu": "npm:^2.2.16" "@radix-ui/react-dropdown-menu": "npm:^2.1.1" "@radix-ui/react-form": "npm:^0.1.0" "@radix-ui/react-progress": "npm:^1.1.0" @@ -6057,12 +6189,12 @@ __metadata: "@fontsource/inconsolata": ^5 "@fontsource/inter": ^5 "@types/react": "*" - "@vector-im/compound-design-tokens": ">=1.6.1 <6.0.0" + "@vector-im/compound-design-tokens": ">=1.6.1 <7.0.0" react: ^18 || ^19.0.0 peerDependenciesMeta: "@types/react": optional: true - checksum: 10c0/4ac4074dcf9611bdff7de4bf66763397c926d6312f31758bcabe3e7bf704cb76bc2ce1023fe5f2cf0d05e97c9c540fef8b63edea7a521a2f7b4b7fbcb883fb17 + checksum: 10c0/44764fa64b5fce2e7181e25b50ee970eda4d921cf650b92bd5e88df0eb60872f3086b8702d18f55c3e39b3751ac19f10bafda8c4306df65c3605bd44b297d95c languageName: node linkType: hard From bf19b388ea4c057749ce5d8938a2c63197f6d0d9 Mon Sep 17 00:00:00 2001 From: Valere Date: Fri, 9 Jan 2026 12:43:33 +0100 Subject: [PATCH 29/46] Add codecov shields.io badge --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 73505a8d..688a7a7f 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ [![Chat](https://img.shields.io/matrix/webrtc:matrix.org)](https://matrix.to/#/#webrtc:matrix.org) [![Localazy](https://img.shields.io/endpoint?url=https%3A%2F%2Fconnect.localazy.com%2Fstatus%2Felement-call%2Fdata%3Fcontent%3Dall%26title%3Dlocalazy%26logo%3Dtrue)](https://localazy.com/p/element-call) [![License](https://img.shields.io/github/license/element-hq/element-call)](LICENSE-AGPL-3.0) +[![Codecov](https://img.shields.io/codecov/c/github/element-hq/element-call)](https://app.codecov.io/gh/element-hq/element-call) [🎬 Live Demo 🎬](https://call.element.io) From f89a7a821108c7c8f96a1e9e12cd326b5db7f36a Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 9 Jan 2026 12:58:06 +0100 Subject: [PATCH 30/46] Fix tests --- src/__snapshots__/AppBar.test.tsx.snap | 2 +- .../ReactionToggleButton.test.tsx.snap | 10 +++--- .../GroupCallErrorBoundary.test.tsx.snap | 32 +++++++++---------- .../__snapshots__/InCallView.test.tsx.snap | 14 ++++---- .../DeveloperSettingsTab.test.tsx.snap | 18 +++++------ 5 files changed, 38 insertions(+), 38 deletions(-) diff --git a/src/__snapshots__/AppBar.test.tsx.snap b/src/__snapshots__/AppBar.test.tsx.snap index 25bb54ed..c769ec12 100644 --- a/src/__snapshots__/AppBar.test.tsx.snap +++ b/src/__snapshots__/AppBar.test.tsx.snap @@ -21,7 +21,7 @@ exports[`AppBar > renders 1`] = ` tabindex="0" >
rendering > renders 1`] = ` data-show="false" >
rendering > renders 1`] = ` Only works while using app

renders and matches snapshot 1`] = ` value="legacy" />
@@ -275,11 +275,11 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = ` class="_inline-field-control_19upo_44" >
renders and matches snapshot 1`] = ` value="compatibil" />
@@ -315,11 +315,11 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = ` class="_inline-field-control_19upo_44" >
renders and matches snapshot 1`] = ` value="matrix_2_0" />
From 7dbbd763b965c47e65772ead3d5707e8b6242202 Mon Sep 17 00:00:00 2001 From: Timo K Date: Fri, 9 Jan 2026 13:38:26 +0100 Subject: [PATCH 31/46] Refactor how we aquire the jwt token for the local user. (only fetch it once) The local jwt token needs to be aquired via the right endpoint. The endpoint defines how our rtcBackendIdentity is computed. Based on us using sticky events or state events we also need to use the right endpoint. This cannot be done generically in the connection manager. The jwt token now is computed in the localTransport and the resolved sfu config is passed to the connection manager. Add JWT endpoint version and SFU config support Pin matrix-js-sdk to a specific commit and update dev auth image tag. Propagate SFU config and JWT endpoint choice through local transport, ConnectionManager and Connection; add JwtEndpointVersion enum and LocalTransportWithSFUConfig type. Add NO_MATRIX_2 auth error and locale string, thread rtcBackendIdentity through UI props, and include related test, CSS and minor imports updates --- dev-backend-docker-compose.yml | 4 +- locales/en/app.json | 1 + package.json | 2 +- src/RTCConnectionStats.tsx | 5 + src/livekit/openIDSFU.test.ts | 17 ++- src/livekit/openIDSFU.ts | 43 ++++-- src/room/InCallView.tsx | 18 ++- src/settings/DeveloperSettingsTab.module.css | 11 ++ src/settings/DeveloperSettingsTab.tsx | 22 +++- src/state/CallViewModel/CallViewModel.ts | 46 +++++-- .../localMember/LocalMember.test.ts | 35 +++-- .../CallViewModel/localMember/LocalMember.ts | 9 +- .../localMember/LocalTransport.test.ts | 14 +- .../localMember/LocalTransport.ts | 110 ++++++++++++---- .../remoteMembers/Connection.test.ts | 15 ++- .../CallViewModel/remoteMembers/Connection.ts | 44 ++++--- .../remoteMembers/ConnectionFactory.ts | 21 +-- .../remoteMembers/ConnectionManager.ts | 123 ++++++++++-------- .../remoteMembers/ECConnectionFactory.test.ts | 4 +- src/state/MediaViewModel.ts | 12 ++ src/state/ScreenShare.ts | 3 + src/state/UserMedia.ts | 6 + src/tile/GridTile.tsx | 2 + src/tile/MediaView.tsx | 15 ++- src/utils/errors.ts | 18 +++ src/utils/test.ts | 2 + yarn.lock | 11 +- 27 files changed, 421 insertions(+), 192 deletions(-) diff --git a/dev-backend-docker-compose.yml b/dev-backend-docker-compose.yml index 5e955831..0efefd07 100644 --- a/dev-backend-docker-compose.yml +++ b/dev-backend-docker-compose.yml @@ -3,7 +3,7 @@ networks: services: auth-service: - image: ghcr.io/element-hq/lk-jwt-service:latest-ci + image: ghcr.io/element-hq/lk-jwt-service:pr_139 pull_policy: always hostname: auth-server environment: @@ -25,7 +25,7 @@ services: - ecbackend auth-service-1: - image: ghcr.io/element-hq/lk-jwt-service:latest-ci + image: ghcr.io/element-hq/lk-jwt-service:pr_139 pull_policy: always hostname: auth-server-1 environment: diff --git a/locales/en/app.json b/locales/en/app.json index 1ff066ea..c70a0a49 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -116,6 +116,7 @@ "matrix_rtc_transport_missing": "The server is not configured to work with {{brand}}. Please contact your server admin (Domain: {{domain}}, Error Code: {{ errorCode }}).", "membership_manager": "Membership Manager Error", "membership_manager_description": "The Membership Manager had to shut down. This is caused by many consequtive failed network requests.", + "no_matrix_2_authorization_service": "Your authorization service for you media server (SFU) is not on the newest version", "open_elsewhere": "Opened in another tab", "open_elsewhere_description": "{{brand}} has been opened in another tab. If that doesn't sound right, try reloading the page.", "room_creation_restricted": "Failed to create call", diff --git a/package.json b/package.json index 944fc43b..6a8d3c76 100644 --- a/package.json +++ b/package.json @@ -104,7 +104,7 @@ "livekit-client": "^2.13.0", "lodash-es": "^4.17.21", "loglevel": "^1.9.1", - "matrix-js-sdk": "matrix-org/matrix-js-sdk#develop", + "matrix-js-sdk": "matrix-org/matrix-js-sdk#ff5fab7722e54be2d69a380cd6182ecb262d7859", "matrix-widget-api": "^1.14.0", "node-stdlib-browser": "^1.3.1", "normalize.css": "^8.0.1", diff --git a/src/RTCConnectionStats.tsx b/src/RTCConnectionStats.tsx index d51089cf..f62a1e4d 100644 --- a/src/RTCConnectionStats.tsx +++ b/src/RTCConnectionStats.tsx @@ -20,6 +20,7 @@ interface Props { audio?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; video?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; focusUrl?: string; + rtcBackendIdentity?: string; } const extractDomain = (url: string): string => { @@ -37,6 +38,7 @@ export const RTCConnectionStats: FC = ({ audio, video, focusUrl, + rtcBackendIdentity, ...rest }) => { const [showModal, setShowModal] = useState(false); @@ -71,6 +73,9 @@ export const RTCConnectionStats: FC = ({
+ + rtcBackendIdentity:{rtcBackendIdentity} + {focusUrl && (
diff --git a/src/livekit/openIDSFU.test.ts b/src/livekit/openIDSFU.test.ts index 5f286958..20820748 100644 --- a/src/livekit/openIDSFU.test.ts +++ b/src/livekit/openIDSFU.test.ts @@ -46,7 +46,6 @@ describe("getSFUConfigWithOpenID", () => { matrixClient, ownMemberMock, "https://sfu.example.org", - false, "!example_room_id", ); expect(config).toEqual({ @@ -70,7 +69,6 @@ describe("getSFUConfigWithOpenID", () => { matrixClient, ownMemberMock, "https://sfu.example.org", - false, "!example_room_id", ); } catch (ex) { @@ -101,10 +99,11 @@ describe("getSFUConfigWithOpenID", () => { matrixClient, ownMemberMock, "https://sfu.example.org", - false, "!example_room_id", - "https://matrix.homeserverserver.org", - "mock_delay_id", + { + delayEndpointBaseUrl: "https://matrix.homeserverserver.org", + delayId: "mock_delay_id", + }, ); } catch (ex) { expect((ex as Error).message).toEqual( @@ -154,10 +153,11 @@ describe("getSFUConfigWithOpenID", () => { matrixClient, ownMemberMock, "https://sfu.example.org", - false, "!example_room_id", - "https://matrix.homeserverserver.org", - "mock_delay_id", + { + delayEndpointBaseUrl: "https://matrix.homeserverserver.org", + delayId: "mock_delay_id", + }, ); } catch (ex) { expect((ex as Error).message).toEqual( @@ -203,7 +203,6 @@ describe("getSFUConfigWithOpenID", () => { matrixClient, ownMemberMock, "https://sfu.example.org", - false, "!example_room_id", ); expect(config).toEqual({ diff --git a/src/livekit/openIDSFU.ts b/src/livekit/openIDSFU.ts index df1d02a5..a5a8adc5 100644 --- a/src/livekit/openIDSFU.ts +++ b/src/livekit/openIDSFU.ts @@ -13,9 +13,13 @@ import { import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; import { type Logger } from "matrix-js-sdk/lib/logger"; -import { FailToGetOpenIdToken } from "../utils/errors"; +import { + FailToGetOpenIdToken, + NoMatrix2AuthorizationService, +} from "../utils/errors"; import { doNetworkOperationWithRetry } from "../utils/matrix"; import { Config } from "../config/Config"; +import { JwtEndpointVersion } from "../state/CallViewModel/localMember/LocalTransport"; /** * Configuration and access tokens provided by the SFU on successful authentication. @@ -73,14 +77,15 @@ export type OpenIDClientParts = Pick< * @param client The Matrix client * @param membership Our own membership identity parts used to send to jwt service. * @param serviceUrl The URL of the livekit SFU service - * @param forceOldJwtEndpoint This will use the old jwt endpoint which will create the rtc backend identity based on string concatination + * @param roomId The room id used in the jwt request. This is NOT the livekit_alias. The jwt service will provide the alias. It maps matrix room ids <-> Livekit aliases. + * @param opts Additional options to modify which endpoint with which data will be used to aquire the jwt token. + * @param opts.forceJwtEndpoint This will use the old jwt endpoint which will create the rtc backend identity based on string concatination * instead of a hash. * This function by default uses whatever is possible with the current jwt service installed next to the SFU. * For remote connections this does not matter, since we will not publish there we can rely on the newest option. * For our own connection we can only use the hashed version if we also send the new matrix2.0 sticky events. - * @param roomId The room id used in the jwt request. This is NOT the livekit_alias. The jwt service will provide the alias. It maps matrix room ids <-> Livekit aliases. - * @param delayEndpointBaseUrl The URL of the matrix homeserver. - * @param delayId The delay id used for the jwt service to manage. + * @param opts.delayEndpointBaseUrl The URL of the matrix homeserver. + * @param opts.delayId The delay id used for the jwt service to manage. * @param logger optional logger. * @returns Object containing the token information * @throws FailToGetOpenIdToken @@ -89,10 +94,12 @@ export async function getSFUConfigWithOpenID( client: OpenIDClientParts, membership: CallMembershipIdentityParts, serviceUrl: string, - forceOldJwtEndpoint: boolean, roomId: string, - delayEndpointBaseUrl?: string, - delayId?: string, + opts?: { + forceJwtEndpoint?: JwtEndpointVersion; + delayEndpointBaseUrl?: string; + delayId?: string; + }, logger?: Logger, ): Promise { let openIdToken: IOpenIDToken; @@ -113,15 +120,21 @@ export async function getSFUConfigWithOpenID( // If forceOldJwtEndpoint is set we indicate that we do not want to try the new endpoint, // since we are not sending the new matrix2.0 sticky events (no hashed identity in the event) - if (forceOldJwtEndpoint === false) { + if ( + // we do not force anything. Try with new first (remote connections) + !opts?.forceJwtEndpoint || + // we do force the matrix2.0 endpoint + (opts?.forceJwtEndpoint && + opts?.forceJwtEndpoint === JwtEndpointVersion.Matrix_2_0) + ) { try { sfuConfig = await getLiveKitJWTWithDelayDelegation( membership, serviceUrl, roomId, openIdToken, - delayEndpointBaseUrl, - delayId, + opts?.delayEndpointBaseUrl, + opts?.delayId, ); logger?.info(`Got JWT from call's active focus URL.`); } catch (e) { @@ -137,12 +150,16 @@ export async function getSFUConfigWithOpenID( `(not going to try with legacy endpoint: forceOldJwtEndpoint is set to false, we did not get a not supported error from the sfu)`, e, ); + // Make this throw a hard error in case we force the matrix2.0 endpoint. + if (opts?.forceJwtEndpoint === JwtEndpointVersion.Matrix_2_0) + throw new NoMatrix2AuthorizationService(e as Error); } } } // DEPRECATED - // Either forceOldJwtEndpoint = true or getLiveKitJWTWithDelayDelegation throws -> reset sfuConfig = undefined + // here we either have a sfuConfig or we alredy exited becuause of `if (opts?.forceEndpoint === MatrixRTCMode.Matrix_2_0) throw e;` + // The only case we can get into this if is, if `opts?.forceEndpoint !== MatrixRTCMode.Matrix_2_0` if (sfuConfig === undefined) { sfuConfig = await getLiveKitJWT( membership.deviceId, @@ -235,7 +252,7 @@ export async function getLiveKitJWTWithDelayDelegation( let bodyDalayParts = {}; // Also check for empty string - if (delayId && delayEndpointBaseUrl && false) { + if (delayId && delayEndpointBaseUrl) { const delayTimeoutMs = Config.get().matrix_rtc_session?.delayed_leave_event_delay_ms ?? 1000; bodyDalayParts = { diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index fdbd4461..54f189f0 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -113,10 +113,8 @@ const logger = rootLogger.getChild("[InCallView]"); const maxTapDurationMs = 400; -export interface ActiveCallProps extends Omit< - InCallViewProps, - "vm" | "livekitRoom" | "connState" -> { +export interface ActiveCallProps + extends Omit { e2eeSystem: EncryptionSystem; // TODO refactor those reasons into an enum onLeft: ( @@ -798,6 +796,8 @@ export const InCallView: FC = ({
); + const allConnections = useBehavior(vm.allConnections$); + return (
= ({ onDismiss={closeSettings} tab={settingsTab} onTabChange={setSettingsTab} - // TODO expose correct data to setttings modal - livekitRooms={[]} + livekitRooms={allConnections + .getConnections() + .map((connectionItem) => ({ + room: connectionItem.livekitRoom, + // TODO compute is local or tag it in the livekit room items already + isLocal: undefined, + url: connectionItem.transport.livekit_service_url, + }))} /> )} diff --git a/src/settings/DeveloperSettingsTab.module.css b/src/settings/DeveloperSettingsTab.module.css index 7b83eb6c..29f4211b 100644 --- a/src/settings/DeveloperSettingsTab.module.css +++ b/src/settings/DeveloperSettingsTab.module.css @@ -8,3 +8,14 @@ Please see LICENSE in the repository root for full details. pre { font-size: var(--font-size-micro); } + +.livekit_room_box { + border: 3px solid var(--cpd-color-bg-subtle-secondary); + border-radius: var(--cpd-space-8x); + padding: var(--cpd-space-4x); + margin-bottom: var(--cpd-space-4x); + margin-top: var(--cpd-space-4x); + li { + font-size: var(--font-size-micro); + } +} diff --git a/src/settings/DeveloperSettingsTab.tsx b/src/settings/DeveloperSettingsTab.tsx index c88eadf0..a187e4b5 100644 --- a/src/settings/DeveloperSettingsTab.tsx +++ b/src/settings/DeveloperSettingsTab.tsx @@ -29,6 +29,7 @@ import { Label, RadioControl, } from "@vector-im/compound-web"; +import { type Room as LivekitRoom } from "livekit-client"; import { FieldRow, InputField } from "../input/Input"; import { @@ -42,7 +43,6 @@ import { customLivekitUrl as customLivekitUrlSetting, MatrixRTCMode, } from "./settings"; -import type { Room as LivekitRoom } from "livekit-client"; import styles from "./DeveloperSettingsTab.module.css"; import { useUrlParams } from "../UrlParams"; @@ -304,12 +304,12 @@ export const DeveloperSettingsTab: FC = ({ {livekitRooms?.map((livekitRoom) => ( - <> -

+
+

{t("developer_mode.livekit_sfu", { url: livekitRoom.url || "unknown", })} -

+

{livekitRoom.isLocal &&

ws-url: {localSfuUrl?.href}

}

{t("developer_mode.livekit_server_info")}( @@ -321,7 +321,19 @@ export const DeveloperSettingsTab: FC = ({ : "undefined"} {livekitRoom.room.metadata} - +

Local Participant

+ 
+            {livekitRoom.room.localParticipant.identity}
+
+

Remote Participants

+
    + {Array.from(livekitRoom.room.remoteParticipants.keys()).map( + (id) => ( +
  • {id}
    • + ), + )} +
+
))}

{t("developer_mode.environment_variables")}

{JSON.stringify(env, null, 2)}
diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index c75b3ec4..8fc2d7e6 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -108,13 +108,19 @@ import { enterRTCSession, TransportState, } from "./localMember/LocalMember.ts"; -import { createLocalTransport$ } from "./localMember/LocalTransport.ts"; +import { + createLocalTransport$, + JwtEndpointVersion, +} from "./localMember/LocalTransport.ts"; import { createMemberships$, membershipsAndTransports$, } from "../SessionBehaviors.ts"; import { ECConnectionFactory } from "./remoteMembers/ConnectionFactory.ts"; -import { createConnectionManager$ } from "./remoteMembers/ConnectionManager.ts"; +import { + type ConnectionManagerData, + createConnectionManager$, +} from "./remoteMembers/ConnectionManager.ts"; import { createMatrixLivekitMembers$, type TaggedParticipant, @@ -263,6 +269,7 @@ export interface CallViewModel { * multiple devices. */ participantCount$: Behavior; + allConnections$: Behavior; /** Participants sorted by livekit room so they can be used in the audio rendering */ livekitRoomItems$: Behavior; userMedia$: Behavior; @@ -428,14 +435,6 @@ export function createCallViewModel$( memberId: `${userId}:${deviceId}`, }; - const useOldJwtEndpoint$ = scope.behavior( - matrixRTCMode$.pipe( - map( - (v) => v === MatrixRTCMode.Legacy || v === MatrixRTCMode.Compatibility, - ), - ), - ); - const localTransport$ = createLocalTransport$({ scope: scope, memberships$: memberships$, @@ -451,7 +450,15 @@ export function createCallViewModel$( matrixRTCSession.delayId ?? null, ), roomId: matrixRoom.roomId, - useOldJwtEndpoint$, + forceJwtEndpoint$: scope.behavior( + matrixRTCMode$.pipe( + map((v) => + v === MatrixRTCMode.Matrix_2_0 + ? JwtEndpointVersion.Matrix_2_0 + : JwtEndpointVersion.Legacy, + ), + ), + ), useOldestMember$: scope.behavior( matrixRTCMode$.pipe(map((v) => v === MatrixRTCMode.Legacy)), ), @@ -483,7 +490,6 @@ export function createCallViewModel$( ), ), remoteTransports$: membershipsAndTransports.transports$, - forceOldJwtEndpointForLocalTransport$: useOldJwtEndpoint$, logger: logger, ownMembershipIdentity, }); @@ -628,6 +634,9 @@ export function createCallViewModel$( ), ); + const allConnections$ = scope.behavior( + connectionManager.connectionManagerData$.pipe(map((d) => d.value)), + ); const livekitRoomItems$ = scope.behavior( matrixLivekitMembers$.pipe( switchMap((members) => { @@ -724,6 +733,7 @@ export function createCallViewModel$( userId, participant satisfies TaggedParticipant as TaggedParticipant, // Widen the type safely connection$, + membership$.value, ], data: undefined, }; @@ -742,7 +752,14 @@ export function createCallViewModel$( // const participantId = membership$.value?.identity; for (let dup = 0; dup < 1 + duplicateTiles; dup++) { yield { - keys: [dup, userMediaId, userId, participant, connection$], + keys: [ + dup, + userMediaId, + userId, + participant, + connection$, + membership$.value, + ], data: undefined, }; } @@ -756,6 +773,7 @@ export function createCallViewModel$( userId, participant, connection$, + membership, ) => { const livekitRoom$ = scope.behavior( connection$.pipe(map((c) => c?.livekitRoom)), @@ -773,6 +791,7 @@ export function createCallViewModel$( scope, `${participantId}:${dup}`, userId, + membership, participant, options.encryptionSystem, livekitRoom$, @@ -1523,6 +1542,7 @@ export function createCallViewModel$( ), null, ), + allConnections$, participantCount$: participantCount$, handsRaised$: handsRaised$, reactions$: reactions$, diff --git a/src/state/CallViewModel/localMember/LocalMember.test.ts b/src/state/CallViewModel/localMember/LocalMember.test.ts index 8a7505eb..6b750785 100644 --- a/src/state/CallViewModel/localMember/LocalMember.test.ts +++ b/src/state/CallViewModel/localMember/LocalMember.test.ts @@ -39,6 +39,7 @@ import { constant } from "../../Behavior"; import { ConnectionManagerData } from "../remoteMembers/ConnectionManager"; import { ConnectionState, type Connection } from "../remoteMembers/Connection"; import { type Publisher } from "./Publisher"; +import { type LocalTransportWithSFUConfig } from "./LocalTransport"; const MATRIX_RTC_MODE = MatrixRTCMode.Legacy; const getUrlParams = vi.hoisted(() => vi.fn(() => ({}))); @@ -212,10 +213,11 @@ describe("LocalMembership", () => { it("throws error on missing RTC config error", () => { withTestScheduler(({ scope, hot, expectObservable }) => { - const localTransport$ = scope.behavior( - hot("1ms #", {}, new MatrixRTCTransportMissingError("domain.com")), - null, - ); + const localTransport$ = + scope.behavior( + hot("1ms #", {}, new MatrixRTCTransportMissingError("domain.com")), + null, + ); // we do not need any connection data since we want to fail before reaching that. const mockConnectionManager = { @@ -243,11 +245,23 @@ describe("LocalMembership", () => { }); const aTransport = { - livekit_service_url: "a", - } as LivekitTransport; + transport: { + livekit_service_url: "a", + } as LivekitTransport, + sfuConfig: { + url: "sfu-url", + jwt: "sfu-token", + }, + } as LocalTransportWithSFUConfig; const bTransport = { - livekit_service_url: "b", - } as LivekitTransport; + transport: { + livekit_service_url: "b", + } as LivekitTransport, + sfuConfig: { + url: "sfu-url", + jwt: "sfu-token", + }, + } as LocalTransportWithSFUConfig; const connectionTransportAConnected = { livekitRoom: mockLivekitRoom({ @@ -391,7 +405,8 @@ describe("LocalMembership", () => { const scope = new ObservableScope(); const connectionManagerData = new ConnectionManagerData(); - const localTransport$ = new BehaviorSubject(null); + const localTransport$ = + new BehaviorSubject(null); const connectionManagerData$ = new BehaviorSubject( new Epoch(connectionManagerData), ); @@ -468,7 +483,7 @@ describe("LocalMembership", () => { }); ( - connectionManagerData2.getConnectionForTransport(aTransport)! + connectionManagerData2.getConnectionForTransport(aTransport.transport)! .state$ as BehaviorSubject ).next(ConnectionState.LivekitConnected); expect(localMembership.localMemberState$.value).toStrictEqual({ diff --git a/src/state/CallViewModel/localMember/LocalMember.ts b/src/state/CallViewModel/localMember/LocalMember.ts index eb506132..4749e942 100644 --- a/src/state/CallViewModel/localMember/LocalMember.ts +++ b/src/state/CallViewModel/localMember/LocalMember.ts @@ -61,6 +61,7 @@ import { } from "../remoteMembers/Connection.ts"; import { type HomeserverConnected } from "./HomeserverConnected.ts"; import { and$ } from "../../../utils/observable.ts"; +import { type LocalTransportWithSFUConfig } from "./LocalTransport.ts"; export enum TransportState { /** Not even a transport is available to the LocalMembership */ @@ -126,7 +127,7 @@ interface Props { createPublisherFactory: (connection: Connection) => Publisher; joinMatrixRTC: (transport: LivekitTransport) => void; homeserverConnected: HomeserverConnected; - localTransport$: Behavior; + localTransport$: Behavior; matrixRTCSession: Pick< MatrixRTCSession, "updateCallIntent" | "leaveRoomSession" @@ -234,7 +235,9 @@ export const createLocalMembership$ = ({ return null; } - return connectionData.getConnectionForTransport(localTransport); + return connectionData.getConnectionForTransport( + localTransport.transport, + ); }), tap((connection) => { logger.info( @@ -533,7 +536,7 @@ export const createLocalMembership$ = ({ if (!shouldConnect) return; try { - joinMatrixRTC(transport); + joinMatrixRTC(transport.transport); } catch (error) { logger.error("Error entering RTC session", error); if (error instanceof Error) diff --git a/src/state/CallViewModel/localMember/LocalTransport.test.ts b/src/state/CallViewModel/localMember/LocalTransport.test.ts index 2199ca94..8da0db7f 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.test.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.test.ts @@ -19,7 +19,7 @@ import { BehaviorSubject, lastValueFrom } from "rxjs"; import fetchMock from "fetch-mock"; import { mockConfig, flushPromises, ownMemberMock } from "../../../utils/test"; -import { createLocalTransport$ } from "./LocalTransport"; +import { createLocalTransport$, JwtEndpointVersion } from "./LocalTransport"; import { constant } from "../../Behavior"; import { Epoch, ObservableScope } from "../../ObservableScope"; import { @@ -58,7 +58,7 @@ describe("LocalTransport", () => { getDeviceId: vi.fn(), }, ownMembershipIdentity: ownMemberMock, - useOldJwtEndpoint$: constant(false), + forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy), delayId$: constant("delay_id_mock"), }); await flushPromises(); @@ -98,7 +98,7 @@ describe("LocalTransport", () => { getDeviceId: vi.fn(), }, ownMembershipIdentity: ownMemberMock, - useOldJwtEndpoint$: constant(false), + forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy), delayId$: constant("delay_id_mock"), }); localTransport$.subscribe( @@ -140,7 +140,7 @@ describe("LocalTransport", () => { baseUrl: "https://lk.example.org", }, ownMembershipIdentity: ownMemberMock, - useOldJwtEndpoint$: constant(false), + forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy), delayId$: constant("delay_id_mock"), }); @@ -186,7 +186,7 @@ describe("LocalTransport", () => { baseUrl: "https://lk.example.org", }, ownMembershipIdentity: ownMemberMock, - useOldJwtEndpoint$: constant(false), + forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy), delayId$: constant("delay_id_mock"), }); @@ -216,7 +216,7 @@ describe("LocalTransport", () => { scope, roomId: "!example_room_id", useOldestMember$: constant(false), - useOldJwtEndpoint$: constant(false), + forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy), delayId$: constant(null), memberships$: constant(new Epoch([])), client: { @@ -333,7 +333,7 @@ describe("LocalTransport", () => { ownMembershipIdentity: ownMemberMock, roomId: "!example_room_id", useOldestMember$: constant(false), - useOldJwtEndpoint$: constant(false), + forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy), delayId$: constant(null), memberships$: constant(new Epoch([])), client: { diff --git a/src/state/CallViewModel/localMember/LocalTransport.ts b/src/state/CallViewModel/localMember/LocalTransport.ts index 8a6a750b..6e0e56a3 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.ts @@ -31,9 +31,11 @@ import { Config } from "../../../config/Config.ts"; import { FailToGetOpenIdToken, MatrixRTCTransportMissingError, + NoMatrix2AuthorizationService, } from "../../../utils/errors.ts"; import { getSFUConfigWithOpenID, + type SFUConfig, type OpenIDClientParts, } from "../../../livekit/openIDSFU.ts"; import { areLivekitTransportsEqual } from "../remoteMembers/MatrixLivekitMembers.ts"; @@ -57,10 +59,25 @@ interface Props { OpenIDClientParts; roomId: string; useOldestMember$: Behavior; - useOldJwtEndpoint$: Behavior; + forceJwtEndpoint$: Behavior; delayId$: Behavior; } +export enum JwtEndpointVersion { + Legacy = "legacy", + Matrix_2_0 = "matrix_2_0", +} + +export interface LocalTransportWithSFUConfig { + transport: LivekitTransport; + sfuConfig: SFUConfig; +} +export function isLocalTransportWithSFUConfig( + obj: LivekitTransport | LocalTransportWithSFUConfig, +): obj is LocalTransportWithSFUConfig { + return "transport" in obj && "sfuConfig" in obj; +} + /** * This class is responsible for managing the local transport. * "Which transport is the local member going to use" @@ -81,22 +98,40 @@ export const createLocalTransport$ = ({ client, roomId, useOldestMember$, - useOldJwtEndpoint$, + forceJwtEndpoint$, delayId$, -}: Props): Behavior => { +}: Props): Behavior => { /** * The transport over which we should be actively publishing our media. * undefined when not joined. */ const oldestMemberTransport$ = scope.behavior( - combineLatest([memberships$, useOldJwtEndpoint$]).pipe( - map(([memberships, forceOldJwtEndpoint]) => { + combineLatest([memberships$]).pipe( + map(([memberships]) => { const oldestMember = memberships.value[0]; const transport = oldestMember?.getTransport(memberships.value[0]); if (!transport) return null; return transport; }), first((t) => t != null && isLivekitTransport(t)), + switchMap((transport) => { + // Get the open jwt token to connect to the sfu + const computeLocalTransportWithSFUConfig = + async (): Promise => { + return { + transport, + sfuConfig: await getSFUConfigWithOpenID( + client, + ownMembershipIdentity, + transport.livekit_service_url, + roomId, + { forceJwtEndpoint: JwtEndpointVersion.Legacy }, + logger, + ), + }; + }; + return from(computeLocalTransportWithSFUConfig()); + }), ), null, ); @@ -108,19 +143,29 @@ export const createLocalTransport$ = ({ * @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken */ const preferredTransport$ = scope.behavior( - combineLatest([customLivekitUrl.value$, delayId$, useOldJwtEndpoint$]).pipe( - switchMap(([customUrl, delayId, forceOldJwtEndpoint]) => - from( + // preferredTransport$ (used for multi sfu) needs to know if we are using the old or new + // jwt endpoint (`get_token` vs `sfu/get`) based on that the jwt endpoint will compute the rtcBackendIdentity + // differently. (sha(`${userId}|${deviceId}|${memberId}`) vs `${userId}|${deviceId}|${memberId}`) + // When using sticky events (we need to use the new endpoint). + combineLatest([customLivekitUrl.value$, delayId$, forceJwtEndpoint$]).pipe( + switchMap(([customUrl, delayId, forceEndpoint]) => { + logger.info( + "Creating preferred transport based on: ", + customUrl, + delayId, + forceEndpoint, + ); + return from( makeTransport( client, ownMembershipIdentity, roomId, customUrl, - forceOldJwtEndpoint, + forceEndpoint, delayId ?? undefined, ), - ), - ), + ); + }), ), null, ); @@ -139,7 +184,9 @@ export const createLocalTransport$ = ({ ? (oldestMemberTransport ?? preferredTransport) : preferredTransport, ), - distinctUntilChanged((t1, t2) => areLivekitTransportsEqual(t1, t2)), + distinctUntilChanged((t1, t2) => + areLivekitTransportsEqual(t1?.transport ?? null, t2?.transport ?? null), + ), ), ); }; @@ -161,7 +208,10 @@ const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci"; * @param membership The membership identity of the user. * @param roomId The ID of the room to be connected to. * @param urlFromDevSettings Override URL provided by the user's local config. - * @param forceOldJwtEndpoint Whether to force the old JWT endpoint (not hashing the backendIdentity). + * @param forceJwtEndpoint Whether to force a specific JWT endpoint + * - `Legacy` / `Matrix_2_0` + * - `get_token` / `sfu/get` + * - not hashing / hashing the backendIdentity * @param delayId the delay id passed to the jwt service. * * @returns A fully validated transport config. @@ -176,26 +226,33 @@ async function makeTransport( membership: CallMembershipIdentityParts, roomId: string, urlFromDevSettings: string | null, - forceOldJwtEndpoint: boolean, + forceJwtEndpoint: JwtEndpointVersion, delayId?: string, -): Promise { +): Promise { logger.trace("Searching for a preferred transport"); - async function doOpenIdAndJWTFromUrl(url: string): Promise { - const { livekitAlias } = await getSFUConfigWithOpenID( + async function doOpenIdAndJWTFromUrl( + url: string, + ): Promise { + const sfuConfig = await getSFUConfigWithOpenID( client, membership, url, - forceOldJwtEndpoint, roomId, - client.baseUrl, - delayId, + { + forceJwtEndpoint: forceJwtEndpoint, + delayEndpointBaseUrl: client.baseUrl, + delayId, + }, logger, ); return { - type: "livekit", - livekit_service_url: url, - livekit_alias: livekitAlias, + transport: { + type: "livekit", + livekit_service_url: url, + livekit_alias: sfuConfig.livekitAlias, + }, + sfuConfig, }; } // We will call `getSFUConfigWithOpenID` once per transport here as it's our @@ -217,7 +274,7 @@ async function makeTransport( async function getFirstUsableTransport( transports: Transport[], - ): Promise { + ): Promise { for (const potentialTransport of transports) { if (isLivekitTransportConfig(potentialTransport)) { try { @@ -226,8 +283,11 @@ async function makeTransport( potentialTransport.livekit_service_url, ); } catch (ex) { + // Explictly throw these if (ex instanceof FailToGetOpenIdToken) { - // Explictly throw these + throw ex; + } + if (ex instanceof NoMatrix2AuthorizationService) { throw ex; } logger.debug( diff --git a/src/state/CallViewModel/remoteMembers/Connection.test.ts b/src/state/CallViewModel/remoteMembers/Connection.test.ts index 0130a5ce..cc430645 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.test.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.test.ts @@ -114,6 +114,7 @@ function setupRemoteConnection(): Connection { client: client, transport: livekitFocus, scope: testScope, + ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, }; @@ -138,7 +139,7 @@ function setupRemoteConnection(): Connection { return Promise.resolve(); }); - return new Connection(opts, logger, ownMemberMock); + return new Connection(opts, logger); } afterEach(() => { @@ -155,9 +156,10 @@ describe("Start connection states", () => { client: client, transport: livekitFocus, scope: testScope, + ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, }; - const connection = new Connection(opts, logger, ownMemberMock); + const connection = new Connection(opts, logger); expect(connection.state$.getValue()).toEqual("Initialized"); }); @@ -170,10 +172,11 @@ describe("Start connection states", () => { client: client, transport: livekitFocus, scope: testScope, + ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, }; - const connection = new Connection(opts, logger, ownMemberMock); + const connection = new Connection(opts, logger); const capturedStates: (ConnectionState | Error)[] = []; const s = connection.state$.subscribe((value) => { @@ -220,10 +223,11 @@ describe("Start connection states", () => { client: client, transport: livekitFocus, scope: testScope, + ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, }; - const connection = new Connection(opts, logger, ownMemberMock); + const connection = new Connection(opts, logger); const capturedStates: (ConnectionState | Error)[] = []; const s = connection.state$.subscribe((value) => { @@ -277,10 +281,11 @@ describe("Start connection states", () => { client: client, transport: livekitFocus, scope: testScope, + ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, }; - const connection = new Connection(opts, logger, ownMemberMock); + const connection = new Connection(opts, logger); const capturedStates: (ConnectionState | Error)[] = []; const s = connection.state$.subscribe((value) => { diff --git a/src/state/CallViewModel/remoteMembers/Connection.ts b/src/state/CallViewModel/remoteMembers/Connection.ts index 269fd4f1..f286b0cd 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.ts @@ -33,10 +33,21 @@ import { SFURoomCreationRestrictedError, UnknownCallError, } from "../../../utils/errors.ts"; +import { type JwtEndpointVersion } from "../localMember/LocalTransport.ts"; export interface ConnectionOpts { - /** Whether we always try to connect to this connection via the legacy jwt endpoint. (no hash identity) */ - forceOldJwtEndpoint?: boolean; + /** + * For the local transport we already do know the jwt token and url. We can reuse it. + * On top the local transport will send additional data to the jwt server to use delayed event delegation. + */ + existingSFUConfig?: SFUConfig; + /** + * For local connections that use the oldest member pattern. here we have not prefetched the sfuConfig + * and hence we need to let the connection do the jwt token fetching. + */ + forceJwtEndpoint?: JwtEndpointVersion; + /** The identity parts to use on this connection */ + ownMembershipIdentity: CallMembershipIdentityParts; /** The media transport to connect to. */ transport: LivekitTransport; /** The Matrix client to use for OpenID and SFU config requests. */ @@ -132,8 +143,10 @@ export class Connection { try { this._state$.next(ConnectionState.FetchingConfig); // We should already have this information after creating the localTransport. - // It would probably be better to forward this here. - const { url, jwt } = await this.getSFUConfigWithOpenID(); + // only call getSFUConfigWithOpenID for connections where we do not have a token yet. (existingJwtTokenData === undefined) + const { url, jwt } = + this.existingSFUConfig ?? + (await this.getSFUConfigForRemoteConnection()); // If we were stopped while fetching the config, don't proceed to connect if (this.stopped) return; @@ -189,17 +202,16 @@ export class Connection { } } - protected async getSFUConfigWithOpenID(): Promise { + protected async getSFUConfigForRemoteConnection(): Promise { + // This will only be called for sfu's where we do not publish ourselves. + // For the local connection we will use the existingJwtTokenData return await getSFUConfigWithOpenID( this.client, this.ownMembershipIdentity, this.transport.livekit_service_url, - this.forceOldJwtEndpoint, this.transport.livekit_alias, - // For the remote members we intentionally do not pass a delayEndpointBaseUrl. - undefined, - // and no delayId. - undefined, + // dont pass any custom opts for the subscribe only connections + {}, this.logger, ); } @@ -222,7 +234,8 @@ export class Connection { private readonly client: OpenIDClientParts; private readonly logger: Logger; - private readonly forceOldJwtEndpoint: boolean; + private readonly ownMembershipIdentity: CallMembershipIdentityParts; + private readonly existingSFUConfig?: SFUConfig; /** * Creates a new connection to a matrix RTC LiveKit backend. * @@ -230,12 +243,9 @@ export class Connection { * * @param logger - The logger to use. */ - public constructor( - opts: ConnectionOpts, - logger: Logger, - private ownMembershipIdentity: CallMembershipIdentityParts, - ) { - this.forceOldJwtEndpoint = opts.forceOldJwtEndpoint ?? false; + public constructor(opts: ConnectionOpts, logger: Logger) { + this.ownMembershipIdentity = opts.ownMembershipIdentity; + this.existingSFUConfig = opts.existingSFUConfig; this.logger = logger.getChild("[Connection]"); this.logger.info( `Creating new connection to ${opts.transport.livekit_service_url} ${opts.transport.livekit_alias}`, diff --git a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts index 668538ac..aa20037c 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts @@ -20,7 +20,10 @@ import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc/LivekitTransp import { type ObservableScope } from "../../ObservableScope.ts"; import { Connection } from "./Connection.ts"; -import type { OpenIDClientParts } from "../../../livekit/openIDSFU.ts"; +import type { + OpenIDClientParts, + SFUConfig, +} from "../../../livekit/openIDSFU.ts"; import type { MediaDevices } from "../../MediaDevices.ts"; import type { Behavior } from "../../Behavior.ts"; import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx"; @@ -29,11 +32,11 @@ import { defaultLiveKitOptions } from "../../../livekit/options.ts"; // TODO evaluate if this should be done like the Publisher Factory export interface ConnectionFactory { createConnection( - transport: LivekitTransport, scope: ObservableScope, + transport: LivekitTransport, ownMembershipIdentity: CallMembershipIdentityParts, logger: Logger, - forceOldJwtEndpoint?: boolean, + sfuConfig?: SFUConfig, ): Connection; } @@ -83,30 +86,30 @@ export class ECConnectionFactory implements ConnectionFactory { /** * - * @param transport The transport to use for this connection. * @param scope The observable scope (used for clean-up) + * @param transport The transport to use for this connection. * @param ownMembershipIdentity required to connect (using the jwt service) with the SFU. * @param logger The logger instance to use for this connection. - * @param forceOldJwtEndpoint Use the old JWT endpoint independent of what the sfu supports. + * @param sfuConfig optional config in case we already have a token for this connection. * @returns */ public createConnection( - transport: LivekitTransport, scope: ObservableScope, + transport: LivekitTransport, ownMembershipIdentity: CallMembershipIdentityParts, logger: Logger, - forceOldJwtEndpoint?: boolean, + sfuConfig?: SFUConfig, ): Connection { return new Connection( { + existingSFUConfig: sfuConfig, transport, client: this.client, scope: scope, livekitRoomFactory: this.livekitRoomFactory, - forceOldJwtEndpoint, + ownMembershipIdentity, }, logger, - ownMembershipIdentity, ); } } diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts index 5c50f0cd..4295c5f2 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts @@ -12,12 +12,17 @@ import { type Logger } from "matrix-js-sdk/lib/logger"; import { type RemoteParticipant } from "livekit-client"; import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; -import { constant, type Behavior } from "../../Behavior.ts"; +import { type Behavior } from "../../Behavior.ts"; import { type Connection } from "./Connection.ts"; import { Epoch, type ObservableScope } from "../../ObservableScope.ts"; import { generateItemsWithEpoch } from "../../../utils/observable.ts"; import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts"; import { type ConnectionFactory } from "./ConnectionFactory.ts"; +import { + isLocalTransportWithSFUConfig, + type LocalTransportWithSFUConfig, +} from "../localMember/LocalTransport.ts"; +import { type SFUConfig } from "../../../livekit/openIDSFU.ts"; export class ConnectionManagerData { private readonly store: Map< @@ -66,9 +71,9 @@ export class ConnectionManagerData { interface Props { scope: ObservableScope; connectionFactory: ConnectionFactory; - localTransport$: Behavior; + localTransport$: Behavior; remoteTransports$: Behavior>; - forceOldJwtEndpointForLocalTransport$?: Behavior; + logger: Logger; ownMembershipIdentity: CallMembershipIdentityParts; } @@ -87,7 +92,7 @@ export interface IConnectionManager { * @param props.remoteTransports$ - All other transports. The connection manager will create connections for each transport. (deduplicated with localTransport$) * @param props.ownMembershipIdentity - The own membership identity to use. * @param props.logger - The logger to use. - * @param props.forceOldJwtEndpointForLocalTransport$ - Use the old JWT endpoint independent of what the sfu supports. Only applies for localTransport$. + * * Each of these behaviors can be interpreted as subscribed list of transports. * @@ -103,7 +108,6 @@ export function createConnectionManager$({ connectionFactory, localTransport$, remoteTransports$, - forceOldJwtEndpointForLocalTransport$ = constant(false), logger: parentLogger, ownMembershipIdentity, }: Props): IConnectionManager { @@ -118,42 +122,35 @@ export function createConnectionManager$({ * It is build based on the list of subscribed transports (`transportsSubscriptions$`). * externally this is modified via `registerTransports()`. */ - const transportsWithJwtTag$ = scope.behavior( - combineLatest([ - remoteTransports$, - localTransport$, - forceOldJwtEndpointForLocalTransport$, - ]).pipe( - // combine local and remote transports into one transport array + const localAndRemoteTransports$: Behavior< + Epoch<(LivekitTransport | LocalTransportWithSFUConfig)[]> + > = scope.behavior( + combineLatest([remoteTransports$, localTransport$]).pipe( + // Combine local and remote transports into one transport array // and set the forceOldJwtEndpoint property on the local transport - map( - ([ - remoteTransports, - localTransport, - forceOldJwtEndpointForLocalTransport, - ]) => { - let localTransportAsArray: (LivekitTransport & { - forceOldJwtEndpoint: boolean; - })[] = []; - if (localTransport) { - localTransportAsArray = [ - { - ...localTransport, - forceOldJwtEndpoint: forceOldJwtEndpointForLocalTransport, - }, - ]; - } - return new Epoch( - removeDuplicateTransports([ - ...localTransportAsArray, - ...remoteTransports.value, - ]) as (LivekitTransport & { - forceOldJwtEndpoint?: boolean; - })[], - remoteTransports.epoch, - ); - }, - ), + map(([remoteTransports, localTransport]) => { + let localTransportAsArray: LocalTransportWithSFUConfig[] = []; + if (localTransport) { + localTransportAsArray = [localTransport]; + } + const dedupedRemote = removeDuplicateTransports(remoteTransports.value); + const remoteWithoutLocal = dedupedRemote.filter( + (transport) => + !localTransportAsArray.find((l) => + areLivekitTransportsEqual(l.transport, transport), + ), + ); + logger.debug( + "remoteWithoutLocal", + remoteWithoutLocal, + "localTransportAsArray", + localTransportAsArray, + ); + return new Epoch( + [...localTransportAsArray, ...remoteWithoutLocal], + remoteTransports.epoch, + ); + }), ), ); @@ -161,33 +158,51 @@ export function createConnectionManager$({ * Connections for each transport in use by one or more session members. */ const connections$ = scope.behavior( - transportsWithJwtTag$.pipe( + localAndRemoteTransports$.pipe( generateItemsWithEpoch( function* (transports) { - for (const transport of transports) - yield { - keys: [ - transport.livekit_service_url, - transport.livekit_alias, - transport.forceOldJwtEndpoint, - ], - data: undefined, - }; + for (const transportWithOrWithoutSfuConfig of transports) { + if ( + isLocalTransportWithSFUConfig(transportWithOrWithoutSfuConfig) + ) { + // This is the local transport only the `LocalTransportWithSFUConfig` has a `sfuConfig` field + const { transport, sfuConfig } = transportWithOrWithoutSfuConfig; + yield { + keys: [ + transport.livekit_service_url, + transport.livekit_alias, + sfuConfig, + ], + data: undefined, + }; + } else { + const transport = transportWithOrWithoutSfuConfig; + yield { + keys: [ + transport.livekit_service_url, + transport.livekit_alias, + undefined as undefined | SFUConfig, + ], + data: undefined, + }; + } + } }, - (scope, _data$, serviceUrl, alias, forceOldJwtEndpoint) => { + (scope, _data$, serviceUrl, alias, sfuConfig) => { logger.debug( - `Creating connection to ${serviceUrl} (${alias}, forceOldJwtEndpoint: ${forceOldJwtEndpoint})`, + `Creating connection to ${serviceUrl} (${alias}, withSfuConfig (local connection?): ${JSON.stringify(sfuConfig) ?? "no config->remote connection"})`, ); + const connection = connectionFactory.createConnection( + scope, { type: "livekit", livekit_service_url: serviceUrl, livekit_alias: alias, }, - scope, ownMembershipIdentity, logger, - forceOldJwtEndpoint, + sfuConfig, ); // Start the connection immediately // Use connection state to track connection progress diff --git a/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts b/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts index f28bd158..b56a17f7 100644 --- a/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts +++ b/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts @@ -77,8 +77,8 @@ describe("ECConnectionFactory - Audio inputs options", () => { noise, ); ecConnectionFactory.createConnection( - exampleTransport, testScope, + exampleTransport, ownMemberMock, logger, ); @@ -123,8 +123,8 @@ describe("ECConnectionFactory - ControlledAudioDevice", () => { false, ); ecConnectionFactory.createConnection( - exampleTransport, testScope, + exampleTransport, ownMemberMock, logger, ); diff --git a/src/state/MediaViewModel.ts b/src/state/MediaViewModel.ts index 9888d6bf..6b020057 100644 --- a/src/state/MediaViewModel.ts +++ b/src/state/MediaViewModel.ts @@ -44,6 +44,7 @@ import { throttleTime, distinctUntilChanged, } from "rxjs"; +import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; import { alwaysShowSelf } from "../settings/settings"; import { showConnectionStats } from "../settings/settings"; @@ -257,6 +258,7 @@ abstract class BaseMediaViewModel { * The Matrix user to which this media belongs. */ public readonly userId: string, + public readonly rtcMembership: CallMembership, // We don't necessarily have a participant if a user connects via MatrixRTC but not (yet) through // livekit. protected readonly participant$: Observable< @@ -402,10 +404,13 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { */ public readonly cropVideo$: Behavior = this._cropVideo$; + public readonly rtcBackendIdentity = this.rtcMembership.rtcBackendIdentity; + public constructor( scope: ObservableScope, id: string, userId: string, + rtcMembership: CallMembership, participant$: Observable, encryptionSystem: EncryptionSystem, livekitRoom$: Behavior, @@ -419,6 +424,7 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { scope, id, userId, + rtcMembership, participant$, encryptionSystem, Track.Source.Microphone, @@ -544,6 +550,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { scope: ObservableScope, id: string, userId: string, + rtcMembership: CallMembership, participant$: Behavior, encryptionSystem: EncryptionSystem, livekitRoom$: Behavior, @@ -558,6 +565,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { scope, id, userId, + rtcMembership, participant$, encryptionSystem, livekitRoom$, @@ -671,6 +679,7 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel { scope: ObservableScope, id: string, userId: string, + rtcMembership: CallMembership, participant$: Observable, encryptionSystem: EncryptionSystem, livekitRoom$: Behavior, @@ -685,6 +694,7 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel { scope, id, userId, + rtcMembership, participant$, encryptionSystem, livekitRoom$, @@ -772,6 +782,7 @@ export class ScreenShareViewModel extends BaseMediaViewModel { scope: ObservableScope, id: string, userId: string, + rtcMembership: CallMembership, participant$: Observable, encryptionSystem: EncryptionSystem, livekitRoom$: Behavior, @@ -785,6 +796,7 @@ export class ScreenShareViewModel extends BaseMediaViewModel { scope, id, userId, + rtcMembership, participant$, encryptionSystem, Track.Source.ScreenShareAudio, diff --git a/src/state/ScreenShare.ts b/src/state/ScreenShare.ts index 0a241cdf..67f97a93 100644 --- a/src/state/ScreenShare.ts +++ b/src/state/ScreenShare.ts @@ -10,6 +10,7 @@ import { type RemoteParticipant, type Room as LivekitRoom, } from "livekit-client"; +import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc/CallMembership"; import { type ObservableScope } from "./ObservableScope.ts"; import { ScreenShareViewModel } from "./MediaViewModel.ts"; @@ -28,6 +29,7 @@ export class ScreenShare { private readonly scope: ObservableScope, id: string, userId: string, + rtcMember: CallMembership, participant: LocalParticipant | RemoteParticipant, encryptionSystem: EncryptionSystem, livekitRoom$: Behavior, @@ -40,6 +42,7 @@ export class ScreenShare { this.scope, id, userId, + rtcMember, of(participant), encryptionSystem, livekitRoom$, diff --git a/src/state/UserMedia.ts b/src/state/UserMedia.ts index 690870e6..6435fb17 100644 --- a/src/state/UserMedia.ts +++ b/src/state/UserMedia.ts @@ -13,6 +13,7 @@ import { type Room as LivekitRoom, } from "livekit-client"; import { observeParticipantEvents } from "@livekit/components-core"; +import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc/CallMembership"; import { type ObservableScope } from "./ObservableScope.ts"; import { @@ -75,6 +76,7 @@ export class UserMedia { this.scope, this.id, this.userId, + this.rtcMembership, this.participant.value$, this.encryptionSystem, this.livekitRoom$, @@ -89,6 +91,7 @@ export class UserMedia { this.scope, this.id, this.userId, + this.rtcMembership, this.participant.value$, this.encryptionSystem, this.livekitRoom$, @@ -140,6 +143,7 @@ export class UserMedia { scope, `${this.id}:${key}`, this.userId, + this.rtcMembership, p, this.encryptionSystem, this.livekitRoom$, @@ -191,6 +195,8 @@ export class UserMedia { private readonly scope: ObservableScope, public readonly id: string, private readonly userId: string, + // TODO evaluate if this should just be the rtcBackendIdentity + private readonly rtcMembership: CallMembership, private readonly participant: TaggedParticipant, private readonly encryptionSystem: EncryptionSystem, private readonly livekitRoom$: Behavior, diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index 2f750c50..92262f05 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -113,6 +113,7 @@ const UserMediaTile: FC = ({ }, [vm], ); + const rtcBackendIdentity = vm.rtcBackendIdentity; const handRaised = useBehavior(vm.handRaised$); const reaction = useBehavior(vm.reaction$); @@ -200,6 +201,7 @@ const UserMediaTile: FC = ({ focusUrl={focusUrl} audioStreamStats={audioStreamStats} videoStreamStats={videoStreamStats} + rtcBackendIdentity={rtcBackendIdentity} {...props} /> ); diff --git a/src/tile/MediaView.tsx b/src/tile/MediaView.tsx index 8bb38d94..7847a30a 100644 --- a/src/tile/MediaView.tsx +++ b/src/tile/MediaView.tsx @@ -46,6 +46,7 @@ interface Props extends ComponentProps { waitingForMedia?: boolean; audioStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; videoStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; + rtcBackendIdentity?: string; // The focus url, mainly for debugging purposes focusUrl?: string; } @@ -74,6 +75,7 @@ export const MediaView: FC = ({ waitingForMedia, audioStreamStats, videoStreamStats, + rtcBackendIdentity, focusUrl, ...props }) => { @@ -135,11 +137,14 @@ export const MediaView: FC = ({
)} {(audioStreamStats || videoStreamStats) && ( - + <> + + )} {/* TODO: Bring this back once encryption status is less broken */} {/*encryptionStatus !== EncryptionStatus.Okay && ( diff --git a/src/utils/errors.ts b/src/utils/errors.ts index cddf90de..20a282e6 100644 --- a/src/utils/errors.ts +++ b/src/utils/errors.ts @@ -19,6 +19,7 @@ export enum ErrorCode { INSUFFICIENT_CAPACITY_ERROR = "INSUFFICIENT_CAPACITY_ERROR", E2EE_NOT_SUPPORTED = "E2EE_NOT_SUPPORTED", OPEN_ID_ERROR = "OPEN_ID_ERROR", + NO_MATRIX_2_AUTHORIZATION_SERVICE = "NO_MATRIX_2_0_AUTHORIZATION_SERVICE", SFU_ERROR = "SFU_ERROR", UNKNOWN_ERROR = "UNKNOWN_ERROR", } @@ -171,6 +172,23 @@ export class FailToGetOpenIdToken extends ElementCallError { } } +export class NoMatrix2AuthorizationService extends ElementCallError { + /** + * Creates an instance of NoMatrix2_0AuthorizationService. + * @param error - The underlying error that caused the failure. + */ + public constructor(error: Error) { + super( + t("error.generic"), + ErrorCode.NO_MATRIX_2_AUTHORIZATION_SERVICE, + ErrorCategory.CONFIGURATION_ISSUE, + t("error.no_matrix_2_authorization_service"), + // Properly set it as a cause for a better reporting on sentry + error, + ); + } +} + /** * Error indicating a failure to start publishing on a LiveKit connection. */ diff --git a/src/utils/test.ts b/src/utils/test.ts index d24ad130..9f9ae4b7 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -334,6 +334,7 @@ export function createLocalMedia( testScope(), "local", member.userId, + rtcMember, constant(localParticipant), { kind: E2eeType.PER_PARTICIPANT, @@ -379,6 +380,7 @@ export function createRemoteMedia( testScope(), "remote", member.userId, + rtcMember, constant(participant), { kind: E2eeType.PER_PARTICIPANT, diff --git a/yarn.lock b/yarn.lock index 54832a25..b32cf3c8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8246,7 +8246,7 @@ __metadata: livekit-client: "npm:^2.13.0" lodash-es: "npm:^4.17.21" loglevel: "npm:^1.9.1" - matrix-js-sdk: "matrix-org/matrix-js-sdk#develop" + matrix-js-sdk: "matrix-org/matrix-js-sdk#ff5fab7722e54be2d69a380cd6182ecb262d7859" matrix-widget-api: "npm:^1.14.0" node-stdlib-browser: "npm:^1.3.1" normalize.css: "npm:^8.0.1" @@ -11333,9 +11333,9 @@ __metadata: languageName: node linkType: hard -"matrix-js-sdk@matrix-org/matrix-js-sdk#develop": - version: 39.4.0 - resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=4b89fb23c54aaf7826bd127d8fa21cc7bb87688f" +"matrix-js-sdk@portal:/Users/timo/Projects/matrix-js-sdk::locator=element-call%40workspace%3A.": + version: 0.0.0-use.local + resolution: "matrix-js-sdk@portal:/Users/timo/Projects/matrix-js-sdk::locator=element-call%40workspace%3A." dependencies: "@babel/runtime": "npm:^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm": "npm:^16.0.0" @@ -11351,9 +11351,8 @@ __metadata: sdp-transform: "npm:^3.0.0" unhomoglyph: "npm:^1.0.6" uuid: "npm:13" - checksum: 10c0/bc7443bf67822e9bc7b8e531b4e61e6ebac41c2fd8047ac0567456c264ae0d1911fbef6e437d312a3adeead86cd5e7134944e3fd73d28002777618bc0ebaa1ca languageName: node - linkType: hard + linkType: soft "matrix-widget-api@npm:^1.14.0": version: 1.15.0 From c7c6dc18bf3387a0077de5b539c632f79ad2ccb7 Mon Sep 17 00:00:00 2001 From: Timo K Date: Fri, 9 Jan 2026 14:04:24 +0100 Subject: [PATCH 32/46] fix js-sdk dependency --- package.json | 2 +- yarn.lock | 21 +++++++++++---------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/package.json b/package.json index 6a8d3c76..944fc43b 100644 --- a/package.json +++ b/package.json @@ -104,7 +104,7 @@ "livekit-client": "^2.13.0", "lodash-es": "^4.17.21", "loglevel": "^1.9.1", - "matrix-js-sdk": "matrix-org/matrix-js-sdk#ff5fab7722e54be2d69a380cd6182ecb262d7859", + "matrix-js-sdk": "matrix-org/matrix-js-sdk#develop", "matrix-widget-api": "^1.14.0", "node-stdlib-browser": "^1.3.1", "normalize.css": "^8.0.1", diff --git a/yarn.lock b/yarn.lock index b32cf3c8..d795f1e6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3300,10 +3300,10 @@ __metadata: languageName: node linkType: hard -"@matrix-org/matrix-sdk-crypto-wasm@npm:^16.0.0": - version: 16.0.0 - resolution: "@matrix-org/matrix-sdk-crypto-wasm@npm:16.0.0" - checksum: 10c0/13b4ede3e618da819957abff778afefcf3baf9a2faac04a36bb5a07a44fae2ea05fbfa072eb3408d48b2b7b9aaf27242ce52c594c8ce9bf1fb8b3aade2832be1 +"@matrix-org/matrix-sdk-crypto-wasm@npm:^17.0.0": + version: 17.0.0 + resolution: "@matrix-org/matrix-sdk-crypto-wasm@npm:17.0.0" + checksum: 10c0/fa97e3111099057e0953e7550d6556b6e7553f3badd5b25a6988d2fcc94d22288a27e63cb204771b74ff24388d770c83f2cf5aec583f05c6ecf46509b8020570 languageName: node linkType: hard @@ -8246,7 +8246,7 @@ __metadata: livekit-client: "npm:^2.13.0" lodash-es: "npm:^4.17.21" loglevel: "npm:^1.9.1" - matrix-js-sdk: "matrix-org/matrix-js-sdk#ff5fab7722e54be2d69a380cd6182ecb262d7859" + matrix-js-sdk: "matrix-org/matrix-js-sdk#develop" matrix-widget-api: "npm:^1.14.0" node-stdlib-browser: "npm:^1.3.1" normalize.css: "npm:^8.0.1" @@ -11333,12 +11333,12 @@ __metadata: languageName: node linkType: hard -"matrix-js-sdk@portal:/Users/timo/Projects/matrix-js-sdk::locator=element-call%40workspace%3A.": - version: 0.0.0-use.local - resolution: "matrix-js-sdk@portal:/Users/timo/Projects/matrix-js-sdk::locator=element-call%40workspace%3A." +"matrix-js-sdk@matrix-org/matrix-js-sdk#develop": + version: 39.4.0 + resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=841b654c00c2a47dbd4380b4b2a6fa25156c8af0" dependencies: "@babel/runtime": "npm:^7.12.5" - "@matrix-org/matrix-sdk-crypto-wasm": "npm:^16.0.0" + "@matrix-org/matrix-sdk-crypto-wasm": "npm:^17.0.0" another-json: "npm:^0.2.0" bs58: "npm:^6.0.0" content-type: "npm:^1.0.4" @@ -11351,8 +11351,9 @@ __metadata: sdp-transform: "npm:^3.0.0" unhomoglyph: "npm:^1.0.6" uuid: "npm:13" + checksum: 10c0/532d4adb2ce95a27b4715be518d648740efddc5c88d399e127d95ff962bddd8f62f848a0ea98088672fab03b67a973d63935cd3b5ed371bb9da3131ac9472db6 languageName: node - linkType: soft + linkType: hard "matrix-widget-api@npm:^1.14.0": version: 1.15.0 From c2e377474932d15c7ac8aae8e893aa977112691c Mon Sep 17 00:00:00 2001 From: Timo K Date: Fri, 9 Jan 2026 15:40:39 +0100 Subject: [PATCH 33/46] prettier --- src/room/InCallView.tsx | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 54f189f0..5b80db3d 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -113,8 +113,10 @@ const logger = rootLogger.getChild("[InCallView]"); const maxTapDurationMs = 400; -export interface ActiveCallProps - extends Omit { +export interface ActiveCallProps extends Omit< + InCallViewProps, + "vm" | "livekitRoom" | "connState" +> { e2eeSystem: EncryptionSystem; // TODO refactor those reasons into an enum onLeft: ( From 0eb0169ad9f41af2d0daaf648c2c0a2c9ca1d847 Mon Sep 17 00:00:00 2001 From: Timo K Date: Fri, 9 Jan 2026 16:00:02 +0100 Subject: [PATCH 34/46] update example --- sdk/index.html | 25 +++---------------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/sdk/index.html b/sdk/index.html index 1420c3ea..8883b9a3 100644 --- a/sdk/index.html +++ b/sdk/index.html @@ -1,7 +1,7 @@ - Godot MatrixRTC Widget + MatrixRTC Widget - - -
From b7838e43533ee2af50472cdf9ec2ff1326670c08 Mon Sep 17 00:00:00 2001 From: Timo K Date: Fri, 9 Jan 2026 16:25:06 +0100 Subject: [PATCH 35/46] Update README.md --- sdk/README.md | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/sdk/README.md b/sdk/README.md index 03801b83..27d2de6d 100644 --- a/sdk/README.md +++ b/sdk/README.md @@ -6,15 +6,35 @@ It allows to use matrixRTC in combination with livekit without relying on elemen This is done by instantiating the call view model and exposing some useful behaviors (observables) and methods. -This folder contains an example index.html file that showcases the sdk in use (hosted on localhost:8123 with a webserver ellowing cors (for example `npx serve -l 81234 --cors`)) as a godot engine HTML export template. +This folder contains an example index.html file that showcases the sdk in use (hosted on localhost:8123 with a webserver allowing cors (for example `npx serve -l 81234 --cors`)) as a godot engine HTML export template. + +## Getting started + +To get started run +``` +yarn +yarn build:sdk +``` +in the repository root. + +It will create a `dist` folder containing the compiled js file. + +This file needs to be hosted. Locally (via `npx serve -l 81234 --cors`) or on a remote server. + +Now you just need to add the widget to element web via: +``` +/addwidget http://localhost:3000?widgetId=$matrix_widget_id&perParticipantE2EE=true&userId=$matrix_user_id&deviceId=$org.matrix.msc3819.matrix_device_id&baseUrl=$org.matrix.msc4039.matrix_base_url&roomId=$matrix_room_id +``` ## Widgets -The sdk mode is particularly interesting to be used in widgets where you do not need to pay attention to matrix login/cs api ... -To create a widget see the example index.html file in this folder. And add it to EW via: +The sdk mode is particularly interesting to be used in widgets. In widgets you do not need to pay attention to matrix login/cs api ... +To create a widget see the example `index.html` file in this folder. And add it to EW via: `/addwidget ` (see **url parameters** for more details on ``) ### url parameters +The url parameters are needed to pass initial data to the widget. They will automatically be used +but the matrixRTCSdk to start the postmessage widget api (communication between the client (e.g. Element Web) and the widget) ``` widgetId = $matrix_widget_id From 70b7ebff80cfd83e5b45eda0350b17d2f992b3a8 Mon Sep 17 00:00:00 2001 From: Timo K Date: Fri, 9 Jan 2026 16:32:14 +0100 Subject: [PATCH 36/46] prettier --- sdk/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sdk/README.md b/sdk/README.md index 27d2de6d..9282fc1f 100644 --- a/sdk/README.md +++ b/sdk/README.md @@ -11,10 +11,12 @@ This folder contains an example index.html file that showcases the sdk in use (h ## Getting started To get started run + ``` yarn yarn build:sdk ``` + in the repository root. It will create a `dist` folder containing the compiled js file. @@ -22,6 +24,7 @@ It will create a `dist` folder containing the compiled js file. This file needs to be hosted. Locally (via `npx serve -l 81234 --cors`) or on a remote server. Now you just need to add the widget to element web via: + ``` /addwidget http://localhost:3000?widgetId=$matrix_widget_id&perParticipantE2EE=true&userId=$matrix_user_id&deviceId=$org.matrix.msc3819.matrix_device_id&baseUrl=$org.matrix.msc4039.matrix_base_url&roomId=$matrix_room_id ``` @@ -33,6 +36,7 @@ To create a widget see the example `index.html` file in this folder. And add it `/addwidget ` (see **url parameters** for more details on ``) ### url parameters + The url parameters are needed to pass initial data to the widget. They will automatically be used but the matrixRTCSdk to start the postmessage widget api (communication between the client (e.g. Element Web) and the widget) From ed3d168ac381d623d7c8dc00d57cb16aff8a1e35 Mon Sep 17 00:00:00 2001 From: Timo K Date: Fri, 9 Jan 2026 17:29:48 +0100 Subject: [PATCH 37/46] fix tests --- src/settings/DeveloperSettingsTab.test.tsx | 6 +- .../DeveloperSettingsTab.test.tsx.snap | 96 ++++++++++++------- .../localMember/LocalMember.test.ts | 14 ++- .../localMember/LocalTransport.test.ts | 84 ++++++++++++---- .../remoteMembers/ConnectionManager.test.ts | 4 +- 5 files changed, 146 insertions(+), 58 deletions(-) diff --git a/src/settings/DeveloperSettingsTab.test.tsx b/src/settings/DeveloperSettingsTab.test.tsx index c18cf23b..23fa67a7 100644 --- a/src/settings/DeveloperSettingsTab.test.tsx +++ b/src/settings/DeveloperSettingsTab.test.tsx @@ -7,9 +7,9 @@ Please see LICENSE in the repository root for full details. import { describe, expect, it, vi } from "vitest"; import { render, waitFor } from "@testing-library/react"; +import { type Room as LivekitRoom } from "livekit-client"; import type { MatrixClient } from "matrix-js-sdk"; -import type { Room as LivekitRoom } from "livekit-client"; import { DeveloperSettingsTab } from "./DeveloperSettingsTab"; // Mock url params hook to avoid environment-dependent snapshot churn. @@ -30,6 +30,8 @@ function createMockLivekitRoom( serverInfo, metadata, engine: { client: { ws: { url: wsUrl } } }, + localParticipant: { identity: "localParticipantIdentity" }, + remoteParticipants: new Map(), } as unknown as LivekitRoom; return { @@ -69,6 +71,8 @@ describe("DeveloperSettingsTab", () => { isLocal: false, url: "wss://remote-sfu.example.org", room: { + localParticipant: { identity: "localParticipantIdentity" }, + remoteParticipants: new Map(), serverInfo: { region: "remote", version: "4.5.6" }, metadata: "remote-metadata", engine: { client: { ws: { url: "wss://remote-sfu.example.org" } } }, diff --git a/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap b/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap index 1c82d07b..cb17bcc5 100644 --- a/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap +++ b/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap @@ -349,46 +349,78 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
-

- LiveKit SFU: wss://local-sfu.example.org -

-

- ws-url: - wss://local-sfu.example.org/ -

-

- LiveKit Server Info - ( - local - ) -

- 
-    {
+    

+      LiveKit SFU: wss://local-sfu.example.org
+    

+    

+      ws-url: 
+      wss://local-sfu.example.org/
+    

+    

+      LiveKit Server Info
+      (
+      local
+      )
+    

+    
+      {
   "region": "local",
   "version": "1.2.3"
 }
-    local-metadata
-  

-  

-    LiveKit SFU: wss://remote-sfu.example.org
-  

-  

-    LiveKit Server Info
-    (
-    remote
-    )
-  

-  
+    

+      Local Participant
+    

+    
+      localParticipantIdentity
+    

+    

+      Remote Participants
+    

+    
    
+
+
- { +

+ LiveKit SFU: wss://remote-sfu.example.org +

+

+ LiveKit Server Info + ( + remote + ) +

+ 
+      {
   "region": "remote",
   "version": "4.5.6"
 }
-    remote-metadata
-
+ remote-metadata + +

+ Local Participant +

+ 
+      localParticipantIdentity
+
+

+ Remote Participants +

+
    +

Environment variables

diff --git a/src/state/CallViewModel/localMember/LocalMember.test.ts b/src/state/CallViewModel/localMember/LocalMember.test.ts index 6b750785..af12c98b 100644 --- a/src/state/CallViewModel/localMember/LocalMember.test.ts +++ b/src/state/CallViewModel/localMember/LocalMember.test.ts @@ -271,7 +271,7 @@ describe("LocalMembership", () => { } as unknown as LocalParticipant, }), state$: constant(ConnectionState.LivekitConnected), - transport: aTransport, + transport: aTransport.transport, } as unknown as Connection; const connectionTransportAConnecting = { ...connectionTransportAConnected, @@ -280,11 +280,11 @@ describe("LocalMembership", () => { } as unknown as Connection; const connectionTransportBConnected = { state$: constant(ConnectionState.LivekitConnected), - transport: bTransport, + transport: bTransport.transport, livekitRoom: mockLivekitRoom({}), } as unknown as Connection; - it("recreates publisher if new connection is used and ENDS always unpublish and end tracks", async () => { + it("recreates publisher if new connection is used, always unpublish and end tracks", async () => { const scope = new ObservableScope(); const localTransport$ = new BehaviorSubject(aTransport); @@ -332,8 +332,12 @@ describe("LocalMembership", () => { expect(publishers[1].stopTracks).not.toHaveBeenCalled(); expect(publishers[0].stopPublishing).toHaveBeenCalled(); expect(publishers[1].stopPublishing).not.toHaveBeenCalled(); - expect(publisherFactory.mock.calls[0][0].transport).toBe(aTransport); - expect(publisherFactory.mock.calls[1][0].transport).toBe(bTransport); + expect(publisherFactory.mock.calls[0][0].transport).toBe( + aTransport.transport, + ); + expect(publisherFactory.mock.calls[1][0].transport).toBe( + bTransport.transport, + ); scope.end(); await flushPromises(); // stop all tracks after ending scopes diff --git a/src/state/CallViewModel/localMember/LocalTransport.test.ts b/src/state/CallViewModel/localMember/LocalTransport.test.ts index 8da0db7f..9199b51e 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.test.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.test.ts @@ -154,9 +154,17 @@ describe("LocalTransport", () => { await flushPromises(); // final expect(localTransport$.value).toStrictEqual({ - livekit_alias: "!room:example.org", - livekit_service_url: "https://lk.example.org", - type: "livekit", + transport: { + livekit_alias: "!room:example.org", + livekit_service_url: "https://lk.example.org", + type: "livekit", + }, + sfuConfig: { + jwt: "jwt", + livekitAlias: "!room:example.org", + livekitIdentity: "@alice:example.org:DEVICE", + url: "https://lk.example.org", + }, }); }); @@ -195,9 +203,17 @@ describe("LocalTransport", () => { await flushPromises(); // final expect(localTransport$.value).toStrictEqual({ - livekit_alias: "!example_room_id", - livekit_service_url: "https://lk.example.org", - type: "livekit", + transport: { + livekit_alias: "!example_room_id", + livekit_service_url: "https://lk.example.org", + type: "livekit", + }, + sfuConfig: { + jwt: "e30=.eyJzdWIiOiJAbWU6ZXhhbXBsZS5vcmc6QUJDREVGIiwidmlkZW8iOnsicm9vbSI6IiFleGFtcGxlX3Jvb21faWQifX0=.e30=", + livekitAlias: "!example_room_id", + livekitIdentity: "@lk_user:ABCDEF", + url: "https://lk.example.org", + }, }); }); @@ -247,9 +263,17 @@ describe("LocalTransport", () => { expect(localTransport$.value).toBe(null); await flushPromises(); expect(localTransport$.value).toStrictEqual({ - livekit_alias: "!example_room_id", - livekit_service_url: "https://lk.example.org", - type: "livekit", + transport: { + livekit_alias: "!example_room_id", + livekit_service_url: "https://lk.example.org", + type: "livekit", + }, + sfuConfig: { + jwt: "e30=.eyJzdWIiOiJAbWU6ZXhhbXBsZS5vcmc6QUJDREVGIiwidmlkZW8iOnsicm9vbSI6IiFleGFtcGxlX3Jvb21faWQifX0=.e30=", + livekitAlias: "!example_room_id", + livekitIdentity: "@lk_user:ABCDEF", + url: "https://lk.example.org", + }, }); }); it("supports getting transport via user settings", async () => { @@ -259,9 +283,17 @@ describe("LocalTransport", () => { expect(localTransport$.value).toBe(null); await flushPromises(); expect(localTransport$.value).toStrictEqual({ - livekit_alias: "!example_room_id", - livekit_service_url: "https://lk.example.org", - type: "livekit", + transport: { + livekit_alias: "!example_room_id", + livekit_service_url: "https://lk.example.org", + type: "livekit", + }, + sfuConfig: { + jwt: "e30=.eyJzdWIiOiJAbWU6ZXhhbXBsZS5vcmc6QUJDREVGIiwidmlkZW8iOnsicm9vbSI6IiFleGFtcGxlX3Jvb21faWQifX0=.e30=", + livekitAlias: "!example_room_id", + livekitIdentity: "@lk_user:ABCDEF", + url: "https://lk.example.org", + }, }); }); it("supports getting transport via backend", async () => { @@ -273,9 +305,17 @@ describe("LocalTransport", () => { expect(localTransport$.value).toBe(null); await flushPromises(); expect(localTransport$.value).toStrictEqual({ - livekit_alias: "!example_room_id", - livekit_service_url: "https://lk.example.org", - type: "livekit", + transport: { + livekit_alias: "!example_room_id", + livekit_service_url: "https://lk.example.org", + type: "livekit", + }, + sfuConfig: { + jwt: "e30=.eyJzdWIiOiJAbWU6ZXhhbXBsZS5vcmc6QUJDREVGIiwidmlkZW8iOnsicm9vbSI6IiFleGFtcGxlX3Jvb21faWQifX0=.e30=", + livekitAlias: "!example_room_id", + livekitIdentity: "@lk_user:ABCDEF", + url: "https://lk.example.org", + }, }); }); it("fails fast if the openID request fails for backend config", async () => { @@ -304,9 +344,17 @@ describe("LocalTransport", () => { expect(localTransport$.value).toBe(null); await flushPromises(); expect(localTransport$.value).toStrictEqual({ - livekit_alias: "!example_room_id", - livekit_service_url: "https://lk.example.org", - type: "livekit", + transport: { + livekit_alias: "!example_room_id", + livekit_service_url: "https://lk.example.org", + type: "livekit", + }, + sfuConfig: { + jwt: "e30=.eyJzdWIiOiJAbWU6ZXhhbXBsZS5vcmc6QUJDREVGIiwidmlkZW8iOnsicm9vbSI6IiFleGFtcGxlX3Jvb21faWQifX0=.e30=", + livekitAlias: "!example_room_id", + livekitIdentity: "@lk_user:ABCDEF", + url: "https://lk.example.org", + }, }); expect(fetchMock.done()).toEqual(true); }); diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts index 6a3bd71b..cf930415 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts @@ -49,7 +49,7 @@ beforeEach(() => { vi.mocked(fakeConnectionFactory).createConnection = vi .fn() .mockImplementation( - (transport: LivekitTransport, scope: ObservableScope) => { + (scope: ObservableScope, transport: LivekitTransport) => { const mockConnection = { transport, remoteParticipants$: new BehaviorSubject([]), @@ -229,7 +229,7 @@ describe("connectionManagerData$ stream", () => { vi.mocked(fakeConnectionFactory).createConnection = vi .fn() .mockImplementation( - (transport: LivekitTransport, scope: ObservableScope) => { + (scope: ObservableScope, transport: LivekitTransport) => { const fakeRemoteParticipants$ = new BehaviorSubject< RemoteParticipant[] >([]); From 0b766472ed7017ba04395dc21f6047796354cd44 Mon Sep 17 00:00:00 2001 From: Timo <16718859+toger5@users.noreply.github.com> Date: Fri, 9 Jan 2026 17:30:02 +0100 Subject: [PATCH 38/46] Update sdk/README.md Co-authored-by: Robin --- sdk/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/README.md b/sdk/README.md index 9282fc1f..91337f10 100644 --- a/sdk/README.md +++ b/sdk/README.md @@ -38,7 +38,7 @@ To create a widget see the example `index.html` file in this folder. And add it ### url parameters The url parameters are needed to pass initial data to the widget. They will automatically be used -but the matrixRTCSdk to start the postmessage widget api (communication between the client (e.g. Element Web) and the widget) +by the matrixRTCSdk to start the postmessage widget api (communication between the client (e.g. Element Web) and the widget) ``` widgetId = $matrix_widget_id From 39432310e304322b425b1456256be311bd15b3ef Mon Sep 17 00:00:00 2001 From: Timo K Date: Fri, 9 Jan 2026 17:56:30 +0100 Subject: [PATCH 39/46] use actual uuid for `member.id` review with some renaming and removal of unnecassary if --- src/state/CallViewModel/CallViewModel.ts | 44 +++++++++---------- .../remoteMembers/MatrixLivekitMembers.ts | 2 +- 2 files changed, 22 insertions(+), 24 deletions(-) diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 8fc2d7e6..bf5ea441 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -47,6 +47,7 @@ import { } from "matrix-js-sdk/lib/matrixrtc"; import { type IWidgetApiRequest } from "matrix-widget-api"; import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; +import { v4 as uuidv4 } from "uuid"; import { LocalUserMediaViewModel, @@ -430,9 +431,8 @@ export function createCallViewModel$( const ownMembershipIdentity: CallMembershipIdentityParts = { userId, deviceId, - // This will eventually become the salt for the hash endpoint. - // For now we keep it as the user+device string since it is expected by non matrix matrixRTCMode === Legacy. - memberId: `${userId}:${deviceId}`, + // This will only be consumed by the sticky membership manager. So it has no impact on legacy calls. + memberId: uuidv4(), }; const localTransport$ = createLocalTransport$({ @@ -722,22 +722,20 @@ export function createCallViewModel$( if (localMatrixLivekitMember) { const { userId, participant, connection$, membership$ } = localMatrixLivekitMember; - localUserMediaId = `${userId}:${membership$.value.deviceId}`; // should be membership$.value.membershipID which is not optional - if (localUserMediaId) { - for (let dup = 0; dup < 1 + duplicateTiles; dup++) { - yield { - keys: [ - dup, - localUserMediaId, - userId, - participant satisfies TaggedParticipant as TaggedParticipant, // Widen the type safely - connection$, - membership$.value, - ], - data: undefined, - }; - } + localUserMediaId = `${userId}:${membership$.value.deviceId}`; + for (let dup = 0; dup < 1 + duplicateTiles; dup++) { + yield { + keys: [ + dup, + localUserMediaId, + userId, + participant satisfies TaggedParticipant as TaggedParticipant, // Widen the type safely + connection$, + membership$.value, + ], + data: undefined, + }; } } // add remote members that are available @@ -748,8 +746,8 @@ export function createCallViewModel$( membership$, } of matrixLivekitMembers.value) { const userMediaId = `${userId}:${membership$.value.deviceId}`; + // skip local user as we added them manually before if (userMediaId === localUserMediaId) continue; - // const participantId = membership$.value?.identity; for (let dup = 0; dup < 1 + duplicateTiles; dup++) { yield { keys: [ @@ -769,7 +767,7 @@ export function createCallViewModel$( scope, _data$, dup, - participantId, + userMediaId, userId, participant, connection$, @@ -789,7 +787,7 @@ export function createCallViewModel$( return new UserMedia( scope, - `${participantId}:${dup}`, + `${userMediaId}:${dup}`, userId, membership, participant, @@ -800,8 +798,8 @@ export function createCallViewModel$( localMembership.reconnecting$, displayName$, matrixMemberMetadataStore.createAvatarUrlBehavior$(userId), - handsRaised$.pipe(map((v) => v[participantId]?.time ?? null)), - reactions$.pipe(map((v) => v[participantId] ?? undefined)), + handsRaised$.pipe(map((v) => v[userMediaId]?.time ?? null)), + reactions$.pipe(map((v) => v[userMediaId] ?? undefined)), ); }, ), diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts index 64455e87..10a3e2cb 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts @@ -92,7 +92,7 @@ export function createMatrixLivekitMembers$({ filter((values) => values.every((value) => value.epoch === values[0].epoch), ), - map(([x, y]) => new Epoch([x.value, y.value] as const, x.epoch)), + map(([ms, data]) => new Epoch([ms.value, data.value] as const, ms.epoch)), generateItemsWithEpoch( // Generator function. // creates an array of `{key, data}[]` From 9a88e3dab62d33cbae885e8e7a38efebd0b251da Mon Sep 17 00:00:00 2001 From: Timo K Date: Fri, 9 Jan 2026 18:05:26 +0100 Subject: [PATCH 40/46] review valere (more readable force new endpoint logic) --- src/livekit/openIDSFU.ts | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/src/livekit/openIDSFU.ts b/src/livekit/openIDSFU.ts index a5a8adc5..0b7c2c78 100644 --- a/src/livekit/openIDSFU.ts +++ b/src/livekit/openIDSFU.ts @@ -118,15 +118,14 @@ export async function getSFUConfigWithOpenID( let sfuConfig: { url: string; jwt: string } | undefined; - // If forceOldJwtEndpoint is set we indicate that we do not want to try the new endpoint, - // since we are not sending the new matrix2.0 sticky events (no hashed identity in the event) - if ( - // we do not force anything. Try with new first (remote connections) - !opts?.forceJwtEndpoint || - // we do force the matrix2.0 endpoint - (opts?.forceJwtEndpoint && - opts?.forceJwtEndpoint === JwtEndpointVersion.Matrix_2_0) - ) { + const tryBothJwtEndpoints = opts?.forceJwtEndpoint === undefined; // This is for SFUs where we do not publish. + + const forceMatrix2Jwt = + opts?.forceJwtEndpoint === JwtEndpointVersion.Matrix_2_0; + + // We want to start using the new endpoint (with optional delay delegation) + // if we can use both or if we are forced to use the new one. + if (tryBothJwtEndpoints || forceMatrix2Jwt) { try { sfuConfig = await getLiveKitJWTWithDelayDelegation( membership, @@ -151,15 +150,16 @@ export async function getSFUConfigWithOpenID( e, ); // Make this throw a hard error in case we force the matrix2.0 endpoint. - if (opts?.forceJwtEndpoint === JwtEndpointVersion.Matrix_2_0) + if (forceMatrix2Jwt) throw new NoMatrix2AuthorizationService(e as Error); + // NEVER get bejond this point if we forceMatrix2 and it failed! } } } // DEPRECATED - // here we either have a sfuConfig or we alredy exited becuause of `if (opts?.forceEndpoint === MatrixRTCMode.Matrix_2_0) throw e;` - // The only case we can get into this if is, if `opts?.forceEndpoint !== MatrixRTCMode.Matrix_2_0` + // here we either have a sfuConfig or we alredy exited because of `if (forceMatrix2) throw ...` + // The only case we can get into this condition is, if `forceMatrix2` is `false` if (sfuConfig === undefined) { sfuConfig = await getLiveKitJWT( membership.deviceId, @@ -167,7 +167,6 @@ export async function getSFUConfigWithOpenID( roomId, openIdToken, ); - logger?.info(`Got JWT from call's active focus URL.`); } From 32694fddf62fd35254458d1c5f579f74153ab926 Mon Sep 17 00:00:00 2001 From: Timo K Date: Fri, 9 Jan 2026 19:04:02 +0100 Subject: [PATCH 41/46] improve rtcBackendIdentity debugging --- src/tile/GridTile.tsx | 1 + src/tile/MediaView.tsx | 11 +++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index 92262f05..fa2000d0 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -51,6 +51,7 @@ import { type GridTileViewModel } from "../state/TileViewModel"; import { useMergedRefs } from "../useMergedRefs"; import { useReactionsSender } from "../reactions/useReactionsSender"; import { useBehavior } from "../useBehavior"; +import { showConnectionStats } from "../settings/settings"; interface TileProps { ref?: Ref; diff --git a/src/tile/MediaView.tsx b/src/tile/MediaView.tsx index 7847a30a..02f34462 100644 --- a/src/tile/MediaView.tsx +++ b/src/tile/MediaView.tsx @@ -18,7 +18,11 @@ import styles from "./MediaView.module.css"; import { Avatar } from "../Avatar"; import { type EncryptionStatus } from "../state/MediaViewModel"; import { RaisedHandIndicator } from "../reactions/RaisedHandIndicator"; -import { showHandRaisedTimer, useSetting } from "../settings/settings"; +import { + showConnectionStats, + showHandRaisedTimer, + useSetting, +} from "../settings/settings"; import { type ReactionOption } from "../reactions"; import { ReactionIndicator } from "../reactions/ReactionIndicator"; import { RTCConnectionStats } from "../RTCConnectionStats"; @@ -81,6 +85,7 @@ export const MediaView: FC = ({ }) => { const { t } = useTranslation(); const [handRaiseTimerVisible] = useSetting(showHandRaisedTimer); + const [showConnectioStats] = useSetting(showConnectionStats); const avatarSize = Math.round(Math.min(targetWidth, targetHeight) / 2); @@ -133,7 +138,9 @@ export const MediaView: FC = ({
{waitingForMedia && (
- {t("video_tile.waiting_for_media")} + {t("video_tile.waiting_for_media") + showConnectioStats + ? " " + rtcBackendIdentity + : ""}
)} {(audioStreamStats || videoStreamStats) && ( From 7387a00406e6f32069c8f640892a424883d4ccdc Mon Sep 17 00:00:00 2001 From: Timo K Date: Fri, 9 Jan 2026 19:04:42 +0100 Subject: [PATCH 42/46] lint --- src/tile/GridTile.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index fa2000d0..92262f05 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -51,7 +51,6 @@ import { type GridTileViewModel } from "../state/TileViewModel"; import { useMergedRefs } from "../useMergedRefs"; import { useReactionsSender } from "../reactions/useReactionsSender"; import { useBehavior } from "../useBehavior"; -import { showConnectionStats } from "../settings/settings"; interface TileProps { ref?: Ref; From 1a9e4a1a5ceebcddcc57d675772270e7e6614034 Mon Sep 17 00:00:00 2001 From: Timo K Date: Fri, 9 Jan 2026 19:12:54 +0100 Subject: [PATCH 43/46] bump js-sdk --- yarn.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yarn.lock b/yarn.lock index d795f1e6..5ce82c02 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11335,7 +11335,7 @@ __metadata: "matrix-js-sdk@matrix-org/matrix-js-sdk#develop": version: 39.4.0 - resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=841b654c00c2a47dbd4380b4b2a6fa25156c8af0" + resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=4d0d32307eb4f1ce1fb65080fcca704f5bdedc31" dependencies: "@babel/runtime": "npm:^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm": "npm:^17.0.0" @@ -11351,7 +11351,7 @@ __metadata: sdp-transform: "npm:^3.0.0" unhomoglyph: "npm:^1.0.6" uuid: "npm:13" - checksum: 10c0/532d4adb2ce95a27b4715be518d648740efddc5c88d399e127d95ff962bddd8f62f848a0ea98088672fab03b67a973d63935cd3b5ed371bb9da3131ac9472db6 + checksum: 10c0/59c9d81ccf823584dc783502cb5c928562e3490c63f5ce98ee3232a603545d6278e90dc951c1fd0bae2792ba732ec5171e03596fd396bb2150d596cebb7fbac9 languageName: node linkType: hard From 735c17d3e667c4b755f037ac470b1500342fef04 Mon Sep 17 00:00:00 2001 From: Timo K Date: Fri, 9 Jan 2026 19:31:40 +0100 Subject: [PATCH 44/46] fix tile flicker on video mute/unmute (only pass rtcBackendIdenitity instead of full rtcMember) --- src/state/CallViewModel/CallViewModel.ts | 11 +++++++---- src/state/MediaViewModel.ts | 21 +++++++++------------ src/state/ScreenShare.ts | 5 ++--- src/state/UserMedia.ts | 10 ++++------ src/utils/test.ts | 4 ++-- 5 files changed, 24 insertions(+), 27 deletions(-) diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index bf5ea441..0f212101 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -724,6 +724,7 @@ export function createCallViewModel$( localMatrixLivekitMember; localUserMediaId = `${userId}:${membership$.value.deviceId}`; + const rtcBackendIdentity = membership$.value.rtcBackendIdentity; for (let dup = 0; dup < 1 + duplicateTiles; dup++) { yield { keys: [ @@ -732,7 +733,7 @@ export function createCallViewModel$( userId, participant satisfies TaggedParticipant as TaggedParticipant, // Widen the type safely connection$, - membership$.value, + rtcBackendIdentity, ], data: undefined, }; @@ -746,8 +747,10 @@ export function createCallViewModel$( membership$, } of matrixLivekitMembers.value) { const userMediaId = `${userId}:${membership$.value.deviceId}`; + const rtcBackendIdentity = membership$.value.rtcBackendIdentity; // skip local user as we added them manually before if (userMediaId === localUserMediaId) continue; + for (let dup = 0; dup < 1 + duplicateTiles; dup++) { yield { keys: [ @@ -756,7 +759,7 @@ export function createCallViewModel$( userId, participant, connection$, - membership$.value, + rtcBackendIdentity, ], data: undefined, }; @@ -771,7 +774,7 @@ export function createCallViewModel$( userId, participant, connection$, - membership, + rtcBackendIdentity, ) => { const livekitRoom$ = scope.behavior( connection$.pipe(map((c) => c?.livekitRoom)), @@ -789,7 +792,7 @@ export function createCallViewModel$( scope, `${userMediaId}:${dup}`, userId, - membership, + rtcBackendIdentity, participant, options.encryptionSystem, livekitRoom$, diff --git a/src/state/MediaViewModel.ts b/src/state/MediaViewModel.ts index 6b020057..3da69c46 100644 --- a/src/state/MediaViewModel.ts +++ b/src/state/MediaViewModel.ts @@ -44,7 +44,6 @@ import { throttleTime, distinctUntilChanged, } from "rxjs"; -import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; import { alwaysShowSelf } from "../settings/settings"; import { showConnectionStats } from "../settings/settings"; @@ -258,7 +257,7 @@ abstract class BaseMediaViewModel { * The Matrix user to which this media belongs. */ public readonly userId: string, - public readonly rtcMembership: CallMembership, + public readonly rtcBackendIdentity: string, // We don't necessarily have a participant if a user connects via MatrixRTC but not (yet) through // livekit. protected readonly participant$: Observable< @@ -404,13 +403,11 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { */ public readonly cropVideo$: Behavior = this._cropVideo$; - public readonly rtcBackendIdentity = this.rtcMembership.rtcBackendIdentity; - public constructor( scope: ObservableScope, id: string, userId: string, - rtcMembership: CallMembership, + rtcBackendIdentity: string, participant$: Observable, encryptionSystem: EncryptionSystem, livekitRoom$: Behavior, @@ -424,7 +421,7 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { scope, id, userId, - rtcMembership, + rtcBackendIdentity, participant$, encryptionSystem, Track.Source.Microphone, @@ -550,7 +547,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { scope: ObservableScope, id: string, userId: string, - rtcMembership: CallMembership, + rtcBackendIdentity: string, participant$: Behavior, encryptionSystem: EncryptionSystem, livekitRoom$: Behavior, @@ -565,7 +562,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { scope, id, userId, - rtcMembership, + rtcBackendIdentity, participant$, encryptionSystem, livekitRoom$, @@ -679,7 +676,7 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel { scope: ObservableScope, id: string, userId: string, - rtcMembership: CallMembership, + rtcBackendIdentity: string, participant$: Observable, encryptionSystem: EncryptionSystem, livekitRoom$: Behavior, @@ -694,7 +691,7 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel { scope, id, userId, - rtcMembership, + rtcBackendIdentity, participant$, encryptionSystem, livekitRoom$, @@ -782,7 +779,7 @@ export class ScreenShareViewModel extends BaseMediaViewModel { scope: ObservableScope, id: string, userId: string, - rtcMembership: CallMembership, + rtcBackendIdentity: string, participant$: Observable, encryptionSystem: EncryptionSystem, livekitRoom$: Behavior, @@ -796,7 +793,7 @@ export class ScreenShareViewModel extends BaseMediaViewModel { scope, id, userId, - rtcMembership, + rtcBackendIdentity, participant$, encryptionSystem, Track.Source.ScreenShareAudio, diff --git a/src/state/ScreenShare.ts b/src/state/ScreenShare.ts index 67f97a93..e4f5de1f 100644 --- a/src/state/ScreenShare.ts +++ b/src/state/ScreenShare.ts @@ -10,7 +10,6 @@ import { type RemoteParticipant, type Room as LivekitRoom, } from "livekit-client"; -import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc/CallMembership"; import { type ObservableScope } from "./ObservableScope.ts"; import { ScreenShareViewModel } from "./MediaViewModel.ts"; @@ -29,7 +28,7 @@ export class ScreenShare { private readonly scope: ObservableScope, id: string, userId: string, - rtcMember: CallMembership, + rtcBackendIdentity: string, participant: LocalParticipant | RemoteParticipant, encryptionSystem: EncryptionSystem, livekitRoom$: Behavior, @@ -42,7 +41,7 @@ export class ScreenShare { this.scope, id, userId, - rtcMember, + rtcBackendIdentity, of(participant), encryptionSystem, livekitRoom$, diff --git a/src/state/UserMedia.ts b/src/state/UserMedia.ts index 6435fb17..2a125257 100644 --- a/src/state/UserMedia.ts +++ b/src/state/UserMedia.ts @@ -13,7 +13,6 @@ import { type Room as LivekitRoom, } from "livekit-client"; import { observeParticipantEvents } from "@livekit/components-core"; -import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc/CallMembership"; import { type ObservableScope } from "./ObservableScope.ts"; import { @@ -76,7 +75,7 @@ export class UserMedia { this.scope, this.id, this.userId, - this.rtcMembership, + this.rtcBackendIdentity, this.participant.value$, this.encryptionSystem, this.livekitRoom$, @@ -91,7 +90,7 @@ export class UserMedia { this.scope, this.id, this.userId, - this.rtcMembership, + this.rtcBackendIdentity, this.participant.value$, this.encryptionSystem, this.livekitRoom$, @@ -143,7 +142,7 @@ export class UserMedia { scope, `${this.id}:${key}`, this.userId, - this.rtcMembership, + this.rtcBackendIdentity, p, this.encryptionSystem, this.livekitRoom$, @@ -195,8 +194,7 @@ export class UserMedia { private readonly scope: ObservableScope, public readonly id: string, private readonly userId: string, - // TODO evaluate if this should just be the rtcBackendIdentity - private readonly rtcMembership: CallMembership, + private readonly rtcBackendIdentity: string, private readonly participant: TaggedParticipant, private readonly encryptionSystem: EncryptionSystem, private readonly livekitRoom$: Behavior, diff --git a/src/utils/test.ts b/src/utils/test.ts index 9f9ae4b7..c41743a8 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -334,7 +334,7 @@ export function createLocalMedia( testScope(), "local", member.userId, - rtcMember, + rtcMember.rtcBackendIdentity, constant(localParticipant), { kind: E2eeType.PER_PARTICIPANT, @@ -380,7 +380,7 @@ export function createRemoteMedia( testScope(), "remote", member.userId, - rtcMember, + rtcMember.rtcBackendIdentity, constant(participant), { kind: E2eeType.PER_PARTICIPANT, From 341277932c9670ca8f95a5832ccddb8daaa96910 Mon Sep 17 00:00:00 2001 From: Timo K Date: Fri, 9 Jan 2026 19:34:41 +0100 Subject: [PATCH 45/46] fix tests --- src/tile/MediaView.tsx | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/tile/MediaView.tsx b/src/tile/MediaView.tsx index 02f34462..fadc9d2b 100644 --- a/src/tile/MediaView.tsx +++ b/src/tile/MediaView.tsx @@ -138,9 +138,8 @@ export const MediaView: FC = ({
{waitingForMedia && (
- {t("video_tile.waiting_for_media") + showConnectioStats - ? " " + rtcBackendIdentity - : ""} + {t("video_tile.waiting_for_media")} + {showConnectioStats ? " " + rtcBackendIdentity : ""}
)} {(audioStreamStats || videoStreamStats) && ( From 47e389678b6099c654597c5c336ac078bcce3b11 Mon Sep 17 00:00:00 2001 From: Timo K Date: Fri, 9 Jan 2026 19:48:04 +0100 Subject: [PATCH 46/46] add uuid in our own package.json --- package.json | 1 + yarn.lock | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 944fc43b..04308630 100644 --- a/package.json +++ b/package.json @@ -125,6 +125,7 @@ "typescript": "^5.8.3", "typescript-eslint-language-service": "^5.0.5", "unique-names-generator": "^4.6.0", + "uuid": "^13.0.0", "vaul": "^1.0.0", "vite": "^7.0.0", "vite-plugin-generate-file": "^0.3.0", diff --git a/yarn.lock b/yarn.lock index 5ce82c02..0979bc9b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8267,6 +8267,7 @@ __metadata: typescript: "npm:^5.8.3" typescript-eslint-language-service: "npm:^5.0.5" unique-names-generator: "npm:^4.6.0" + uuid: "npm:^13.0.0" vaul: "npm:^1.0.0" vite: "npm:^7.0.0" vite-plugin-generate-file: "npm:^0.3.0" @@ -15214,7 +15215,7 @@ __metadata: languageName: node linkType: hard -"uuid@npm:13": +"uuid@npm:13, uuid@npm:^13.0.0": version: 13.0.0 resolution: "uuid@npm:13.0.0" bin: