From 7ac5373af936fca04e9cd6d6259fbc81d64d965f Mon Sep 17 00:00:00 2001 From: Johannes Marbach Date: Mon, 12 Oct 2026 10:03:03 +0200 Subject: [PATCH 01/25] Remove stray pause and improve logging for hung e2e tests The list reporter is the only progress a hung CI run leaves behind, and a global timeout under the job's 60 minute limit means a hung run still ends with a report instead of being cancelled by GitHub. --- playwright.config.ts | 5 ++++- playwright/widget/federated-call.test.ts | 4 ---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/playwright.config.ts b/playwright.config.ts index e6dcc5249..b0cf928f4 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -39,7 +39,10 @@ export default defineConfig({ /* Opt out of parallel tests on CI. */ workers: process.env.CI ? 1 : undefined, /* Reporter to use. See https://playwright.dev/docs/test-reporters */ - reporter: "html", + // The list reporter is the only progress a hung CI run leaves behind. + reporter: process.env.CI ? [["list"], ["html"]] : "html", + // Under the job's 60 minute limit, so a hung run still ends with a report. + globalTimeout: process.env.CI ? 50 * 60 * 1000 : undefined, /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { /* Base URL to use in actions like `await page.goto('/')`. */ diff --git a/playwright/widget/federated-call.test.ts b/playwright/widget/federated-call.test.ts index 61f4750b3..8359dd233 100644 --- a/playwright/widget/federated-call.test.ts +++ b/playwright/widget/federated-call.test.ts @@ -37,8 +37,6 @@ modePairs.forEach(([rtcMode1, rtcMode2]) => { await TestHelpers.acceptRoomInvite(roomName, timo.page); - await florian.page.pause(); - await TestHelpers.openWidgetSetEmbeddedElementCallRtcModeCloseWidget( florian.page, rtcMode1, @@ -74,8 +72,6 @@ modePairs.forEach(([rtcMode1, rtcMode2]) => { await TestHelpers.expectVisibleVideoCount(frame, 2); } - - // await florian.page.pause(); }, ); }); From 994cd36c4425a26926dab3802ec3f2e2ccd53a12 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 12 Oct 2026 10:03:03 +0200 Subject: [PATCH 02/25] Fix typo in the name of the createJWTToken Playwright fixture --- playwright/errors.spec.ts | 4 ++-- playwright/fixtures/jwt-token.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/playwright/errors.spec.ts b/playwright/errors.spec.ts index 23f6e29f3..e7eff6592 100644 --- a/playwright/errors.spec.ts +++ b/playwright/errors.spec.ts @@ -7,7 +7,7 @@ Please see LICENSE in the repository root for full details. import { expect, test } from "@playwright/test"; -import { createJTWToken } from "./fixtures/jwt-token"; +import { createJWTToken } from "./fixtures/jwt-token"; test("Should show error screen if fails to get JWT token", async ({ page }) => { await page.goto("/"); @@ -95,7 +95,7 @@ test("Should show error screen if call creation is restricted", async ({ contentType: "application/json", body: JSON.stringify({ url: "wss://badurltotricktest/livekit/sfu", - jwt: createJTWToken("@fake:user", "!fake:room"), + jwt: createJWTToken("@fake:user", "!fake:room"), }), }), ); diff --git a/playwright/fixtures/jwt-token.ts b/playwright/fixtures/jwt-token.ts index 18119c7e0..dff715f85 100644 --- a/playwright/fixtures/jwt-token.ts +++ b/playwright/fixtures/jwt-token.ts @@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -export function createJTWToken(sub: string, room: string): string { +export function createJWTToken(sub: string, room: string): string { return [ {}, // header { From d8becaf868d1aebf6336631d6e7c348fe477ad8a Mon Sep 17 00:00:00 2001 From: Johannes Marbach Date: Mon, 12 Oct 2026 10:03:03 +0200 Subject: [PATCH 03/25] Advertise the loopback address from the dev LiveKit SFUs The container IP is not routable from the host when Docker runs in a VM (colima, Docker Desktop), so calls against the dev stack drop once ICE fails. Advertising 127.0.0.1 instead lets the browser reach the SFU through the published ports. Firefox ignores loopback candidates, so this only works in Chromium-based browsers. --- backend/dev_livekit-othersite.yaml | 5 +++++ backend/dev_livekit.yaml | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/backend/dev_livekit-othersite.yaml b/backend/dev_livekit-othersite.yaml index 53fc9ce94..476ecc45c 100644 --- a/backend/dev_livekit-othersite.yaml +++ b/backend/dev_livekit-othersite.yaml @@ -6,6 +6,11 @@ rtc: port_range_start: 50300 port_range_end: 50400 use_external_ip: false + # Advertise the loopback address instead of the container IP, which is not + # routable from the host when Docker runs in a VM (colima, Docker Desktop). + # The browser then reaches the SFU through the published ports. Firefox + # ignores loopback candidates, so this only works in Chromium-based browsers. + node_ip: 127.0.0.1 turn: enabled: false domain: localhost diff --git a/backend/dev_livekit.yaml b/backend/dev_livekit.yaml index 6cef4241a..576c67589 100644 --- a/backend/dev_livekit.yaml +++ b/backend/dev_livekit.yaml @@ -6,6 +6,11 @@ rtc: port_range_start: 50100 port_range_end: 50200 use_external_ip: false + # Advertise the loopback address instead of the container IP, which is not + # routable from the host when Docker runs in a VM (colima, Docker Desktop). + # The browser then reaches the SFU through the published ports. Firefox + # ignores loopback candidates, so this only works in Chromium-based browsers. + node_ip: 127.0.0.1 turn: enabled: false domain: localhost From 3f030e64ddd56d77b95757721fedd91943e36d00 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 12 Oct 2026 10:04:47 +0200 Subject: [PATCH 04/25] Match LiveKit participants against all backend identities A member authenticated through the legacy JWT endpoint publishes under a `user:device` identity, while one authenticated through the default endpoint publishes under a hashed identity. Both are listed in the membership's backend identities, so match against the whole list rather than only the primary one. --- .../CallViewModel/remoteMembers/MatrixLivekitMembers.ts | 4 ++-- src/utils/test.ts | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts index 6c9fff026..e7bd9e7ae 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts @@ -111,8 +111,8 @@ export function createRemoteMatrixLivekitMembers$({ const participants = transport ? managerData.getParticipantsForTransport(transport) : []; - const matches = participants.filter( - (p) => p.identity == membership.rtcBackendIdentity, + const matches = participants.filter((p) => + membership.backendIdentities.includes(p.identity), ); const participant = matches[0] ?? null; const connection = transport diff --git a/src/utils/test.ts b/src/utils/test.ts index 6af9fd8c7..32e5cce17 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -261,11 +261,12 @@ export function mockRtcMembership( content: data, }); - const membershipData = CallMembership.membershipDataFromMatrixEvent(event); + const backendIdentity = rtcBackendIdentity ?? `${userId}:${deviceId}`; const cms = new CallMembership( event, - membershipData, - rtcBackendIdentity ?? `${userId}:${deviceId}`, + CallMembership.membershipDataFromMatrixEvent(event), + [backendIdentity], + backendIdentity, ); vi.mocked(cms).getTransport = vi.fn().mockReturnValue(fociPreferred[0]); From 4e4ab23cdad7ba1e25ddc4f2cd699626ecb20682 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 12 Oct 2026 10:08:22 +0200 Subject: [PATCH 05/25] Model ConnectionState as a discriminated union The enum mixed our own lifecycle states with a copy of livekit-client's connection states, and errors were smuggled through the same observable as a separate union member. A discriminated union keeps the LiveKit state nested under "authenticated", carries the error as data, and lets the local member represent the time before a connection exists as an "initializing" state instead of null. --- src/state/CallViewModel/CallViewModel.ts | 4 +- .../localMember/LocalMember.test.ts | 63 ++++++++++++---- .../CallViewModel/localMember/LocalMember.ts | 20 +++-- .../remoteMembers/Connection.test.ts | 70 ++++++++++-------- .../CallViewModel/remoteMembers/Connection.ts | 74 ++++++++----------- 5 files changed, 134 insertions(+), 97 deletions(-) diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index a58b0837f..3b457f719 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -1662,9 +1662,9 @@ export function createCallViewModel$( returnObject.publishError = value.media; else if ( typeof value.media === "object" && - value.media.connection instanceof ElementCallError + value.media.connection.state === "error" ) - returnObject.connectionError = value.media.connection; + returnObject.connectionError = value.media.connection.error; return returnObject; }), ), diff --git a/src/state/CallViewModel/localMember/LocalMember.test.ts b/src/state/CallViewModel/localMember/LocalMember.test.ts index 282759fdd..da4da45aa 100644 --- a/src/state/CallViewModel/localMember/LocalMember.test.ts +++ b/src/state/CallViewModel/localMember/LocalMember.test.ts @@ -23,7 +23,11 @@ import { } from "vitest"; import { BehaviorSubject, map, of, Subject } from "rxjs"; import { logger } from "matrix-js-sdk/lib/logger"; -import { type LocalParticipant, type LocalTrack } from "livekit-client"; +import { + ConnectionState as LivekitConnectionState, + type LocalParticipant, + type LocalTrack, +} from "livekit-client"; import fetchMock from "fetch-mock"; import { PosthogAnalytics } from "../../../analytics/PosthogAnalytics"; @@ -54,7 +58,10 @@ import { MatrixRTCTransportMissingError } from "../../../utils/errors"; import { Epoch, ObservableScope } from "../../ObservableScope"; import { constant } from "../../Behavior"; import { ConnectionManagerData } from "../remoteMembers/ConnectionManager"; -import { ConnectionState, type Connection } from "../remoteMembers/Connection"; +import { + type ConnectionState, + type Connection, +} from "../remoteMembers/Connection"; import { type Publisher } from "./Publisher"; import { initializeWidget } from "../../../widget"; import { nullHostBridge } from "../../../HostBridge"; @@ -336,12 +343,18 @@ describe("LocalMembership", () => { trackPublications: [], } as unknown as LocalParticipant, }), - state$: constant(ConnectionState.LivekitConnected), + state$: constant({ + state: "authenticated", + livekitState: LivekitConnectionState.Connected, + }), transport: mockTransportConfig, } as Connection; const connectionTransportAConnecting = { ...connectionTransportAConnected, - state$: constant(ConnectionState.LivekitConnecting), + state$: constant({ + state: "authenticated", + livekitState: LivekitConnectionState.Connecting, + }), livekitRoom: mockLivekitRoom({}), } as unknown as Connection; @@ -532,7 +545,10 @@ describe("LocalMembership", () => { await flushPromises(); expect(localMembership.localMemberState$.value).toStrictEqual({ matrix: RTCMemberStatus.Connected, - media: { connection: null, tracks: TrackState.WaitingForUser }, + media: { + connection: { state: "initializing" }, + tracks: TrackState.WaitingForUser, + }, }); const connectionManagerData2 = new ConnectionManagerData(); @@ -547,7 +563,10 @@ describe("LocalMembership", () => { expect(localMembership.localMemberState$.value).toStrictEqual({ matrix: RTCMemberStatus.Connected, media: { - connection: ConnectionState.LivekitConnecting, + connection: { + state: "authenticated", + livekitState: LivekitConnectionState.Connecting, + }, tracks: TrackState.WaitingForUser, }, }); @@ -555,11 +574,17 @@ describe("LocalMembership", () => { ( connectionManagerData2.getConnectionForTransport(mockTransportConfig)! .state$ as BehaviorSubject - ).next(ConnectionState.LivekitConnected); + ).next({ + state: "authenticated", + livekitState: LivekitConnectionState.Connected, + }); expect(localMembership.localMemberState$.value).toStrictEqual({ matrix: RTCMemberStatus.Connected, media: { - connection: ConnectionState.LivekitConnected, + connection: { + state: "authenticated", + livekitState: LivekitConnectionState.Connected, + }, tracks: TrackState.WaitingForUser, }, }); @@ -717,9 +742,10 @@ describe("LocalMembership", () => { "track", ); - const connectionState$ = new BehaviorSubject( - ConnectionState.LivekitConnected, - ); + const connectionState$ = new BehaviorSubject({ + state: "authenticated", + livekitState: LivekitConnectionState.Connected, + }); const mutableConnection = { ...connectionTransportAConnected, state$: connectionState$, @@ -746,8 +772,14 @@ describe("LocalMembership", () => { await flushPromises(); - connectionState$.next(ConnectionState.LivekitDisconnected); - connectionState$.next(ConnectionState.LivekitConnected); + connectionState$.next({ + state: "authenticated", + livekitState: LivekitConnectionState.Disconnected, + }); + connectionState$.next({ + state: "authenticated", + livekitState: LivekitConnectionState.Connected, + }); expect(trackSpy).toHaveBeenCalledWith( defaultCreateLocalMemberValues.roomId, @@ -857,7 +889,10 @@ describe("LocalMembership", () => { const error = new Error("NotReadableError"); const setScreenShareEnabled = vi.fn().mockRejectedValue(error); const connection = { - state$: constant(ConnectionState.LivekitConnected), + state$: constant({ + state: "authenticated", + livekitState: LivekitConnectionState.Connected, + }), transport: mockTransportConfig, livekitRoom: mockLivekitRoom({ localParticipant: mockLocalParticipant({ diff --git a/src/state/CallViewModel/localMember/LocalMember.ts b/src/state/CallViewModel/localMember/LocalMember.ts index feb49554c..c06190d3f 100644 --- a/src/state/CallViewModel/localMember/LocalMember.ts +++ b/src/state/CallViewModel/localMember/LocalMember.ts @@ -6,6 +6,7 @@ Please see LICENSE in the repository root for full details. */ import { + ConnectionState as LivekitConnectionState, type Participant, ParticipantEvent, type LocalParticipant, @@ -74,9 +75,8 @@ import { } from "../../../config/ConfigOptions.ts"; import { Config } from "../../../config/Config.ts"; import { - ConnectionState, + type ConnectionState, type Connection, - type FailedToStartError, } from "../remoteMembers/Connection.ts"; import { type HomeserverConnected } from "./HomeserverConnected.ts"; import { type LocalTransport } from "./LocalTransport.ts"; @@ -113,7 +113,7 @@ export enum TrackState { export type LocalMemberMediaState = | { tracks: TrackState; - connection: ConnectionState | FailedToStartError; + connection: ConnectionState; } | PublishState | ElementCallError; @@ -475,7 +475,9 @@ export const createLocalMembership$ = ({ }; const localConnectionState$ = localConnection$.pipe( - switchMap((connection) => (connection ? connection.state$ : of(null))), + switchMap((connection) => + connection ? connection.state$ : of({ state: "initializing" as const }), + ), ); const mediaState$: Behavior = scope.behavior( @@ -493,7 +495,9 @@ export const createLocalMembership$ = ({ : TrackState.WaitingForUser; if ( - localConnectionState !== ConnectionState.LivekitConnected || + localConnectionState.state !== "authenticated" || + localConnectionState.livekitState !== + LivekitConnectionState.Connected || trackState !== TrackState.Ready ) return { @@ -561,7 +565,11 @@ export const createLocalMembership$ = ({ combineLatest([ homeserverConnected.combined$, localConnectionState$.pipe( - map((state) => state === ConnectionState.LivekitConnected), + map( + (state) => + state.state === "authenticated" && + state.livekitState === LivekitConnectionState.Connected, + ), ), ]).pipe( map(([[hsConnected, hsReason], livekitConnected]) => { diff --git a/src/state/CallViewModel/remoteMembers/Connection.test.ts b/src/state/CallViewModel/remoteMembers/Connection.test.ts index 62dc17a31..aa2429f88 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.test.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.test.ts @@ -34,15 +34,12 @@ import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { Connection, - ConnectionState, + type ConnectionState, type ConnectionOpts, } from "./Connection.ts"; import { ObservableScope } from "../../ObservableScope.ts"; import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts"; -import { - ElementCallError, - FailToGetOpenIdToken, -} from "../../../utils/errors.ts"; +import { FailToGetOpenIdToken } from "../../../utils/errors.ts"; import { testJWTToken } from "../../../utils/test-fixtures.ts"; import { mockRemoteParticipant, ownMemberMock } from "../../../utils/test.ts"; @@ -181,7 +178,7 @@ describe("Start connection states", () => { }; const connection = new Connection(opts, logger); - expect(connection.state$.getValue()).toEqual("Initialized"); + expect(connection.state$.getValue()).toEqual({ state: "initialized" }); }); it("fail to getOpenId token then error state", async () => { @@ -199,7 +196,7 @@ describe("Start connection states", () => { const connection = new Connection(opts, logger); - const capturedStates: (ConnectionState | Error)[] = []; + const capturedStates: ConnectionState[] = []; const s = connection.state$.subscribe((value) => { capturedStates.push(value); }); @@ -219,18 +216,20 @@ describe("Start connection states", () => { let capturedState = capturedStates.pop(); expect(capturedState).toBeDefined(); - expect(capturedState!).toEqual("FetchingConfig"); + expect(capturedState).toEqual({ state: "authenticating" }); deferred.reject(new FailToGetOpenIdToken(new Error("Failed to get token"))); await vi.runAllTimersAsync(); capturedState = capturedStates.pop(); - if (capturedState instanceof Error) { - expect(capturedState.message).toEqual("Something went wrong"); + if (capturedState?.state === "error") { + expect(capturedState.error.message).toEqual("Something went wrong"); expect(connection.transport).toEqual(livekitFocus); } else { - expect.fail("Expected FailedToStart state but got " + capturedState); + expect.fail( + "Expected FailedToStart state but got " + JSON.stringify(capturedState), + ); } }); @@ -249,7 +248,7 @@ describe("Start connection states", () => { const connection = new Connection(opts, logger); - const capturedStates: (ConnectionState | Error)[] = []; + const capturedStates: ConnectionState[] = []; const s = connection.state$.subscribe((value) => { capturedStates.push(value); }); @@ -274,7 +273,7 @@ describe("Start connection states", () => { let capturedState = capturedStates.pop(); expect(capturedState).toBeDefined(); - expect(capturedState).toEqual(ConnectionState.FetchingConfig); + expect(capturedState).toEqual({ state: "authenticating" }); deferredSFU.resolve(); await vi.runAllTimersAsync(); @@ -282,15 +281,17 @@ describe("Start connection states", () => { capturedState = capturedStates.pop(); if ( - capturedState instanceof ElementCallError && - capturedState.cause instanceof Error + capturedState?.state === "error" && + capturedState.error.cause instanceof Error ) { - expect(capturedState.cause.message).toContain( + expect(capturedState.error.cause.message).toContain( "Failed to look up user info from homeserver", ); expect(connection.transport).toEqual(livekitFocus); } else { - expect.fail("Expected FailedToStart state but got " + capturedState); + expect.fail( + "Expected FailedToStart state but got " + JSON.stringify(capturedState), + ); } }); @@ -309,7 +310,7 @@ describe("Start connection states", () => { const connection = new Connection(opts, logger); - const capturedStates: (ConnectionState | Error)[] = []; + const capturedStates: ConnectionState[] = []; const s = connection.state$.subscribe((value) => { capturedStates.push(value); }); @@ -339,7 +340,7 @@ describe("Start connection states", () => { let capturedState = capturedStates.pop(); expect(capturedState).toBeDefined(); - expect(capturedState).toEqual(ConnectionState.FetchingConfig); + expect(capturedState).toEqual({ state: "authenticating" }); deferredSFU.resolve(); await vi.runAllTimersAsync(); @@ -347,10 +348,10 @@ describe("Start connection states", () => { capturedState = capturedStates.pop(); if ( - capturedState instanceof ElementCallError && - capturedState.cause instanceof Error + capturedState?.state === "error" && + capturedState.error.cause instanceof Error ) { - expect(capturedState.cause.message).toContain( + expect(capturedState.error.cause.message).toContain( "Failed to connect to livekit", ); expect(connection.transport).toEqual(livekitFocus); @@ -367,7 +368,7 @@ describe("Start connection states", () => { const connection = setupRemoteConnection(); - const capturedStates: (ConnectionState | Error)[] = []; + const capturedStates: ConnectionState[] = []; const s = connection.state$.subscribe((value) => { capturedStates.push(value); }); @@ -377,22 +378,31 @@ describe("Start connection states", () => { await vi.runAllTimersAsync(); const initialState = capturedStates.shift(); - expect(initialState).toEqual(ConnectionState.Initialized); + expect(initialState).toEqual({ state: "initialized" }); const fetchingState = capturedStates.shift(); - expect(fetchingState).toEqual(ConnectionState.FetchingConfig); + expect(fetchingState).toEqual({ state: "authenticating" }); const disconnectedState = capturedStates.shift(); - expect(disconnectedState).toEqual(ConnectionState.LivekitDisconnected); + expect(disconnectedState).toEqual({ + state: "authenticated", + livekitState: LivekitConnectionState.Disconnected, + }); const connectingState = capturedStates.shift(); - expect(connectingState).toEqual(ConnectionState.LivekitConnecting); + expect(connectingState).toEqual({ + state: "authenticated", + livekitState: LivekitConnectionState.Connecting, + }); const connectedState = capturedStates.shift(); - expect(connectedState).toEqual(ConnectionState.LivekitConnected); + expect(connectedState).toEqual({ + state: "authenticated", + livekitState: LivekitConnectionState.Connected, + }); }); it("stopping while connecting does not report an error", async () => { setupTest(); const connection = setupRemoteConnection(); - const capturedStates: (ConnectionState | Error)[] = []; + const capturedStates: ConnectionState[] = []; const s = connection.state$.subscribe((value) => { capturedStates.push(value); }); @@ -412,7 +422,7 @@ describe("Start connection states", () => { // start() resolves rather than rejecting (it is not awaited by the // ConnectionManager, so a rejection would be unhandled). await expect(started).resolves.toBeUndefined(); - expect(capturedStates.at(-1)).toEqual(ConnectionState.Stopped); + expect(capturedStates.at(-1)).toEqual({ state: "stopped" }); expect(capturedStates.some((st) => st instanceof Error)).toBe(false); }); diff --git a/src/state/CallViewModel/remoteMembers/Connection.ts b/src/state/CallViewModel/remoteMembers/Connection.ts index cc84142d9..7face646a 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.ts @@ -13,6 +13,7 @@ import { import { ConnectionError, ConnectionErrorReason, + type ConnectionState as LivekitConnectionState, type Participant, type RemoteParticipant, type RemoteTrackPublication, @@ -23,7 +24,7 @@ import { type TrackPublication, } from "livekit-client"; import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; -import { BehaviorSubject, map } from "rxjs"; +import { BehaviorSubject } from "rxjs"; import { type Logger } from "matrix-js-sdk/lib/logger"; import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; @@ -63,30 +64,15 @@ export interface ConnectionOpts { /** Optional factory to create the LiveKit room, mainly for testing purposes. */ livekitRoomFactory: () => LivekitRoom; } -export class FailedToStartError extends Error { - public constructor(message: string) { - super(message); - this.name = "FailedToStartError"; - } -} -export enum ConnectionState { - /** The start state of a connection. It has been created but nothing has loaded yet. */ - Initialized = "Initialized", - /** `start` has been called on the connection. It aquires the jwt info to conenct to the LK Room */ - FetchingConfig = "FetchingConfig", - Stopped = "Stopped", - /** The same as ConnectionState.Disconnected from `livekit-client` */ - LivekitDisconnected = "disconnected", - /** The same as ConnectionState.Connecting from `livekit-client` */ - LivekitConnecting = "connecting", - /** The same as ConnectionState.Connected from `livekit-client` */ - LivekitConnected = "connected", - /** The same as ConnectionState.Reconnecting from `livekit-client` */ - LivekitReconnecting = "reconnecting", - /** The same as ConnectionState.SignalReconnecting from `livekit-client` */ - LivekitSignalReconnecting = "signalReconnecting", -} +export type ConnectionState = + // The start state of a connection. It has been created but nothing has loaded yet. + | { state: "initialized" } + // `start` has been called on the connection. It acquires the JWT token to connect to the LK room. + | { state: "authenticating" } + | { state: "authenticated"; livekitState: LivekitConnectionState } + | { state: "error"; error: ElementCallError } + | { state: "stopped" }; /** * A connection to a Matrix RTC LiveKit backend. @@ -95,14 +81,14 @@ export enum ConnectionState { */ export class Connection { // Private Behavior - private readonly _state$ = new BehaviorSubject< - ConnectionState | ElementCallError - >(ConnectionState.Initialized); + private readonly _state$ = new BehaviorSubject({ + state: "initialized", + }); /** * The current state of the connection to the media transport. */ - public readonly state$: Behavior = this._state$; + public readonly state$: Behavior = this._state$; /** * The media transport to connect to. @@ -304,7 +290,7 @@ export class Connection { this.logger.debug("Starting Connection"); this.stopped = false; try { - this._state$.next(ConnectionState.FetchingConfig); + this._state$.next({ state: "authenticating" }); // We should already have this information after creating the localTransport. // only call getSFUConfigWithOpenID for connections where we do not have a token yet. (existingJwtTokenData === undefined) const { url, jwt, livekitAlias } = @@ -326,14 +312,10 @@ export class Connection { // Setup observer once we are done with getSFUConfigWithOpenID connectionStateObserver(this.livekitRoom) - .pipe( - this.scope.bind(), - map((s) => s as unknown as ConnectionState), - ) - .subscribe((lkState) => { - // It is save to cast lkState to ConnectionState as they are fully overlapping. - this._state$.next(lkState); - }); + .pipe(this.scope.bind()) + .subscribe((livekitState) => + this._state$.next({ state: "authenticated", livekitState }), + ); try { this.logger.info(`livekitRoom.connect ${url}`); @@ -383,13 +365,15 @@ export class Connection { return; } this.logger.debug(`Failed to connect to LiveKit room: ${error}`); - this._state$.next( - error instanceof ElementCallError - ? error - : error instanceof Error - ? new UnknownCallError(error) - : new UnknownCallError(new Error(`${error}`)), - ); + this._state$.next({ + state: "error", + error: + error instanceof ElementCallError + ? error + : error instanceof Error + ? new UnknownCallError(error) + : new UnknownCallError(new Error(`${error}`)), + }); // Its okay to ignore the throw. The error is part of the state. throw error; } @@ -424,7 +408,7 @@ export class Connection { // disconnect sees the flag and does not report the abort as an error. this.stopped = true; await this.livekitRoom.disconnect(); - this._state$.next(ConnectionState.Stopped); + this._state$.next({ state: "stopped" }); this.logger.debug( `stop: DONE disconnecing from lk room ${this.transport.livekit_service_url}`, ); From 3927c2b7dc5c90ab4ea1b1b6f812e9515efbb752 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 12 Oct 2026 10:09:50 +0200 Subject: [PATCH 06/25] Show the whole transport in debug UI and logs A transport may soon carry a LiveKit SFU `url` instead of, or as well as, a `livekit_service_url`, so a service URL alone no longer identifies it. Pass the whole transport to the tiles' debug stats and print it as JSON in log lines instead of picking out the service URL. --- src/RTCConnectionStats.tsx | 23 ++++++------------- src/state/CallViewModel/CallViewModel.ts | 8 +++---- .../CallViewModel/localMember/LocalMember.ts | 4 ++-- .../localMember/RtcTransportAutoDiscovery.ts | 4 ++-- .../CallViewModel/remoteMembers/Connection.ts | 16 ++++--------- .../remoteMembers/MatrixLivekitMembers.ts | 4 ++-- src/state/media/MemberMediaViewModel.ts | 13 ++++++----- src/tile/GridTile.tsx | 15 ++++++------ src/tile/MediaView.tsx | 9 ++++---- src/tile/SpotlightTile.tsx | 7 +++--- src/utils/test.ts | 6 ++--- 11 files changed, 48 insertions(+), 61 deletions(-) diff --git a/src/RTCConnectionStats.tsx b/src/RTCConnectionStats.tsx index ea9df3f5e..90fefb656 100644 --- a/src/RTCConnectionStats.tsx +++ b/src/RTCConnectionStats.tsx @@ -16,30 +16,21 @@ import classNames from "classnames"; import { Modal } from "./Modal"; import styles from "./RTCConnectionStats.module.css"; import mediaViewStyles from "../src/tile/MediaView.module.css"; +import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; + interface Props { audio?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; video?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; - focusUrl?: string; + transport?: LivekitTransport; rtcBackendIdentity?: string; } -const extractDomain = (url: string): string => { - try { - const parsedUrl = new URL(url); - return parsedUrl.hostname; // Returns "kdk.cpm" - } catch (error) { - console.error("Invalid URL:", error); - return url; - } -}; - // This is only used in developer mode for debugging purposes, so we don't need full localization export const RTCConnectionStats: FC = ({ audio, video, - focusUrl, + transport, rtcBackendIdentity, - ...rest }) => { const [showModal, setShowModal] = useState(false); const [modalContents, setModalContents] = useState< @@ -76,10 +67,10 @@ export const RTCConnectionStats: FC = ({ rtcBackendIdentity:{rtcBackendIdentity} - {focusUrl && ( + {transport && (
- -  {extractDomain(focusUrl)} + +  {JSON.stringify(transport)}
)} diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 3b457f719..d84953206 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -635,9 +635,7 @@ export function createCallViewModel$( mediaDevices, muteStates, trackProcessorState$, - logger.getChild( - "[Publisher " + connection.transport.livekit_service_url + "]", - ), + logger.getChild(`[Publisher ${JSON.stringify(connection.transport)}]`), controlledAudioDevices, ); }, @@ -829,8 +827,8 @@ export function createCallViewModel$( livekitRoom$: scope.behavior( connection$.pipe(map((c) => c?.livekitRoom)), ), - focusUrl$: scope.behavior( - connection$.pipe(map((c) => c?.transport.livekit_service_url)), + transport$: scope.behavior( + connection$.pipe(map((c) => c?.transport)), ), mediaDevices, pretendToBeDisconnected$: localMembership.reconnecting$, diff --git a/src/state/CallViewModel/localMember/LocalMember.ts b/src/state/CallViewModel/localMember/LocalMember.ts index c06190d3f..29c0da6eb 100644 --- a/src/state/CallViewModel/localMember/LocalMember.ts +++ b/src/state/CallViewModel/localMember/LocalMember.ts @@ -334,7 +334,7 @@ export const createLocalMembership$ = ({ ), tap((connection) => { logger.info( - `Local connection updated: ${connection?.transport?.livekit_service_url}`, + `Local connection updated: ${JSON.stringify(connection?.transport)}`, ); }), ), @@ -406,7 +406,7 @@ export const createLocalMembership$ = ({ scope.reconcile(localConnection$, async (connection) => { logger.info( "reconcile based on new localConnection:", - connection?.transport.livekit_service_url, + JSON.stringify(connection?.transport), ); if (connection !== null) { const publisher = createPublisherFactory(connection); diff --git a/src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.ts b/src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.ts index aeb6b051a..58a5f1b19 100644 --- a/src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.ts +++ b/src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.ts @@ -45,7 +45,7 @@ export class RtcTransportAutoDiscovery { const backendTransport = await this.tryBackendTransports(); if (backendTransport) { this.logger.info( - `Found backend transport: ${backendTransport.livekit_service_url}`, + `Found backend transport: ${JSON.stringify(backendTransport)}`, ); return backendTransport; } @@ -54,7 +54,7 @@ export class RtcTransportAutoDiscovery { const configTransport = this.tryConfigTransport(); if (configTransport) { this.logger.info( - `Found app config transport: ${configTransport.livekit_service_url}`, + `Found app config transport: ${JSON.stringify(configTransport)}`, ); return configTransport; } diff --git a/src/state/CallViewModel/remoteMembers/Connection.ts b/src/state/CallViewModel/remoteMembers/Connection.ts index 7face646a..ea9447e99 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.ts @@ -140,10 +140,10 @@ export class Connection { this.existingSFUConfig = opts.existingSFUConfig; this.roomId = opts.roomId; this.logger = logger.getChild( - "[Connection " + opts.transport.livekit_service_url + "]", + `[Connection ${JSON.stringify(opts.transport)}]`, ); this.logger.info( - `constructor: ${opts.transport.livekit_service_url} roomId: ${this.roomId} withSfuConfig?: ${opts.existingSFUConfig ? JSON.stringify(opts.existingSFUConfig) : "undefined"}`, + `constructor: roomId: ${this.roomId} withSfuConfig?: ${opts.existingSFUConfig ? JSON.stringify(opts.existingSFUConfig) : "undefined"}`, ); const { transport, client, scope } = opts; @@ -297,9 +297,7 @@ export class Connection { this.existingSFUConfig ?? (await this.getSFUConfigForRemoteConnection()); this.logger.debug( - "Starting Connection to: ", - this.transport.livekit_service_url, - "jwt: ", + "Starting Connection - jwt: ", jwt, "wss: ", url, @@ -400,17 +398,13 @@ export class Connection { * If the connection is already stopped, this is a no-op. */ public async stop(): Promise { - this.logger.debug( - `stop: disconnecing from lk room ${this.transport.livekit_service_url}`, - ); + this.logger.debug("stop: disconnecing from lk room"); if (this.stopped) return; // Mark as stopped before disconnecting so that a connect() aborted by the // disconnect sees the flag and does not report the abort as an error. this.stopped = true; await this.livekitRoom.disconnect(); this._state$.next({ state: "stopped" }); - this.logger.debug( - `stop: DONE disconnecing from lk room ${this.transport.livekit_service_url}`, - ); + this.logger.debug("stop: DONE disconnecing from lk room"); } } diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts index e7bd9e7ae..d899fd9e2 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts @@ -143,9 +143,9 @@ export function createRemoteMatrixLivekitMembers$({ // Log whether the member could be matched to a LiveKit participant, // since a tile shows "waiting for media" for as long as it cannot. participant$.pipe(scope.bind()).subscribe((p) => { - const url = data$.value.connection?.transport.livekit_service_url; + const transport = data$.value.connection?.transport; logger.info( - `[RemoteMatrixLivekitMembers] ${rtcBackendIdentity}: LiveKit participant ${p ? `matched (${p.sid})` : "missing"} on ${url ?? "no connection"}`, + `[RemoteMatrixLivekitMembers] ${rtcBackendIdentity}: LiveKit participant ${p ? `matched (${p.sid})` : "missing"} on ${transport ? JSON.stringify(transport) : "no connection"}`, ); }); // will only get called once per backend identity. diff --git a/src/state/media/MemberMediaViewModel.ts b/src/state/media/MemberMediaViewModel.ts index a7f4612db..4891809e2 100644 --- a/src/state/media/MemberMediaViewModel.ts +++ b/src/state/media/MemberMediaViewModel.ts @@ -40,6 +40,7 @@ import { E2eeType } from "../../e2ee/e2eeType"; import { observeInboundRtpStreamStats$ } from "./observeRtpStreamStats"; import { type UserMediaViewModel } from "./UserMediaViewModel"; import { type ScreenShareViewModel } from "./ScreenShareViewModel"; +import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; // TODO: Encryption status is kinda broken and thus unused right now. Remove? export enum EncryptionStatus { @@ -59,10 +60,10 @@ export interface BaseMemberMediaViewModel extends BaseMediaViewModel { */ video$: Behavior; /** - * The URL of the LiveKit focus on which this member should be publishing. - * Exposed for debugging. + * The LiveKit transport on which this member should be publishing. Exposed + * for debugging. */ - focusUrl$: Behavior; + transport$: Behavior; /** * Whether there should be a warning that this media is unencrypted. */ @@ -75,7 +76,7 @@ export interface MemberMediaInputs extends BaseMediaViewModel { livekitRoom$: Behavior; audioSource: AudioSource; videoSource: VideoSource; - focusUrl$: Behavior; + transport$: Behavior; encryptionSystem: EncryptionSystem; } @@ -86,7 +87,7 @@ export function createMemberMedia( livekitRoom$, audioSource, videoSource, - focusUrl$, + transport$, encryptionSystem, ...inputs }: MemberMediaInputs, @@ -109,7 +110,7 @@ export function createMemberMedia( return { ...createBaseMedia(inputs), video$, - focusUrl$, + transport$, unencryptedWarning$: scope.behavior( combineLatest( [audio$, video$], diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index e00186320..b83d5f5ea 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -51,6 +51,7 @@ import { type RemoteUserMediaViewModel } from "../state/media/RemoteUserMediaVie import { type UserMediaViewModel } from "../state/media/UserMediaViewModel"; import { type RingingMediaViewModel } from "../state/media/RingingMediaViewModel"; import { RingingStatus } from "./RingingStatus"; +import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; interface TileProps { ref?: Ref; @@ -103,7 +104,7 @@ interface UserMediaTileProps extends TileProps { playbackMuted: boolean; waitingForMedia?: boolean; primaryButton?: ReactNode; - focusUrl: string | undefined; + transport: LivekitTransport | undefined; } /** @@ -120,7 +121,7 @@ const UserMediaTileInner: FC = ({ primaryButton, menu, className, - focusUrl, + transport, displayName, mxcAvatarUrl, focusable, @@ -214,7 +215,7 @@ const UserMediaTileInner: FC = ({ currentReaction={reaction ?? undefined} raisedHandOnClick={raisedHandOnClick} waitingForMedia={waitingForMedia} - focusUrl={focusUrl} + transport={transport} setVideoAspectRatio={vm.setVideoAspectRatio} audioStreamStats={audioStreamStats} videoStreamStats={videoStreamStats} @@ -279,7 +280,7 @@ const LocalUserMediaTile: FC = ({ const mirror = useBehavior(vm.mirror$); const alwaysShow = useBehavior(vm.alwaysShow$); const switchCamera = useBehavior(vm.switchCamera$); - const focusUrl = useBehavior(vm.focusUrl$); + const transport = useBehavior(vm.transport$); const latestAlwaysShow = useLatest(alwaysShow); const onSelectAlwaysShow = useCallback( @@ -334,7 +335,7 @@ const LocalUserMediaTile: FC = ({ menuStart={menuStart} menuEnd={menuEnd} focusable={focusable} - focusUrl={focusUrl} + transport={transport} {...props} /> ); @@ -356,7 +357,7 @@ const RemoteUserMediaTile: FC = ({ const waitingForMedia = useBehavior(vm.waitingForMedia$); const playbackMuted = useBehavior(vm.playbackMuted$); const playbackVolume = useBehavior(vm.playbackVolume$); - const focusUrl = useBehavior(vm.focusUrl$); + const transport = useBehavior(vm.transport$); const onSelectMute = useCallback( (e: Event) => { @@ -398,7 +399,7 @@ const RemoteUserMediaTile: FC = ({ } - focusUrl={focusUrl} + transport={transport} {...props} /> ); diff --git a/src/tile/MediaView.tsx b/src/tile/MediaView.tsx index d57441687..7b22764d5 100644 --- a/src/tile/MediaView.tsx +++ b/src/tile/MediaView.tsx @@ -33,6 +33,7 @@ import { ReactionIndicator } from "../reactions/ReactionIndicator"; import { RTCConnectionStats } from "../RTCConnectionStats"; import videoPlaceholder from "../graphics/video-placeholder.gif"; import { autoVideoFit } from "../utils/videoFit"; +import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; interface Props extends ComponentProps { className?: string; @@ -67,9 +68,9 @@ interface Props extends ComponentProps { videoStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; rtcBackendIdentity?: string; /** - * The focus url, mainly for debugging purposes. + * The LiveKit transport, mainly for debugging purposes. */ - focusUrl?: string; + transport?: LivekitTransport; /** * Called whenever the aspect ratio of the video content becomes known or * otherwise changes. @@ -106,7 +107,7 @@ export const MediaView: FC = ({ audioStreamStats, videoStreamStats, rtcBackendIdentity, - focusUrl, + transport, setVideoAspectRatio: setTheirVideoAspectRatio, ...props }) => { @@ -227,7 +228,7 @@ export const MediaView: FC = ({ diff --git a/src/tile/SpotlightTile.tsx b/src/tile/SpotlightTile.tsx index 71b98e8d8..46cab2566 100644 --- a/src/tile/SpotlightTile.tsx +++ b/src/tile/SpotlightTile.tsx @@ -54,6 +54,7 @@ import { platform } from "../Platform"; import { type RingingMediaViewModel } from "../state/media/RingingMediaViewModel"; import { RingingStatus } from "./RingingStatus"; import { useRootElement } from "../RootElementContext"; +import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; interface SpotlightItemBaseProps { ref?: Ref; @@ -74,7 +75,7 @@ interface SpotlightItemBaseProps { interface SpotlightMemberMediaItemBaseProps extends SpotlightItemBaseProps { video: TrackReferenceOrPlaceholder | undefined; unencryptedWarning: boolean; - focusUrl: string | undefined; + transport: LivekitTransport | undefined; } interface SpotlightUserMediaItemBaseProps extends SpotlightMemberMediaItemBaseProps { @@ -177,13 +178,13 @@ const SpotlightMemberMediaItem: FC = ({ }) => { const video = useBehavior(vm.video$); const unencryptedWarning = useBehavior(vm.unencryptedWarning$); - const focusUrl = useBehavior(vm.focusUrl$); + const transport = useBehavior(vm.transport$); const baseProps: SpotlightMemberMediaItemBaseProps & RefAttributes = { video: video ?? undefined, unencryptedWarning, - focusUrl, + transport, ...props, }; diff --git a/src/utils/test.ts b/src/utils/test.ts index 32e5cce17..48527184e 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -358,7 +358,7 @@ export function mockLocalMedia( participant$: constant(localParticipant), encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, livekitRoom$: constant(mockLivekitRoom({ localParticipant })), - focusUrl$: constant("https://rtc-example.org"), + transport$: constant(exampleTransport), mediaDevices, displayName$: constant(member.rawDisplayName ?? "nodisplayname"), mxcAvatarUrl$: constant(member.getMxcAvatarUrl()), @@ -401,7 +401,7 @@ export function mockRemoteMedia( participant$: constant(participant), encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, livekitRoom$: constant(livekitRoom), - focusUrl$: constant("https://rtc-example.org"), + transport$: constant(exampleTransport), pretendToBeDisconnected$: constant(false), displayName$: constant(member.rawDisplayName ?? "nodisplayname"), mxcAvatarUrl$: constant(member.getMxcAvatarUrl()), @@ -428,7 +428,7 @@ export function mockRemoteScreenShare( participant$: constant(participant), encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, livekitRoom$: constant(livekitRoom), - focusUrl$: constant("https://rtc-example.org"), + transport$: constant(exampleTransport), pretendToBeDisconnected$: constant(false), displayName$: constant(member.rawDisplayName ?? "nodisplayname"), mxcAvatarUrl$: constant(member.getMxcAvatarUrl()), From f260755f46880f1bc8d06bd266b7c7c529895cbe Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 12 Oct 2026 10:11:54 +0200 Subject: [PATCH 07/25] Group livekitRoomItems$ by connection The old reducer keyed items by service URL and mutated the accumulator's entries in place, so a transport without a service URL could not be represented and items were not safe to compare. Group members by their Connection object instead and carry the whole transport on the item, which the audio renderer now receives in place of a URL. --- sdk/main.ts | 2 +- src/livekit/MatrixAudioRenderer.test.tsx | 7 +-- src/livekit/MatrixAudioRenderer.tsx | 10 ++-- src/room/InCallView.tsx | 6 +-- src/state/CallViewModel/CallViewModel.ts | 66 +++++++++++------------- 5 files changed, 42 insertions(+), 49 deletions(-) diff --git a/sdk/main.ts b/sdk/main.ts index 6cd37032d..3932d1190 100644 --- a/sdk/main.ts +++ b/sdk/main.ts @@ -231,7 +231,7 @@ export async function createMatrixRTCSdk( complete: () => { logger.info("Livekit room items subscription completed"); for (const item of callViewModel.livekitRoomItems$.value) { - logger.info("unregistering room item from room", item.url); + logger.info("unregistering room item from room", item.transport); item.livekitRoom.unregisterTextStreamHandler(TEXT_LK_TOPIC); } }, diff --git a/src/livekit/MatrixAudioRenderer.test.tsx b/src/livekit/MatrixAudioRenderer.test.tsx index e4573cda9..21183f866 100644 --- a/src/livekit/MatrixAudioRenderer.test.tsx +++ b/src/livekit/MatrixAudioRenderer.test.tsx @@ -24,6 +24,7 @@ import { testAudioContext } from "../useAudioContext.test"; import * as MediaDevicesContext from "../MediaDevicesContext"; import { LivekitRoomAudioRenderer } from "./MatrixAudioRenderer"; import { + exampleTransport, mockLocalParticipant, mockMediaDevices, mockRemoteParticipant, @@ -118,7 +119,7 @@ function renderTestComponent( p.identity)} livekitRoom={livekitRoom} - url={""} + transport={exampleTransport} /> , ); @@ -150,7 +151,7 @@ it("should not render or warn for the local participant", () => { , ); @@ -177,7 +178,7 @@ it("should warn only once per unexpected participant", () => { , ); diff --git a/src/livekit/MatrixAudioRenderer.tsx b/src/livekit/MatrixAudioRenderer.tsx index 95d68c056..c9c4bbb36 100644 --- a/src/livekit/MatrixAudioRenderer.tsx +++ b/src/livekit/MatrixAudioRenderer.tsx @@ -19,12 +19,10 @@ import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { useEarpieceAudioConfig } from "../MediaDevicesContext"; import { useReactiveState } from "../useReactiveState"; import * as controls from "../controls"; +import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; export interface MatrixAudioRendererProps { - /** - * The service URL of the LiveKit room. - */ - url: string; + transport: LivekitTransport; livekitRoom: LivekitRoom; /** * The list of participant identities to render audio for. @@ -54,7 +52,7 @@ export interface MatrixAudioRendererProps { * @public */ export function LivekitRoomAudioRenderer({ - url, + transport, livekitRoom, validIdentities, muted, @@ -88,7 +86,7 @@ export function LivekitRoomAudioRenderer({ warnedIdentities.current.add(identity); // Log that there is an invalid identity, that means that someone is publishing audio that is not expected to be in the call. logger.warn( - `Audio track ${identity} from ${url} has no matching matrix call member`, + `Audio track ${identity} from ${JSON.stringify(transport)} has no matching matrix call member`, `current members: ${validIdentities.join()}`, `track will not get rendered`, ); diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index b8d7ab89d..51c916f92 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -665,10 +665,10 @@ export const InCallView: FC = ({ onPointerOut={onPointerOut} > {header} - {audioParticipants.map(({ livekitRoom, url, participants }) => ( + {audioParticipants.map(({ livekitRoom, transport, participants }) => ( d.value)), ); - const livekitRoomItems$ = scope.behavior( - remoteMatrixLivekitMembers$.pipe( - switchMap((members) => { - const a$ = combineLatest( - members.value.map((member) => - combineLatest([member.connection$, member.participant.value$]).pipe( - map(([connection, participant]) => { - // do not render audio for local participant - if (!connection || !participant || participant.isLocal) - return null; - const livekitRoom = connection.livekitRoom; - const url = connection.transport.livekit_service_url; - return { - url, - livekitRoom, - participant: participant.identity, - }; - }), + const livekitRoomItems$ = scope.behavior( + remoteMatrixLivekitMembers$.pipe( + // Flatten the members to a snapshot of their current connections and + // participant objects + switchMap(({ value: members }) => { + if (members.length === 0) return of([]); + return combineLatest( + members.map((m) => + combineLatest( + [m.connection$, m.participant.value$], + (connection, participant) => ({ connection, participant }), ), ), ); - return a$; }), + // Keep only those with a defined connection and participant map((members) => - members.reduce((acc, curr) => { - if (!curr) return acc; - - const existing = acc.find((item) => item.url === curr.url); - if (existing) { - existing.participants.push(curr.participant); - } else { - acc.push({ - livekitRoom: curr.livekitRoom, - participants: [curr.participant], - url: curr.url, - }); - } - return acc; - }, []), + members.filter( + ( + m, + ): m is { connection: Connection; participant: RemoteParticipant } => + m.connection !== null && m.participant !== null, + ), + ), + // Group members by connection + map((members) => + [...Map.groupBy(members, (m) => m.connection)].map( + ([connection, members]) => ({ + transport: connection.transport, + livekitRoom: connection.livekitRoom, + participants: members.map((m) => m.participant.identity), + }), + ), ), ), - [], ); const handsRaised$ = scope.behavior( From 17c3223883e68ce32b28c60dc9d27b1c0b36b560 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 12 Oct 2026 10:17:39 +0200 Subject: [PATCH 08/25] Move SFU authentication into livekit/auth and choose the endpoint by role The OpenID flow moves to src/livekit/auth/openID.ts with the SFU config types in a sibling file, takes an options object, and replaces the MatrixRTC mode with the role we take on the transport plus the session's slot ID instead of a hard-coded m.call#ROOM. This changes behaviour: publishers on a livekit_service_url transport always use the legacy /sfu/get endpoint, so that subscribers can predict their user:device participant identity, and subscribers always use the default /get_token endpoint. The MatrixRTC mode no longer influences which endpoint is used. --- src/livekit/auth/index.ts | 9 + src/livekit/auth/openID.test.ts | 200 +++++++++++ src/livekit/auth/openID.ts | 262 ++++++++++++++ src/livekit/auth/types.ts | 79 +++++ src/livekit/openIDSFU.test.ts | 329 ------------------ src/livekit/openIDSFU.ts | 315 ----------------- src/settings/DeveloperSettingsTab.test.tsx | 24 +- src/settings/DeveloperSettingsTab.tsx | 13 +- src/state/CallViewModel/CallViewModel.ts | 4 +- .../CallViewModel/CallViewModelTestUtils.ts | 1 + .../localMember/LocalMember.test.ts | 28 +- .../CallViewModel/localMember/LocalMember.ts | 25 +- .../localMember/LocalTransport.test.ts | 38 +- .../localMember/LocalTransport.ts | 34 +- .../remoteMembers/Connection.test.ts | 71 ++-- .../CallViewModel/remoteMembers/Connection.ts | 42 ++- .../remoteMembers/ConnectionFactory.ts | 12 +- .../remoteMembers/ConnectionManager.ts | 2 +- .../remoteMembers/ECConnectionFactory.test.ts | 10 +- .../remoteMembers/integration.test.ts | 12 +- src/utils/test.ts | 2 +- 21 files changed, 708 insertions(+), 804 deletions(-) create mode 100644 src/livekit/auth/index.ts create mode 100644 src/livekit/auth/openID.test.ts create mode 100644 src/livekit/auth/openID.ts create mode 100644 src/livekit/auth/types.ts delete mode 100644 src/livekit/openIDSFU.test.ts delete mode 100644 src/livekit/openIDSFU.ts diff --git a/src/livekit/auth/index.ts b/src/livekit/auth/index.ts new file mode 100644 index 000000000..6c11768f7 --- /dev/null +++ b/src/livekit/auth/index.ts @@ -0,0 +1,9 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +export * from "./openID"; +export type * from "./types"; diff --git a/src/livekit/auth/openID.test.ts b/src/livekit/auth/openID.test.ts new file mode 100644 index 000000000..a722dcd0a --- /dev/null +++ b/src/livekit/auth/openID.test.ts @@ -0,0 +1,200 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + beforeEach, + afterEach, + describe, + expect, + it, + type MockedObject, + vitest, +} from "vitest"; +import fetchMock from "fetch-mock"; +import { MatrixError } from "matrix-js-sdk"; + +import { + getSFUConfig, + type GetSFUConfigParams, + type ClientOpenIDParts, +} from "./openID"; +import { testJWTToken } from "../../utils/test-fixtures"; +import { ownMemberMock } from "../../utils/test"; +import { FailToGetOpenIdToken } from "../../utils/errors"; +import { logger } from "matrix-js-sdk/lib/logger"; + +const serviceUrl = "https://sfu.example.org"; + +const delayParams = { + delayEndpointBaseUrl: "https://matrix.homeserverserver.org", + delayId: "mock_delay_id", +}; + +describe("getSFUConfig", () => { + let matrixClient: MockedObject; + let params: GetSFUConfigParams; + beforeEach(() => { + fetchMock.catch(404); + matrixClient = { getOpenIdToken: vitest.fn() }; + params = { + client: matrixClient, + membership: ownMemberMock, + serviceUrl: "https://sfu.example.org", + roomId: "!example_room_id", + slotId: "m.call#ROOM", + role: "publisher", + logger, + }; + }); + afterEach(async () => { + void (await fetchMock.flush()); + vitest.clearAllMocks(); + fetchMock.reset(); + }); + + it("should handle fetching a token", async () => { + fetchMock.post("https://sfu.example.org/sfu/get", () => { + return { + status: 200, + body: { url: serviceUrl, jwt: testJWTToken }, + }; + }); + const config = await getSFUConfig(params); + expect(config).toEqual({ + jwt: testJWTToken, + url: serviceUrl, + livekitIdentity: "@me:example.org:ABCDEF", + livekitAlias: "!example_room_id", + }); + }); + + it("should fail if the SFU errors", async () => { + fetchMock.post("https://sfu.example.org/sfu/get", () => { + return { + status: 500, + body: { + errcode: "M_LOOKUP_FAILED", + error: "Failed to look up user info from homeserver", + }, + }; + }); + try { + await getSFUConfig(params); + } catch (ex: unknown) { + expect(ex).toBeInstanceOf(FailToGetOpenIdToken); + expect((ex as FailToGetOpenIdToken).cause).toBeInstanceOf(MatrixError); + const mxError = (ex as Error).cause as MatrixError; + expect(mxError.message).toEqual( + "MatrixError: [500] Failed to look up user info from homeserver", + ); + + return; + } + expect.fail("Expected test to throw;"); + }); + + it("should retry without delay params if the JWT service legacy endpoint returns M_BAD_JSON 400", async () => { + let callCount = 0; + + fetchMock.post( + "https://sfu.example.org/sfu/get", + (url, opts) => { + callCount++; + const body = JSON.parse(opts.body as string); + + // First call: check if it has delay parts and return 400 + if (callCount === 1) { + expect(body).toHaveProperty("delay_id", "mock_delay_id"); + return { + status: 400, + body: { errcode: "M_BAD_JSON", error: "Unsupported parameters" }, + }; + } + + // Second call: check if delay parts were stripped and return success + expect(body).not.toHaveProperty("delay_id"); + expect(body).not.toHaveProperty("delay_timeout"); + expect(body).not.toHaveProperty("delay_cs_api_url"); + + return { + status: 200, + body: { url: serviceUrl, jwt: testJWTToken }, + }; + }, + { overwriteRoutes: true }, + ); + + // Note: Assuming getSFUConfig eventually calls getLiveKitJWT + const config = await getSFUConfig({ ...params, ...delayParams }); + + expect(config.jwt).toBe(testJWTToken); + expect(callCount).toBe(2); + }); + + it("should successfully send delay parameters to the JWT service legacy endpoint", async () => { + fetchMock.post( + "https://sfu.example.org/sfu/get", + (url, opts) => { + const body = JSON.parse(opts.body as string); + + // Verify, that the request contains the expected delay parameters + if ( + body.delay_id === "mock_delay_id" && + body.delay_timeout === 3600000 && + body.delay_cs_api_url === "https://matrix.homeserverserver.org" + ) { + return { + status: 200, + body: { url: serviceUrl, jwt: testJWTToken }, + }; + } + return { + status: 400, + body: { error: "Missing expected delay params" }, + }; + }, + { overwriteRoutes: true }, + ); + + const config = await getSFUConfig({ ...params, ...delayParams }); + + // Prüfe das Ergebnis + expect(config).toMatchObject({ + jwt: testJWTToken, + url: serviceUrl, + }); + }); + + it("should retry fetching the openid token", async () => { + let count = 0; + matrixClient.getOpenIdToken.mockImplementation(async () => { + count++; + if (count < 2) { + throw Error("Test failure"); + } + return Promise.resolve({ + token_type: "Bearer", + access_token: "foobar", + matrix_server_name: "example.org", + expires_in: 30, + }); + }); + fetchMock.post("https://sfu.example.org/sfu/get", () => { + return { + status: 200, + body: { url: serviceUrl, jwt: testJWTToken }, + }; + }); + const config = await getSFUConfig({ ...params, ...delayParams }); + expect(config).toEqual({ + jwt: testJWTToken, + url: serviceUrl, + livekitIdentity: "@me:example.org:ABCDEF", + livekitAlias: "!example_room_id", + }); + }); +}); diff --git a/src/livekit/auth/openID.ts b/src/livekit/auth/openID.ts new file mode 100644 index 000000000..4479d4261 --- /dev/null +++ b/src/livekit/auth/openID.ts @@ -0,0 +1,262 @@ +/* +Copyright 2023, 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + type IOpenIDToken, + type MatrixClient, + parseErrorResponse, +} from "matrix-js-sdk"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; +import { type Logger } from "matrix-js-sdk/lib/logger"; + +import { FailToGetOpenIdToken } from "../../utils/errors"; +import { doNetworkOperationWithRetry } from "../../utils/matrix"; +import { Config } from "../../config/Config"; +import { extractFullConfigFromToken, type SFUConfig } from "./types"; + +// The bits we need from MatrixClient +export type ClientOpenIDParts = Pick; + +export interface GetSFUConfigParams { + client: ClientOpenIDParts; + /** + * Data identifying the local user's session membership. + */ + membership: CallMembershipIdentityParts; + /** + * The base URL of the LiveKit JWT service. + */ + serviceUrl: string; + /** + * The ID of the Matrix room in which the session takes place. + */ + roomId: string; + /** + * The ID of the MatrixRTC slot in which the session takes place. + */ + slotId: string; + /** + * Whether we want to publish or only subscribe on the {@link transport}. + */ + role: "publisher" | "subscriber"; + /** + * The base URL of the Matrix homeserver, for delayed leave delegation. + */ + delayEndpointBaseUrl?: string; + /** + * The delay ID of the leave event to be delegated to the JWT service. + */ + delayId?: string; + logger: Logger; +} + +/** + * Gets an {@link SFUConfig} appropriate for connecting to the SFU behind a + * given service URL, using the legacy OpenID flow. + * @throws {@link FailToGetOpenIdToken} + */ +export async function getSFUConfig({ + client, + membership, + serviceUrl, + roomId, + slotId, + role, + delayEndpointBaseUrl, + delayId, + logger, +}: GetSFUConfigParams): Promise { + let openIdToken: IOpenIDToken; + try { + openIdToken = await doNetworkOperationWithRetry(async () => + client.getOpenIdToken(), + ); + } catch (error) { + throw new FailToGetOpenIdToken( + error instanceof Error ? error : new Error("Unknown error"), + ); + } + logger.debug("Got openID token", openIdToken); + let sfuConfig: { url: string; jwt: string } | undefined; + + let endpoint: "default" | "legacy"; + if (role === "publisher") { + // When publishing a legacy transport (one with a `livekit_service_url`), + // subscribers will expect our participant identity to use the legacy + // `@user_id:device_id` format. Only the legacy JWT service endpoint assigns + // identities in this format, so we must use it. + endpoint = "legacy"; + } else { + // Use the default endpoint as it affords us slightly better metadata + // protection (pseudonymous participant identities). + endpoint = "default"; + } + + try { + logger.info( + `Trying to get JWT with ${endpoint} endpoint for focus ${serviceUrl}...`, + ); + sfuConfig = await ( + endpoint === "legacy" ? getLiveKitJWTLegacy : getLiveKitJWT + )( + membership, + serviceUrl, + roomId, + slotId, + openIdToken, + delayEndpointBaseUrl, + delayId, + ); + logger.info(`Got JWT from call's active focus URL.`); + return extractFullConfigFromToken(sfuConfig); + } catch (e) { + logger.error(`Failed fetching jwt with ${endpoint} endpoint:`, e); + throw new FailToGetOpenIdToken( + e instanceof Error ? e : new Error(`Unknown error ${e}`), + ); + } +} + +/** + * Gets a JWT token appropriate for connecting to the SFU behind a given service + * URL, using the legacy `/sfu/get` endpoint, which assigns us a legacy + * (`:`) participant identity. + */ +async function getLiveKitJWTLegacy( + membership: CallMembershipIdentityParts, + livekitServiceURL: string, + matrixRoomId: string, + _slotId: string, // Unused - the legacy endpoint is from before slots existed. + openIDToken: IOpenIDToken, + delayEndpointBaseUrl?: string, + delayId?: string, +): Promise<{ url: string; jwt: string }> { + interface IDelayParams { + delay_id?: string; + delay_timeout?: number; + delay_cs_api_url?: string; + } + let bodyDalayParts: IDelayParams = {}; + // Also check for empty string + if (delayId && delayEndpointBaseUrl) { + bodyDalayParts = { + delay_id: delayId, + delay_timeout: + Config.get().matrix_rtc_session.delegated_delayed_leave.delay_ms, + delay_cs_api_url: delayEndpointBaseUrl, + }; + } + + const makeRequest = async (delayParts: IDelayParams): Promise => { + return await fetch(livekitServiceURL + "/sfu/get", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + // The legacy JWT endpoint uses only the matrix room id to calculate the livekit room alias. + // However, the livekit room alias is provided as part of the JWT payload. + room: matrixRoomId, + openid_token: openIDToken, + device_id: membership.deviceId, + ...delayParts, + }), + }); + }; + + const res = await doNetworkOperationWithRetry(async () => { + let response = await makeRequest(bodyDalayParts); + + // Old service compatibility check + const oldServiceDoesNotSupportDelayParts = + response.status === 400 && Object.keys(bodyDalayParts).length > 0; + // If http status 400 with M_BAD_JSON and we sent delay parts, retry without them + if (oldServiceDoesNotSupportDelayParts) { + try { + const errorBody = await response.json(); + if (errorBody.errcode === "M_BAD_JSON") { + response = await makeRequest({}); + } + } catch { + // If we can't parse the error, treat as real error + } + } + + return response; + }); + + if (!res.ok) { + throw parseErrorResponse(res, await res.text()); + } + return await res.json(); +} + +class NotSupportedError extends Error { + public constructor(message: string) { + super(message); + this.name = "NotSupported"; + } +} + +/** + * Gets a JWT token appropriate for connecting to the SFU behind a given service + * URL, using the default `/get_token` endpoint, which assigns us a hashed + * (pseudonymised) participant identity. + */ +async function getLiveKitJWT( + membership: CallMembershipIdentityParts, + livekitServiceURL: string, + matrixRoomId: string, + slotId: string, + openIDToken: IOpenIDToken, + delayEndpointBaseUrl?: string, + delayId?: string, +): Promise<{ url: string; jwt: string }> { + const { userId, deviceId, memberId } = membership; + + const body = { + room_id: matrixRoomId, + slot_id: slotId, + openid_token: openIDToken, + member: { + id: memberId, + claimed_user_id: userId, + claimed_device_id: deviceId, + }, + }; + + let bodyDalayParts = {}; + // Also check for empty string + if (delayId && delayEndpointBaseUrl) { + bodyDalayParts = { + delay_id: delayId, + delay_timeout: + Config.get().matrix_rtc_session.delegated_delayed_leave.delay_ms, + delay_cs_api_url: delayEndpointBaseUrl, + }; + } + + const res = await doNetworkOperationWithRetry(async () => { + return await fetch(livekitServiceURL + "/get_token", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ ...body, ...bodyDalayParts }), + }); + }); + + if (!res.ok) { + const msg = "SFU Config fetch failed with status code " + res.status; + if (res.status === 404) { + throw new NotSupportedError(msg); + } else { + throw parseErrorResponse(res, await res.text()); + } + } + return await res.json(); +} diff --git a/src/livekit/auth/types.ts b/src/livekit/auth/types.ts new file mode 100644 index 000000000..81982d689 --- /dev/null +++ b/src/livekit/auth/types.ts @@ -0,0 +1,79 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +/** + * Configuration and access tokens provided by the SFU on successful authentication. + */ +export interface SFUConfig { + /** + * The WebSocket URL of the LiveKit SFU. This is what we connect the LiveKit + * room to. Note that this is NOT the JWT service URL + * (`livekit_service_url`): the legacy JWT service returns this URL as part of + * its response, while the CS-Api based flow requires us to know it upfront. + */ + url: string; + jwt: string; + livekitAlias: string; + // NOTE: Currently unused. + livekitIdentity: string; +} + +/** + * Decoded details from the JWT. + */ +interface SFUJWTPayload { + /** + * Expiration time for the JWT. + * Note: This value is in seconds since Unix epoch. + */ + exp: number; + /** + * Name of the instance which authored the JWT + */ + iss: string; + /** + * Time at which the JWT can start to be used. + * Note: This value is in seconds since Unix epoch. + */ + nbf: number; + /** + * Subject. The Livekit alias in this context. + */ + sub: string; + /** + * The set of permissions for the user. + */ + video: { + canPublish: boolean; + canSubscribe: boolean; + room: string; + roomJoin: boolean; + }; +} + +/** + * Complements the SFU websocket url and the JWT with the information encoded in + * the JWT payload itself. + * @param sfuConfig The SFU websocket url and the JWT to connect with. + * @returns The full SFU config, including the LiveKit alias and identity. + */ +export function extractFullConfigFromToken(sfuConfig: { + url: string; + jwt: string; +}): SFUConfig { + const [, payloadStr] = sfuConfig.jwt.split("."); + const payload = JSON.parse(global.atob(payloadStr)) as SFUJWTPayload; + return { + jwt: sfuConfig.jwt, + url: sfuConfig.url, + livekitAlias: payload.video.room, + // NOTE: Currently unused. + // Probably also not helpful since we now compute the backendIdentity on joining the call so we can use it for the encryption manager. + // The only reason for us to know it locally is to connect the right users with the lk world. (and to set our own keys) + livekitIdentity: payload.sub, + }; +} diff --git a/src/livekit/openIDSFU.test.ts b/src/livekit/openIDSFU.test.ts deleted file mode 100644 index f759a3aac..000000000 --- a/src/livekit/openIDSFU.test.ts +++ /dev/null @@ -1,329 +0,0 @@ -/* -Copyright 2025 Element Creations Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - beforeEach, - afterEach, - describe, - expect, - it, - type MockedObject, - vitest, -} from "vitest"; -import fetchMock from "fetch-mock"; -import { MatrixError } from "matrix-js-sdk"; - -import { getSFUConfigWithOpenID, type OpenIDClientParts } from "./openIDSFU"; -import { testJWTToken } from "../utils/test-fixtures"; -import { ownMemberMock } from "../utils/test"; -import { FailToGetOpenIdToken } from "../utils/errors"; - -const sfuUrl = "https://sfu.example.org"; - -describe("getSFUConfigWithOpenID", () => { - let matrixClient: MockedObject; - beforeEach(() => { - fetchMock.catch(404); - matrixClient = { - getOpenIdToken: vitest.fn(), - getDeviceId: vitest.fn(), - }; - }); - afterEach(() => { - vitest.clearAllMocks(); - fetchMock.reset(); - }); - - it("should handle fetching a token", async () => { - fetchMock.post("https://sfu.example.org/sfu/get", () => { - return { - status: 200, - body: { url: sfuUrl, jwt: testJWTToken }, - }; - }); - const config = await getSFUConfigWithOpenID( - matrixClient, - ownMemberMock, - "https://sfu.example.org", - "!example_room_id", - ); - expect(config).toEqual({ - jwt: testJWTToken, - url: sfuUrl, - livekitIdentity: "@me:example.org:ABCDEF", - livekitAlias: "!example_room_id", - }); - void (await fetchMock.flush()); - }); - - it("should fail if the SFU errors", async () => { - fetchMock.post("https://sfu.example.org/sfu/get", () => { - return { - status: 500, - body: { - errcode: "M_LOOKUP_FAILED", - error: "Failed to look up user info from homeserver", - }, - }; - }); - try { - await getSFUConfigWithOpenID( - matrixClient, - ownMemberMock, - "https://sfu.example.org", - "!example_room_id", - ); - } catch (ex: unknown) { - expect(ex).toBeInstanceOf(FailToGetOpenIdToken); - expect((ex as FailToGetOpenIdToken).cause).toBeInstanceOf(MatrixError); - const mxError = (ex as Error).cause as MatrixError; - expect(mxError.message).toEqual( - "MatrixError: [500] Failed to look up user info from homeserver", - ); - - void (await fetchMock.flush()); - return; - } - expect.fail("Expected test to throw;"); - }); - - it("should retry without delay params if the JWT service legacy endpoint returns M_BAD_JSON 400", async () => { - let callCount = 0; - - fetchMock.post( - "https://sfu.example.org/sfu/get", - (url, opts) => { - callCount++; - const body = JSON.parse(opts.body as string); - - // First call: check if it has delay parts and return 400 - if (callCount === 1) { - expect(body).toHaveProperty("delay_id", "mock_delay_id"); - return { - status: 400, - body: { errcode: "M_BAD_JSON", error: "Unsupported parameters" }, - }; - } - - // Second call: check if delay parts were stripped and return success - expect(body).not.toHaveProperty("delay_id"); - expect(body).not.toHaveProperty("delay_timeout"); - expect(body).not.toHaveProperty("delay_cs_api_url"); - - return { - status: 200, - body: { url: sfuUrl, jwt: testJWTToken }, - }; - }, - { overwriteRoutes: true }, - ); - - // Note: Assuming getSFUConfigWithOpenID eventually calls getLiveKitJWT - const config = await getSFUConfigWithOpenID( - matrixClient, - ownMemberMock, - "https://sfu.example.org", - "!example_room_id", - { - delayEndpointBaseUrl: "https://matrix.homeserverserver.org", - delayId: "mock_delay_id", - }, - ); - - expect(config.jwt).toBe(testJWTToken); - expect(callCount).toBe(2); - void (await fetchMock.flush()); - }); - - it("should successfully send delay parameters to the JWT service legacy endpoint", async () => { - fetchMock.post( - "https://sfu.example.org/sfu/get", - (url, opts) => { - const body = JSON.parse(opts.body as string); - - // Verify, that the request contains the expected delay parameters - if ( - body.delay_id === "mock_delay_id" && - body.delay_timeout === 3600000 && - body.delay_cs_api_url === "https://homeserverserver.org/cs_api" - ) { - return { - status: 200, - body: { url: sfuUrl, jwt: testJWTToken }, - }; - } - return { - status: 400, - body: { error: "Missing expected delay params" }, - }; - }, - { overwriteRoutes: true }, - ); - - const config = await getSFUConfigWithOpenID( - matrixClient, - ownMemberMock, - "https://sfu.example.org", - "!example_room_id", - { - delayEndpointBaseUrl: "https://homeserverserver.org/cs_api", - delayId: "mock_delay_id", - }, - ); - - // Prüfe das Ergebnis - expect(config).toMatchObject({ - jwt: testJWTToken, - url: sfuUrl, - }); - - void (await fetchMock.flush()); - }); - - it("should try legacy and then new endpoint with delay delegation", async () => { - fetchMock.post("https://sfu.example.org/get_token", () => { - return { - status: 500, - body: { - errcode: "M_LOOKUP_FAILED", - error: "Failed to look up user info from homeserver", - }, - }; - }); - fetchMock.post("https://sfu.example.org/sfu/get", () => { - return { - status: 500, - body: { - errcode: "M_LOOKUP_FAILED", - error: "Failed to look up user info from homeserver", - }, - }; - }); - try { - await getSFUConfigWithOpenID( - matrixClient, - ownMemberMock, - "https://sfu.example.org", - "!example_room_id", - { - delayEndpointBaseUrl: "https://matrix.homeserverserver.org", - delayId: "mock_delay_id", - }, - ); - } catch (ex) { - expect(ex).toBeInstanceOf(FailToGetOpenIdToken); - expect((ex as FailToGetOpenIdToken).cause).toBeInstanceOf(MatrixError); - const mxError = (ex as Error).cause as MatrixError; - expect(mxError.message).toEqual( - "MatrixError: [500] Failed to look up user info from homeserver", - ); - void (await fetchMock.flush()); - } - const calls = fetchMock.calls(); - expect(calls.length).toBe(2); - - expect(calls[0][0]).toStrictEqual("https://sfu.example.org/get_token"); - expect(calls[0][1]).toStrictEqual({ - // check if it uses correct delayID! - body: '{"room_id":"!example_room_id","slot_id":"m.call#ROOM","member":{"id":"@alice:example.org:DEVICE","claimed_user_id":"@alice:example.org","claimed_device_id":"DEVICE"},"delay_id":"mock_delay_id","delay_timeout":3600000,"delay_cs_api_url":"https://matrix.homeserverserver.org"}', - method: "POST", - headers: { - "Content-Type": "application/json", - }, - }); - - expect(calls[1][0]).toStrictEqual("https://sfu.example.org/sfu/get"); - - expect(calls[1][1]).toStrictEqual({ - body: '{"room":"!example_room_id","device_id":"DEVICE","delay_id":"mock_delay_id","delay_timeout":3600000,"delay_cs_api_url":"https://matrix.homeserverserver.org"}', - headers: { - "Content-Type": "application/json", - }, - method: "POST", - }); - }); - - it("dont try legacy if endpoint with delay delegation is sucessful", async () => { - fetchMock.post("https://sfu.example.org/get_token", () => { - return { - status: 200, - body: { url: sfuUrl, jwt: testJWTToken }, - }; - }); - fetchMock.post("https://sfu.example.org/sfu/get", () => { - return { - status: 500, - body: { error: "Test failure" }, - }; - }); - try { - await getSFUConfigWithOpenID( - matrixClient, - ownMemberMock, - "https://sfu.example.org", - "!example_room_id", - { - delayEndpointBaseUrl: "https://matrix.homeserverserver.org", - delayId: "mock_delay_id", - }, - ); - } catch (ex) { - expect(ex).toBeInstanceOf(FailToGetOpenIdToken); - expect((ex as FailToGetOpenIdToken).cause).toEqual( - new Error("SFU Config fetch failed with status code 500"), - ); - void (await fetchMock.flush()); - } - const calls = fetchMock.calls(); - expect(calls.length).toBe(1); - - expect(calls[0][0]).toStrictEqual("https://sfu.example.org/get_token"); - expect(calls[0][1]).toStrictEqual({ - // check if it uses correct delayID! - body: '{"room_id":"!example_room_id","slot_id":"m.call#ROOM","member":{"id":"@alice:example.org:DEVICE","claimed_user_id":"@alice:example.org","claimed_device_id":"DEVICE"},"delay_id":"mock_delay_id","delay_timeout":3600000,"delay_cs_api_url":"https://matrix.homeserverserver.org"}', - method: "POST", - headers: { - "Content-Type": "application/json", - }, - }); - }); - - it("should retry fetching the openid token", async () => { - let count = 0; - matrixClient.getOpenIdToken.mockImplementation(async () => { - count++; - if (count < 2) { - throw Error("Test failure"); - } - return Promise.resolve({ - token_type: "Bearer", - access_token: "foobar", - matrix_server_name: "example.org", - expires_in: 30, - }); - }); - fetchMock.post("https://sfu.example.org/sfu/get", () => { - return { - status: 200, - body: { url: sfuUrl, jwt: testJWTToken }, - }; - }); - const config = await getSFUConfigWithOpenID( - matrixClient, - ownMemberMock, - "https://sfu.example.org", - "!example_room_id", - ); - expect(config).toEqual({ - jwt: testJWTToken, - url: sfuUrl, - livekitIdentity: "@me:example.org:ABCDEF", - livekitAlias: "!example_room_id", - }); - void (await fetchMock.flush()); - }); -}); diff --git a/src/livekit/openIDSFU.ts b/src/livekit/openIDSFU.ts deleted file mode 100644 index cf1f3c951..000000000 --- a/src/livekit/openIDSFU.ts +++ /dev/null @@ -1,315 +0,0 @@ -/* -Copyright 2023, 2024 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - type IOpenIDToken, - type MatrixClient, - parseErrorResponse, -} from "matrix-js-sdk"; -import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; -import { type Logger } from "matrix-js-sdk/lib/logger"; - -import { - FailToGetOpenIdToken, - NoMatrix2AuthorizationService, -} from "../utils/errors"; -import { doNetworkOperationWithRetry } from "../utils/matrix"; -import { Config } from "../config/Config"; -import { MatrixRTCMode } from "../config/ConfigOptions"; - -/** - * Configuration and access tokens provided by the SFU on successful authentication. - */ -export interface SFUConfig { - url: string; - jwt: string; - livekitAlias: string; - // NOTE: Currently unused. - livekitIdentity: string; -} - -/** - * Decoded details from the JWT. - */ -interface SFUJWTPayload { - /** - * Expiration time for the JWT. - * Note: This value is in seconds since Unix epoch. - */ - exp: number; - /** - * Name of the instance which authored the JWT - */ - iss: string; - /** - * Time at which the JWT can start to be used. - * Note: This value is in seconds since Unix epoch. - */ - nbf: number; - /** - * Subject. The Livekit alias in this context. - */ - sub: string; - /** - * The set of permissions for the user. - */ - video: { - canPublish: boolean; - canSubscribe: boolean; - room: string; - roomJoin: boolean; - }; -} - -// The bits we need from MatrixClient -export type OpenIDClientParts = Pick< - MatrixClient, - "getOpenIdToken" | "getDeviceId" ->; - -/** - * Gets a bearer token from the homeserver and then use it to authenticate - * to the matrix RTC backend in order to get acces to the SFU. - * It has built-in retry for calls to the homeserver with a backoff policy. - * @param client The Matrix client - * @param membership Our own membership identity parts used to send to jwt service. - * @param serviceUrl The URL of the livekit SFU service - * @param roomId The room id used in the jwt request. This is NOT the livekit_alias. The jwt service will provide the alias. It maps matrix room ids <-> Livekit aliases. - * @param opts Additional options to modify which endpoint with which data will be used to acquire the jwt token. - * @param opts.matrixRTCMode Determines which version of the JWT endpoint to use, which affects whether the - * RTC backend identity is based on string concatenation (legacy) or a hash (Matrix 2.0). - * This function by default uses whatever is possible with the current jwt service installed next to the SFU. - * For remote connections this does not matter, since we will not publish there we can rely on the newest option. - * @param opts.delayEndpointBaseUrl The URL of the matrix homeserver. - * @param opts.delayId The delay id used for the jwt service to manage. - * @param logger optional logger. - * @returns Object containing the token information - * @throws FailToGetOpenIdToken - */ -export async function getSFUConfigWithOpenID( - client: OpenIDClientParts, - membership: CallMembershipIdentityParts, - serviceUrl: string, - roomId: string, - opts?: { - matrixRTCMode?: MatrixRTCMode; - delayEndpointBaseUrl?: string; - delayId?: string; - }, - logger?: Logger, -): Promise { - let openIdToken: IOpenIDToken; - try { - openIdToken = await doNetworkOperationWithRetry(async () => - client.getOpenIdToken(), - ); - } catch (error) { - throw new FailToGetOpenIdToken( - error instanceof Error ? error : new Error("Unknown error"), - ); - } - logger?.debug("Got openID token", openIdToken); - let sfuConfig: { url: string; jwt: string } | undefined; - - const tryBothJwtEndpoints = opts?.matrixRTCMode === undefined; // This is for SFUs where we do not publish. - - const forceMatrix2Jwt = opts?.matrixRTCMode === MatrixRTCMode.Matrix_2_0; - - // We want to start using the new endpoint (with optional delay delegation) - // if we can use both or if we are forced to use the new one. - if (tryBothJwtEndpoints || forceMatrix2Jwt) { - try { - logger?.info( - `Trying to get JWT via default endpoint for focus ${serviceUrl}...`, - ); - const sfuConfig = await getLiveKitJWT( - membership, - serviceUrl, - roomId, - openIdToken, - opts?.delayEndpointBaseUrl, - opts?.delayId, - ); - - return extractFullConfigFromToken(sfuConfig); - } catch (e) { - logger?.debug(`Failed fetching jwt with matrix 2.0 endpoint:`, e); - // Make this throw a hard error in case we force the matrix2.0 endpoint. - if (forceMatrix2Jwt) { - throw new NoMatrix2AuthorizationService(e as Error); - } - } - } - - // DEPRECATED - // here we either have a sfuConfig or we already exited because of `if (forceMatrix2) throw ...` - // The only case we can get into this condition is, if `forceMatrix2` is `false` - try { - logger?.info( - `Trying to get JWT with legacy endpoint for focus ${serviceUrl}...`, - ); - sfuConfig = await getLiveKitJWTLegacy( - membership.deviceId, - serviceUrl, - roomId, - openIdToken, - opts?.delayEndpointBaseUrl, - opts?.delayId, - ); - logger?.info(`Got JWT from call's active focus URL.`); - return extractFullConfigFromToken(sfuConfig); - } catch (ex) { - throw new FailToGetOpenIdToken( - ex instanceof Error ? ex : new Error(`Unknown error ${ex}`), - ); - } -} - -function extractFullConfigFromToken(sfuConfig: { - url: string; - jwt: string; -}): SFUConfig { - const [, payloadStr] = sfuConfig.jwt.split("."); - const payload = JSON.parse(global.atob(payloadStr)) as SFUJWTPayload; - return { - jwt: sfuConfig.jwt, - url: sfuConfig.url, - livekitAlias: payload.video.room, - // NOTE: Currently unused. - // Probably also not helpful since we now compute the backendIdentity on joining the call so we can use it for the encryption manager. - // The only reason for us to know it locally is to connect the right users with the lk world. (and to set our own keys) - livekitIdentity: payload.sub, - }; -} - -async function getLiveKitJWTLegacy( - deviceId: string, - livekitServiceURL: string, - matrixRoomId: string, - openIDToken: IOpenIDToken, - delayEndpointBaseUrl?: string, - delayId?: string, -): Promise<{ url: string; jwt: string }> { - interface IDelayParams { - delay_id?: string; - delay_timeout?: number; - delay_cs_api_url?: string; - } - let bodyDalayParts: IDelayParams = {}; - // Also check for empty string - if (delayId && delayEndpointBaseUrl) { - bodyDalayParts = { - delay_id: delayId, - delay_timeout: - Config.get().matrix_rtc_session.delegated_delayed_leave.delay_ms, - delay_cs_api_url: delayEndpointBaseUrl, - }; - } - - const makeRequest = async (delayParts: IDelayParams): Promise => { - return await fetch(livekitServiceURL + "/sfu/get", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ - // The legacy JWT endpoint uses only the matrix room id to calculate the livekit room alias. - // However, the livekit room alias is provided as part of the JWT payload. - room: matrixRoomId, - openid_token: openIDToken, - device_id: deviceId, - ...delayParts, - }), - }); - }; - - const res = await doNetworkOperationWithRetry(async () => { - let response = await makeRequest(bodyDalayParts); - - // Old service compatibility check - const oldServiceDoesNotSupportDelayParts = - response.status === 400 && Object.keys(bodyDalayParts).length > 0; - // If http status 400 with M_BAD_JSON and we sent delay parts, retry without them - if (oldServiceDoesNotSupportDelayParts) { - try { - const errorBody = await response.json(); - if (errorBody.errcode === "M_BAD_JSON") { - response = await makeRequest({}); - } - } catch { - // If we can't parse the error, treat as real error - } - } - - return response; - }); - - if (!res.ok) { - throw parseErrorResponse(res, await res.text()); - } - return await res.json(); -} - -class NotSupportedError extends Error { - public constructor(message: string) { - super(message); - this.name = "NotSupported"; - } -} - -export async function getLiveKitJWT( - membership: CallMembershipIdentityParts, - livekitServiceURL: string, - matrixRoomId: string, - openIDToken: IOpenIDToken, - delayEndpointBaseUrl?: string, - delayId?: string, -): Promise<{ url: string; jwt: string }> { - const { userId, deviceId, memberId } = membership; - - const body = { - room_id: matrixRoomId, - slot_id: "m.call#ROOM", - openid_token: openIDToken, - member: { - id: memberId, - claimed_user_id: userId, - claimed_device_id: deviceId, - }, - }; - - let bodyDalayParts = {}; - // Also check for empty string - if (delayId && delayEndpointBaseUrl) { - bodyDalayParts = { - delay_id: delayId, - delay_timeout: - Config.get().matrix_rtc_session.delegated_delayed_leave.delay_ms, - delay_cs_api_url: delayEndpointBaseUrl, - }; - } - - const res = await doNetworkOperationWithRetry(async () => { - return await fetch(livekitServiceURL + "/get_token", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ ...body, ...bodyDalayParts }), - }); - }); - - if (!res.ok) { - const msg = "SFU Config fetch failed with status code " + res.status; - if (res.status === 404) { - throw new NotSupportedError(msg); - } else { - throw parseErrorResponse(res, await res.text()); - } - } - return await res.json(); -} diff --git a/src/settings/DeveloperSettingsTab.test.tsx b/src/settings/DeveloperSettingsTab.test.tsx index bd2b40b7d..bb0808761 100644 --- a/src/settings/DeveloperSettingsTab.test.tsx +++ b/src/settings/DeveloperSettingsTab.test.tsx @@ -18,7 +18,6 @@ import { } from "./DeveloperSettingsTab"; import { outOfCallDeveloperSettingsTabViewModel } from "./DeveloperSettingsTabViewModel"; import { createStaticViewModel } from "../state/ViewModel"; -import { getSFUConfigWithOpenID } from "../livekit/openIDSFU"; import { customLivekitUrl as customLivekitUrlSetting, enableExtendedLivekitLogs as enableExtendedLivekitLogsSetting, @@ -26,6 +25,7 @@ import { } from "./settings"; import { MatrixRTCMode } from "../config/ConfigOptions"; import { mockConfig } from "../utils/test"; +import { getSFUConfig } from "../livekit/auth"; // Mock url params hook to avoid environment-dependent snapshot churn. vi.mock("../UrlParams", () => ({ @@ -36,8 +36,8 @@ vi.mock("../UrlParams", () => ({ })); // IMPORTANT: mock the same specifier used by DeveloperSettingsTab -vi.mock("../livekit/openIDSFU", () => ({ - getSFUConfigWithOpenID: vi.fn().mockResolvedValue({ +vi.mock("../livekit/auth", () => ({ + getSFUConfig: vi.fn().mockResolvedValue({ url: "mock-url", jwt: "mock-jwt", }), @@ -153,7 +153,7 @@ describe("DeveloperSettingsTab", () => { const saveButton = screen.getByRole("button", { name: "Save" }); await user.click(saveButton); - expect(getSFUConfigWithOpenID).not.toHaveBeenCalled(); + expect(getSFUConfig).not.toHaveBeenCalled(); expect(customLivekitUrlSetting.getValue()).toBe(null); }); @@ -176,7 +176,7 @@ describe("DeveloperSettingsTab", () => { const saveButton = screen.getByRole("button", { name: "Save" }); await user.click(saveButton); - expect(getSFUConfigWithOpenID).not.toHaveBeenCalled(); + expect(getSFUConfig).not.toHaveBeenCalled(); expect(customLivekitUrlSetting.getValue()).toBe(null); }); @@ -202,7 +202,7 @@ describe("DeveloperSettingsTab", () => { name: "Reset overwrite", }); await user.click(cancelButton); - expect(getSFUConfigWithOpenID).not.toHaveBeenCalled(); + expect(getSFUConfig).not.toHaveBeenCalled(); expect(customLivekitUrlSetting.getValue()).toBe(null); }); @@ -226,11 +226,11 @@ describe("DeveloperSettingsTab", () => { const saveButton = screen.getByRole("button", { name: "Save" }); await user.click(saveButton); - expect(getSFUConfigWithOpenID).toHaveBeenCalledWith( - expect.anything(), - expect.anything(), - "wss://example.livekit.valid", - "#testRoom", + expect(getSFUConfig).toHaveBeenCalledWith( + expect.objectContaining({ + serviceUrl: "wss://example.livekit.valid", + roomId: "#testRoom", + }), ); expect(customLivekitUrlSetting.getValue()).toBe( @@ -256,7 +256,7 @@ describe("DeveloperSettingsTab", () => { await user.type(input, "wss://example.livekit.valid"); const saveButton = screen.getByRole("button", { name: "Save" }); - (getSFUConfigWithOpenID as Mock).mockImplementation(() => { + (getSFUConfig as Mock).mockImplementation(() => { throw new Error("Invalid URL"); }); await user.click(saveButton); diff --git a/src/settings/DeveloperSettingsTab.tsx b/src/settings/DeveloperSettingsTab.tsx index ec5bf7b08..c8cc2a524 100644 --- a/src/settings/DeveloperSettingsTab.tsx +++ b/src/settings/DeveloperSettingsTab.tsx @@ -67,9 +67,9 @@ import styles from "./DeveloperSettingsTab.module.css"; import settingsStyles from "./SettingsModal.module.css"; import { Slider } from "../Slider"; import { useUrlParams } from "../UrlParams"; -import { getSFUConfigWithOpenID } from "../livekit/openIDSFU"; import { useBehavior } from "../useBehavior"; import { type ViewModel } from "../state/ViewModel.ts"; +import { getSFUConfig } from "../livekit/auth/index.ts"; /** * The state of MatrixRTC's media key rotation. @@ -518,12 +518,15 @@ export const DeveloperSettingsTab: FC = ({ if (userId === null || deviceId === null) { throw new Error("Invalid user or device ID"); } - await getSFUConfigWithOpenID( + await getSFUConfig({ client, - { userId, deviceId, memberId: "" }, - customLivekitUrlTextBuffer, + membership: { userId, deviceId, memberId: "" }, + serviceUrl: customLivekitUrlTextBuffer, roomId, - ); + slotId: "org.example.fake-slot#room", + role: "publisher", + logger, + }); setCustomLivekitUrlUpdateError(null); setCustomLivekitUrl(customLivekitUrlTextBuffer); } catch { diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index a9e3abc26..13eb0f8ed 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -570,7 +570,7 @@ export function createCallViewModel$( ownMembershipIdentity, client, roomId: matrixRoom.roomId, - matrixRTCMode, + slotId: matrixRTCSession.slotId, }), ); @@ -579,6 +579,7 @@ export function createCallViewModel$( new ECConnectionFactory( client, matrixRoom.roomId, + matrixRTCSession.slotId, mediaDevices, trackProcessorState$, livekitKeyProvider, @@ -659,7 +660,6 @@ export function createCallViewModel$( ).pipe(map(([delayId]) => delayId ?? null)), matrixRTCSession.delayId ?? null, ), - matrixRTCMode, logger: logger.getChild(`[${Date.now()}]`), }); diff --git a/src/state/CallViewModel/CallViewModelTestUtils.ts b/src/state/CallViewModel/CallViewModelTestUtils.ts index 0863ea2b7..a38d12f3d 100644 --- a/src/state/CallViewModel/CallViewModelTestUtils.ts +++ b/src/state/CallViewModel/CallViewModelTestUtils.ts @@ -229,6 +229,7 @@ export function withCallViewModel(mode: MatrixRTCMode) { existingSFUConfig: sfuConfig, client: room.client, roomId: room.roomId, + slotId: "m.call#ROOM", livekitRoomFactory, }, logger, diff --git a/src/state/CallViewModel/localMember/LocalMember.test.ts b/src/state/CallViewModel/localMember/LocalMember.test.ts index da4da45aa..2dd5dfdce 100644 --- a/src/state/CallViewModel/localMember/LocalMember.test.ts +++ b/src/state/CallViewModel/localMember/LocalMember.test.ts @@ -66,7 +66,7 @@ import { type Publisher } from "./Publisher"; import { initializeWidget } from "../../../widget"; import { nullHostBridge } from "../../../HostBridge"; import { type LocalTransport } from "./LocalTransport"; -import * as openIDSFU from "../../../livekit/openIDSFU"; +import * as livekitAuth from "../../../livekit/auth"; initializeWidget(); @@ -237,7 +237,6 @@ describe("LocalMembership", () => { ownMembershipIdentity: ownMemberMock, client: mockedClient, delayId$: constant(null), - matrixRTCMode: MATRIX_RTC_MODE, }; beforeEach(() => { @@ -306,6 +305,7 @@ describe("LocalMembership", () => { scope, ...defaultCreateLocalMemberValues, matrixRTCSession: { + slotId: "m.call#ROOM", updateCallIntent: vi.fn().mockImplementation(reject), leave: vi.fn(), }, @@ -359,7 +359,7 @@ describe("LocalMembership", () => { } as unknown as Connection; const authCallSpy = vi - .spyOn(openIDSFU, "getSFUConfigWithOpenID") + .spyOn(livekitAuth, "getSFUConfig") .mockImplementation(() => mockedClient.getOpenIdToken()); afterEach(() => authCallSpy.mockClear()); @@ -412,18 +412,16 @@ describe("LocalMembership", () => { // Delegation is supported in this test case, so go on to check that // LocalMember actually performs delegation const expectDelegation = (delayId: string) => - expect(authCallSpy).toHaveBeenLastCalledWith( - mockedClient, - ownMemberMock, - "a", - "!test-room-id:example.org", - { - matrixRTCMode: MATRIX_RTC_MODE, - delayEndpointBaseUrl: "https://matrix.example.org", - delayId, - }, - expect.anything(), - ); + expect(authCallSpy).toHaveBeenLastCalledWith({ + client: mockedClient, + membership: ownMemberMock, + serviceUrl: "a", + roomId: "!test-room-id:example.org", + role: "publisher", + delayEndpointBaseUrl: "https://matrix.example.org", + delayId, + logger: expect.anything(), + }); expectDelegation("leave1"); delayId$.next("leave2"); // Can change delegated leaves diff --git a/src/state/CallViewModel/localMember/LocalMember.ts b/src/state/CallViewModel/localMember/LocalMember.ts index 29c0da6eb..4aad67b08 100644 --- a/src/state/CallViewModel/localMember/LocalMember.ts +++ b/src/state/CallViewModel/localMember/LocalMember.ts @@ -16,7 +16,6 @@ import { MediaDeviceFailure, } from "livekit-client"; import { observeParticipantEvents } from "@livekit/components-core"; -import { type MatrixClient } from "matrix-js-sdk"; import { Status as RTCSessionStatus, type LivekitTransport, @@ -80,7 +79,7 @@ import { } from "../remoteMembers/Connection.ts"; import { type HomeserverConnected } from "./HomeserverConnected.ts"; import { type LocalTransport } from "./LocalTransport.ts"; -import { getSFUConfigWithOpenID } from "../../../livekit/openIDSFU.ts"; +import { type ClientOpenIDParts, getSFUConfig } from "../../../livekit/auth"; export enum TransportState { /** Not even a transport is available to the LocalMembership */ @@ -150,15 +149,17 @@ interface Props { roomId: string; ownMembershipIdentity: CallMembershipIdentityParts; localTransport$: Observable; - client: Pick; - matrixRTCSession: Pick; + client: ClientOpenIDParts; + matrixRTCSession: Pick< + MatrixRTCSession, + "slotId" | "updateCallIntent" | "leave" + >; /** Whether to hide the screen-sharing button. */ hideScreensharing: boolean; /** The application hosting Element Call, to be kept informed of join/leave. */ hostBridge: HostBridge; baseUrl: string; delayId$: Behavior; - matrixRTCMode: MatrixRTCMode; logger: Logger; } @@ -206,7 +207,6 @@ export const createLocalMembership$ = ({ hostBridge, ownMembershipIdentity, delayId$, - matrixRTCMode, }: Props): { /** * This request to start audio and video tracks. @@ -713,14 +713,17 @@ export const createLocalMembership$ = ({ // This will technically cause the service to issue a new JWT token, // but it's safe to discard. We're only interested in triggering // delegation. - await getSFUConfigWithOpenID( + await getSFUConfig({ client, - ownMembershipIdentity, - joinParams.transport.livekit_service_url, + membership: ownMembershipIdentity, + serviceUrl: joinParams.transport.livekit_service_url, roomId, - { matrixRTCMode, delayEndpointBaseUrl: baseUrl, delayId }, + slotId: matrixRTCSession.slotId, + role: "publisher", + delayEndpointBaseUrl: baseUrl, + delayId, logger, - ); + }); } catch (e) { // TODO: Surface this to the user as a service interruption? logger.error( diff --git a/src/state/CallViewModel/localMember/LocalTransport.test.ts b/src/state/CallViewModel/localMember/LocalTransport.test.ts index 90d0d15e5..805142176 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.test.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.test.ts @@ -22,13 +22,12 @@ import { MatrixRTCTransportMissingError, FailToGetOpenIdToken, } from "../../../utils/errors"; -import * as openIDSFU from "../../../livekit/openIDSFU"; +import * as livekitAuth from "../../../livekit/auth"; import { customLivekitUrl } from "../../../settings/settings"; import { testJWTToken } from "../../../utils/test-fixtures"; -import { MatrixRTCMode } from "../../../config/ConfigOptions"; describe("LocalTransport", () => { - const openIdResponse: openIDSFU.SFUConfig = { + const openIdResponse: livekitAuth.SFUConfig = { url: "https://lk.example.org", jwt: testJWTToken, livekitAlias: "Akph4alDMhen", @@ -47,10 +46,9 @@ describe("LocalTransport", () => { getDomain: () => "example.org", // These won't be called in this error path but satisfy the type getOpenIdToken: vi.fn(), - getDeviceId: vi.fn(), }, ownMembershipIdentity: ownMemberMock, - matrixRTCMode: MatrixRTCMode.Compatibility, + slotId: "m.call#ROOM", }), ).rejects.toThrow(new MatrixRTCTransportMissingError("example.org")); }); @@ -60,7 +58,7 @@ describe("LocalTransport", () => { mockConfig({ livekit: { livekit_service_url: "https://lk.example.org" }, }); - vi.spyOn(openIDSFU, "getSFUConfigWithOpenID").mockRejectedValue( + vi.spyOn(livekitAuth, "getSFUConfig").mockRejectedValue( new FailToGetOpenIdToken(new Error("no openid")), ); @@ -72,10 +70,9 @@ describe("LocalTransport", () => { // eslint-disable-next-line @typescript-eslint/naming-convention _unstable_getRTCTransports: async () => Promise.resolve([]), getOpenIdToken: vi.fn(), - getDeviceId: vi.fn(), }, ownMembershipIdentity: ownMemberMock, - matrixRTCMode: MatrixRTCMode.Compatibility, + slotId: "m.call#ROOM", }), ).rejects.toThrow(new FailToGetOpenIdToken(new Error("no openid"))); }); @@ -86,7 +83,7 @@ describe("LocalTransport", () => { livekit: { livekit_service_url: "https://lk.example.org" }, }); - vi.spyOn(openIDSFU, "getSFUConfigWithOpenID").mockResolvedValue({ + vi.spyOn(livekitAuth, "getSFUConfig").mockResolvedValue({ url: "https://lk.example.org", jwt: "jwt", livekitAlias: "Akph4alDMhen", @@ -101,10 +98,9 @@ describe("LocalTransport", () => { _unstable_getRTCTransports: async () => Promise.resolve([]), getDomain: () => "example.org", getOpenIdToken: vi.fn(), - getDeviceId: vi.fn(), }, ownMembershipIdentity: ownMemberMock, - matrixRTCMode: MatrixRTCMode.Compatibility, + slotId: "m.call#ROOM", }), ).toStrictEqual({ transport: { @@ -132,13 +128,12 @@ describe("LocalTransport", () => { localTransportOpts = { ownMembershipIdentity: ownMemberMock, roomId: "!example_room_id", - matrixRTCMode: MatrixRTCMode.Compatibility, + slotId: "m.call#ROOM", client: { getDomain: vi.fn().mockReturnValue("example.org"), // eslint-disable-next-line @typescript-eslint/naming-convention _unstable_getRTCTransports: vi.fn().mockResolvedValue([]), getOpenIdToken: vi.fn(), - getDeviceId: vi.fn(), }, }; }); @@ -151,9 +146,7 @@ describe("LocalTransport", () => { mockConfig({ livekit: { livekit_service_url: "https://lk.example.org" }, }); - vi.spyOn(openIDSFU, "getSFUConfigWithOpenID").mockResolvedValue( - openIdResponse, - ); + vi.spyOn(livekitAuth, "getSFUConfig").mockResolvedValue(openIdResponse); expect(await getLocalTransport(localTransportOpts)).toStrictEqual({ transport: { @@ -171,9 +164,7 @@ describe("LocalTransport", () => { it("supports getting transport via user settings", async () => { customLivekitUrl.setValue("https://lk.example.org"); - vi.spyOn(openIDSFU, "getSFUConfigWithOpenID").mockResolvedValue( - openIdResponse, - ); + vi.spyOn(livekitAuth, "getSFUConfig").mockResolvedValue(openIdResponse); expect(await getLocalTransport(localTransportOpts)).toStrictEqual({ transport: { @@ -193,9 +184,7 @@ describe("LocalTransport", () => { localTransportOpts.client._unstable_getRTCTransports.mockResolvedValue([ { type: "livekit", livekit_service_url: "https://lk.example.org" }, ]); - vi.spyOn(openIDSFU, "getSFUConfigWithOpenID").mockResolvedValue( - openIdResponse, - ); + vi.spyOn(livekitAuth, "getSFUConfig").mockResolvedValue(openIdResponse); expect(await getLocalTransport(localTransportOpts)).toStrictEqual({ transport: { @@ -215,7 +204,7 @@ describe("LocalTransport", () => { localTransportOpts.client._unstable_getRTCTransports.mockResolvedValue([ { type: "livekit", livekit_service_url: "https://lk.example.org" }, ]); - vi.spyOn(openIDSFU, "getSFUConfigWithOpenID").mockRejectedValue( + vi.spyOn(livekitAuth, "getSFUConfig").mockRejectedValue( new FailToGetOpenIdToken(new Error("Test driven error")), ); @@ -229,14 +218,13 @@ describe("LocalTransport", () => { getLocalTransport({ ownMembershipIdentity: ownMemberMock, roomId: "!example_room_id", - matrixRTCMode: MatrixRTCMode.Compatibility, + slotId: "m.call#ROOM", client: { getDomain: () => "example.org", // eslint-disable-next-line @typescript-eslint/naming-convention _unstable_getRTCTransports: async () => Promise.resolve([]), // These won't be called in this error path but satisfy the type getOpenIdToken: vi.fn(), - getDeviceId: vi.fn(), }, }), ).rejects.toThrow(new MatrixRTCTransportMissingError("example.org")); diff --git a/src/state/CallViewModel/localMember/LocalTransport.ts b/src/state/CallViewModel/localMember/LocalTransport.ts index 3534b3fec..09880c034 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.ts @@ -17,21 +17,20 @@ import { NoMatrix2AuthorizationService, } from "../../../utils/errors.ts"; import { - getSFUConfigWithOpenID, + getSFUConfig, type SFUConfig, - type OpenIDClientParts, -} from "../../../livekit/openIDSFU.ts"; + type ClientOpenIDParts, +} from "../../../livekit/auth"; import { customLivekitUrl } from "../../../settings/settings.ts"; import { RtcTransportAutoDiscovery } from "./RtcTransportAutoDiscovery.ts"; -import { type MatrixRTCMode } from "../../../config/ConfigOptions.ts"; interface Props { ownMembershipIdentity: CallMembershipIdentityParts; client: Pick & - OpenIDClientParts; + ClientOpenIDParts; // Used by the jwt service to create the livekit room and compute the livekit alias. roomId: string; - matrixRTCMode: MatrixRTCMode; + slotId: string; } export interface LocalTransport { @@ -58,7 +57,7 @@ export async function getLocalTransport({ ownMembershipIdentity, client, roomId, - matrixRTCMode, + slotId, }: Props): Promise { const logger = rootLogger.getChild("[LocalTransport]"); const discovery = new RtcTransportAutoDiscovery({ @@ -79,9 +78,9 @@ export async function getLocalTransport({ try { return await doOpenIdAndJWTFromUrl( transport, - matrixRTCMode, ownMembershipIdentity, roomId, + slotId, client, logger, ); @@ -96,35 +95,36 @@ export async function getLocalTransport({ /** * Utility to ensure the user can authenticate with the SFU. - * We will call `getSFUConfigWithOpenID` once per transport here as it's our + * We will call `getSFUConfig` once per transport here as it's our * only mechanism of validation. This means we will also ask the * homeserver for a OpenID token a few times. Since OpenID tokens are single * use we don't want to risk any issues by re-using a token. * * @param transport The transport to authenticate with. - * @param matrixRTCMode Whether to force the JWT endpoint to be used. * @param membership The identity of the local member. * @param roomId The room ID to use for the JWT. + * @param slotId The slot ID to use for the JWT. * @param client The client to use for the OpenID token. * * @throws FailToGetOpenIdToken, NoMatrix2AuthorizationService */ async function doOpenIdAndJWTFromUrl( transport: LivekitTransport, - matrixRTCMode: MatrixRTCMode, membership: CallMembershipIdentityParts, roomId: string, - client: Pick & OpenIDClientParts, - logger?: Logger, + slotId: string, + client: Pick & ClientOpenIDParts, + logger: Logger, ): Promise { - const sfuConfig = await getSFUConfigWithOpenID( + const sfuConfig = await getSFUConfig({ client, membership, - transport.livekit_service_url, + serviceUrl: transport.livekit_service_url, roomId, - { matrixRTCMode }, + slotId, + role: "publisher", logger, - ); + }); return { transport, sfuConfig, diff --git a/src/state/CallViewModel/remoteMembers/Connection.test.ts b/src/state/CallViewModel/remoteMembers/Connection.test.ts index aa2429f88..1c0e8ee20 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.test.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.test.ts @@ -8,7 +8,6 @@ Please see LICENSE in the repository root for full details. import { afterEach, - beforeEach, describe, expect, it, @@ -38,14 +37,14 @@ import { type ConnectionOpts, } from "./Connection.ts"; import { ObservableScope } from "../../ObservableScope.ts"; -import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts"; import { FailToGetOpenIdToken } from "../../../utils/errors.ts"; import { testJWTToken } from "../../../utils/test-fixtures.ts"; import { mockRemoteParticipant, ownMemberMock } from "../../../utils/test.ts"; +import { type ClientOpenIDParts } from "../../../livekit/auth"; let testScope: ObservableScope; -let client: MockedObject; +let client: MockedObject; let fakeLivekitRoom: MockedObject; @@ -53,6 +52,7 @@ let localParticipantEventEmiter: EventEmitter; let fakeLocalParticipant: MockedObject; const ROOM_ID = "!roomID:example.org"; +const SLOT_ID = "m.call#ROOM"; const livekitFocus: LivekitTransport = { livekit_service_url: "https://matrix-rtc.example.org/livekit/jwt", @@ -61,15 +61,14 @@ const livekitFocus: LivekitTransport = { function setupTest(): void { testScope = new ObservableScope(); - client = vi.mocked({ + client = vi.mocked({ getOpenIdToken: vi.fn().mockResolvedValue({ access_token: "rYsmGUEwNjKgJYyeNUkZseJN", token_type: "Bearer", matrix_server_name: "example.org", expires_in: 3600, }), - getDeviceId: vi.fn().mockReturnValue("ABCDEF"), - } as unknown as OpenIDClientParts); + }); localParticipantEventEmiter = new EventEmitter(); @@ -113,15 +112,16 @@ function setupTest(): void { function setupRemoteConnection(): Connection { const opts: ConnectionOpts = { - client: client, + client, roomId: ROOM_ID, + slotId: SLOT_ID, transport: livekitFocus, scope: testScope, ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, }; - fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, () => { + fetchMock.post(`${livekitFocus.livekit_service_url}/get_token`, () => { return { status: 200, body: { @@ -152,25 +152,13 @@ afterEach(() => { }); describe("Start connection states", () => { - beforeEach(() => { - fetchMock.post( - `https://matrix-rtc.example.org/livekit/jwt/get_token`, - () => { - return { - // Return a non-retryable error, if not, the retry logic will - // wait and fail the test with a timeout. - status: 404, - }; - }, - ); - }); - it("start in initialized state", () => { setupTest(); const opts: ConnectionOpts = { - client: client, + client, roomId: ROOM_ID, + slotId: SLOT_ID, transport: livekitFocus, scope: testScope, ownMembershipIdentity: ownMemberMock, @@ -186,8 +174,9 @@ describe("Start connection states", () => { vi.useFakeTimers(); const opts: ConnectionOpts = { - client: client, + client, roomId: ROOM_ID, + slotId: SLOT_ID, transport: livekitFocus, scope: testScope, ownMembershipIdentity: ownMemberMock, @@ -238,8 +227,9 @@ describe("Start connection states", () => { vi.useFakeTimers(); const opts: ConnectionOpts = { - client: client, + client, roomId: ROOM_ID, + slotId: SLOT_ID, transport: livekitFocus, scope: testScope, ownMembershipIdentity: ownMemberMock, @@ -255,17 +245,20 @@ describe("Start connection states", () => { onTestFinished(() => s.unsubscribe()); const deferredSFU = Promise.withResolvers(); - // mock the /sfu/get call - fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, async () => { - await deferredSFU.promise; - return { - status: 500, - body: { - errcode: "M_LOOKUP_FAILED", - error: "Failed to look up user info from homeserver", - }, - }; - }); + // mock the /get_token call + fetchMock.post( + `${livekitFocus.livekit_service_url}/get_token`, + async () => { + await deferredSFU.promise; + return { + status: 500, + body: { + errcode: "M_LOOKUP_FAILED", + error: "Failed to look up user info from homeserver", + }, + }; + }, + ); connection.start().catch(() => { // expected to throw @@ -300,8 +293,9 @@ describe("Start connection states", () => { vi.useFakeTimers(); const opts: ConnectionOpts = { - client: client, + client, roomId: ROOM_ID, + slotId: SLOT_ID, transport: livekitFocus, scope: testScope, ownMembershipIdentity: ownMemberMock, @@ -317,8 +311,8 @@ describe("Start connection states", () => { onTestFinished(() => s.unsubscribe()); const deferredSFU = Promise.withResolvers(); - // mock the /sfu/get call - fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, () => { + // mock the /get_token call + fetchMock.post(`${livekitFocus.livekit_service_url}/get_token`, () => { return { status: 200, body: { @@ -560,6 +554,7 @@ describe("remote track logging", () => { { client, roomId: ROOM_ID, + slotId: SLOT_ID, transport: livekitFocus, scope: testScope, ownMembershipIdentity: ownMemberMock, diff --git a/src/state/CallViewModel/remoteMembers/Connection.ts b/src/state/CallViewModel/remoteMembers/Connection.ts index ea9447e99..da8c34daa 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.ts @@ -28,11 +28,6 @@ import { BehaviorSubject } from "rxjs"; import { type Logger } from "matrix-js-sdk/lib/logger"; import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; -import { - getSFUConfigWithOpenID, - type OpenIDClientParts, - type SFUConfig, -} from "../../../livekit/openIDSFU.ts"; import { type Behavior } from "../../Behavior.ts"; import { type ObservableScope } from "../../ObservableScope.ts"; import { @@ -43,6 +38,11 @@ import { SFURoomCreationRestrictedError, UnknownCallError, } from "../../../utils/errors.ts"; +import { + getSFUConfig, + type ClientOpenIDParts, + type SFUConfig, +} from "../../../livekit/auth"; export interface ConnectionOpts { /** @@ -55,9 +55,11 @@ export interface ConnectionOpts { /** The media transport to connect to. */ transport: LivekitTransport; /** The Matrix client to use for OpenID and SFU config requests. */ - client: OpenIDClientParts; - /** The room ID this connection is associated with. */ + client: ClientOpenIDParts; + /** The ID of the Matrix room in which the session takes place. */ roomId: string; + /** The ID of the MatrixRTC slot in which the session takes place. */ + slotId: string; /** The observable scope to use for this connection. */ scope: ObservableScope; @@ -123,8 +125,9 @@ export class Connection { protected stopped = false; // TODO: can we just keep the ConnectionOpts object instead of spreading? - private readonly client: OpenIDClientParts; + private readonly client: ClientOpenIDParts; private readonly roomId: string; + private readonly slotId: string; private readonly logger: Logger; private readonly ownMembershipIdentity: CallMembershipIdentityParts; private readonly existingSFUConfig?: SFUConfig; @@ -139,6 +142,7 @@ export class Connection { this.ownMembershipIdentity = opts.ownMembershipIdentity; this.existingSFUConfig = opts.existingSFUConfig; this.roomId = opts.roomId; + this.slotId = opts.slotId; this.logger = logger.getChild( `[Connection ${JSON.stringify(opts.transport)}]`, ); @@ -292,7 +296,7 @@ export class Connection { try { this._state$.next({ state: "authenticating" }); // We should already have this information after creating the localTransport. - // only call getSFUConfigWithOpenID for connections where we do not have a token yet. (existingJwtTokenData === undefined) + // only call getSFUConfig for connections where we do not have a token yet. (existingJwtTokenData === undefined) const { url, jwt, livekitAlias } = this.existingSFUConfig ?? (await this.getSFUConfigForRemoteConnection()); @@ -308,7 +312,7 @@ export class Connection { // If we were stopped while fetching the config, don't proceed to connect if (this.stopped) return; - // Setup observer once we are done with getSFUConfigWithOpenID + // Setup observer once we are done with getSFUConfig connectionStateObserver(this.livekitRoom) .pipe(this.scope.bind()) .subscribe((livekitState) => @@ -380,15 +384,15 @@ export class Connection { protected async getSFUConfigForRemoteConnection(): Promise { // This will only be called for sfu's where we do not publish ourselves. // For the local connection we will use the existingJwtTokenData - return await getSFUConfigWithOpenID( - this.client, - this.ownMembershipIdentity, - this.transport.livekit_service_url, - this.roomId, - // dont pass any custom opts for the subscribe only connections - {}, - this.logger, - ); + return await getSFUConfig({ + client: this.client, + membership: this.ownMembershipIdentity, + serviceUrl: this.transport.livekit_service_url, + roomId: this.roomId, + slotId: this.slotId, + role: "subscriber", + logger: this.logger, + }); } /** diff --git a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts index 96a28d5ad..4cb1371e1 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts @@ -20,10 +20,6 @@ import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { type ObservableScope } from "../../ObservableScope.ts"; import { Connection } from "./Connection.ts"; -import type { - OpenIDClientParts, - SFUConfig, -} from "../../../livekit/openIDSFU.ts"; import type { MediaDevices } from "../../MediaDevices.ts"; import type { Behavior } from "../../Behavior.ts"; import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx"; @@ -39,6 +35,7 @@ import { noiseSuppressionSetting, autoGainControlSetting, } from "../../../settings/settings.ts"; +import { type ClientOpenIDParts, type SFUConfig } from "../../../livekit/auth"; // TODO evaluate if this should be done like the Publisher Factory export interface ConnectionFactory { @@ -58,7 +55,8 @@ export class ECConnectionFactory implements ConnectionFactory { * Creates a ConnectionFactory for LiveKit connections. * * @param client - The OpenID client parts for authentication, needed to get openID and JWT tokens. - * @param roomId - The current room ID. + * @param roomId - The ID of the Matrix room in which the session takes place. + * @param slotId - The ID of the MatrixRTC slot in which the session takes place. * @param devices - Used for video/audio out/in capture options. * @param processorState$ - Effects like background blur (only for publishing connection?) * @param livekitKeyProvider - Optional key provider for end-to-end encryption. @@ -66,8 +64,9 @@ export class ECConnectionFactory implements ConnectionFactory { * @param livekitRoomFactory - Optional factory function (for testing) to create LivekitRoom instances. If not provided, a default factory is used. */ public constructor( - private client: OpenIDClientParts, + private client: ClientOpenIDParts, private readonly roomId: string, + private readonly slotId: string, private devices: MediaDevices, private processorState$: Behavior, livekitKeyProvider: BaseKeyProvider | undefined, @@ -112,6 +111,7 @@ export class ECConnectionFactory implements ConnectionFactory { { existingSFUConfig: sfuConfig, roomId: this.roomId, + slotId: this.slotId, transport, client: this.client, scope: scope, diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts index 18f54f021..2897cbe0e 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts @@ -31,7 +31,7 @@ import { isLocalTransport, type LocalTransport, } from "../localMember/LocalTransport.ts"; -import { type SFUConfig } from "../../../livekit/openIDSFU.ts"; +import { type SFUConfig } from "../../../livekit/auth"; export class ConnectionManagerData { private readonly store: Map< diff --git a/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts b/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts index 411ae2001..d954c2e39 100644 --- a/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts +++ b/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts @@ -14,7 +14,7 @@ import EventEmitter from "events"; import { ObservableScope } from "../../ObservableScope.ts"; import { ECConnectionFactory } from "./ConnectionFactory.ts"; -import type { OpenIDClientParts } from "../../../livekit/openIDSFU.ts"; +import { type ClientOpenIDParts } from "../../../livekit/auth"; import { exampleTransport, mockMediaDevices, @@ -51,13 +51,12 @@ vi.mock("livekit-client", async (importOriginal) => { }); let testScope: ObservableScope; -let mockClient: OpenIDClientParts; +let mockClient: ClientOpenIDParts; beforeEach(() => { testScope = new ObservableScope(); mockClient = { getOpenIdToken: vi.fn().mockReturnValue(""), - getDeviceId: vi.fn().mockReturnValue("DEV000"), }; }); @@ -79,6 +78,7 @@ describe("ECConnectionFactory - Audio inputs options", () => { const ecConnectionFactory = new ECConnectionFactory( mockClient, "!roomid:example.org", + "m.call#ROOM", mockMediaDevices({}), new BehaviorSubject({ supported: true, @@ -121,6 +121,7 @@ describe("ECConnectionFactory - ControlledAudioDevice", () => { const ecConnectionFactory = new ECConnectionFactory( mockClient, "!roomid:example.org", + "m.call#ROOM", mockMediaDevices({ audioOutput: { available$: constant(new Map()), @@ -162,6 +163,7 @@ describe("ECConnectionFactory - Camera quality settings", () => { const ecConnectionFactory = new ECConnectionFactory( mockClient, "!roomid:example.org", + "m.call#ROOM", mockMediaDevices({}), new BehaviorSubject({ supported: true, @@ -199,6 +201,7 @@ describe("ECConnectionFactory - Camera quality settings", () => { const ecConnectionFactory = new ECConnectionFactory( mockClient, "!roomid:example.org", + "m.call#ROOM", mockMediaDevices({}), new BehaviorSubject({ supported: true, @@ -237,6 +240,7 @@ describe("ECConnectionFactory - Camera quality settings", () => { const ecConnectionFactory = new ECConnectionFactory( mockClient, "!roomid:example.org", + "m.call#ROOM", mockMediaDevices({}), new BehaviorSubject({ supported: true, diff --git a/src/state/CallViewModel/remoteMembers/integration.test.ts b/src/state/CallViewModel/remoteMembers/integration.test.ts index 4c0f4fea7..07b8d6f58 100644 --- a/src/state/CallViewModel/remoteMembers/integration.test.ts +++ b/src/state/CallViewModel/remoteMembers/integration.test.ts @@ -19,7 +19,7 @@ import { trackEpoch, } from "../../ObservableScope.ts"; import { ECConnectionFactory } from "./ConnectionFactory.ts"; -import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts"; +import { type ClientOpenIDParts } from "../../../livekit/auth"; import { mockMediaDevices, mockRtcMembership, @@ -40,7 +40,7 @@ import { localRtcMember, testJWTToken } from "../../../utils/test-fixtures.ts"; let testScope: ObservableScope; let ecConnectionFactory: ECConnectionFactory; -let mockClient: OpenIDClientParts; +let mockClient: ClientOpenIDParts; let lkRoomFactory: () => LivekitRoom; const createdMockLivekitRooms: Map = new Map(); @@ -49,7 +49,6 @@ beforeEach(() => { testScope = new ObservableScope(); mockClient = { getOpenIdToken: vi.fn().mockReturnValue(""), - getDeviceId: vi.fn().mockReturnValue("DEV000"), }; lkRoomFactory = vi.fn().mockImplementation(() => { @@ -71,6 +70,7 @@ beforeEach(() => { ecConnectionFactory = new ECConnectionFactory( mockClient, "!roomid:example.org", + "m.call#ROOM", mockMediaDevices({}), new BehaviorSubject({ supported: true, @@ -82,7 +82,7 @@ beforeEach(() => { ); //TODO a bit annoying to have to do a http mock? - fetchMock.post(`path:/sfu/get`, (url) => { + const sfuConfigResponse = (url: string): { status: number; body: object } => { const domain = new URL(url).hostname; // Extract the domain from the URL return { status: 200, @@ -91,7 +91,9 @@ beforeEach(() => { jwt: testJWTToken, }, }; - }); + }; + fetchMock.post(`path:/sfu/get`, sfuConfigResponse); + fetchMock.post(`path:/get_token`, sfuConfigResponse); }); afterEach(() => { diff --git a/src/utils/test.ts b/src/utils/test.ts index 48527184e..9f7f14b80 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -78,7 +78,7 @@ import { type RemoteScreenShareViewModel, } from "../state/media/RemoteScreenShareViewModel"; import { Connection } from "../state/CallViewModel/remoteMembers/Connection"; -import { type SFUConfig } from "../livekit/openIDSFU"; +import { type SFUConfig } from "../livekit/auth"; export function withFakeTimers(continuation: () => void): void { vi.useFakeTimers(); From 24b6a632dde26b047c6e7878a8bcc331e4d72893 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 12 Oct 2026 10:22:33 +0200 Subject: [PATCH 09/25] Let the publishing Connection authenticate itself getLocalTransport no longer authenticates with the SFU up front and hands the resulting config to the connection. Instead the ConnectionManager tags the connection whose transport equals the local one as the publisher, and every Connection fetches its own SFU config for its role. The local member now only joins the MatrixRTC session once its connection is authenticated, so that the LiveKit room exists before remote members try to get a subscriber token for it. Subscribers authenticating with the OpenID flow lack permission to create the room and would otherwise fail without retrying, causing a cascade of leaves and rejoins at the start of a call. --- src/state/CallViewModel/CallViewModel.ts | 16 +- .../CallViewModel/CallViewModelTestUtils.ts | 10 +- .../localMember/LocalMember.test.ts | 35 ++-- .../CallViewModel/localMember/LocalMember.ts | 63 ++++--- .../localMember/LocalTransport.test.ts | 178 +++--------------- .../localMember/LocalTransport.ts | 126 +------------ .../remoteMembers/Connection.test.ts | 6 + .../CallViewModel/remoteMembers/Connection.ts | 33 ++-- .../remoteMembers/ConnectionFactory.ts | 8 +- .../remoteMembers/ConnectionManager.test.ts | 8 +- .../remoteMembers/ConnectionManager.ts | 57 +++--- .../remoteMembers/ECConnectionFactory.test.ts | 5 + src/utils/test.ts | 8 - 13 files changed, 149 insertions(+), 404 deletions(-) diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 13eb0f8ed..216a1e617 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -112,10 +112,7 @@ import { enterRTCSession, TransportState, } from "./localMember/LocalMember.ts"; -import { - getLocalTransport, - type LocalTransport, -} from "./localMember/LocalTransport.ts"; +import { getLocalTransport } from "./localMember/LocalTransport.ts"; import { createKeyRotationSuppressed$, createMemberships$, @@ -216,7 +213,7 @@ export interface CallViewModelOptions { */ windowSize$: Behavior<{ width: number; height: number }>; /** Optional value overriding the local transport, for testing purposes. */ - localTransport?: LocalTransport; + localTransport?: LivekitTransport; /** Optional value overriding the connection factory, for testing purposes. */ connectionFactory?: ConnectionFactory; /** The version & compatibility mode of MatrixRTC that we should use. */ @@ -565,14 +562,7 @@ export function createCallViewModel$( const localTransport$ = options.localTransport ? constant(options.localTransport) - : from( - getLocalTransport({ - ownMembershipIdentity, - client, - roomId: matrixRoom.roomId, - slotId: matrixRTCSession.slotId, - }), - ); + : from(getLocalTransport(client)); const connectionFactory = options.connectionFactory ?? diff --git a/src/state/CallViewModel/CallViewModelTestUtils.ts b/src/state/CallViewModel/CallViewModelTestUtils.ts index a38d12f3d..95920eee7 100644 --- a/src/state/CallViewModel/CallViewModelTestUtils.ts +++ b/src/state/CallViewModel/CallViewModelTestUtils.ts @@ -30,7 +30,6 @@ import { type CallViewModelOptions, } from "./CallViewModel"; import { - exampleSfuConfig, exampleTransport, mockConfig, MockConnection, @@ -209,24 +208,21 @@ export function withCallViewModel(mode: MatrixRTCMode) { livekitRoomFactory, connectionState$, windowSize$, - localTransport: { - transport: exampleTransport, - sfuConfig: exampleSfuConfig, - }, + localTransport: exampleTransport, connectionFactory: { createConnection( scope, + role, transport, ownMembershipIdentity, logger, - sfuConfig, ) { return new MockConnection( { scope, + role, transport, ownMembershipIdentity, - existingSFUConfig: sfuConfig, client: room.client, roomId: room.roomId, slotId: "m.call#ROOM", diff --git a/src/state/CallViewModel/localMember/LocalMember.test.ts b/src/state/CallViewModel/localMember/LocalMember.test.ts index 2dd5dfdce..b1e309d58 100644 --- a/src/state/CallViewModel/localMember/LocalMember.test.ts +++ b/src/state/CallViewModel/localMember/LocalMember.test.ts @@ -65,7 +65,6 @@ import { import { type Publisher } from "./Publisher"; import { initializeWidget } from "../../../widget"; import { nullHostBridge } from "../../../HostBridge"; -import { type LocalTransport } from "./LocalTransport"; import * as livekitAuth from "../../../livekit/auth"; initializeWidget(); @@ -221,7 +220,10 @@ describe("LocalMembership", () => { processor: undefined, }), logger: logger, - createPublisherFactory: vi.fn(), + createPublisherFactory: vi.fn().mockReturnValue({ + destroy: vi.fn(), + createAndSetupTracks: vi.fn().mockResolvedValue(undefined), + }), joinMatrixRTC: async (): Promise => {}, homeserverConnected: { combined$: constant<[boolean, HomeserverDisconnectReason | null]>([ @@ -254,6 +256,7 @@ describe("LocalMembership", () => { afterEach(async () => { void (await fetchMock.flush()); fetchMock.reset(); + vi.clearAllMocks(); }); it("throws error on missing RTC config error", () => { @@ -322,20 +325,10 @@ describe("LocalMembership", () => { scope.end(); }); - const mockTransportConfig = { + const mockTransport = { livekit_service_url: "a", } as LivekitTransport; - const mockTransport = { - transport: mockTransportConfig, - sfuConfig: { - jwt: "foo", - livekitAlias: "bar", - livekitIdentity: "baz", - url: "bro", - }, - } as LocalTransport; - const connectionTransportAConnected = { livekitRoom: mockLivekitRoom({ localParticipant: { @@ -347,7 +340,7 @@ describe("LocalMembership", () => { state: "authenticated", livekitState: LivekitConnectionState.Connected, }), - transport: mockTransportConfig, + transport: mockTransport, } as Connection; const connectionTransportAConnecting = { ...connectionTransportAConnected, @@ -381,13 +374,13 @@ describe("LocalMembership", () => { if (delegationUrl !== null) fetchMock.post(delegationUrl, () => ({ status: 401, body: {} })); + const connectionManagerData = new ConnectionManagerData(); + connectionManagerData.add(connectionTransportAConnected, []); const localMembership = createLocalMembership$({ scope, ...defaultCreateLocalMemberValues, connectionManager: { - connectionManagerData$: constant( - new Epoch(new ConnectionManagerData()), - ), + connectionManagerData$: constant(new Epoch(connectionManagerData)), }, joinMatrixRTC, localTransport$: constant(mockTransport), @@ -399,7 +392,7 @@ describe("LocalMembership", () => { await flushPromises(); // Joins with timings appropriate for the level of delegation support expect(joinMatrixRTC).toHaveBeenCalledWith( - mockTransportConfig, + mockTransport, delayedLeaveTimings, ); @@ -491,7 +484,7 @@ describe("LocalMembership", () => { it("tracks livekit state correctly", async () => { const scope = new ObservableScope(); const connectionManagerData = new ConnectionManagerData(); - const localTransport$ = new Subject(); + const localTransport$ = new Subject(); const connectionManagerData$ = new BehaviorSubject( new Epoch(connectionManagerData), @@ -570,7 +563,7 @@ describe("LocalMembership", () => { }); ( - connectionManagerData2.getConnectionForTransport(mockTransportConfig)! + connectionManagerData2.getConnectionForTransport(mockTransport)! .state$ as BehaviorSubject ).next({ state: "authenticated", @@ -891,7 +884,7 @@ describe("LocalMembership", () => { state: "authenticated", livekitState: LivekitConnectionState.Connected, }), - transport: mockTransportConfig, + transport: mockTransport, livekitRoom: mockLivekitRoom({ localParticipant: mockLocalParticipant({ isScreenShareEnabled: false, diff --git a/src/state/CallViewModel/localMember/LocalMember.ts b/src/state/CallViewModel/localMember/LocalMember.ts index 4aad67b08..2fb6e0229 100644 --- a/src/state/CallViewModel/localMember/LocalMember.ts +++ b/src/state/CallViewModel/localMember/LocalMember.ts @@ -78,7 +78,6 @@ import { type Connection, } from "../remoteMembers/Connection.ts"; import { type HomeserverConnected } from "./HomeserverConnected.ts"; -import { type LocalTransport } from "./LocalTransport.ts"; import { type ClientOpenIDParts, getSFUConfig } from "../../../livekit/auth"; export enum TransportState { @@ -148,7 +147,7 @@ interface Props { homeserverConnected: HomeserverConnected; roomId: string; ownMembershipIdentity: CallMembershipIdentityParts; - localTransport$: Observable; + localTransport$: Observable; client: ClientOpenIDParts; matrixRTCSession: Pick< MatrixRTCSession, @@ -307,7 +306,7 @@ export const createLocalMembership$ = ({ // to whether delayed event delegation is supported const joinParams$ = scope.behavior( localTransport$.pipe( - switchMap(async ({ transport }) => { + switchMap(async (transport) => { const transportSupportsDelegation = checkDelegationSupport( transport.livekit_service_url + "/delegate_delayed_leave", `transport ${transport.livekit_service_url}`, @@ -329,7 +328,7 @@ export const createLocalMembership$ = ({ connectionManager.connectionManagerData$, localTransport$, ]).pipe( - map(([{ value: connectionData }, { transport }]) => + map(([{ value: connectionData }, transport]) => connectionData.getConnectionForTransport(transport), ), tap((connection) => { @@ -480,6 +479,10 @@ export const createLocalMembership$ = ({ ), ); + const authenticated$ = scope.behavior( + localConnectionState$.pipe(map(({ state }) => state === "authenticated")), + ); + const mediaState$: Behavior = scope.behavior( combineLatest([ localConnectionState$, @@ -664,34 +667,38 @@ export const createLocalMembership$ = ({ // Join and leave the session as needed scope.reconcile( - scope.behavior(combineLatest([joinParams$, joinAndPublishRequested$])), - async ([joinParams, shouldConnect]) => { - if (!joinParams) return; - // if shouldConnect=false we will do the disconnect as the cleanup from the previous reconcile iteration. - if (!shouldConnect) return; + scope.behavior( + combineLatest([joinParams$, joinAndPublishRequested$, authenticated$]), + ), + async ([joinParams, shouldConnect, authenticated]) => { const sessionConfig = Config.get().matrix_rtc_session; - try { - joinMatrixRTC( - joinParams.transport, - joinParams.delegationSupported - ? sessionConfig.delegated_delayed_leave - : sessionConfig.delayed_leave, - ); - } catch (error) { - logger.error("Error entering RTC session", error); - if (error instanceof Error) - setMatrixError(new MembershipManagerError(error)); - } - - return Promise.resolve(async (): Promise => { + // Only join once authenticated, to ensure the LiveKit room exists first. + // Otherwise, subscribers authenticating with the OpenID flow may fail to + // join (they lack permission to create the room) and never retry. + if (joinParams && shouldConnect && authenticated) { try { - // TODO Update matrixRTCSession to allow udpating the transport without leaving the session! - await matrixRTCSession.leave(1000); - } catch (e) { - logger.error("Error leaving RTC session", e); + joinMatrixRTC( + joinParams.transport, + joinParams.delegationSupported + ? sessionConfig.delegated_delayed_leave + : sessionConfig.delayed_leave, + ); + } catch (error) { + logger.error("Error entering RTC session", error); + if (error instanceof Error) + setMatrixError(new MembershipManagerError(error)); } - }); + + // Clean-up callback to leave the session + return Promise.resolve(async (): Promise => { + try { + await matrixRTCSession.leave(1000); + } catch (e) { + logger.error("Error leaving RTC session", e); + } + }); + } }, ); diff --git a/src/state/CallViewModel/localMember/LocalTransport.test.ts b/src/state/CallViewModel/localMember/LocalTransport.test.ts index 805142176..178b5bde4 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.test.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.test.ts @@ -16,125 +16,51 @@ import { } from "vitest"; import fetchMock from "fetch-mock"; -import { mockConfig, ownMemberMock } from "../../../utils/test"; +import { mockConfig } from "../../../utils/test"; import { getLocalTransport } from "./LocalTransport"; -import { - MatrixRTCTransportMissingError, - FailToGetOpenIdToken, -} from "../../../utils/errors"; -import * as livekitAuth from "../../../livekit/auth"; +import { MatrixRTCTransportMissingError } from "../../../utils/errors"; import { customLivekitUrl } from "../../../settings/settings"; -import { testJWTToken } from "../../../utils/test-fixtures"; +import { type MatrixClient } from "matrix-js-sdk"; describe("LocalTransport", () => { - const openIdResponse: livekitAuth.SFUConfig = { - url: "https://lk.example.org", - jwt: testJWTToken, - livekitAlias: "Akph4alDMhen", - livekitIdentity: "@lk_user:ABCDEF", - }; - beforeEach(() => vi.clearAllMocks()); it("throws if config is missing", async () => { await expect( getLocalTransport({ - roomId: "!room:example.org", - client: { - // eslint-disable-next-line @typescript-eslint/naming-convention - _unstable_getRTCTransports: async () => Promise.resolve([]), - getDomain: () => "example.org", - // These won't be called in this error path but satisfy the type - getOpenIdToken: vi.fn(), - }, - ownMembershipIdentity: ownMemberMock, - slotId: "m.call#ROOM", + _unstable_getRTCTransports: async () => Promise.resolve([]), + getDomain: () => "example.org", }), ).rejects.toThrow(new MatrixRTCTransportMissingError("example.org")); }); - it("throws FailToGetOpenIdToken when OpenID fetch fails", async () => { - // Provide a valid config so makeTransportInternal resolves a transport - mockConfig({ - livekit: { livekit_service_url: "https://lk.example.org" }, - }); - vi.spyOn(livekitAuth, "getSFUConfig").mockRejectedValue( - new FailToGetOpenIdToken(new Error("no openid")), - ); - - await expect( - getLocalTransport({ - roomId: "!example_room_id", - client: { - getDomain: () => "example.org", - // eslint-disable-next-line @typescript-eslint/naming-convention - _unstable_getRTCTransports: async () => Promise.resolve([]), - getOpenIdToken: vi.fn(), - }, - ownMembershipIdentity: ownMemberMock, - slotId: "m.call#ROOM", - }), - ).rejects.toThrow(new FailToGetOpenIdToken(new Error("no openid"))); - }); - it("returns preferred transport", async () => { - // Use config so transport discovery succeeds, but delay OpenID JWT fetch + // Use config so transport discovery succeeds mockConfig({ livekit: { livekit_service_url: "https://lk.example.org" }, }); - vi.spyOn(livekitAuth, "getSFUConfig").mockResolvedValue({ - url: "https://lk.example.org", - jwt: "jwt", - livekitAlias: "Akph4alDMhen", - livekitIdentity: ownMemberMock.userId + ":" + ownMemberMock.deviceId, - }); - expect( await getLocalTransport({ - roomId: "!room:example.org", - client: { - // eslint-disable-next-line @typescript-eslint/naming-convention - _unstable_getRTCTransports: async () => Promise.resolve([]), - getDomain: () => "example.org", - getOpenIdToken: vi.fn(), - }, - ownMembershipIdentity: ownMemberMock, - slotId: "m.call#ROOM", + _unstable_getRTCTransports: async () => Promise.resolve([]), + getDomain: () => "example.org", }), ).toStrictEqual({ - transport: { - livekit_service_url: "https://lk.example.org", - type: "livekit", - }, - sfuConfig: { - jwt: "jwt", - livekitAlias: "Akph4alDMhen", - livekitIdentity: "@alice:example.org:DEVICE", - url: "https://lk.example.org", - }, + livekit_service_url: "https://lk.example.org", + type: "livekit", }); }); - type LocalTransportProps = Parameters[0]; - describe("transport configuration mechanisms", () => { - let localTransportOpts: LocalTransportProps & { - client: MockedObject; - }; + let client: MockedObject< + Pick + >; beforeEach(() => { mockConfig({}); customLivekitUrl.setValue(customLivekitUrl.defaultValue); - localTransportOpts = { - ownMembershipIdentity: ownMemberMock, - roomId: "!example_room_id", - slotId: "m.call#ROOM", - client: { - getDomain: vi.fn().mockReturnValue("example.org"), - // eslint-disable-next-line @typescript-eslint/naming-convention - _unstable_getRTCTransports: vi.fn().mockResolvedValue([]), - getOpenIdToken: vi.fn(), - }, + client = { + getDomain: vi.fn().mockReturnValue("example.org"), + _unstable_getRTCTransports: vi.fn().mockResolvedValue([]), }; }); @@ -146,86 +72,38 @@ describe("LocalTransport", () => { mockConfig({ livekit: { livekit_service_url: "https://lk.example.org" }, }); - vi.spyOn(livekitAuth, "getSFUConfig").mockResolvedValue(openIdResponse); - expect(await getLocalTransport(localTransportOpts)).toStrictEqual({ - transport: { - livekit_service_url: "https://lk.example.org", - type: "livekit", - }, - sfuConfig: { - jwt: "e30=.eyJzdWIiOiJAbWU6ZXhhbXBsZS5vcmc6QUJDREVGIiwidmlkZW8iOnsicm9vbSI6IiFleGFtcGxlX3Jvb21faWQifX0=.e30=", - livekitAlias: "Akph4alDMhen", - livekitIdentity: "@lk_user:ABCDEF", - url: "https://lk.example.org", - }, + expect(await getLocalTransport(client)).toStrictEqual({ + livekit_service_url: "https://lk.example.org", + type: "livekit", }); }); it("supports getting transport via user settings", async () => { customLivekitUrl.setValue("https://lk.example.org"); - vi.spyOn(livekitAuth, "getSFUConfig").mockResolvedValue(openIdResponse); - expect(await getLocalTransport(localTransportOpts)).toStrictEqual({ - transport: { - livekit_service_url: "https://lk.example.org", - type: "livekit", - }, - sfuConfig: { - jwt: "e30=.eyJzdWIiOiJAbWU6ZXhhbXBsZS5vcmc6QUJDREVGIiwidmlkZW8iOnsicm9vbSI6IiFleGFtcGxlX3Jvb21faWQifX0=.e30=", - livekitAlias: "Akph4alDMhen", - livekitIdentity: "@lk_user:ABCDEF", - url: "https://lk.example.org", - }, + expect(await getLocalTransport(client)).toStrictEqual({ + livekit_service_url: "https://lk.example.org", + type: "livekit", }); }); it("supports getting transport via backend", async () => { - localTransportOpts.client._unstable_getRTCTransports.mockResolvedValue([ + client._unstable_getRTCTransports.mockResolvedValue([ { type: "livekit", livekit_service_url: "https://lk.example.org" }, ]); - vi.spyOn(livekitAuth, "getSFUConfig").mockResolvedValue(openIdResponse); - expect(await getLocalTransport(localTransportOpts)).toStrictEqual({ - transport: { - livekit_service_url: "https://lk.example.org", - type: "livekit", - }, - sfuConfig: { - jwt: "e30=.eyJzdWIiOiJAbWU6ZXhhbXBsZS5vcmc6QUJDREVGIiwidmlkZW8iOnsicm9vbSI6IiFleGFtcGxlX3Jvb21faWQifX0=.e30=", - livekitAlias: "Akph4alDMhen", - livekitIdentity: "@lk_user:ABCDEF", - url: "https://lk.example.org", - }, + expect(await getLocalTransport(client)).toStrictEqual({ + livekit_service_url: "https://lk.example.org", + type: "livekit", }); }); - it("fails fast if the openID request fails for backend config", async () => { - localTransportOpts.client._unstable_getRTCTransports.mockResolvedValue([ - { type: "livekit", livekit_service_url: "https://lk.example.org" }, - ]); - vi.spyOn(livekitAuth, "getSFUConfig").mockRejectedValue( - new FailToGetOpenIdToken(new Error("Test driven error")), - ); - - await expect(getLocalTransport(localTransportOpts)).rejects.toThrow( - expect.any(FailToGetOpenIdToken), - ); - }); - it("throws if no options are available", async () => { await expect( getLocalTransport({ - ownMembershipIdentity: ownMemberMock, - roomId: "!example_room_id", - slotId: "m.call#ROOM", - client: { - getDomain: () => "example.org", - // eslint-disable-next-line @typescript-eslint/naming-convention - _unstable_getRTCTransports: async () => Promise.resolve([]), - // These won't be called in this error path but satisfy the type - getOpenIdToken: vi.fn(), - }, + getDomain: () => "example.org", + _unstable_getRTCTransports: async () => Promise.resolve([]), }), ).rejects.toThrow(new MatrixRTCTransportMissingError("example.org")); }); diff --git a/src/state/CallViewModel/localMember/LocalTransport.ts b/src/state/CallViewModel/localMember/LocalTransport.ts index 09880c034..891f70ccf 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.ts @@ -7,63 +7,24 @@ Please see LICENSE in the repository root for full details. import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { type MatrixClient } from "matrix-js-sdk"; -import { logger as rootLogger, type Logger } from "matrix-js-sdk/lib/logger"; -import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; +import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { Config } from "../../../config/Config.ts"; -import { - FailToGetOpenIdToken, - MatrixRTCTransportMissingError, - NoMatrix2AuthorizationService, -} from "../../../utils/errors.ts"; -import { - getSFUConfig, - type SFUConfig, - type ClientOpenIDParts, -} from "../../../livekit/auth"; +import { MatrixRTCTransportMissingError } from "../../../utils/errors.ts"; import { customLivekitUrl } from "../../../settings/settings.ts"; import { RtcTransportAutoDiscovery } from "./RtcTransportAutoDiscovery.ts"; -interface Props { - ownMembershipIdentity: CallMembershipIdentityParts; - client: Pick & - ClientOpenIDParts; - // Used by the jwt service to create the livekit room and compute the livekit alias. - roomId: string; - slotId: string; -} - -export interface LocalTransport { - transport: LivekitTransport; - sfuConfig: SFUConfig; -} - -export function isLocalTransport( - obj: LivekitTransport | LocalTransport, -): obj is LocalTransport { - return "transport" in obj && "sfuConfig" in obj; -} - /** - * Connects to the JWT service and determines the transport that the local member should use. - * - * @prop useOldJwtEndpoint Whether to set forceOldJwtEndpoint on the returned transport and to use the old JWT endpoint. - * This is used when the connection manager needs to know if it has to use the legacy endpoint which implies a string concatenated rtcBackendIdentity. - * (which is expected for non sticky event based rtc member events) - * @returns The transport to advertise in our MatrixRTC membership and publish media on. - * @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken + * Determines the transport to advertise in our MatrixRTC membership and publish + * media on. */ -export async function getLocalTransport({ - ownMembershipIdentity, - client, - roomId, - slotId, -}: Props): Promise { - const logger = rootLogger.getChild("[LocalTransport]"); +export async function getLocalTransport( + client: Pick, +): Promise { const discovery = new RtcTransportAutoDiscovery({ - client: client, + client, resolvedConfig: Config.get(), - logger: logger, + logger: rootLogger.getChild("[LocalTransport]"), }); const customUrl = customLivekitUrl.value$.value; @@ -75,72 +36,5 @@ export async function getLocalTransport({ if (transport === null) throw new MatrixRTCTransportMissingError(client.getDomain() ?? ""); - try { - return await doOpenIdAndJWTFromUrl( - transport, - ownMembershipIdentity, - roomId, - slotId, - client, - logger, - ); - } catch (e) { - logger.error( - `Failed to authenticate to transport ${transport.livekit_service_url}`, - e, - ); - throw mapAuthErrorToUserFriendlyError(e); - } -} - -/** - * Utility to ensure the user can authenticate with the SFU. - * We will call `getSFUConfig` once per transport here as it's our - * only mechanism of validation. This means we will also ask the - * homeserver for a OpenID token a few times. Since OpenID tokens are single - * use we don't want to risk any issues by re-using a token. - * - * @param transport The transport to authenticate with. - * @param membership The identity of the local member. - * @param roomId The room ID to use for the JWT. - * @param slotId The slot ID to use for the JWT. - * @param client The client to use for the OpenID token. - * - * @throws FailToGetOpenIdToken, NoMatrix2AuthorizationService - */ -async function doOpenIdAndJWTFromUrl( - transport: LivekitTransport, - membership: CallMembershipIdentityParts, - roomId: string, - slotId: string, - client: Pick & ClientOpenIDParts, - logger: Logger, -): Promise { - const sfuConfig = await getSFUConfig({ - client, - membership, - serviceUrl: transport.livekit_service_url, - roomId, - slotId, - role: "publisher", - logger, - }); - return { - transport, - sfuConfig, - }; -} - -function mapAuthErrorToUserFriendlyError(e: unknown): Error { - if ( - e instanceof FailToGetOpenIdToken || - e instanceof NoMatrix2AuthorizationService - ) { - // rethrow as is - return e; - } - // Catch others and rethrow as FailToGetOpenIdToken that has user friendly message. - return new FailToGetOpenIdToken( - e instanceof Error ? e : new Error(String(e)), - ); + return transport; } diff --git a/src/state/CallViewModel/remoteMembers/Connection.test.ts b/src/state/CallViewModel/remoteMembers/Connection.test.ts index 1c0e8ee20..21e79ef97 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.test.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.test.ts @@ -115,6 +115,7 @@ function setupRemoteConnection(): Connection { client, roomId: ROOM_ID, slotId: SLOT_ID, + role: "subscriber", transport: livekitFocus, scope: testScope, ownMembershipIdentity: ownMemberMock, @@ -159,6 +160,7 @@ describe("Start connection states", () => { client, roomId: ROOM_ID, slotId: SLOT_ID, + role: "subscriber", transport: livekitFocus, scope: testScope, ownMembershipIdentity: ownMemberMock, @@ -177,6 +179,7 @@ describe("Start connection states", () => { client, roomId: ROOM_ID, slotId: SLOT_ID, + role: "subscriber", transport: livekitFocus, scope: testScope, ownMembershipIdentity: ownMemberMock, @@ -230,6 +233,7 @@ describe("Start connection states", () => { client, roomId: ROOM_ID, slotId: SLOT_ID, + role: "subscriber", transport: livekitFocus, scope: testScope, ownMembershipIdentity: ownMemberMock, @@ -296,6 +300,7 @@ describe("Start connection states", () => { client, roomId: ROOM_ID, slotId: SLOT_ID, + role: "subscriber", transport: livekitFocus, scope: testScope, ownMembershipIdentity: ownMemberMock, @@ -555,6 +560,7 @@ describe("remote track logging", () => { client, roomId: ROOM_ID, slotId: SLOT_ID, + role: "subscriber", transport: livekitFocus, scope: testScope, ownMembershipIdentity: ownMemberMock, diff --git a/src/state/CallViewModel/remoteMembers/Connection.ts b/src/state/CallViewModel/remoteMembers/Connection.ts index da8c34daa..35c2e2c75 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.ts @@ -45,13 +45,10 @@ import { } from "../../../livekit/auth"; export interface ConnectionOpts { - /** - * For the local transport we already do know the jwt token and url. We can reuse it. - * On top the local transport will send additional data to the jwt server to use delayed event delegation. - */ - existingSFUConfig?: SFUConfig; /** The identity parts to use on this connection */ ownMembershipIdentity: CallMembershipIdentityParts; + /** Whether we want to publish or only subscribe on this connection. */ + role: "publisher" | "subscriber"; /** The media transport to connect to. */ transport: LivekitTransport; /** The Matrix client to use for OpenID and SFU config requests. */ @@ -92,6 +89,11 @@ export class Connection { */ public readonly state$: Behavior = this._state$; + /** + * Whether we want to publish or only subscribe on this connection. + */ + public readonly role: "publisher" | "subscriber"; + /** * The media transport to connect to. */ @@ -130,7 +132,6 @@ export class Connection { private readonly slotId: string; private readonly logger: Logger; private readonly ownMembershipIdentity: CallMembershipIdentityParts; - private readonly existingSFUConfig?: SFUConfig; /** * Creates a new connection to a matrix RTC LiveKit backend. * @@ -140,19 +141,17 @@ export class Connection { */ public constructor(opts: ConnectionOpts, logger: Logger) { this.ownMembershipIdentity = opts.ownMembershipIdentity; - this.existingSFUConfig = opts.existingSFUConfig; this.roomId = opts.roomId; this.slotId = opts.slotId; this.logger = logger.getChild( `[Connection ${JSON.stringify(opts.transport)}]`, ); - this.logger.info( - `constructor: roomId: ${this.roomId} withSfuConfig?: ${opts.existingSFUConfig ? JSON.stringify(opts.existingSFUConfig) : "undefined"}`, - ); - const { transport, client, scope } = opts; + this.logger.debug(`constructor called`); + const { role, transport, client, scope } = opts; this.scope = scope; this.livekitRoom = opts.livekitRoomFactory(); + this.role = role; this.transport = transport; this.client = client; @@ -295,11 +294,7 @@ export class Connection { this.stopped = false; try { this._state$.next({ state: "authenticating" }); - // We should already have this information after creating the localTransport. - // only call getSFUConfig for connections where we do not have a token yet. (existingJwtTokenData === undefined) - const { url, jwt, livekitAlias } = - this.existingSFUConfig ?? - (await this.getSFUConfigForRemoteConnection()); + const { url, jwt, livekitAlias } = await this.getSFUConfig(); this.logger.debug( "Starting Connection - jwt: ", jwt, @@ -381,16 +376,14 @@ export class Connection { } } - protected async getSFUConfigForRemoteConnection(): Promise { - // This will only be called for sfu's where we do not publish ourselves. - // For the local connection we will use the existingJwtTokenData + protected async getSFUConfig(): Promise { return await getSFUConfig({ client: this.client, membership: this.ownMembershipIdentity, + role: this.role, serviceUrl: this.transport.livekit_service_url, roomId: this.roomId, slotId: this.slotId, - role: "subscriber", logger: this.logger, }); } diff --git a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts index 4cb1371e1..b062e8d20 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts @@ -35,16 +35,16 @@ import { noiseSuppressionSetting, autoGainControlSetting, } from "../../../settings/settings.ts"; -import { type ClientOpenIDParts, type SFUConfig } from "../../../livekit/auth"; +import { type ClientOpenIDParts } from "../../../livekit/auth"; // TODO evaluate if this should be done like the Publisher Factory export interface ConnectionFactory { createConnection( scope: ObservableScope, + role: "publisher" | "subscriber", transport: LivekitTransport, ownMembershipIdentity: CallMembershipIdentityParts, logger: Logger, - sfuConfig?: SFUConfig, ): Connection; } @@ -102,16 +102,16 @@ export class ECConnectionFactory implements ConnectionFactory { */ public createConnection( scope: ObservableScope, + role: "publisher" | "subscriber", transport: LivekitTransport, ownMembershipIdentity: CallMembershipIdentityParts, logger: Logger, - sfuConfig?: SFUConfig, ): Connection { return new Connection( { - existingSFUConfig: sfuConfig, roomId: this.roomId, slotId: this.slotId, + role, transport, client: this.client, scope: scope, diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts index e7d91c633..89772fcf4 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts @@ -47,7 +47,7 @@ beforeEach(() => { vi.mocked(fakeConnectionFactory).createConnection = vi .fn() .mockImplementation( - (scope: ObservableScope, transport: LivekitTransport) => { + (scope: ObservableScope, _role: unknown, transport: LivekitTransport) => { const mockConnection = { transport, remoteParticipants$: new BehaviorSubject([]), @@ -245,7 +245,11 @@ describe("connectionManagerData$ stream", () => { vi.mocked(fakeConnectionFactory).createConnection = vi .fn() .mockImplementation( - (scope: ObservableScope, transport: LivekitTransport) => { + ( + scope: ObservableScope, + _role: unknown, + transport: LivekitTransport, + ) => { const fakeRemoteParticipants$ = new BehaviorSubject< RemoteParticipant[] >([]); diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts index 2897cbe0e..3aa53d449 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts @@ -27,11 +27,6 @@ import { Epoch, type ObservableScope } from "../../ObservableScope.ts"; import { generateItemsWithEpoch } from "../../../utils/observable.ts"; import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts"; import { type ConnectionFactory } from "./ConnectionFactory.ts"; -import { - isLocalTransport, - type LocalTransport, -} from "../localMember/LocalTransport.ts"; -import { type SFUConfig } from "../../../livekit/auth"; export class ConnectionManagerData { private readonly store: Map< @@ -87,7 +82,7 @@ export class ConnectionManagerData { interface Props { scope: ObservableScope; connectionFactory: ConnectionFactory; - localTransport$: Observable; + localTransport$: Observable; remoteTransports$: Behavior>; logger: Logger; @@ -147,17 +142,17 @@ export function createConnectionManager$({ * externally this is modified via `registerTransports()`. */ const localAndRemoteTransports$: Behavior< - Epoch<(LivekitTransport | LocalTransport)[]> + Epoch<{ transport: LivekitTransport; role: "publisher" | "subscriber" }[]> > = scope.behavior( combineLatest([localTransportAsArray$, remoteTransports$]).pipe( - // Combine local and remote transports into one transport array - // and set the forceOldJwtEndpoint property on the local transport + // Combine local and remote transports into one transport array, tagged + // with the role we take on each of them map(([localTransportAsArray, remoteTransports]) => { const dedupedRemote = removeDuplicateTransports(remoteTransports.value); const remoteWithoutLocal = dedupedRemote.filter( (transport) => !localTransportAsArray.find((l) => - areLivekitTransportsEqual(l.transport, transport), + areLivekitTransportsEqual(l, transport), ), ); logger.debug( @@ -167,7 +162,16 @@ export function createConnectionManager$({ localTransportAsArray, ); return new Epoch( - [...localTransportAsArray, ...remoteWithoutLocal], + [ + ...localTransportAsArray.map((transport) => ({ + transport, + role: "publisher" as const, + })), + ...remoteWithoutLocal.map((transport) => ({ + transport, + role: "subscriber" as const, + })), + ], remoteTransports.epoch, ); }), @@ -182,40 +186,23 @@ export function createConnectionManager$({ generateItemsWithEpoch( "ConnectionManager connections$", function* (transports) { - for (const transport of transports) { - if (isLocalTransport(transport)) { - // This is the local transport; only the `LocalTransport` - // interface has a `sfuConfig` field. - yield { - keys: [ - transport.transport.livekit_service_url, - transport.sfuConfig, - ], - data: undefined, - }; - } else { - yield { - keys: [ - transport.livekit_service_url, - undefined as SFUConfig | undefined, - ], - data: undefined, - }; - } + for (const { transport, role } of transports) { + yield { + keys: [role, transport.livekit_service_url], + data: undefined, + }; } }, - (scope, _data$, serviceUrl, sfuConfig) => { + (scope, _data$, role, serviceUrl) => { const connection = connectionFactory.createConnection( scope, + role, { type: "livekit", livekit_service_url: serviceUrl, }, ownMembershipIdentity, logger, - // TODO: This whole optional SFUConfig parameter is not particularly elegant. - // I would like it if connections always fetched the SFUConfig by themselves. - sfuConfig, ); // Start the connection immediately // Use connection state to track connection progress diff --git a/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts b/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts index d954c2e39..1fdc7e8a3 100644 --- a/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts +++ b/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts @@ -89,6 +89,7 @@ describe("ECConnectionFactory - Audio inputs options", () => { ); ecConnectionFactory.createConnection( testScope, + "subscriber", exampleTransport, ownMemberMock, logger, @@ -138,6 +139,7 @@ describe("ECConnectionFactory - ControlledAudioDevice", () => { ); ecConnectionFactory.createConnection( testScope, + "subscriber", exampleTransport, ownMemberMock, logger, @@ -174,6 +176,7 @@ describe("ECConnectionFactory - Camera quality settings", () => { ); ecConnectionFactory.createConnection( testScope, + "subscriber", exampleTransport, ownMemberMock, logger, @@ -212,6 +215,7 @@ describe("ECConnectionFactory - Camera quality settings", () => { ); ecConnectionFactory.createConnection( testScope, + "subscriber", exampleTransport, ownMemberMock, logger, @@ -251,6 +255,7 @@ describe("ECConnectionFactory - Camera quality settings", () => { ); ecConnectionFactory.createConnection( testScope, + "subscriber", exampleTransport, ownMemberMock, logger, diff --git a/src/utils/test.ts b/src/utils/test.ts index 9f7f14b80..6949c2171 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -78,7 +78,6 @@ import { type RemoteScreenShareViewModel, } from "../state/media/RemoteScreenShareViewModel"; import { Connection } from "../state/CallViewModel/remoteMembers/Connection"; -import { type SFUConfig } from "../livekit/auth"; export function withFakeTimers(continuation: () => void): void { vi.useFakeTimers(); @@ -214,13 +213,6 @@ export const exampleTransport: LivekitTransport = { livekit_service_url: "https://lk.example.org", }; -export const exampleSfuConfig: SFUConfig = { - jwt: "foo", - livekitAlias: "bar", - livekitIdentity: "baz", - url: "bro", -}; - export function mockRtcMembership( user: string | RoomMember, deviceId: string, From d5bbb43e3a0c6e1b00a8f9ad186a9771527eefd8 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 12 Oct 2026 10:26:54 +0200 Subject: [PATCH 10/25] Reduce ConnectionManager to a ConnectionMap createConnectionMap$ now returns the behavior of connections and their participants directly and takes an already resolved local transport instead of an observable of it. The CallViewModel wraps it with mapScoped so that a new map is built, in a fresh scope, whenever the local transport changes, and swallows local transport errors there since the local member already surfaces them in the UI. --- src/room/InCallView.tsx | 16 +- src/state/CallViewModel/CallViewModel.ts | 46 +-- .../localMember/LocalMember.test.ts | 109 +++---- .../CallViewModel/localMember/LocalMember.ts | 22 +- .../remoteMembers/Connection.test.ts | 4 +- .../CallViewModel/remoteMembers/Connection.ts | 2 +- .../remoteMembers/ConnectionManager.ts | 269 ------------------ ...nManager.test.ts => ConnectionMap.test.ts} | 75 +++-- .../remoteMembers/ConnectionMap.ts | 193 +++++++++++++ .../MatrixLivekitMembers.test.ts | 60 ++-- .../remoteMembers/MatrixLivekitMembers.ts | 13 +- .../remoteMembers/integration.test.ts | 13 +- src/utils/observable.ts | 22 ++ 13 files changed, 370 insertions(+), 474 deletions(-) delete mode 100644 src/state/CallViewModel/remoteMembers/ConnectionManager.ts rename src/state/CallViewModel/remoteMembers/{ConnectionManager.test.ts => ConnectionMap.test.ts} (85%) create mode 100644 src/state/CallViewModel/remoteMembers/ConnectionMap.ts diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 51c916f92..8b66eb160 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -694,15 +694,13 @@ export const InCallView: FC = ({ tab={settingsTab} onTabChange={setSettingsTab} developerSettingsVm={developerSettingsVm} - livekitRooms={allConnections - .getConnections() - .map((connectionItem) => ({ - room: connectionItem.livekitRoom, - livekitAlias: connectionItem.livekitAlias, - // TODO compute is local or tag it in the livekit room items already - isLocal: undefined, - url: connectionItem.transport.livekit_service_url, - }))} + livekitRooms={allConnections.map((connectionItem) => ({ + room: connectionItem.livekitRoom, + livekitAlias: connectionItem.livekitAlias, + // TODO compute is local or tag it in the livekit room items already + isLocal: undefined, + url: connectionItem.transport.livekit_service_url, + }))} /> )} diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 216a1e617..a005c05ed 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -39,6 +39,9 @@ import { timer, takeUntil, from, + concatWith, + catchError, + switchAll, } from "rxjs"; import { type Logger, logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { @@ -56,6 +59,7 @@ import { createToggle$, filterBehavior, generateItems, + mapScoped, pauseWhen, } from "../../utils/observable"; import { @@ -105,7 +109,7 @@ import { type OneOnOneMedia, } from "../layoutMedia.ts"; import { ElementCallError, UnknownCallError } from "../../utils/errors.ts"; -import { type Epoch, type ObservableScope } from "../ObservableScope.ts"; +import { Epoch, type ObservableScope } from "../ObservableScope.ts"; import { createHomeserverConnected$ } from "./localMember/HomeserverConnected.ts"; import { createLocalMembership$, @@ -123,9 +127,9 @@ import { ECConnectionFactory, } from "./remoteMembers/ConnectionFactory.ts"; import { - type ConnectionManagerData, - createConnectionManager$, -} from "./remoteMembers/ConnectionManager.ts"; + ConnectionMap, + createConnectionMap$, +} from "./remoteMembers/ConnectionMap.ts"; import { createRemoteMatrixLivekitMembers$, type LocalMatrixLivekitMember, @@ -371,7 +375,7 @@ export interface CallViewModel { * key is generated when someone joins or leaves. */ keyRotationSuppressed$: Behavior; - allConnections$: Behavior; + allConnections$: Behavior; /** Participants sorted by livekit room so they can be used in the audio rendering */ livekitRoomItems$: Behavior; /** use the layout instead, this is just for the sdk export. */ @@ -577,14 +581,24 @@ export function createCallViewModel$( options.livekitRoomFactory, ); - const connectionManager = createConnectionManager$({ - scope: scope, - connectionFactory: connectionFactory, - localTransport$, - remoteTransports$: membershipsAndTransports.transports$, - logger: logger, - ownMembershipIdentity, - }); + const connectionMap$ = scope.behavior>( + localTransport$.pipe( + catchError(() => NEVER), // Swallow errors + concatWith(NEVER), // So the Observable doesn't complete prematurely + mapScoped("connectionMap$", (localTransport, scope) => + createConnectionMap$({ + scope, + connectionFactory, + localTransport, + remoteTransports$: membershipsAndTransports.transports$, + logger, + ownMembershipIdentity, + }), + ), + switchAll(), + ), + new Epoch(new ConnectionMap()), + ); const remoteMatrixLivekitMembers$: Behavior< Epoch @@ -592,7 +606,7 @@ export function createCallViewModel$( scope: scope, membershipsWithTransport$: membershipsAndTransports.membershipsWithTransport$, - connectionManager: connectionManager, + connectionMap$, localUser: { userId, deviceId }, }); @@ -631,7 +645,7 @@ export function createCallViewModel$( controlledAudioDevices, ); }, - connectionManager, + connectionMap$, client, matrixRTCSession, localTransport$, @@ -717,7 +731,7 @@ export function createCallViewModel$( }); const allConnections$ = scope.behavior( - connectionManager.connectionManagerData$.pipe(map((d) => d.value)), + connectionMap$.pipe(map((d) => d.value.getConnections())), ); const livekitRoomItems$ = scope.behavior( diff --git a/src/state/CallViewModel/localMember/LocalMember.test.ts b/src/state/CallViewModel/localMember/LocalMember.test.ts index b1e309d58..543cada8c 100644 --- a/src/state/CallViewModel/localMember/LocalMember.test.ts +++ b/src/state/CallViewModel/localMember/LocalMember.test.ts @@ -21,7 +21,7 @@ import { beforeEach, afterEach, } from "vitest"; -import { BehaviorSubject, map, of, Subject } from "rxjs"; +import { BehaviorSubject, of, Subject } from "rxjs"; import { logger } from "matrix-js-sdk/lib/logger"; import { ConnectionState as LivekitConnectionState, @@ -57,7 +57,7 @@ import { import { MatrixRTCTransportMissingError } from "../../../utils/errors"; import { Epoch, ObservableScope } from "../../ObservableScope"; import { constant } from "../../Behavior"; -import { ConnectionManagerData } from "../remoteMembers/ConnectionManager"; +import { ConnectionMap } from "../remoteMembers/ConnectionMap"; import { type ConnectionState, type Connection, @@ -261,25 +261,11 @@ describe("LocalMembership", () => { it("throws error on missing RTC config error", () => { withTestScheduler(({ scope, hot, expectObservable }) => { - const localTransport$ = scope.behavior( - hot("1ms #", {}, new MatrixRTCTransportMissingError("domain.com")), - null, - ); - - // we do not need any connection data since we want to fail before reaching that. - const mockConnectionManager = { - transports$: scope.behavior( - localTransport$.pipe(map((t) => new Epoch([t]))), - ), - connectionManagerData$: constant( - new Epoch(new ConnectionManagerData()), - ), - }; - const localMembership = createLocalMembership$({ scope, ...defaultCreateLocalMemberValues, - connectionManager: mockConnectionManager, + // we do not need any connection data since we want to fail before reaching that. + connectionMap$: constant(new Epoch(new ConnectionMap())), localTransport$: hot( "1ms #", {}, @@ -295,24 +281,18 @@ describe("LocalMembership", () => { }); it("logs if callIntent cannot be updated", async () => { - const scope = new ObservableScope(); - - const mockConnectionManager = { - transports$: constant(new Epoch([])), - connectionManagerData$: constant(new Epoch(new ConnectionManagerData())), - }; async function reject(): Promise { return Promise.reject(new Error("Not connected yet")); } const localMembership = createLocalMembership$({ - scope, + scope: testScope(), ...defaultCreateLocalMemberValues, matrixRTCSession: { slotId: "m.call#ROOM", updateCallIntent: vi.fn().mockImplementation(reject), leave: vi.fn(), }, - connectionManager: mockConnectionManager, + connectionMap$: constant(new Epoch(new ConnectionMap())), localTransport$: constant(mockTransport), }); const expextedLog = @@ -322,7 +302,6 @@ describe("LocalMembership", () => { await flushPromises(); defaultCreateLocalMemberValues.muteStates.video.setEnabled$.value?.(true); expect(internalLogger).toHaveBeenCalledWith(expextedLog); - scope.end(); }); const mockTransport = { @@ -374,14 +353,12 @@ describe("LocalMembership", () => { if (delegationUrl !== null) fetchMock.post(delegationUrl, () => ({ status: 401, body: {} })); - const connectionManagerData = new ConnectionManagerData(); - connectionManagerData.add(connectionTransportAConnected, []); + const connectionMap = new ConnectionMap(); + connectionMap.add(connectionTransportAConnected, []); const localMembership = createLocalMembership$({ scope, ...defaultCreateLocalMemberValues, - connectionManager: { - connectionManagerData$: constant(new Epoch(connectionManagerData)), - }, + connectionMap$: constant(new Epoch(connectionMap)), joinMatrixRTC, localTransport$: constant(mockTransport), delayId$, @@ -453,15 +430,13 @@ describe("LocalMembership", () => { typeof vi.fn >; - const connectionManagerData = new ConnectionManagerData(); - connectionManagerData.add(connectionTransportAConnected, []); - // connectionManagerData.add(connectionTransportB, []); + const connectionMap = new ConnectionMap(); + connectionMap.add(connectionTransportAConnected, []); + // connectionMap.add(connectionTransportB, []); const localMembership = createLocalMembership$({ scope, ...defaultCreateLocalMemberValues, - connectionManager: { - connectionManagerData$: constant(new Epoch(connectionManagerData)), - }, + connectionMap$: constant(new Epoch(connectionMap)), localTransport$: constant(mockTransport), }); await flushPromises(); @@ -483,12 +458,10 @@ describe("LocalMembership", () => { // it("tracks livekit state correctly", async () => { const scope = new ObservableScope(); - const connectionManagerData = new ConnectionManagerData(); + const connectionMap = new ConnectionMap(); const localTransport$ = new Subject(); - const connectionManagerData$ = new BehaviorSubject( - new Epoch(connectionManagerData), - ); + const connectionMap$ = new BehaviorSubject(new Epoch(connectionMap)); const publishers: Publisher[] = []; const publishing$ = new BehaviorSubject(false); @@ -522,9 +495,7 @@ describe("LocalMembership", () => { const localMembership = createLocalMembership$({ scope, ...defaultCreateLocalMemberValues, - connectionManager: { - connectionManagerData$, - }, + connectionMap$: connectionMap$, localTransport$, }); @@ -542,14 +513,14 @@ describe("LocalMembership", () => { }, }); - const connectionManagerData2 = new ConnectionManagerData(); - connectionManagerData2.add( + const connectionMap2 = new ConnectionMap(); + connectionMap2.add( // clone because we will mutate this later. { ...connectionTransportAConnecting } as unknown as Connection, [], ); - connectionManagerData$.next(new Epoch(connectionManagerData2)); + connectionMap$.next(new Epoch(connectionMap2)); await flushPromises(); expect(localMembership.localMemberState$.value).toStrictEqual({ matrix: RTCMemberStatus.Connected, @@ -563,7 +534,7 @@ describe("LocalMembership", () => { }); ( - connectionManagerData2.getConnectionForTransport(mockTransport)! + connectionMap2.getConnectionForTransport(mockTransport)! .state$ as BehaviorSubject ).next({ state: "authenticated", @@ -659,8 +630,8 @@ describe("LocalMembership", () => { [boolean, HomeserverDisconnectReason | null] >([false, "membership"]); - const connectionManagerData = new ConnectionManagerData(); - connectionManagerData.add(connectionTransportAConnected, []); + const connectionMap = new ConnectionMap(); + connectionMap.add(connectionTransportAConnected, []); createLocalMembership$({ scope, @@ -669,9 +640,7 @@ describe("LocalMembership", () => { combined$: hsReason$, rtsSession$: constant(RTCMemberStatus.Connected), }, - connectionManager: { - connectionManagerData$: constant(new Epoch(connectionManagerData)), - }, + connectionMap$: constant(new Epoch(connectionMap)), localTransport$: constant(mockTransport), }); @@ -696,8 +665,8 @@ describe("LocalMembership", () => { [boolean, HomeserverDisconnectReason | null] >([true, null]); - const connectionManagerData = new ConnectionManagerData(); - connectionManagerData.add(connectionTransportAConnected, []); + const connectionMap = new ConnectionMap(); + connectionMap.add(connectionTransportAConnected, []); createLocalMembership$({ scope, @@ -706,9 +675,7 @@ describe("LocalMembership", () => { combined$: hsReason$, rtsSession$: constant(RTCMemberStatus.Connected), }, - connectionManager: { - connectionManagerData$: constant(new Epoch(connectionManagerData)), - }, + connectionMap$: constant(new Epoch(connectionMap)), localTransport$: constant(mockTransport), }); @@ -742,8 +709,8 @@ describe("LocalMembership", () => { state$: connectionState$, } as unknown as Connection; - const connectionManagerData = new ConnectionManagerData(); - connectionManagerData.add(mutableConnection, []); + const connectionMap = new ConnectionMap(); + connectionMap.add(mutableConnection, []); createLocalMembership$({ scope, @@ -755,9 +722,7 @@ describe("LocalMembership", () => { ]), rtsSession$: constant(RTCMemberStatus.Connected), }, - connectionManager: { - connectionManagerData$: constant(new Epoch(connectionManagerData)), - }, + connectionMap$: constant(new Epoch(connectionMap)), localTransport$: constant(mockTransport), }); @@ -792,8 +757,8 @@ describe("LocalMembership", () => { [boolean, HomeserverDisconnectReason | null] >([true, null]); - const connectionManagerData = new ConnectionManagerData(); - connectionManagerData.add(connectionTransportAConnected, []); + const connectionMap = new ConnectionMap(); + connectionMap.add(connectionTransportAConnected, []); createLocalMembership$({ scope, @@ -802,9 +767,7 @@ describe("LocalMembership", () => { combined$: hsReason$, rtsSession$: constant(RTCMemberStatus.Connected), }, - connectionManager: { - connectionManagerData$: constant(new Epoch(connectionManagerData)), - }, + connectionMap$: constant(new Epoch(connectionMap)), localTransport$: constant(mockTransport), }); @@ -863,14 +826,12 @@ describe("LocalMembership", () => { localMembership: ReturnType; } => { const scope = new ObservableScope(); - const connectionManagerData = new ConnectionManagerData(); - if (connection) connectionManagerData.add(connection, []); + const connectionMap = new ConnectionMap(); + if (connection) connectionMap.add(connection, []); const localMembership = createLocalMembership$({ scope, ...defaultCreateLocalMemberValues, - connectionManager: { - connectionManagerData$: constant(new Epoch(connectionManagerData)), - }, + connectionMap$: constant(new Epoch(connectionMap)), localTransport$: constant(mockTransport), }); return { scope, localMembership }; diff --git a/src/state/CallViewModel/localMember/LocalMember.ts b/src/state/CallViewModel/localMember/LocalMember.ts index 2fb6e0229..482627ee2 100644 --- a/src/state/CallViewModel/localMember/LocalMember.ts +++ b/src/state/CallViewModel/localMember/LocalMember.ts @@ -47,8 +47,8 @@ import { deepCompare } from "matrix-js-sdk/lib/utils"; import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; import { type Behavior } from "../../Behavior.ts"; -import { type IConnectionManager } from "../remoteMembers/ConnectionManager.ts"; -import { type ObservableScope } from "../../ObservableScope.ts"; +import { type ConnectionMap } from "../remoteMembers/ConnectionMap.ts"; +import { type Epoch, type ObservableScope } from "../../ObservableScope.ts"; import { type Publisher } from "./Publisher.ts"; import { type MuteStates } from "../../MuteStates.ts"; import { @@ -138,7 +138,7 @@ interface Props { // that the inputs for those createSomething$() functions should NOT contain any js-sdk objectes scope: ObservableScope; muteStates: MuteStates; - connectionManager: IConnectionManager; + connectionMap$: Behavior>; createPublisherFactory: (connection: Connection) => Publisher; joinMatrixRTC: ( transport: LivekitTransport, @@ -169,7 +169,7 @@ interface Props { * - * @param props The properties required to create the local membership. * @param props.scope The observable scope to use. - * @param props.connectionManager The connection manager to get connections from. + * @param props.connectionMap$ The connection map to get connections from. * @param props.createPublisherFactory Factory to create a publisher once we have a connection. * @param props.joinMatrixRTC Callback to join the matrix RTC session once we have a transport. * @param props.homeserverConnected The homeserver connected state. @@ -191,7 +191,7 @@ interface Props { */ export const createLocalMembership$ = ({ scope, - connectionManager, + connectionMap$, localTransport$: localTransportWithErrors$, homeserverConnected, createPublisherFactory, @@ -324,13 +324,11 @@ export const createLocalMembership$ = ({ // Drop Epoch data here since we will not combine this anymore const localConnection$ = scope.behavior( - combineLatest([ - connectionManager.connectionManagerData$, - localTransport$, - ]).pipe( - map(([{ value: connectionData }, transport]) => - connectionData.getConnectionForTransport(transport), - ), + combineLatest( + [connectionMap$, localTransport$], + ({ value: connectionMap }, transport) => + connectionMap.getConnectionForTransport(transport), + ).pipe( tap((connection) => { logger.info( `Local connection updated: ${JSON.stringify(connection?.transport)}`, diff --git a/src/state/CallViewModel/remoteMembers/Connection.test.ts b/src/state/CallViewModel/remoteMembers/Connection.test.ts index 21e79ef97..5afd79dec 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.test.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.test.ts @@ -418,8 +418,8 @@ describe("Start connection states", () => { pendingConnect.reject(new Error("Client initiated disconnect")); await stopping; - // start() resolves rather than rejecting (it is not awaited by the - // ConnectionManager, so a rejection would be unhandled). + // start() resolves rather than rejecting (it is not awaited by the caller, + // so a rejection would be unhandled). await expect(started).resolves.toBeUndefined(); expect(capturedStates.at(-1)).toEqual({ state: "stopped" }); expect(capturedStates.some((st) => st instanceof Error)).toBe(false); diff --git a/src/state/CallViewModel/remoteMembers/Connection.ts b/src/state/CallViewModel/remoteMembers/Connection.ts index 35c2e2c75..15b0559cb 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.ts @@ -356,7 +356,7 @@ export class Connection { // stop() was called while we were connecting, which makes the pending // connect reject. That is the abort we asked for, not a failure, so // don't record an error state on a stopped connection or rethrow it - // (start() is not awaited by the ConnectionManager, so a throw here + // (start() is not necessarily awaited by the caller, so a throw here // becomes an unhandled promise rejection). this.logger.debug(`Connect aborted because the connection was stopped`); return; diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts deleted file mode 100644 index 3aa53d449..000000000 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts +++ /dev/null @@ -1,269 +0,0 @@ -/* -Copyright 2025 Element Creations Ltd. -Copyright 2025 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; -import { - combineLatest, - map, - type Observable, - of, - switchMap, - startWith, - catchError, - NEVER, -} from "rxjs"; -import { type Logger } from "matrix-js-sdk/lib/logger"; -import { type RemoteParticipant } from "livekit-client"; -import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; - -import { type Behavior } from "../../Behavior.ts"; -import { type Connection } from "./Connection.ts"; -import { Epoch, type ObservableScope } from "../../ObservableScope.ts"; -import { generateItemsWithEpoch } from "../../../utils/observable.ts"; -import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts"; -import { type ConnectionFactory } from "./ConnectionFactory.ts"; - -export class ConnectionManagerData { - private readonly store: Map< - string, - { connection: Connection; participants: RemoteParticipant[] } - > = new Map(); - - public constructor(private readonly logger?: Logger) {} - - public add(connection: Connection, participants: RemoteParticipant[]): void { - const key = this.getKey(connection.transport); - const existing = this.store.get(key); - if (!existing) { - this.store.set(key, { connection, participants }); - } else { - // Transports are deduplicated by URL upstream, so this should never - // happen; if it does, members may be matched against the wrong room. - this.logger?.warn( - `Merging participants from a second connection to ${key}: existing [${existing.participants.map((p) => p.identity).join(", ")}], adding [${participants.map((p) => p.identity).join(", ")}]`, - ); - existing.participants.push(...participants); - } - } - - private getKey(transport: LivekitTransport): string { - // This is enough as a key because the ConnectionManager is already scoped by room. - // We also do not need to consider the slotId at this point since each `MatrixRTCSession` is already scoped by `slotDescription: {id, application}`. - return transport.livekit_service_url; - } - - public getConnections(): Connection[] { - return Array.from(this.store.values()).map(({ connection }) => connection); - } - - public getConnectionForTransport( - transport: LivekitTransport, - ): Connection | null { - return this.store.get(this.getKey(transport))?.connection ?? null; - } - - public getParticipantsForTransport( - transport: LivekitTransport, - ): RemoteParticipant[] { - const key = this.getKey(transport); - const existing = this.store.get(key); - if (existing) { - return existing.participants; - } - return []; - } -} - -interface Props { - scope: ObservableScope; - connectionFactory: ConnectionFactory; - localTransport$: Observable; - remoteTransports$: Behavior>; - - logger: Logger; - ownMembershipIdentity: CallMembershipIdentityParts; -} - -// TODO - write test for scopes (do we really need to bind scope) -export interface IConnectionManager { - connectionManagerData$: Behavior>; -} - -/** - * Crete a `ConnectionManager` - * @param props - Configuration object - * @param props.scope - The observable scope used by this object - * @param props.connectionFactory - Used to create new connections - * @param props.localTransport$ - The transport to publish local media on. (deduplicated with remoteTransports$) - * @param props.remoteTransports$ - All other transports. The connection manager will create connections for each transport. (deduplicated with localTransport$) - * @param props.ownMembershipIdentity - The own membership identity to use. - * @param props.logger - The logger to use. - - * - * Each of these behaviors can be interpreted as subscribed list of transports. - * - * Using `registerTransports` independent external modules can control what connections - * are created by the ConnectionManager. - * - * The connection manager will remove all duplicate transports in each subscibed list. - * - * See `unregisterAllTransports` and `unregisterTransport` for details on how to unsubscribe. - */ -export function createConnectionManager$({ - scope, - connectionFactory, - localTransport$, - remoteTransports$, - logger: parentLogger, - ownMembershipIdentity, -}: Props): IConnectionManager { - const logger = parentLogger.getChild("[ConnectionManager]"); - // TODO logger: only construct one logger from the client and make it compatible via a EC specific sing - - const localTransportAsArray$ = localTransport$.pipe( - // LocalMember already surfaces local transport errors properly in the UI, - // here we can just swallow them - catchError(() => NEVER), - map((transport) => [transport]), - startWith([]), - ); - - /** - * All transports currently managed by the ConnectionManager. - * - * This list does not include duplicate transports. - * - * It is build based on the list of subscribed transports (`transportsSubscriptions$`). - * externally this is modified via `registerTransports()`. - */ - const localAndRemoteTransports$: Behavior< - Epoch<{ transport: LivekitTransport; role: "publisher" | "subscriber" }[]> - > = scope.behavior( - combineLatest([localTransportAsArray$, remoteTransports$]).pipe( - // Combine local and remote transports into one transport array, tagged - // with the role we take on each of them - map(([localTransportAsArray, remoteTransports]) => { - const dedupedRemote = removeDuplicateTransports(remoteTransports.value); - const remoteWithoutLocal = dedupedRemote.filter( - (transport) => - !localTransportAsArray.find((l) => - areLivekitTransportsEqual(l, transport), - ), - ); - logger.debug( - "remoteWithoutLocal", - remoteWithoutLocal, - "localTransportAsArray", - localTransportAsArray, - ); - return new Epoch( - [ - ...localTransportAsArray.map((transport) => ({ - transport, - role: "publisher" as const, - })), - ...remoteWithoutLocal.map((transport) => ({ - transport, - role: "subscriber" as const, - })), - ], - remoteTransports.epoch, - ); - }), - ), - ); - - /** - * Connections for each transport in use by one or more session members. - */ - const connections$ = scope.behavior( - localAndRemoteTransports$.pipe( - generateItemsWithEpoch( - "ConnectionManager connections$", - function* (transports) { - for (const { transport, role } of transports) { - yield { - keys: [role, transport.livekit_service_url], - data: undefined, - }; - } - }, - (scope, _data$, role, serviceUrl) => { - const connection = connectionFactory.createConnection( - scope, - role, - { - type: "livekit", - livekit_service_url: serviceUrl, - }, - ownMembershipIdentity, - logger, - ); - // Start the connection immediately - // Use connection state to track connection progress - void connection.start(); - // TODO subscribe to connection state to retry or log issues? - return connection; - }, - ), - ), - ); - - const connectionManagerData$ = scope.behavior( - connections$.pipe( - switchMap((connections) => { - const epoch = connections.epoch; - - // Map the connections to list of {connection, participants}[] - const listOfConnectionsWithRemoteParticipants = connections.value.map( - (connection) => { - return connection.remoteParticipants$.pipe( - map((participants) => ({ - connection, - participants, - })), - ); - }, - ); - - // probably not required - - if (listOfConnectionsWithRemoteParticipants.length === 0) { - return of(new Epoch(new ConnectionManagerData(), epoch)); - } - - // combineLatest the several streams into a single stream with the ConnectionManagerData - return combineLatest(listOfConnectionsWithRemoteParticipants).pipe( - map( - (lists) => - new Epoch( - lists.reduce((data, { connection, participants }) => { - data.add(connection, participants); - return data; - }, new ConnectionManagerData(logger)), - epoch, - ), - ), - ); - }), - ), - new Epoch(new ConnectionManagerData(), -1), - ); - - return { connectionManagerData$ }; -} - -function removeDuplicateTransports( - transports: T[], -): T[] { - return transports.reduce((acc, transport) => { - if (!acc.some((t) => areLivekitTransportsEqual(t, transport))) - acc.push(transport); - return acc; - }, [] as T[]); -} diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts b/src/state/CallViewModel/remoteMembers/ConnectionMap.test.ts similarity index 85% rename from src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts rename to src/state/CallViewModel/remoteMembers/ConnectionMap.test.ts index 89772fcf4..e0f7add1c 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionMap.test.ts @@ -6,16 +6,13 @@ Please see LICENSE in the repository root for full details. */ import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; -import { BehaviorSubject, NEVER } from "rxjs"; +import { BehaviorSubject } from "rxjs"; import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { type RemoteParticipant } from "livekit-client"; import { logger } from "matrix-js-sdk/lib/logger"; import { Epoch, mapEpoch, ObservableScope } from "../../ObservableScope.ts"; -import { - createConnectionManager$, - ConnectionManagerData, -} from "./ConnectionManager.ts"; +import { createConnectionMap$, ConnectionMap } from "./ConnectionMap.ts"; import { type ConnectionFactory } from "./ConnectionFactory.ts"; import { type Connection } from "./Connection.ts"; import { ownMemberMock, withTestScheduler } from "../../../utils/test.ts"; @@ -34,6 +31,15 @@ const TRANSPORT_2: LivekitTransport = { livekit_service_url: "https://lk.sample.com", }; +function findConnection( + transport: LivekitTransport, + connections: Connection[], +): Connection | undefined { + return connections.find((c) => + areLivekitTransportsEqual(c.transport, transport), + ); +} + let fakeConnectionFactory: ConnectionFactory; let testScope: ObservableScope; @@ -71,10 +77,10 @@ afterEach(() => { describe("connections$ stream", () => { test("Should create and start new connections for each transports", () => { withTestScheduler(({ behavior, expectObservable }) => { - const { connectionManagerData$ } = createConnectionManager$({ + const connectionMap$ = createConnectionMap$({ scope: testScope, connectionFactory: fakeConnectionFactory, - localTransport$: NEVER, + localTransport: TRANSPORT_1, remoteTransports$: behavior("a", { a: new Epoch([TRANSPORT_1, TRANSPORT_2], 0), }), @@ -83,7 +89,7 @@ describe("connections$ stream", () => { }); expectObservable( - connectionManagerData$.pipe(mapEpoch((d) => d.getConnections())), + connectionMap$.pipe(mapEpoch((d) => d.getConnections())), ).toBe("a", { a: expect.toSatisfy((e: Epoch) => { const connections = e.value; @@ -93,15 +99,11 @@ describe("connections$ stream", () => { vi.mocked(fakeConnectionFactory).createConnection, ).toHaveBeenCalledTimes(2); - const conn1 = connections.find((c) => - areLivekitTransportsEqual(c.transport, TRANSPORT_1), - ); + const conn1 = findConnection(TRANSPORT_1, connections); expect(conn1).toBeDefined(); expect(conn1!.start).toHaveBeenCalled(); - const conn2 = connections.find((c) => - areLivekitTransportsEqual(c.transport, TRANSPORT_2), - ); + const conn2 = findConnection(TRANSPORT_2, connections); expect(conn2).toBeDefined(); expect(conn2!.start).toHaveBeenCalled(); return true; @@ -112,10 +114,10 @@ describe("connections$ stream", () => { test("Should start connection only once", () => { withTestScheduler(({ behavior, expectObservable }) => { - const { connectionManagerData$ } = createConnectionManager$({ + const connectionMap$ = createConnectionMap$({ scope: testScope, connectionFactory: fakeConnectionFactory, - localTransport$: NEVER, + localTransport: TRANSPORT_1, remoteTransports$: behavior("abcdef", { a: new Epoch([TRANSPORT_1], 0), b: new Epoch([TRANSPORT_1], 1), @@ -129,7 +131,7 @@ describe("connections$ stream", () => { }); expectObservable( - connectionManagerData$.pipe(mapEpoch((d) => d.getConnections())), + connectionMap$.pipe(mapEpoch((d) => d.getConnections())), ).toBe("xxxxxa", { x: expect.anything(), a: expect.toSatisfy((e: Epoch) => { @@ -140,14 +142,10 @@ describe("connections$ stream", () => { vi.mocked(fakeConnectionFactory).createConnection, ).toHaveBeenCalledTimes(2); - const conn2 = connections.find((c) => - areLivekitTransportsEqual(c.transport, TRANSPORT_2), - ); + const conn2 = findConnection(TRANSPORT_2, connections); expect(conn2).toBeDefined(); - const conn1 = connections.find((c) => - areLivekitTransportsEqual(c.transport, TRANSPORT_1), - ); + const conn1 = findConnection(TRANSPORT_1, connections); expect(conn1).toBeDefined(); expect(conn1!.start).toHaveBeenCalledOnce(); @@ -159,10 +157,10 @@ describe("connections$ stream", () => { test("Should cleanup connections when not needed anymore", () => { withTestScheduler(({ behavior, expectObservable }) => { - const { connectionManagerData$ } = createConnectionManager$({ + const connectionMap$ = createConnectionMap$({ scope: testScope, connectionFactory: fakeConnectionFactory, - localTransport$: NEVER, + localTransport: TRANSPORT_1, remoteTransports$: behavior("abc", { a: new Epoch([TRANSPORT_1], 0), b: new Epoch([TRANSPORT_1, TRANSPORT_2], 1), @@ -173,7 +171,7 @@ describe("connections$ stream", () => { }); expectObservable( - connectionManagerData$.pipe(mapEpoch((d) => d.getConnections())), + connectionMap$.pipe(mapEpoch((d) => d.getConnections())), ).toBe("xab", { x: expect.anything(), a: expect.toSatisfy((e: Epoch) => { @@ -186,8 +184,9 @@ describe("connections$ stream", () => { expect(connections.length).toBe(1); // The second connection should have been stopped has it is no longer needed. - const connection2 = allCreatedConnections.find((c) => - areLivekitTransportsEqual(c.transport, TRANSPORT_2), + const connection2 = findConnection( + TRANSPORT_2, + allCreatedConnections, ); expect(connection2).toBeDefined(); expect(connection2!.stop).toHaveBeenCalled(); @@ -203,10 +202,10 @@ describe("connections$ stream", () => { }); }); -describe("ConnectionManagerData", () => { +describe("ConnectionMap", () => { test("warns when a second connection to the same URL is merged", () => { const warn = vi.spyOn(logger, "warn").mockImplementation(() => {}); - const data = new ConnectionManagerData(logger); + const data = new ConnectionMap(logger); const connection = { transport: TRANSPORT_1 } as unknown as Connection; const p = (identity: string): RemoteParticipant => ({ identity }) as unknown as RemoteParticipant; @@ -221,7 +220,7 @@ describe("ConnectionManagerData", () => { }); }); -describe("connectionManagerData$ stream", () => { +describe("connectionMap$", () => { // Used in test to control fake connections' remoteParticipants$ streams let fakeRemoteParticipantsStreams: Map>; @@ -298,10 +297,10 @@ describe("connectionManagerData$ stream", () => { ); // ============================== - const { connectionManagerData$ } = createConnectionManager$({ + const connectionMap$ = createConnectionMap$({ scope: testScope, connectionFactory: fakeConnectionFactory, - localTransport$: NEVER, + localTransport: TRANSPORT_1, remoteTransports$: behavior("a", { a: new Epoch([TRANSPORT_1, TRANSPORT_2], 0), }), @@ -309,16 +308,16 @@ describe("connectionManagerData$ stream", () => { ownMembershipIdentity: ownMemberMock, }); - expectObservable(connectionManagerData$).toBe("abcd", { + expectObservable(connectionMap$).toBe("abcd", { a: expect.toSatisfy((e) => { - const data: ConnectionManagerData = e.value; + const data: ConnectionMap = e.value; expect(data.getConnections().length).toBe(2); expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(0); expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(0); return true; }), b: expect.toSatisfy((e) => { - const data: ConnectionManagerData = e.value; + const data: ConnectionMap = e.value; expect(data.getConnections().length).toBe(2); expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(1); expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(0); @@ -328,7 +327,7 @@ describe("connectionManagerData$ stream", () => { return true; }), c: expect.toSatisfy((e) => { - const data: ConnectionManagerData = e.value; + const data: ConnectionMap = e.value; expect(data.getConnections().length).toBe(2); expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(1); expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(1); @@ -341,7 +340,7 @@ describe("connectionManagerData$ stream", () => { return true; }), d: expect.toSatisfy((e) => { - const data: ConnectionManagerData = e.value; + const data: ConnectionMap = e.value; expect(data.getConnections().length).toBe(2); expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(2); expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(1); diff --git a/src/state/CallViewModel/remoteMembers/ConnectionMap.ts b/src/state/CallViewModel/remoteMembers/ConnectionMap.ts new file mode 100644 index 000000000..67253bf38 --- /dev/null +++ b/src/state/CallViewModel/remoteMembers/ConnectionMap.ts @@ -0,0 +1,193 @@ +/* +Copyright 2025-2026 Element Creations Ltd. +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { combineLatest, map, of, switchMap, tap } from "rxjs"; +import { type Logger } from "matrix-js-sdk/lib/logger"; +import { type RemoteParticipant } from "livekit-client"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; + +import { type Behavior } from "../../Behavior.ts"; +import { type Connection } from "./Connection.ts"; +import { + Epoch, + mapEpoch, + type ObservableScope, +} from "../../ObservableScope.ts"; +import { generateItemsWithEpoch } from "../../../utils/observable.ts"; +import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts"; +import { type ConnectionFactory } from "./ConnectionFactory.ts"; + +function removeDuplicateTransports( + transports: T[], +): T[] { + return transports.reduce((acc, transport) => { + if (!acc.some((t) => areLivekitTransportsEqual(t, transport))) + acc.push(transport); + return acc; + }, [] as T[]); +} + +export class ConnectionMap { + private readonly map = new Map< + string, + { connection: Connection; participants: RemoteParticipant[] } + >(); + + public constructor(private readonly logger?: Logger) {} + + public add(connection: Connection, participants: RemoteParticipant[]): void { + const key = this.getKey(connection.transport); + const existing = this.map.get(key); + if (!existing) { + this.map.set(key, { connection, participants }); + } else { + // Transports are deduplicated by URL upstream, so this should never + // happen; if it does, members may be matched against the wrong room. + this.logger?.warn( + `Merging participants from a second connection to ${key}: existing [${existing.participants.map((p) => p.identity).join(", ")}], adding [${participants.map((p) => p.identity).join(", ")}]`, + ); + existing.participants.push(...participants); + } + } + + private getKey(transport: LivekitTransport): string { + // This is enough as a key because the ConnectionMap is already scoped by room. + // We also do not need to consider the slotId at this point since each `MatrixRTCSession` is already scoped by `slotDescription: {id, application}`. + return transport.livekit_service_url; + } + + public getConnections(): Connection[] { + return Array.from(this.map.values()).map(({ connection }) => connection); + } + + public getConnectionForTransport( + transport: LivekitTransport, + ): Connection | null { + return this.map.get(this.getKey(transport))?.connection ?? null; + } + + public getParticipantsForTransport( + transport: LivekitTransport, + ): RemoteParticipant[] { + const existing = this.map.get(this.getKey(transport)); + if (existing) { + return existing.participants; + } + return []; + } +} + +interface Props { + scope: ObservableScope; + connectionFactory: ConnectionFactory; + localTransport: LivekitTransport; + remoteTransports$: Behavior>; + + logger: Logger; + ownMembershipIdentity: CallMembershipIdentityParts; +} + +/** + * Create a dynamic map of connections and their associated backend participants. + * @param props - Configuration object + * @param props.scope - The observable scope used by this object + * @param props.connectionFactory - Used to create new connections + * @param props.localTransport - The transport to publish local media on. (deduplicated with remoteTransports$) + * @param props.remoteTransports$ - All other transports. The connection manager will create connections for each transport. (deduplicated with localTransport) + * @param props.ownMembershipIdentity - The own membership identity to use. + * @param props.logger - The logger to use. + */ +export function createConnectionMap$({ + scope, + connectionFactory, + localTransport, + remoteTransports$, + logger: parentLogger, + ownMembershipIdentity, +}: Props): Behavior> { + const logger = parentLogger.getChild("[ConnectionMap]"); + // TODO logger: only construct one logger from the client and make it compatible via a EC specific sing + + /** + * All transports currently represented in the connection map. This list does + * not include duplicate transports. + */ + const localAndRemoteTransports$: Behavior> = + scope.behavior( + remoteTransports$.pipe( + mapEpoch((remoteTransports) => + removeDuplicateTransports([localTransport, ...remoteTransports]), + ), + tap((transports) => + logger.debug("localAndRemoteTransports$ = ", transports), + ), + ), + ); + + /** + * Connections for each transport in use by one or more session members. + */ + const connections$ = scope.behavior( + localAndRemoteTransports$.pipe( + generateItemsWithEpoch( + "ConnectionMap connections$", + function* (transports) { + for (const transport of transports) { + const role = areLivekitTransportsEqual(transport, localTransport) + ? ("publisher" as const) + : ("subscriber" as const); + yield { + keys: [role, transport.livekit_service_url], + data: undefined, + }; + } + }, + (scope, _data$, role, serviceUrl) => { + const connection = connectionFactory.createConnection( + scope, + role, + { type: "livekit", livekit_service_url: serviceUrl }, + ownMembershipIdentity, + logger, + ); + // Start the connection immediately + // Use connection state to track connection progress + void connection.start(); + // TODO subscribe to connection state to retry or log issues? + return connection; + }, + ), + ), + ); + + return scope.behavior( + connections$.pipe( + switchMap(({ value: connections, epoch }) => { + if (connections.length === 0) + return of(new Epoch(new ConnectionMap(), epoch)); + + return combineLatest( + // Map the connections to list of Observable<{connection, participants}> + connections.map((connection) => + connection.remoteParticipants$.pipe( + map((participants) => ({ connection, participants })), + ), + ), + // Collect into a single ConnectionMap + (...list) => { + const map = new ConnectionMap(logger); + for (const item of list) + map.add(item.connection, item.participants); + return new Epoch(map, epoch); + }, + ); + }), + ), + ); +} diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts index 2e0051928..42548a79a 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts @@ -13,7 +13,6 @@ import { } from "matrix-js-sdk/lib/matrixrtc"; import { BehaviorSubject, combineLatest, map, type Observable } from "rxjs"; -import { type IConnectionManager } from "./ConnectionManager.ts"; import { type RemoteMatrixLivekitMember, createRemoteMatrixLivekitMembers$, @@ -24,7 +23,7 @@ import { ObservableScope, trackEpoch, } from "../../ObservableScope.ts"; -import { ConnectionManagerData } from "./ConnectionManager.ts"; +import { ConnectionMap } from "./ConnectionMap.ts"; import { flushPromises, mockRtcMembership, @@ -78,23 +77,18 @@ function epochMeWith$( test("should signal participant not yet connected to livekit", async () => { const info = vi.spyOn(logger, "info"); const mockedMemberships$ = new BehaviorSubject([bobMembership]); - const mockConnectionManagerData$ = new BehaviorSubject( - new ConnectionManagerData(), - ); + const mockConnectionMap$ = new BehaviorSubject(new ConnectionMap()); const { memberships$, membershipsWithTransport$ } = createEpochedMemberships$(mockedMemberships$); - const connectionManagerData$ = epochMeWith$( - memberships$, - mockConnectionManagerData$, + const connectionMap$ = testScope.behavior( + epochMeWith$(memberships$, mockConnectionMap$), ); const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({ scope: testScope, membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), - connectionManager: { - connectionManagerData$: connectionManagerData$, - } as unknown as IConnectionManager, + connectionMap$, localUser: localRtcMember, }); @@ -152,22 +146,19 @@ test("should signal participant on a connection that is publishing", async () => const connection = { transport: bobMembership.getTransport(), } as unknown as Connection; - const dataWithPublisher = new ConnectionManagerData(); + const dataWithPublisher = new ConnectionMap(); dataWithPublisher.add(connection, [ mockRemoteParticipant({ identity: bobParticipantId }), ]); - const connectionManagerData$ = epochMeWith$( - memberships$, - constant(dataWithPublisher), + const connectionMap$ = testScope.behavior( + epochMeWith$(memberships$, constant(dataWithPublisher)), ); const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({ scope: testScope, membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), - connectionManager: { - connectionManagerData$: connectionManagerData$, - } as unknown as IConnectionManager, + connectionMap$, localUser: localRtcMember, }); @@ -195,20 +186,17 @@ test("should signal participant on a connection that is not publishing", async ( const connection = { transport: bobMembership.getTransport(), } as unknown as Connection; - const dataWithPublisher = new ConnectionManagerData(); + const dataWithPublisher = new ConnectionMap(); dataWithPublisher.add(connection, []); - const connectionManagerData$ = epochMeWith$( - memberships$, - constant(dataWithPublisher), + const connectionMap$ = testScope.behavior( + epochMeWith$(memberships$, constant(dataWithPublisher)), ); const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({ scope: testScope, membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), - connectionManager: { - connectionManagerData$: connectionManagerData$, - } as unknown as IConnectionManager, + connectionMap$, localUser: localRtcMember, }); await flushPromises(); @@ -228,7 +216,7 @@ describe("Publication edge case", () => { const { memberships$, membershipsWithTransport$ } = createEpochedMemberships$(constant([bobMembership, carlMembership])); - const connectionWithPublisher = new ConnectionManagerData(); + const connectionWithPublisher = new ConnectionMap(); const bobParticipantId = fallbackMemberId( bobMembership.userId, bobMembership.deviceId, @@ -247,17 +235,14 @@ describe("Publication edge case", () => { mockRemoteParticipant({ identity: bobParticipantId }), ]); - const connectionManagerData$ = epochMeWith$( - memberships$, - constant(connectionWithPublisher), + const connectionMap$ = testScope.behavior( + epochMeWith$(memberships$, constant(connectionWithPublisher)), ); const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({ scope: testScope, membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), - connectionManager: { - connectionManagerData$: connectionManagerData$, - } as unknown as IConnectionManager, + connectionMap$, localUser: localRtcMember, }); await flushPromises(); @@ -287,7 +272,7 @@ test("bob is publishing in the wrong connection", async () => { const { memberships$, membershipsWithTransport$ } = createEpochedMemberships$(mockedMemberships$); - const connectionWithPublisher = new ConnectionManagerData(); + const connectionWithPublisher = new ConnectionMap(); const bobParticipantId = fallbackMemberId( bobMembership.userId, @@ -306,17 +291,14 @@ test("bob is publishing in the wrong connection", async () => { const connectionsWithPublisher$ = new BehaviorSubject( connectionWithPublisher, ); - const connectionManagerData$ = epochMeWith$( - memberships$, - connectionsWithPublisher$, + const connectionMap$ = testScope.behavior( + epochMeWith$(memberships$, connectionsWithPublisher$), ); const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({ scope: testScope, membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), - connectionManager: { - connectionManagerData$: connectionManagerData$, - } as unknown as IConnectionManager, + connectionMap$, localUser: localRtcMember, }); diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts index d899fd9e2..8c5f21d77 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts @@ -14,7 +14,7 @@ import { combineLatest, filter, map } from "rxjs"; import { logger } from "matrix-js-sdk/lib/logger"; import { type Behavior } from "../../Behavior"; -import { type IConnectionManager } from "./ConnectionManager"; +import { type ConnectionMap } from "./ConnectionMap"; import { Epoch, type ObservableScope } from "../../ObservableScope"; import { type Connection } from "./Connection"; import { generateItemsWithEpoch } from "../../../utils/observable"; @@ -62,7 +62,7 @@ interface Props { membershipsWithTransport$: Behavior< Epoch<{ membership: CallMembership; transport?: LivekitTransport }[]> >; - connectionManager: IConnectionManager; + connectionMap$: Behavior>; localUser: { deviceId: string; userId: string }; } @@ -72,24 +72,21 @@ interface Props { * It has a small public interface: * - in (via constructor): * - an observable of CallMembership[] to track the call members (The matrix side) - * - a `ConnectionManager` for the lk rooms (The livekit side) + * - a `ConnectionMap` for the lk rooms (The livekit side) * - out (via public Observable): * - `remoteMatrixLivekitMember` an observable of MatrixLivekitMember[] to track the remote members and associated livekit data. */ export function createRemoteMatrixLivekitMembers$({ scope, membershipsWithTransport$, - connectionManager, + connectionMap$, localUser, }: Props): Behavior> { /** * Behavior of all the remote call members and their associated livekit data (if available). */ return scope.behavior( - combineLatest([ - membershipsWithTransport$, - connectionManager.connectionManagerData$, - ]).pipe( + combineLatest([membershipsWithTransport$, connectionMap$]).pipe( filter((values) => values.every((value) => value.epoch === values[0].epoch), ), diff --git a/src/state/CallViewModel/remoteMembers/integration.test.ts b/src/state/CallViewModel/remoteMembers/integration.test.ts index 07b8d6f58..df3ff8dc3 100644 --- a/src/state/CallViewModel/remoteMembers/integration.test.ts +++ b/src/state/CallViewModel/remoteMembers/integration.test.ts @@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details. */ import { test, vi, expect, beforeEach, afterEach } from "vitest"; -import { BehaviorSubject, NEVER } from "rxjs"; +import { BehaviorSubject } from "rxjs"; import { type Room as LivekitRoom } from "livekit-client"; import EventEmitter from "events"; import fetchMock from "fetch-mock"; @@ -21,6 +21,7 @@ import { import { ECConnectionFactory } from "./ConnectionFactory.ts"; import { type ClientOpenIDParts } from "../../../livekit/auth"; import { + exampleTransport, mockMediaDevices, mockRtcMembership, ownMemberMock, @@ -32,11 +33,11 @@ import { createRemoteMatrixLivekitMembers$, type RemoteMatrixLivekitMember, } from "./MatrixLivekitMembers.ts"; -import { createConnectionManager$ } from "./ConnectionManager.ts"; +import { createConnectionMap$ } from "./ConnectionMap.ts"; import { membershipsAndTransports$ } from "../../SessionBehaviors.ts"; import { localRtcMember, testJWTToken } from "../../../utils/test-fixtures.ts"; -// Test the integration of ConnectionManager and MatrixLivekitMerger +// Test the integration of ConnectionMap and MatrixLivekitMembers let testScope: ObservableScope; let ecConnectionFactory: ECConnectionFactory; @@ -122,10 +123,10 @@ test("bob, carl, then bob joining no tracks yet", () => { memberships$, ); - const connectionManager = createConnectionManager$({ + const connectionMap$ = createConnectionMap$({ scope: testScope, connectionFactory: ecConnectionFactory, - localTransport$: NEVER, + localTransport: exampleTransport, remoteTransports$: membershipsAndTransports.transports$, logger: logger, ownMembershipIdentity: ownMemberMock, @@ -135,7 +136,7 @@ test("bob, carl, then bob joining no tracks yet", () => { scope: testScope, membershipsWithTransport$: membershipsAndTransports.membershipsWithTransport$, - connectionManager, + connectionMap$, localUser: localRtcMember, }); diff --git a/src/utils/observable.ts b/src/utils/observable.ts index ca51ad196..b35e523d6 100644 --- a/src/utils/observable.ts +++ b/src/utils/observable.ts @@ -218,6 +218,28 @@ export function filterBehavior( ); } +/** + * Applies a given projection to each value emitted by the source Observable. + * The `project` function additionally receives an {@link ObservableScope}. + */ +export function mapScoped( + name: string, + project: (a: A, scope: ObservableScope) => B, +): OperatorFunction { + return (a$) => + a$.pipe( + generateItemsInternal( + name, + function* (a) { + yield { keys: [a], data: undefined }; + }, + (scope, _data$, a) => project(a, scope), + (items) => items, + ), + map(([item]) => item), + ); +} + function generateItemsInternal< Input, Keys extends [unknown, ...unknown[]], From 83e1396bdd199711cdbc0d1c74118304bdb28aa0 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 12 Oct 2026 10:30:52 +0200 Subject: [PATCH 11/25] Use the unstable LiveKit transport type UnstableLivekitTransport covers every shape a homeserver may advertise while MSC4195 evolves: a LiveKit SFU `url`, a `livekit_service_url`, or both. Switching to it means a service URL is no longer guaranteed, so the remaining code that relies on one checks for it explicitly: delegation probing skips transports without one, the connection map keys transports by whichever URL they carry, and the OpenID flow rejects transports without a service URL until the client-server API flow lands. --- src/RTCConnectionStats.tsx | 4 +- src/livekit/MatrixAudioRenderer.tsx | 4 +- src/room/CallEventAudioRenderer.test.tsx | 4 +- src/room/InCallView.tsx | 7 +++- src/state/CallViewModel/CallViewModel.test.ts | 4 +- src/state/CallViewModel/CallViewModel.ts | 8 ++-- .../localMember/LocalMember.test.ts | 8 ++-- .../CallViewModel/localMember/LocalMember.ts | 30 +++++++-------- .../localMember/LocalTransport.ts | 6 +-- .../RtcTransportAutoDiscovery.test.ts | 6 +-- .../localMember/RtcTransportAutoDiscovery.ts | 12 +++--- .../remoteMembers/Connection.test.ts | 34 ++++++++--------- .../CallViewModel/remoteMembers/Connection.ts | 8 ++-- .../remoteMembers/ConnectionFactory.ts | 6 +-- .../remoteMembers/ConnectionMap.test.ts | 24 +++++++----- .../remoteMembers/ConnectionMap.ts | 37 +++++++++++-------- .../MatrixLivekitMembers.test.ts | 14 ++++--- .../remoteMembers/MatrixLivekitMembers.ts | 11 ++++-- .../remoteMembers/integration.test.ts | 8 ++-- src/state/SessionBehaviors.ts | 27 ++++++++------ src/state/media/MemberMediaViewModel.ts | 6 +-- src/tile/GridTile.tsx | 4 +- src/tile/MediaView.tsx | 4 +- src/tile/SpotlightTile.tsx | 4 +- src/utils/test.ts | 4 +- 25 files changed, 157 insertions(+), 127 deletions(-) diff --git a/src/RTCConnectionStats.tsx b/src/RTCConnectionStats.tsx index 90fefb656..a19cdd977 100644 --- a/src/RTCConnectionStats.tsx +++ b/src/RTCConnectionStats.tsx @@ -16,12 +16,12 @@ import classNames from "classnames"; import { Modal } from "./Modal"; import styles from "./RTCConnectionStats.module.css"; import mediaViewStyles from "../src/tile/MediaView.module.css"; -import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; interface Props { audio?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; video?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; - transport?: LivekitTransport; + transport?: UnstableLivekitTransport; rtcBackendIdentity?: string; } diff --git a/src/livekit/MatrixAudioRenderer.tsx b/src/livekit/MatrixAudioRenderer.tsx index c9c4bbb36..7f5e8b8fa 100644 --- a/src/livekit/MatrixAudioRenderer.tsx +++ b/src/livekit/MatrixAudioRenderer.tsx @@ -19,10 +19,10 @@ import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { useEarpieceAudioConfig } from "../MediaDevicesContext"; import { useReactiveState } from "../useReactiveState"; import * as controls from "../controls"; -import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; export interface MatrixAudioRendererProps { - transport: LivekitTransport; + transport: UnstableLivekitTransport; livekitRoom: LivekitRoom; /** * The list of participant identities to render audio for. diff --git a/src/room/CallEventAudioRenderer.test.tsx b/src/room/CallEventAudioRenderer.test.tsx index 9ab64dfd6..1d8fd33d2 100644 --- a/src/room/CallEventAudioRenderer.test.tsx +++ b/src/room/CallEventAudioRenderer.test.tsx @@ -17,7 +17,7 @@ import { import { act } from "react"; import { type RoomMember } from "matrix-js-sdk"; import { - type LivekitTransport, + type UnstableLivekitTransport, type CallMembership, } from "matrix-js-sdk/lib/matrixrtc"; @@ -46,7 +46,7 @@ vitest.mock("../useAudioContext"); vitest.mock("../soundUtils"); vitest.mock("../rtcSessionHelpers", async (importOriginal) => ({ ...(await importOriginal()), - makeTransport: (): [LivekitTransport] => [exampleTransport], + makeTransport: (): [UnstableLivekitTransport] => [exampleTransport], })); afterEach(() => { diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 8b66eb160..7f35c3887 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -699,7 +699,12 @@ export const InCallView: FC = ({ livekitAlias: connectionItem.livekitAlias, // TODO compute is local or tag it in the livekit room items already isLocal: undefined, - url: connectionItem.transport.livekit_service_url, + url: + // It should be relatively clear to the user which kind of URL + // this is, based on the scheme (ws:// vs. http://) + "url" in connectionItem.transport + ? connectionItem.transport.url + : connectionItem.transport.livekit_service_url, }))} /> diff --git a/src/state/CallViewModel/CallViewModel.test.ts b/src/state/CallViewModel/CallViewModel.test.ts index c7783d152..1b1c38d4a 100644 --- a/src/state/CallViewModel/CallViewModel.test.ts +++ b/src/state/CallViewModel/CallViewModel.test.ts @@ -31,7 +31,7 @@ import { type CallMembership, type IRTCNotificationContent, MatrixRTCSessionEvent, - type LivekitTransport, + type UnstableLivekitTransport, } from "matrix-js-sdk/lib/matrixrtc"; import { deepCompare } from "matrix-js-sdk/lib/utils"; @@ -97,7 +97,7 @@ vi.mock( "../state/CallViewModel/localMember/localTransport", async (importOriginal) => ({ ...(await importOriginal()), - makeTransport: async (): Promise => + makeTransport: async (): Promise => Promise.resolve(exampleTransport), }), ); diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index a005c05ed..d6791c995 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -46,7 +46,7 @@ import { import { type Logger, logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { MembershipManagerEvent, - type LivekitTransport, + type UnstableLivekitTransport, type MatrixRTCSession, type RTCCallIntent, type RTCNotificationType, @@ -217,7 +217,7 @@ export interface CallViewModelOptions { */ windowSize$: Behavior<{ width: number; height: number }>; /** Optional value overriding the local transport, for testing purposes. */ - localTransport?: LivekitTransport; + localTransport?: UnstableLivekitTransport; /** Optional value overriding the connection factory, for testing purposes. */ connectionFactory?: ConnectionFactory; /** The version & compatibility mode of MatrixRTC that we should use. */ @@ -278,7 +278,7 @@ interface LayoutScanState { export type LivekitRoomItem = { livekitRoom: LivekitRoom; participants: string[]; - transport: LivekitTransport; + transport: UnstableLivekitTransport; }; /** @@ -619,7 +619,7 @@ export function createCallViewModel$( ), muteStates, joinMatrixRTC: ( - transport: LivekitTransport, + transport: UnstableLivekitTransport, delayedLeaveTimings: ResolvedDelayedLeaveTimings, ) => { return enterRTCSession( diff --git a/src/state/CallViewModel/localMember/LocalMember.test.ts b/src/state/CallViewModel/localMember/LocalMember.test.ts index 543cada8c..437cdd20e 100644 --- a/src/state/CallViewModel/localMember/LocalMember.test.ts +++ b/src/state/CallViewModel/localMember/LocalMember.test.ts @@ -8,7 +8,7 @@ Please see LICENSE in the repository root for full details. import { Status as RTCMemberStatus, - type LivekitTransport, + type UnstableLivekitTransport, type MatrixRTCSession, } from "matrix-js-sdk/lib/matrixrtc"; import { @@ -127,7 +127,7 @@ const mockedClient = { }; describe("enterRTCSession", () => { - const transport: LivekitTransport = { + const transport: UnstableLivekitTransport = { livekit_service_url: "http://my-livekit-service-url.com", type: "livekit", }; @@ -306,7 +306,7 @@ describe("LocalMembership", () => { const mockTransport = { livekit_service_url: "a", - } as LivekitTransport; + } as UnstableLivekitTransport; const connectionTransportAConnected = { livekitRoom: mockLivekitRoom({ @@ -459,7 +459,7 @@ describe("LocalMembership", () => { it("tracks livekit state correctly", async () => { const scope = new ObservableScope(); const connectionMap = new ConnectionMap(); - const localTransport$ = new Subject(); + const localTransport$ = new Subject(); const connectionMap$ = new BehaviorSubject(new Epoch(connectionMap)); const publishers: Publisher[] = []; diff --git a/src/state/CallViewModel/localMember/LocalMember.ts b/src/state/CallViewModel/localMember/LocalMember.ts index 482627ee2..1b1e8aae4 100644 --- a/src/state/CallViewModel/localMember/LocalMember.ts +++ b/src/state/CallViewModel/localMember/LocalMember.ts @@ -18,7 +18,7 @@ import { import { observeParticipantEvents } from "@livekit/components-core"; import { Status as RTCSessionStatus, - type LivekitTransport, + type UnstableLivekitTransport, type MatrixRTCSession, type RTCCallIntent, type RTCNotificationType, @@ -141,13 +141,13 @@ interface Props { connectionMap$: Behavior>; createPublisherFactory: (connection: Connection) => Publisher; joinMatrixRTC: ( - transport: LivekitTransport, + transport: UnstableLivekitTransport, delayedLeaveTimings: ResolvedDelayedLeaveTimings, ) => void; homeserverConnected: HomeserverConnected; roomId: string; ownMembershipIdentity: CallMembershipIdentityParts; - localTransport$: Observable; + localTransport$: Observable; client: ClientOpenIDParts; matrixRTCSession: Pick< MatrixRTCSession, @@ -307,15 +307,16 @@ export const createLocalMembership$ = ({ const joinParams$ = scope.behavior( localTransport$.pipe( switchMap(async (transport) => { - const transportSupportsDelegation = checkDelegationSupport( - transport.livekit_service_url + "/delegate_delayed_leave", - `transport ${transport.livekit_service_url}`, - ); + const transportSupportsDelegation = + "livekit_service_url" in transport && + (await checkDelegationSupport( + transport.livekit_service_url + "/delegate_delayed_leave", + `transport ${transport.livekit_service_url}`, + )); return { transport, delegationSupported: - (await homeserverSupportsDelegation) || - (await transportSupportsDelegation), + transportSupportsDelegation || (await homeserverSupportsDelegation), }; }), ), @@ -715,6 +716,8 @@ export const createLocalMembership$ = ({ async ([joinParams, delayId]) => { if (joinParams?.delegationSupported && delayId !== null) { try { + if (!("livekit_service_url" in joinParams.transport)) + throw new Error("Transport has no livekit_service_url"); // This will technically cause the service to issue a new JWT token, // but it's safe to discard. We're only interested in triggering // delegation. @@ -731,10 +734,7 @@ export const createLocalMembership$ = ({ }); } catch (e) { // TODO: Surface this to the user as a service interruption? - logger.error( - `Failed to delegate leave to ${joinParams.transport.livekit_service_url}`, - e, - ); + logger.error(`Failed to delegate leave`, e); } } }, @@ -967,7 +967,7 @@ interface EnterRTCSessionOptions { * * @param rtcSession - The MatrixRTCSession to join. * @param ownMembershipIdentity - Options for entering the RTC session. - * @param transport - The LivekitTransport to use for this session. + * @param transport - The UnstableLivekitTransport to use for this session. * @param options - `encryptMedia`: Whether to encrypt media. `matrixRTCMode`: The * Matrix RTC mode to use. `delayedLeaveTimings`: The preferred timings for * delayed leave events. `sendNotificationType`: Whether and what kind of @@ -978,7 +978,7 @@ interface EnterRTCSessionOptions { export function enterRTCSession( rtcSession: MatrixRTCSession, ownMembershipIdentity: CallMembershipIdentityParts, - transport: LivekitTransport, + transport: UnstableLivekitTransport, { encryptMedia, matrixRTCMode, diff --git a/src/state/CallViewModel/localMember/LocalTransport.ts b/src/state/CallViewModel/localMember/LocalTransport.ts index 891f70ccf..f2572bf30 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.ts @@ -5,7 +5,7 @@ SPDX-License-IdFentifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { type MatrixClient } from "matrix-js-sdk"; import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; @@ -20,7 +20,7 @@ import { RtcTransportAutoDiscovery } from "./RtcTransportAutoDiscovery.ts"; */ export async function getLocalTransport( client: Pick, -): Promise { +): Promise { const discovery = new RtcTransportAutoDiscovery({ client, resolvedConfig: Config.get(), @@ -29,7 +29,7 @@ export async function getLocalTransport( const customUrl = customLivekitUrl.value$.value; // Respect the user's custom URL, if set - const transport: LivekitTransport | null = customUrl + const transport: UnstableLivekitTransport | null = customUrl ? { type: "livekit", livekit_service_url: customUrl } : await discovery.discoverPreferredTransport(); diff --git a/src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.test.ts b/src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.test.ts index 61df88a73..d3f246c97 100644 --- a/src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.test.ts +++ b/src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.test.ts @@ -16,7 +16,7 @@ import { import { MatrixError } from "matrix-js-sdk"; import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { - type LivekitTransport, + type UnstableLivekitTransport, type Transport, } from "matrix-js-sdk/lib/matrixrtc"; @@ -28,12 +28,12 @@ import { type DiscoveryClient = RtcTransportAutoDiscoveryProps["client"]; -const backendTransport: LivekitTransport = { +const backendTransport: UnstableLivekitTransport = { type: "livekit", livekit_service_url: "https://backend.example.org", }; -const configTransport: LivekitTransport = { +const configTransport: UnstableLivekitTransport = { type: "livekit", livekit_service_url: "https://config.example.org", }; diff --git a/src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.ts b/src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.ts index 58a5f1b19..a5bedcd80 100644 --- a/src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.ts +++ b/src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.ts @@ -5,8 +5,8 @@ SPDX-License-IdFentifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ import { - isLivekitTransport, - type LivekitTransport, + isUnstableLivekitTransport, + type UnstableLivekitTransport, } from "matrix-js-sdk/lib/matrixrtc"; import { type MatrixClient } from "matrix-js-sdk"; import { type Logger } from "matrix-js-sdk/lib/logger"; @@ -40,7 +40,7 @@ export class RtcTransportAutoDiscovery { this.logger = logger.getChild("[RtcTransportAutoDiscovery]"); } - public async discoverPreferredTransport(): Promise { + public async discoverPreferredTransport(): Promise { // 1) backend transports const backendTransport = await this.tryBackendTransports(); if (backendTransport) { @@ -67,7 +67,7 @@ export class RtcTransportAutoDiscovery { * This will not throw errors, but instead just log them and return null if the expected config is not found or malformed. * @private */ - private async tryBackendTransports(): Promise { + private async tryBackendTransports(): Promise { const client = this.client; // MSC4143: Attempt to fetch transports from backend. this.logger.info("First try to use getRTCTransports end point ..."); @@ -75,7 +75,7 @@ export class RtcTransportAutoDiscovery { const transportList = await doNetworkOperationWithRetry(async () => client._unstable_getRTCTransports(), ); - const first = transportList.find(isLivekitTransport); + const first = transportList.find(isUnstableLivekitTransport); if (first) { return first; } else { @@ -90,7 +90,7 @@ export class RtcTransportAutoDiscovery { return null; } - private tryConfigTransport(): LivekitTransport | null { + private tryConfigTransport(): UnstableLivekitTransport | null { const url = this.resolvedConfig.livekit?.livekit_service_url; if (url) { return { diff --git a/src/state/CallViewModel/remoteMembers/Connection.test.ts b/src/state/CallViewModel/remoteMembers/Connection.test.ts index 5afd79dec..98a1aed87 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.test.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.test.ts @@ -29,7 +29,7 @@ import fetchMock from "fetch-mock"; import EventEmitter from "events"; import { type IOpenIDToken } from "matrix-js-sdk"; import { logger, type Logger } from "matrix-js-sdk/lib/logger"; -import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { Connection, @@ -54,8 +54,9 @@ let fakeLocalParticipant: MockedObject; const ROOM_ID = "!roomID:example.org"; const SLOT_ID = "m.call#ROOM"; -const livekitFocus: LivekitTransport = { - livekit_service_url: "https://matrix-rtc.example.org/livekit/jwt", +const livekitServiceUrl = "https://matrix-rtc.example.org/livekit/jwt"; +const livekitFocus: UnstableLivekitTransport = { + livekit_service_url: livekitServiceUrl, type: "livekit", }; @@ -122,7 +123,7 @@ function setupRemoteConnection(): Connection { livekitRoomFactory: () => fakeLivekitRoom, }; - fetchMock.post(`${livekitFocus.livekit_service_url}/get_token`, () => { + fetchMock.post(`${livekitServiceUrl}/get_token`, () => { return { status: 200, body: { @@ -250,19 +251,16 @@ describe("Start connection states", () => { const deferredSFU = Promise.withResolvers(); // mock the /get_token call - fetchMock.post( - `${livekitFocus.livekit_service_url}/get_token`, - async () => { - await deferredSFU.promise; - return { - status: 500, - body: { - errcode: "M_LOOKUP_FAILED", - error: "Failed to look up user info from homeserver", - }, - }; - }, - ); + fetchMock.post(`${livekitServiceUrl}/get_token`, async () => { + await deferredSFU.promise; + return { + status: 500, + body: { + errcode: "M_LOOKUP_FAILED", + error: "Failed to look up user info from homeserver", + }, + }; + }); connection.start().catch(() => { // expected to throw @@ -317,7 +315,7 @@ describe("Start connection states", () => { const deferredSFU = Promise.withResolvers(); // mock the /get_token call - fetchMock.post(`${livekitFocus.livekit_service_url}/get_token`, () => { + fetchMock.post(`${livekitServiceUrl}/get_token`, () => { return { status: 200, body: { diff --git a/src/state/CallViewModel/remoteMembers/Connection.ts b/src/state/CallViewModel/remoteMembers/Connection.ts index 15b0559cb..575ddfad6 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.ts @@ -23,7 +23,7 @@ import { type Track, type TrackPublication, } from "livekit-client"; -import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { BehaviorSubject } from "rxjs"; import { type Logger } from "matrix-js-sdk/lib/logger"; import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; @@ -50,7 +50,7 @@ export interface ConnectionOpts { /** Whether we want to publish or only subscribe on this connection. */ role: "publisher" | "subscriber"; /** The media transport to connect to. */ - transport: LivekitTransport; + transport: UnstableLivekitTransport; /** The Matrix client to use for OpenID and SFU config requests. */ client: ClientOpenIDParts; /** The ID of the Matrix room in which the session takes place. */ @@ -97,7 +97,7 @@ export class Connection { /** * The media transport to connect to. */ - public readonly transport: LivekitTransport; + public readonly transport: UnstableLivekitTransport; public readonly livekitRoom: LivekitRoom; @@ -377,6 +377,8 @@ export class Connection { } protected async getSFUConfig(): Promise { + if (!("livekit_service_url" in this.transport)) + throw new Error("Transport has no livekit_service_url"); return await getSFUConfig({ client: this.client, membership: this.ownMembershipIdentity, diff --git a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts index b062e8d20..b2a4833d7 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts @@ -16,7 +16,7 @@ import { logger, type Logger } from "matrix-js-sdk/lib/logger"; // imported as inline to support worker when loaded from a cdn (cross domain) import E2EEWorker from "livekit-client/e2ee-worker?worker&inline"; import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; -import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { type ObservableScope } from "../../ObservableScope.ts"; import { Connection } from "./Connection.ts"; @@ -42,7 +42,7 @@ export interface ConnectionFactory { createConnection( scope: ObservableScope, role: "publisher" | "subscriber", - transport: LivekitTransport, + transport: UnstableLivekitTransport, ownMembershipIdentity: CallMembershipIdentityParts, logger: Logger, ): Connection; @@ -103,7 +103,7 @@ export class ECConnectionFactory implements ConnectionFactory { public createConnection( scope: ObservableScope, role: "publisher" | "subscriber", - transport: LivekitTransport, + transport: UnstableLivekitTransport, ownMembershipIdentity: CallMembershipIdentityParts, logger: Logger, ): Connection { diff --git a/src/state/CallViewModel/remoteMembers/ConnectionMap.test.ts b/src/state/CallViewModel/remoteMembers/ConnectionMap.test.ts index e0f7add1c..dba1cb0be 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionMap.test.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionMap.test.ts @@ -7,7 +7,7 @@ Please see LICENSE in the repository root for full details. import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; import { BehaviorSubject } from "rxjs"; -import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { type RemoteParticipant } from "livekit-client"; import { logger } from "matrix-js-sdk/lib/logger"; @@ -21,18 +21,18 @@ import { type Behavior } from "../../Behavior.ts"; // Some test constants -const TRANSPORT_1: LivekitTransport = { +const TRANSPORT_1: UnstableLivekitTransport = { type: "livekit", livekit_service_url: "https://lk.example.org", }; -const TRANSPORT_2: LivekitTransport = { +const TRANSPORT_2: UnstableLivekitTransport = { type: "livekit", livekit_service_url: "https://lk.sample.com", }; function findConnection( - transport: LivekitTransport, + transport: UnstableLivekitTransport, connections: Connection[], ): Connection | undefined { return connections.find((c) => @@ -53,7 +53,11 @@ beforeEach(() => { vi.mocked(fakeConnectionFactory).createConnection = vi .fn() .mockImplementation( - (scope: ObservableScope, _role: unknown, transport: LivekitTransport) => { + ( + scope: ObservableScope, + _role: unknown, + transport: UnstableLivekitTransport, + ) => { const mockConnection = { transport, remoteParticipants$: new BehaviorSubject([]), @@ -224,15 +228,17 @@ describe("connectionMap$", () => { // Used in test to control fake connections' remoteParticipants$ streams let fakeRemoteParticipantsStreams: Map>; - function keyForTransport(transport: LivekitTransport): string { - return `${transport.livekit_service_url}`; + function keyForTransport(transport: UnstableLivekitTransport): string { + return "livekit_service_url" in transport + ? transport.livekit_service_url + : transport.url; } beforeEach(() => { fakeRemoteParticipantsStreams = new Map(); function getRemoteParticipantsFor( - transport: LivekitTransport, + transport: UnstableLivekitTransport, ): Behavior { return ( fakeRemoteParticipantsStreams.get(keyForTransport(transport)) ?? @@ -247,7 +253,7 @@ describe("connectionMap$", () => { ( scope: ObservableScope, _role: unknown, - transport: LivekitTransport, + transport: UnstableLivekitTransport, ) => { const fakeRemoteParticipants$ = new BehaviorSubject< RemoteParticipant[] diff --git a/src/state/CallViewModel/remoteMembers/ConnectionMap.ts b/src/state/CallViewModel/remoteMembers/ConnectionMap.ts index 67253bf38..698d9ad97 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionMap.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionMap.ts @@ -6,7 +6,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { combineLatest, map, of, switchMap, tap } from "rxjs"; import { type Logger } from "matrix-js-sdk/lib/logger"; import { type RemoteParticipant } from "livekit-client"; @@ -23,7 +23,17 @@ import { generateItemsWithEpoch } from "../../../utils/observable.ts"; import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts"; import { type ConnectionFactory } from "./ConnectionFactory.ts"; -function removeDuplicateTransports( +/** + * The URL by which a transport is identified: its JWT service URL, or failing + * that, its SFU URL. + */ +function urlOf(transport: UnstableLivekitTransport): string { + return "livekit_service_url" in transport + ? transport.livekit_service_url + : transport.url; +} + +function removeDuplicateTransports( transports: T[], ): T[] { return transports.reduce((acc, transport) => { @@ -56,10 +66,10 @@ export class ConnectionMap { } } - private getKey(transport: LivekitTransport): string { + private getKey(transport: UnstableLivekitTransport): string { // This is enough as a key because the ConnectionMap is already scoped by room. // We also do not need to consider the slotId at this point since each `MatrixRTCSession` is already scoped by `slotDescription: {id, application}`. - return transport.livekit_service_url; + return urlOf(transport); } public getConnections(): Connection[] { @@ -67,13 +77,13 @@ export class ConnectionMap { } public getConnectionForTransport( - transport: LivekitTransport, + transport: UnstableLivekitTransport, ): Connection | null { return this.map.get(this.getKey(transport))?.connection ?? null; } public getParticipantsForTransport( - transport: LivekitTransport, + transport: UnstableLivekitTransport, ): RemoteParticipant[] { const existing = this.map.get(this.getKey(transport)); if (existing) { @@ -86,8 +96,8 @@ export class ConnectionMap { interface Props { scope: ObservableScope; connectionFactory: ConnectionFactory; - localTransport: LivekitTransport; - remoteTransports$: Behavior>; + localTransport: UnstableLivekitTransport; + remoteTransports$: Behavior>; logger: Logger; ownMembershipIdentity: CallMembershipIdentityParts; @@ -118,7 +128,7 @@ export function createConnectionMap$({ * All transports currently represented in the connection map. This list does * not include duplicate transports. */ - const localAndRemoteTransports$: Behavior> = + const localAndRemoteTransports$: Behavior> = scope.behavior( remoteTransports$.pipe( mapEpoch((remoteTransports) => @@ -142,17 +152,14 @@ export function createConnectionMap$({ const role = areLivekitTransportsEqual(transport, localTransport) ? ("publisher" as const) : ("subscriber" as const); - yield { - keys: [role, transport.livekit_service_url], - data: undefined, - }; + yield { keys: [role, urlOf(transport)], data: transport }; } }, - (scope, _data$, role, serviceUrl) => { + (scope, transport$, role, _url) => { const connection = connectionFactory.createConnection( scope, role, - { type: "livekit", livekit_service_url: serviceUrl }, + transport$.value, ownMembershipIdentity, logger, ); diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts index 42548a79a..af98b2342 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts @@ -9,7 +9,7 @@ import { describe, test, expect, beforeEach, afterEach, vi } from "vitest"; import { logger } from "matrix-js-sdk/lib/logger"; import { type CallMembership, - type LivekitTransport, + type UnstableLivekitTransport, } from "matrix-js-sdk/lib/matrixrtc"; import { BehaviorSubject, combineLatest, map, type Observable } from "rxjs"; @@ -38,12 +38,12 @@ let testScope: ObservableScope; const fallbackMemberId = (userId: string, deviceId: string): string => `${userId}:${deviceId}`; -const transportA: LivekitTransport = { +const transportA: UnstableLivekitTransport = { type: "livekit", livekit_service_url: "https://lk.example.org", }; -const transportB: LivekitTransport = { +const transportB: UnstableLivekitTransport = { type: "livekit", livekit_service_url: "https://lk.sample.com", }; @@ -111,7 +111,9 @@ test("should signal participant not yet connected to livekit", async () => { function createEpochedMemberships$(m$: Observable): { memberships$: Observable>; membershipsWithTransport$: Observable< - Epoch<{ membership: CallMembership; transport?: LivekitTransport }[]> + Epoch< + { membership: CallMembership; transport?: UnstableLivekitTransport }[] + > >; } { const memberships$ = m$.pipe(trackEpoch()); @@ -122,7 +124,9 @@ function createEpochedMemberships$(m$: Observable): { return { membership: m, transport: - tr?.type === "livekit" ? (tr as LivekitTransport) : undefined, + tr?.type === "livekit" + ? (tr as UnstableLivekitTransport) + : undefined, }; }); }), diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts index 8c5f21d77..46735bb2f 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts @@ -8,10 +8,11 @@ Please see LICENSE in the repository root for full details. import { type LocalParticipant, type RemoteParticipant } from "livekit-client"; import { type CallMembership, - type LivekitTransport, + type UnstableLivekitTransport, } from "matrix-js-sdk/lib/matrixrtc"; import { combineLatest, filter, map } from "rxjs"; import { logger } from "matrix-js-sdk/lib/logger"; +import { deepCompare } from "matrix-js-sdk/lib/utils"; import { type Behavior } from "../../Behavior"; import { type ConnectionMap } from "./ConnectionMap"; @@ -60,7 +61,9 @@ export interface RemoteMatrixLivekitMember extends MatrixLivekitMember { interface Props { scope: ObservableScope; membershipsWithTransport$: Behavior< - Epoch<{ membership: CallMembership; transport?: LivekitTransport }[]> + Epoch< + { membership: CallMembership; transport?: UnstableLivekitTransport }[] + > >; connectionMap$: Behavior>; localUser: { deviceId: string; userId: string }; @@ -162,12 +165,12 @@ export function createRemoteMatrixLivekitMembers$({ // TODO add back in the callviewmodel pauseWhen(this.pretendToBeDisconnected$) // TODO add this to the JS-SDK -export function areLivekitTransportsEqual( +export function areLivekitTransportsEqual( t1: T | null, t2: T | null, ): boolean { if (t1 && t2) { - return t1.livekit_service_url === t2.livekit_service_url; + return deepCompare(t1, t2); } return !t1 && !t2; } diff --git a/src/state/CallViewModel/remoteMembers/integration.test.ts b/src/state/CallViewModel/remoteMembers/integration.test.ts index df3ff8dc3..1a965cc02 100644 --- a/src/state/CallViewModel/remoteMembers/integration.test.ts +++ b/src/state/CallViewModel/remoteMembers/integration.test.ts @@ -10,7 +10,7 @@ import { BehaviorSubject } from "rxjs"; import { type Room as LivekitRoom } from "livekit-client"; import EventEmitter from "events"; import fetchMock from "fetch-mock"; -import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { logger } from "matrix-js-sdk/lib/logger"; import { @@ -152,7 +152,7 @@ test("bob, carl, then bob joining no tracks yet", () => { a: expect.toSatisfy((co) => areLivekitTransportsEqual( co.transport, - bobMembership.transports[0]! as LivekitTransport, + bobMembership.transports[0]! as UnstableLivekitTransport, ), ), }); @@ -189,7 +189,7 @@ test("bob, carl, then bob joining no tracks yet", () => { expect( areLivekitTransportsEqual( connection.transport, - carlMembership.transports[0]! as LivekitTransport, + carlMembership.transports[0]! as UnstableLivekitTransport, ), ).toBe(true); return true; @@ -219,7 +219,7 @@ test("bob, carl, then bob joining no tracks yet", () => { expect( areLivekitTransportsEqual( connection.transport, - daveMembership.transports[0]! as LivekitTransport, + daveMembership.transports[0]! as UnstableLivekitTransport, ), ).toBe(true); return true; diff --git a/src/state/SessionBehaviors.ts b/src/state/SessionBehaviors.ts index 4ca1c5861..435e77021 100644 --- a/src/state/SessionBehaviors.ts +++ b/src/state/SessionBehaviors.ts @@ -7,10 +7,10 @@ Please see LICENSE in the repository root for full details. import { type CallMembership, - type LivekitTransport, + type UnstableLivekitTransport, type MatrixRTCSession, MatrixRTCSessionEvent, - isLivekitTransport, + isUnstableLivekitTransport, } from "matrix-js-sdk/lib/matrixrtc"; import { fromEvent } from "rxjs"; @@ -31,15 +31,17 @@ export const membershipsAndTransports$ = ( memberships$: Behavior>, ): { membershipsWithTransport$: Behavior< - Epoch<{ membership: CallMembership; transport?: LivekitTransport }[]> + Epoch< + { membership: CallMembership; transport?: UnstableLivekitTransport }[] + > >; - transports$: Behavior>; + transports$: Behavior>; } => { const membershipsWithTransport$: Behavior< Epoch< { membership: CallMembership; - transport: LivekitTransport | undefined; + transport: UnstableLivekitTransport | undefined; }[] > > = scope.behavior( @@ -49,18 +51,21 @@ export const membershipsAndTransports$ = ( const transport = membership.getTransport(); return { membership, - transport: isLivekitTransport(transport) ? transport : undefined, + transport: isUnstableLivekitTransport(transport) + ? transport + : undefined, }; }); }), ), ); - const transports$: Behavior> = scope.behavior( - membershipsWithTransport$.pipe( - mapEpoch((mts) => mts.flatMap(({ transport: t }) => (t ? [t] : []))), - ), - ); + const transports$: Behavior> = + scope.behavior( + membershipsWithTransport$.pipe( + mapEpoch((mts) => mts.flatMap(({ transport: t }) => (t ? [t] : []))), + ), + ); return { membershipsWithTransport$, diff --git a/src/state/media/MemberMediaViewModel.ts b/src/state/media/MemberMediaViewModel.ts index 4891809e2..e57dcc636 100644 --- a/src/state/media/MemberMediaViewModel.ts +++ b/src/state/media/MemberMediaViewModel.ts @@ -40,7 +40,7 @@ import { E2eeType } from "../../e2ee/e2eeType"; import { observeInboundRtpStreamStats$ } from "./observeRtpStreamStats"; import { type UserMediaViewModel } from "./UserMediaViewModel"; import { type ScreenShareViewModel } from "./ScreenShareViewModel"; -import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; // TODO: Encryption status is kinda broken and thus unused right now. Remove? export enum EncryptionStatus { @@ -63,7 +63,7 @@ export interface BaseMemberMediaViewModel extends BaseMediaViewModel { * The LiveKit transport on which this member should be publishing. Exposed * for debugging. */ - transport$: Behavior; + transport$: Behavior; /** * Whether there should be a warning that this media is unencrypted. */ @@ -76,7 +76,7 @@ export interface MemberMediaInputs extends BaseMediaViewModel { livekitRoom$: Behavior; audioSource: AudioSource; videoSource: VideoSource; - transport$: Behavior; + transport$: Behavior; encryptionSystem: EncryptionSystem; } diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index b83d5f5ea..b0625a4d7 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -51,7 +51,7 @@ import { type RemoteUserMediaViewModel } from "../state/media/RemoteUserMediaVie import { type UserMediaViewModel } from "../state/media/UserMediaViewModel"; import { type RingingMediaViewModel } from "../state/media/RingingMediaViewModel"; import { RingingStatus } from "./RingingStatus"; -import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; interface TileProps { ref?: Ref; @@ -104,7 +104,7 @@ interface UserMediaTileProps extends TileProps { playbackMuted: boolean; waitingForMedia?: boolean; primaryButton?: ReactNode; - transport: LivekitTransport | undefined; + transport: UnstableLivekitTransport | undefined; } /** diff --git a/src/tile/MediaView.tsx b/src/tile/MediaView.tsx index 7b22764d5..6776317df 100644 --- a/src/tile/MediaView.tsx +++ b/src/tile/MediaView.tsx @@ -33,7 +33,7 @@ import { ReactionIndicator } from "../reactions/ReactionIndicator"; import { RTCConnectionStats } from "../RTCConnectionStats"; import videoPlaceholder from "../graphics/video-placeholder.gif"; import { autoVideoFit } from "../utils/videoFit"; -import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; interface Props extends ComponentProps { className?: string; @@ -70,7 +70,7 @@ interface Props extends ComponentProps { /** * The LiveKit transport, mainly for debugging purposes. */ - transport?: LivekitTransport; + transport?: UnstableLivekitTransport; /** * Called whenever the aspect ratio of the video content becomes known or * otherwise changes. diff --git a/src/tile/SpotlightTile.tsx b/src/tile/SpotlightTile.tsx index 46cab2566..344eed00e 100644 --- a/src/tile/SpotlightTile.tsx +++ b/src/tile/SpotlightTile.tsx @@ -54,7 +54,7 @@ import { platform } from "../Platform"; import { type RingingMediaViewModel } from "../state/media/RingingMediaViewModel"; import { RingingStatus } from "./RingingStatus"; import { useRootElement } from "../RootElementContext"; -import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; interface SpotlightItemBaseProps { ref?: Ref; @@ -75,7 +75,7 @@ interface SpotlightItemBaseProps { interface SpotlightMemberMediaItemBaseProps extends SpotlightItemBaseProps { video: TrackReferenceOrPlaceholder | undefined; unencryptedWarning: boolean; - transport: LivekitTransport | undefined; + transport: UnstableLivekitTransport | undefined; } interface SpotlightUserMediaItemBaseProps extends SpotlightMemberMediaItemBaseProps { diff --git a/src/utils/test.ts b/src/utils/test.ts index 6949c2171..c3efc6b89 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -26,7 +26,7 @@ import { import { CallMembership, type LivekitFocusSelection, - type LivekitTransport, + type UnstableLivekitTransport, type MatrixRTCSession, MatrixRTCSessionEvent, type MatrixRTCSessionEventHandlerMap, @@ -208,7 +208,7 @@ export function mockEmitter(): EmitterMock { }; } -export const exampleTransport: LivekitTransport = { +export const exampleTransport: UnstableLivekitTransport = { type: "livekit", livekit_service_url: "https://lk.example.org", }; From 53a0e273d65e673c984b94f150a7d0a0e779f5af Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 12 Oct 2026 10:31:36 +0200 Subject: [PATCH 12/25] Identify transports by server name as well as URLs A transport with only a LiveKit SFU `url` has to be authenticated through our own homeserver, which needs to know the name of the server owning the SFU. Carry that name alongside the transport as a TransportLocator, derived from the member's user ID for remote transports and from the client's domain for our own, and key the connection map on the SFU URL, the service URL and the server name together so that `url`, service URL and combined transport shapes are deduplicated consistently. --- src/livekit/MatrixAudioRenderer.test.tsx | 8 +- src/livekit/MatrixAudioRenderer.tsx | 4 +- src/livekit/auth/types.ts | 11 ++ src/state/CallViewModel/CallViewModel.ts | 10 +- .../CallViewModel/CallViewModelTestUtils.ts | 6 +- .../localMember/LocalMember.test.ts | 29 +++-- .../CallViewModel/localMember/LocalMember.ts | 10 +- .../localMember/LocalTransport.test.ts | 28 +++-- .../localMember/LocalTransport.ts | 11 +- .../remoteMembers/Connection.test.ts | 6 ++ .../CallViewModel/remoteMembers/Connection.ts | 10 +- .../remoteMembers/ConnectionFactory.ts | 4 + .../remoteMembers/ConnectionMap.test.ts | 62 ++++++----- .../remoteMembers/ConnectionMap.ts | 101 +++++++++++------- .../remoteMembers/ECConnectionFactory.test.ts | 5 + .../MatrixLivekitMembers.test.ts | 43 ++++---- .../remoteMembers/MatrixLivekitMembers.ts | 20 +--- .../remoteMembers/integration.test.ts | 36 ++----- src/state/SessionBehaviors.ts | 27 +++-- src/utils/test-viewmodel.ts | 2 +- src/utils/test.ts | 6 ++ 21 files changed, 257 insertions(+), 182 deletions(-) diff --git a/src/livekit/MatrixAudioRenderer.test.tsx b/src/livekit/MatrixAudioRenderer.test.tsx index 21183f866..ab252a0e7 100644 --- a/src/livekit/MatrixAudioRenderer.test.tsx +++ b/src/livekit/MatrixAudioRenderer.test.tsx @@ -24,7 +24,7 @@ import { testAudioContext } from "../useAudioContext.test"; import * as MediaDevicesContext from "../MediaDevicesContext"; import { LivekitRoomAudioRenderer } from "./MatrixAudioRenderer"; import { - exampleTransport, + exampleTransportLocator, mockLocalParticipant, mockMediaDevices, mockRemoteParticipant, @@ -119,7 +119,7 @@ function renderTestComponent( p.identity)} livekitRoom={livekitRoom} - transport={exampleTransport} + transport={exampleTransportLocator} /> , ); @@ -151,7 +151,7 @@ it("should not render or warn for the local participant", () => { , ); @@ -178,7 +178,7 @@ it("should warn only once per unexpected participant", () => { , ); diff --git a/src/livekit/MatrixAudioRenderer.tsx b/src/livekit/MatrixAudioRenderer.tsx index 7f5e8b8fa..4ca1d3bae 100644 --- a/src/livekit/MatrixAudioRenderer.tsx +++ b/src/livekit/MatrixAudioRenderer.tsx @@ -19,10 +19,10 @@ import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { useEarpieceAudioConfig } from "../MediaDevicesContext"; import { useReactiveState } from "../useReactiveState"; import * as controls from "../controls"; -import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { type TransportLocator } from "./auth"; export interface MatrixAudioRendererProps { - transport: UnstableLivekitTransport; + transport: TransportLocator; livekitRoom: LivekitRoom; /** * The list of participant identities to render audio for. diff --git a/src/livekit/auth/types.ts b/src/livekit/auth/types.ts index 81982d689..409d88a6b 100644 --- a/src/livekit/auth/types.ts +++ b/src/livekit/auth/types.ts @@ -5,6 +5,17 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; + +/** + * Data identifying a transport and the homeserver to which it belongs, so + * consumers know how to reach it. + */ +export interface TransportLocator { + transport: UnstableLivekitTransport; + serverName: string; +} + /** * Configuration and access tokens provided by the SFU on successful authentication. */ diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index d6791c995..169abb98c 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -165,6 +165,7 @@ import { type RingingMediaViewModel, } from "../media/RingingMediaViewModel.ts"; import { type GridTileViewModel } from "../TileViewModel.ts"; +import { type TransportLocator } from "../../livekit/auth"; //TODO // Larger rename @@ -217,7 +218,7 @@ export interface CallViewModelOptions { */ windowSize$: Behavior<{ width: number; height: number }>; /** Optional value overriding the local transport, for testing purposes. */ - localTransport?: UnstableLivekitTransport; + localTransport?: TransportLocator; /** Optional value overriding the connection factory, for testing purposes. */ connectionFactory?: ConnectionFactory; /** The version & compatibility mode of MatrixRTC that we should use. */ @@ -278,7 +279,7 @@ interface LayoutScanState { export type LivekitRoomItem = { livekitRoom: LivekitRoom; participants: string[]; - transport: UnstableLivekitTransport; + transport: TransportLocator; }; /** @@ -762,7 +763,10 @@ export function createCallViewModel$( map((members) => [...Map.groupBy(members, (m) => m.connection)].map( ([connection, members]) => ({ - transport: connection.transport, + transport: { + transport: connection.transport, + serverName: connection.serverName, + }, livekitRoom: connection.livekitRoom, participants: members.map((m) => m.participant.identity), }), diff --git a/src/state/CallViewModel/CallViewModelTestUtils.ts b/src/state/CallViewModel/CallViewModelTestUtils.ts index 95920eee7..809dc75a5 100644 --- a/src/state/CallViewModel/CallViewModelTestUtils.ts +++ b/src/state/CallViewModel/CallViewModelTestUtils.ts @@ -30,7 +30,7 @@ import { type CallViewModelOptions, } from "./CallViewModel"; import { - exampleTransport, + exampleTransportLocator, mockConfig, MockConnection, mockLivekitRoom, @@ -208,12 +208,13 @@ export function withCallViewModel(mode: MatrixRTCMode) { livekitRoomFactory, connectionState$, windowSize$, - localTransport: exampleTransport, + localTransport: exampleTransportLocator, connectionFactory: { createConnection( scope, role, transport, + serverName, ownMembershipIdentity, logger, ) { @@ -222,6 +223,7 @@ export function withCallViewModel(mode: MatrixRTCMode) { scope, role, transport, + serverName, ownMembershipIdentity, client: room.client, roomId: room.roomId, diff --git a/src/state/CallViewModel/localMember/LocalMember.test.ts b/src/state/CallViewModel/localMember/LocalMember.test.ts index 437cdd20e..f1c95d1cb 100644 --- a/src/state/CallViewModel/localMember/LocalMember.test.ts +++ b/src/state/CallViewModel/localMember/LocalMember.test.ts @@ -293,7 +293,7 @@ describe("LocalMembership", () => { leave: vi.fn(), }, connectionMap$: constant(new Epoch(new ConnectionMap())), - localTransport$: constant(mockTransport), + localTransport$: constant(mockTransportLocator), }); const expextedLog = "'not connected yet' while updating the call intent (this is expected on startup)"; @@ -308,6 +308,11 @@ describe("LocalMembership", () => { livekit_service_url: "a", } as UnstableLivekitTransport; + const mockTransportLocator: livekitAuth.TransportLocator = { + transport: mockTransport, + serverName: "example.org", + }; + const connectionTransportAConnected = { livekitRoom: mockLivekitRoom({ localParticipant: { @@ -320,6 +325,7 @@ describe("LocalMembership", () => { livekitState: LivekitConnectionState.Connected, }), transport: mockTransport, + serverName: "example.org", } as Connection; const connectionTransportAConnecting = { ...connectionTransportAConnected, @@ -360,7 +366,7 @@ describe("LocalMembership", () => { ...defaultCreateLocalMemberValues, connectionMap$: constant(new Epoch(connectionMap)), joinMatrixRTC, - localTransport$: constant(mockTransport), + localTransport$: constant(mockTransportLocator), delayId$, }); @@ -437,7 +443,7 @@ describe("LocalMembership", () => { scope, ...defaultCreateLocalMemberValues, connectionMap$: constant(new Epoch(connectionMap)), - localTransport$: constant(mockTransport), + localTransport$: constant(mockTransportLocator), }); await flushPromises(); expect(publisherFactory).toHaveBeenCalledOnce(); @@ -459,7 +465,7 @@ describe("LocalMembership", () => { it("tracks livekit state correctly", async () => { const scope = new ObservableScope(); const connectionMap = new ConnectionMap(); - const localTransport$ = new Subject(); + const localTransport$ = new Subject(); const connectionMap$ = new BehaviorSubject(new Epoch(connectionMap)); const publishers: Publisher[] = []; @@ -503,7 +509,7 @@ describe("LocalMembership", () => { expect(localMembership.localMemberState$.value).toStrictEqual( TransportState.Waiting, ); - localTransport$.next(mockTransport); + localTransport$.next(mockTransportLocator); await flushPromises(); expect(localMembership.localMemberState$.value).toStrictEqual({ matrix: RTCMemberStatus.Connected, @@ -534,7 +540,7 @@ describe("LocalMembership", () => { }); ( - connectionMap2.getConnectionForTransport(mockTransport)! + connectionMap2.getConnectionForTransport(mockTransportLocator)! .state$ as BehaviorSubject ).next({ state: "authenticated", @@ -641,7 +647,7 @@ describe("LocalMembership", () => { rtsSession$: constant(RTCMemberStatus.Connected), }, connectionMap$: constant(new Epoch(connectionMap)), - localTransport$: constant(mockTransport), + localTransport$: constant(mockTransportLocator), }); await flushPromises(); @@ -676,7 +682,7 @@ describe("LocalMembership", () => { rtsSession$: constant(RTCMemberStatus.Connected), }, connectionMap$: constant(new Epoch(connectionMap)), - localTransport$: constant(mockTransport), + localTransport$: constant(mockTransportLocator), }); await flushPromises(); @@ -723,7 +729,7 @@ describe("LocalMembership", () => { rtsSession$: constant(RTCMemberStatus.Connected), }, connectionMap$: constant(new Epoch(connectionMap)), - localTransport$: constant(mockTransport), + localTransport$: constant(mockTransportLocator), }); await flushPromises(); @@ -768,7 +774,7 @@ describe("LocalMembership", () => { rtsSession$: constant(RTCMemberStatus.Connected), }, connectionMap$: constant(new Epoch(connectionMap)), - localTransport$: constant(mockTransport), + localTransport$: constant(mockTransportLocator), }); await flushPromises(); @@ -832,7 +838,7 @@ describe("LocalMembership", () => { scope, ...defaultCreateLocalMemberValues, connectionMap$: constant(new Epoch(connectionMap)), - localTransport$: constant(mockTransport), + localTransport$: constant(mockTransportLocator), }); return { scope, localMembership }; }; @@ -846,6 +852,7 @@ describe("LocalMembership", () => { livekitState: LivekitConnectionState.Connected, }), transport: mockTransport, + serverName: "example.org", livekitRoom: mockLivekitRoom({ localParticipant: mockLocalParticipant({ isScreenShareEnabled: false, diff --git a/src/state/CallViewModel/localMember/LocalMember.ts b/src/state/CallViewModel/localMember/LocalMember.ts index 1b1e8aae4..14f853403 100644 --- a/src/state/CallViewModel/localMember/LocalMember.ts +++ b/src/state/CallViewModel/localMember/LocalMember.ts @@ -78,7 +78,11 @@ import { type Connection, } from "../remoteMembers/Connection.ts"; import { type HomeserverConnected } from "./HomeserverConnected.ts"; -import { type ClientOpenIDParts, getSFUConfig } from "../../../livekit/auth"; +import { + type ClientOpenIDParts, + getSFUConfig, + type TransportLocator, +} from "../../../livekit/auth"; export enum TransportState { /** Not even a transport is available to the LocalMembership */ @@ -147,7 +151,7 @@ interface Props { homeserverConnected: HomeserverConnected; roomId: string; ownMembershipIdentity: CallMembershipIdentityParts; - localTransport$: Observable; + localTransport$: Observable; client: ClientOpenIDParts; matrixRTCSession: Pick< MatrixRTCSession, @@ -306,7 +310,7 @@ export const createLocalMembership$ = ({ // to whether delayed event delegation is supported const joinParams$ = scope.behavior( localTransport$.pipe( - switchMap(async (transport) => { + switchMap(async ({ transport }) => { const transportSupportsDelegation = "livekit_service_url" in transport && (await checkDelegationSupport( diff --git a/src/state/CallViewModel/localMember/LocalTransport.test.ts b/src/state/CallViewModel/localMember/LocalTransport.test.ts index 178b5bde4..0b3caa42e 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.test.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.test.ts @@ -46,8 +46,11 @@ describe("LocalTransport", () => { getDomain: () => "example.org", }), ).toStrictEqual({ - livekit_service_url: "https://lk.example.org", - type: "livekit", + transport: { + livekit_service_url: "https://lk.example.org", + type: "livekit", + }, + serverName: "example.org", }); }); @@ -74,8 +77,11 @@ describe("LocalTransport", () => { }); expect(await getLocalTransport(client)).toStrictEqual({ - livekit_service_url: "https://lk.example.org", - type: "livekit", + transport: { + livekit_service_url: "https://lk.example.org", + type: "livekit", + }, + serverName: "example.org", }); }); @@ -83,8 +89,11 @@ describe("LocalTransport", () => { customLivekitUrl.setValue("https://lk.example.org"); expect(await getLocalTransport(client)).toStrictEqual({ - livekit_service_url: "https://lk.example.org", - type: "livekit", + transport: { + livekit_service_url: "https://lk.example.org", + type: "livekit", + }, + serverName: "example.org", }); }); @@ -94,8 +103,11 @@ describe("LocalTransport", () => { ]); expect(await getLocalTransport(client)).toStrictEqual({ - livekit_service_url: "https://lk.example.org", - type: "livekit", + transport: { + livekit_service_url: "https://lk.example.org", + type: "livekit", + }, + serverName: "example.org", }); }); diff --git a/src/state/CallViewModel/localMember/LocalTransport.ts b/src/state/CallViewModel/localMember/LocalTransport.ts index f2572bf30..feb8aee6f 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.ts @@ -13,6 +13,7 @@ import { Config } from "../../../config/Config.ts"; import { MatrixRTCTransportMissingError } from "../../../utils/errors.ts"; import { customLivekitUrl } from "../../../settings/settings.ts"; import { RtcTransportAutoDiscovery } from "./RtcTransportAutoDiscovery.ts"; +import { type TransportLocator } from "../../../livekit/auth/types.ts"; /** * Determines the transport to advertise in our MatrixRTC membership and publish @@ -20,7 +21,10 @@ import { RtcTransportAutoDiscovery } from "./RtcTransportAutoDiscovery.ts"; */ export async function getLocalTransport( client: Pick, -): Promise { +): Promise { + const serverName = client.getDomain(); + if (serverName === null) throw new Error("No server name"); + const discovery = new RtcTransportAutoDiscovery({ client, resolvedConfig: Config.get(), @@ -33,8 +37,7 @@ export async function getLocalTransport( ? { type: "livekit", livekit_service_url: customUrl } : await discovery.discoverPreferredTransport(); - if (transport === null) - throw new MatrixRTCTransportMissingError(client.getDomain() ?? ""); + if (transport === null) throw new MatrixRTCTransportMissingError(serverName); - return transport; + return { transport, serverName }; } diff --git a/src/state/CallViewModel/remoteMembers/Connection.test.ts b/src/state/CallViewModel/remoteMembers/Connection.test.ts index 98a1aed87..f0fc6bfd5 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.test.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.test.ts @@ -118,6 +118,7 @@ function setupRemoteConnection(): Connection { slotId: SLOT_ID, role: "subscriber", transport: livekitFocus, + serverName: "m.localhost", scope: testScope, ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, @@ -163,6 +164,7 @@ describe("Start connection states", () => { slotId: SLOT_ID, role: "subscriber", transport: livekitFocus, + serverName: "m.localhost", scope: testScope, ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, @@ -182,6 +184,7 @@ describe("Start connection states", () => { slotId: SLOT_ID, role: "subscriber", transport: livekitFocus, + serverName: "m.localhost", scope: testScope, ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, @@ -236,6 +239,7 @@ describe("Start connection states", () => { slotId: SLOT_ID, role: "subscriber", transport: livekitFocus, + serverName: "m.localhost", scope: testScope, ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, @@ -300,6 +304,7 @@ describe("Start connection states", () => { slotId: SLOT_ID, role: "subscriber", transport: livekitFocus, + serverName: "m.localhost", scope: testScope, ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, @@ -560,6 +565,7 @@ describe("remote track logging", () => { slotId: SLOT_ID, role: "subscriber", transport: livekitFocus, + serverName: "m.localhost", scope: testScope, ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, diff --git a/src/state/CallViewModel/remoteMembers/Connection.ts b/src/state/CallViewModel/remoteMembers/Connection.ts index 575ddfad6..c0fe4fd9a 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.ts @@ -51,6 +51,8 @@ export interface ConnectionOpts { role: "publisher" | "subscriber"; /** The media transport to connect to. */ transport: UnstableLivekitTransport; + /** The name of the homeserver to which the {@link transport} belongs. */ + serverName: string; /** The Matrix client to use for OpenID and SFU config requests. */ client: ClientOpenIDParts; /** The ID of the Matrix room in which the session takes place. */ @@ -99,6 +101,11 @@ export class Connection { */ public readonly transport: UnstableLivekitTransport; + /** + * The name of the homeserver to which the {@link transport} belongs. + */ + public readonly serverName: string; + public readonly livekitRoom: LivekitRoom; private scope: ObservableScope; @@ -147,12 +154,13 @@ export class Connection { `[Connection ${JSON.stringify(opts.transport)}]`, ); this.logger.debug(`constructor called`); - const { role, transport, client, scope } = opts; + const { role, transport, serverName, client, scope } = opts; this.scope = scope; this.livekitRoom = opts.livekitRoomFactory(); this.role = role; this.transport = transport; + this.serverName = serverName; this.client = client; this.remoteParticipants$ = scope.behavior( diff --git a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts index b2a4833d7..86cc01b51 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts @@ -43,6 +43,7 @@ export interface ConnectionFactory { scope: ObservableScope, role: "publisher" | "subscriber", transport: UnstableLivekitTransport, + serverName: string, ownMembershipIdentity: CallMembershipIdentityParts, logger: Logger, ): Connection; @@ -95,6 +96,7 @@ export class ECConnectionFactory implements ConnectionFactory { * * @param scope The observable scope (used for clean-up) * @param transport The transport to use for this connection. + * @param serverName The name of the homeserver to which the {@link transport} belongs. * @param ownMembershipIdentity required to connect (using the jwt service) with the SFU. * @param logger The logger instance to use for this connection. * @param sfuConfig optional config in case we already have a token for this connection. @@ -104,6 +106,7 @@ export class ECConnectionFactory implements ConnectionFactory { scope: ObservableScope, role: "publisher" | "subscriber", transport: UnstableLivekitTransport, + serverName: string, ownMembershipIdentity: CallMembershipIdentityParts, logger: Logger, ): Connection { @@ -113,6 +116,7 @@ export class ECConnectionFactory implements ConnectionFactory { slotId: this.slotId, role, transport, + serverName, client: this.client, scope: scope, livekitRoomFactory: this.livekitRoomFactory, diff --git a/src/state/CallViewModel/remoteMembers/ConnectionMap.test.ts b/src/state/CallViewModel/remoteMembers/ConnectionMap.test.ts index dba1cb0be..90c0e4c67 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionMap.test.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionMap.test.ts @@ -12,32 +12,41 @@ import { type RemoteParticipant } from "livekit-client"; import { logger } from "matrix-js-sdk/lib/logger"; import { Epoch, mapEpoch, ObservableScope } from "../../ObservableScope.ts"; -import { createConnectionMap$, ConnectionMap } from "./ConnectionMap.ts"; +import { + createConnectionMap$, + ConnectionMap, + keyFor, +} from "./ConnectionMap.ts"; import { type ConnectionFactory } from "./ConnectionFactory.ts"; import { type Connection } from "./Connection.ts"; import { ownMemberMock, withTestScheduler } from "../../../utils/test.ts"; -import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts"; import { type Behavior } from "../../Behavior.ts"; +import { type TransportLocator } from "../../../livekit/auth"; // Some test constants -const TRANSPORT_1: UnstableLivekitTransport = { - type: "livekit", - livekit_service_url: "https://lk.example.org", +const TRANSPORT_1: TransportLocator = { + transport: { + type: "livekit", + livekit_service_url: "https://lk.example.org", + }, + serverName: "example.org", }; -const TRANSPORT_2: UnstableLivekitTransport = { - type: "livekit", - livekit_service_url: "https://lk.sample.com", +const TRANSPORT_2: TransportLocator = { + transport: { + type: "livekit", + livekit_service_url: "https://lk.sample.com", + }, + serverName: "sample.com", }; function findConnection( - transport: UnstableLivekitTransport, + transport: TransportLocator, connections: Connection[], ): Connection | undefined { - return connections.find((c) => - areLivekitTransportsEqual(c.transport, transport), - ); + const key = keyFor(transport); + return connections.find((c) => keyFor(c) === key); } let fakeConnectionFactory: ConnectionFactory; @@ -57,9 +66,11 @@ beforeEach(() => { scope: ObservableScope, _role: unknown, transport: UnstableLivekitTransport, + serverName: string, ) => { const mockConnection = { transport, + serverName, remoteParticipants$: new BehaviorSubject([]), } as unknown as Connection; vi.mocked(mockConnection).start = vi.fn(); @@ -210,14 +221,14 @@ describe("ConnectionMap", () => { test("warns when a second connection to the same URL is merged", () => { const warn = vi.spyOn(logger, "warn").mockImplementation(() => {}); const data = new ConnectionMap(logger); - const connection = { transport: TRANSPORT_1 } as unknown as Connection; + const connection = TRANSPORT_1 as unknown as Connection; const p = (identity: string): RemoteParticipant => ({ identity }) as unknown as RemoteParticipant; data.add(connection, [p("a")]); data.add({ ...connection } as Connection, [p("b")]); expect(warn).toHaveBeenCalledWith( expect.stringContaining( - "second connection to https://lk.example.org: existing [a], adding [b]", + 'second connection to [null,"https://lk.example.org","example.org"]: existing [a], adding [b]', ), ); expect(data.getParticipantsForTransport(TRANSPORT_1)).toHaveLength(2); @@ -228,20 +239,14 @@ describe("connectionMap$", () => { // Used in test to control fake connections' remoteParticipants$ streams let fakeRemoteParticipantsStreams: Map>; - function keyForTransport(transport: UnstableLivekitTransport): string { - return "livekit_service_url" in transport - ? transport.livekit_service_url - : transport.url; - } - beforeEach(() => { fakeRemoteParticipantsStreams = new Map(); function getRemoteParticipantsFor( - transport: UnstableLivekitTransport, + transport: TransportLocator, ): Behavior { return ( - fakeRemoteParticipantsStreams.get(keyForTransport(transport)) ?? + fakeRemoteParticipantsStreams.get(keyFor(transport)) ?? new BehaviorSubject([]) ); } @@ -254,13 +259,18 @@ describe("connectionMap$", () => { scope: ObservableScope, _role: unknown, transport: UnstableLivekitTransport, + serverName: string, ) => { const fakeRemoteParticipants$ = new BehaviorSubject< RemoteParticipant[] >([]); const mockConnection = { transport, - remoteParticipants$: getRemoteParticipantsFor(transport), + serverName, + remoteParticipants$: getRemoteParticipantsFor({ + transport, + serverName, + }), } as unknown as Connection; vi.mocked(mockConnection).start = vi.fn(); vi.mocked(mockConnection).stop = vi.fn(); @@ -270,7 +280,7 @@ describe("connectionMap$", () => { }); fakeRemoteParticipantsStreams.set( - keyForTransport(transport), + keyFor({ transport, serverName }), fakeRemoteParticipants$, ); return mockConnection; @@ -283,7 +293,7 @@ describe("connectionMap$", () => { // Setup the fake participants streams behavior // ============================== fakeRemoteParticipantsStreams.set( - keyForTransport(TRANSPORT_1), + keyFor(TRANSPORT_1), behavior("oa-b", { o: [], a: [{ identity: "user1A" } as RemoteParticipant], @@ -295,7 +305,7 @@ describe("connectionMap$", () => { ); fakeRemoteParticipantsStreams.set( - keyForTransport(TRANSPORT_2), + keyFor(TRANSPORT_2), behavior("o-a", { o: [], a: [{ identity: "user2A" } as RemoteParticipant], diff --git a/src/state/CallViewModel/remoteMembers/ConnectionMap.ts b/src/state/CallViewModel/remoteMembers/ConnectionMap.ts index 698d9ad97..4adc92b04 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionMap.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionMap.ts @@ -20,44 +20,67 @@ import { type ObservableScope, } from "../../ObservableScope.ts"; import { generateItemsWithEpoch } from "../../../utils/observable.ts"; -import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts"; import { type ConnectionFactory } from "./ConnectionFactory.ts"; +import { type TransportLocator } from "../../../livekit/auth/types.ts"; + +// Branded string type, to enforce that `fromKey` must only be called with +// values returned by `keyFor` +export type TransportKey = string & { __transportKey: true }; /** - * The URL by which a transport is identified: its JWT service URL, or failing - * that, its SFU URL. + * Converts a {@link TransportLocator} to a key suitable for identifying it in a + * {@link Map} or {@link Set}. */ -function urlOf(transport: UnstableLivekitTransport): string { - return "livekit_service_url" in transport - ? transport.livekit_service_url - : transport.url; +export function keyFor({ + transport, + serverName, +}: TransportLocator): TransportKey { + // Serialize the fields in a consistent order + return JSON.stringify([ + "url" in transport ? transport.url : null, + "livekit_service_url" in transport ? transport.livekit_service_url : null, + serverName, + ]) as TransportKey; } -function removeDuplicateTransports( - transports: T[], -): T[] { - return transports.reduce((acc, transport) => { - if (!acc.some((t) => areLivekitTransportsEqual(t, transport))) - acc.push(transport); - return acc; - }, [] as T[]); +/** + * Inverse of {@link keyFor}: turns a key back into a {@link TransportLocator}. + */ +export function fromKey(key: TransportKey): TransportLocator { + const [url, serviceUrl, serverName] = JSON.parse(key); + return { + transport: { + type: "livekit", + ...(url === null ? {} : { url }), + ...(serviceUrl === null ? {} : { livekit_service_url: serviceUrl }), + } as UnstableLivekitTransport, + serverName, + }; +} + +function removeDuplicateTransports( + transports: TransportLocator[], +): TransportLocator[] { + const map = new Map(); + for (const transport of transports) map.set(keyFor(transport), transport); + return [...map.values()]; } export class ConnectionMap { private readonly map = new Map< - string, + TransportKey, { connection: Connection; participants: RemoteParticipant[] } >(); public constructor(private readonly logger?: Logger) {} public add(connection: Connection, participants: RemoteParticipant[]): void { - const key = this.getKey(connection.transport); + const key = keyFor(connection); const existing = this.map.get(key); if (!existing) { this.map.set(key, { connection, participants }); } else { - // Transports are deduplicated by URL upstream, so this should never + // Transports are deduplicated by key upstream, so this should never // happen; if it does, members may be matched against the wrong room. this.logger?.warn( `Merging participants from a second connection to ${key}: existing [${existing.participants.map((p) => p.identity).join(", ")}], adding [${participants.map((p) => p.identity).join(", ")}]`, @@ -66,26 +89,20 @@ export class ConnectionMap { } } - private getKey(transport: UnstableLivekitTransport): string { - // This is enough as a key because the ConnectionMap is already scoped by room. - // We also do not need to consider the slotId at this point since each `MatrixRTCSession` is already scoped by `slotDescription: {id, application}`. - return urlOf(transport); - } - public getConnections(): Connection[] { return Array.from(this.map.values()).map(({ connection }) => connection); } public getConnectionForTransport( - transport: UnstableLivekitTransport, + transport: TransportLocator, ): Connection | null { - return this.map.get(this.getKey(transport))?.connection ?? null; + return this.map.get(keyFor(transport))?.connection ?? null; } public getParticipantsForTransport( - transport: UnstableLivekitTransport, + transport: TransportLocator, ): RemoteParticipant[] { - const existing = this.map.get(this.getKey(transport)); + const existing = this.map.get(keyFor(transport)); if (existing) { return existing.participants; } @@ -96,8 +113,8 @@ export class ConnectionMap { interface Props { scope: ObservableScope; connectionFactory: ConnectionFactory; - localTransport: UnstableLivekitTransport; - remoteTransports$: Behavior>; + localTransport: TransportLocator; + remoteTransports$: Behavior>; logger: Logger; ownMembershipIdentity: CallMembershipIdentityParts; @@ -128,7 +145,7 @@ export function createConnectionMap$({ * All transports currently represented in the connection map. This list does * not include duplicate transports. */ - const localAndRemoteTransports$: Behavior> = + const localAndRemoteTransports$: Behavior> = scope.behavior( remoteTransports$.pipe( mapEpoch((remoteTransports) => @@ -140,6 +157,8 @@ export function createConnectionMap$({ ), ); + const localTransportKey = keyFor(localTransport); + /** * Connections for each transport in use by one or more session members. */ @@ -149,17 +168,25 @@ export function createConnectionMap$({ "ConnectionMap connections$", function* (transports) { for (const transport of transports) { - const role = areLivekitTransportsEqual(transport, localTransport) - ? ("publisher" as const) - : ("subscriber" as const); - yield { keys: [role, urlOf(transport)], data: transport }; + const role = + keyFor(transport) === localTransportKey + ? ("publisher" as const) + : ("subscriber" as const); + yield { + // In order to maintain a static set of map keys, we must + // temporarily serialize `transport` into a stable key format + keys: [role, keyFor(transport)], + data: undefined, + }; } }, - (scope, transport$, role, _url) => { + (scope, _data$, role, transportKey) => { + const { transport, serverName } = fromKey(transportKey); const connection = connectionFactory.createConnection( scope, role, - transport$.value, + transport, + serverName, ownMembershipIdentity, logger, ); diff --git a/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts b/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts index 1fdc7e8a3..5d9abd0fe 100644 --- a/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts +++ b/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts @@ -91,6 +91,7 @@ describe("ECConnectionFactory - Audio inputs options", () => { testScope, "subscriber", exampleTransport, + "example.org", ownMemberMock, logger, ); @@ -141,6 +142,7 @@ describe("ECConnectionFactory - ControlledAudioDevice", () => { testScope, "subscriber", exampleTransport, + "example.org", ownMemberMock, logger, ); @@ -178,6 +180,7 @@ describe("ECConnectionFactory - Camera quality settings", () => { testScope, "subscriber", exampleTransport, + "example.org", ownMemberMock, logger, ); @@ -217,6 +220,7 @@ describe("ECConnectionFactory - Camera quality settings", () => { testScope, "subscriber", exampleTransport, + "example.org", ownMemberMock, logger, ); @@ -257,6 +261,7 @@ describe("ECConnectionFactory - Camera quality settings", () => { testScope, "subscriber", exampleTransport, + "example.org", ownMemberMock, logger, ); diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts index af98b2342..85f21b478 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts @@ -17,12 +17,7 @@ import { type RemoteMatrixLivekitMember, createRemoteMatrixLivekitMembers$, } from "./MatrixLivekitMembers.ts"; -import { - Epoch, - mapEpoch, - ObservableScope, - trackEpoch, -} from "../../ObservableScope.ts"; +import { Epoch, ObservableScope, trackEpoch } from "../../ObservableScope.ts"; import { ConnectionMap } from "./ConnectionMap.ts"; import { flushPromises, @@ -32,6 +27,8 @@ import { import { type Connection } from "./Connection.ts"; import { constant } from "../../Behavior.ts"; import { localRtcMember } from "../../../utils/test-fixtures.ts"; +import { type TransportLocator } from "../../../livekit/auth"; +import { membershipsAndTransports$ } from "../../SessionBehaviors.ts"; let testScope: ObservableScope; @@ -112,24 +109,14 @@ function createEpochedMemberships$(m$: Observable): { memberships$: Observable>; membershipsWithTransport$: Observable< Epoch< - { membership: CallMembership; transport?: UnstableLivekitTransport }[] + { membership: CallMembership; transport: TransportLocator | undefined }[] > >; } { - const memberships$ = m$.pipe(trackEpoch()); - const membershipsWithTransport$ = memberships$.pipe( - mapEpoch((members) => { - return members.map((m) => { - const tr = m.getTransport(); - return { - membership: m, - transport: - tr?.type === "livekit" - ? (tr as UnstableLivekitTransport) - : undefined, - }; - }); - }), + const memberships$ = testScope.behavior(m$.pipe(trackEpoch())); + const { membershipsWithTransport$ } = membershipsAndTransports$( + testScope, + memberships$, ); return { memberships$, @@ -149,6 +136,7 @@ test("should signal participant on a connection that is publishing", async () => const connection = { transport: bobMembership.getTransport(), + serverName: "example.org", } as unknown as Connection; const dataWithPublisher = new ConnectionMap(); dataWithPublisher.add(connection, [ @@ -189,6 +177,7 @@ test("should signal participant on a connection that is not publishing", async ( const connection = { transport: bobMembership.getTransport(), + serverName: "example.org", } as unknown as Connection; const dataWithPublisher = new ConnectionMap(); dataWithPublisher.add(connection, []); @@ -227,9 +216,11 @@ describe("Publication edge case", () => { ); const connectionA = { transport: transportA, + serverName: "example.org", } as unknown as Connection; const connectionB = { transport: transportB, + serverName: "sample.com", } as unknown as Connection; connectionWithPublisher.add(connectionA, [ @@ -282,8 +273,14 @@ test("bob is publishing in the wrong connection", async () => { bobMembership.userId, bobMembership.deviceId, ); - const connectionA = { transport: transportA } as unknown as Connection; - const connectionB = { transport: transportB } as unknown as Connection; + const connectionA = { + transport: transportA, + serverName: "example.org", + } as unknown as Connection; + const connectionB = { + transport: transportB, + serverName: "sample.com", + } as unknown as Connection; // Bob is not publishing on A connectionWithPublisher.add(connectionA, []); diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts index 46735bb2f..2f2aa8e0c 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts @@ -6,19 +6,16 @@ Please see LICENSE in the repository root for full details. */ import { type LocalParticipant, type RemoteParticipant } from "livekit-client"; -import { - type CallMembership, - type UnstableLivekitTransport, -} from "matrix-js-sdk/lib/matrixrtc"; +import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; import { combineLatest, filter, map } from "rxjs"; import { logger } from "matrix-js-sdk/lib/logger"; -import { deepCompare } from "matrix-js-sdk/lib/utils"; import { type Behavior } from "../../Behavior"; import { type ConnectionMap } from "./ConnectionMap"; import { Epoch, type ObservableScope } from "../../ObservableScope"; import { type Connection } from "./Connection"; import { generateItemsWithEpoch } from "../../../utils/observable"; +import { type TransportLocator } from "../../../livekit/auth"; interface LocalTaggedParticipant { type: "local"; @@ -62,7 +59,7 @@ interface Props { scope: ObservableScope; membershipsWithTransport$: Behavior< Epoch< - { membership: CallMembership; transport?: UnstableLivekitTransport }[] + { membership: CallMembership; transport: TransportLocator | undefined }[] > >; connectionMap$: Behavior>; @@ -163,14 +160,3 @@ export function createRemoteMatrixLivekitMembers$({ } // TODO add back in the callviewmodel pauseWhen(this.pretendToBeDisconnected$) - -// TODO add this to the JS-SDK -export function areLivekitTransportsEqual( - t1: T | null, - t2: T | null, -): boolean { - if (t1 && t2) { - return deepCompare(t1, t2); - } - return !t1 && !t2; -} diff --git a/src/state/CallViewModel/remoteMembers/integration.test.ts b/src/state/CallViewModel/remoteMembers/integration.test.ts index 1a965cc02..39737ae6a 100644 --- a/src/state/CallViewModel/remoteMembers/integration.test.ts +++ b/src/state/CallViewModel/remoteMembers/integration.test.ts @@ -10,7 +10,6 @@ import { BehaviorSubject } from "rxjs"; import { type Room as LivekitRoom } from "livekit-client"; import EventEmitter from "events"; import fetchMock from "fetch-mock"; -import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { logger } from "matrix-js-sdk/lib/logger"; import { @@ -21,7 +20,7 @@ import { import { ECConnectionFactory } from "./ConnectionFactory.ts"; import { type ClientOpenIDParts } from "../../../livekit/auth"; import { - exampleTransport, + exampleTransportLocator, mockMediaDevices, mockRtcMembership, ownMemberMock, @@ -29,13 +28,13 @@ import { } from "../../../utils/test.ts"; import { type ProcessorState } from "../../../livekit/TrackProcessorContext.tsx"; import { - areLivekitTransportsEqual, createRemoteMatrixLivekitMembers$, type RemoteMatrixLivekitMember, } from "./MatrixLivekitMembers.ts"; import { createConnectionMap$ } from "./ConnectionMap.ts"; import { membershipsAndTransports$ } from "../../SessionBehaviors.ts"; import { localRtcMember, testJWTToken } from "../../../utils/test-fixtures.ts"; +import { deepCompare } from "matrix-js-sdk/lib/utils"; // Test the integration of ConnectionMap and MatrixLivekitMembers @@ -126,7 +125,7 @@ test("bob, carl, then bob joining no tracks yet", () => { const connectionMap$ = createConnectionMap$({ scope: testScope, connectionFactory: ecConnectionFactory, - localTransport: exampleTransport, + localTransport: exampleTransportLocator, remoteTransports$: membershipsAndTransports.transports$, logger: logger, ownMembershipIdentity: ownMemberMock, @@ -150,10 +149,7 @@ test("bob, carl, then bob joining no tracks yet", () => { }); expectObservable(item.connection$).toBe("a", { a: expect.toSatisfy((co) => - areLivekitTransportsEqual( - co.transport, - bobMembership.transports[0]! as UnstableLivekitTransport, - ), + deepCompare(co.transport, bobMembership.transports[0]), ), }); expectObservable(item.participant.value$).toBe("a", { @@ -185,15 +181,9 @@ test("bob, carl, then bob joining no tracks yet", () => { a: null, }); expectObservable(item.connection$).toBe("a", { - a: expect.toSatisfy((connection) => { - expect( - areLivekitTransportsEqual( - connection.transport, - carlMembership.transports[0]! as UnstableLivekitTransport, - ), - ).toBe(true); - return true; - }), + a: expect.toSatisfy((connection) => + deepCompare(connection.transport, carlMembership.transports[0]), + ), }); } return true; @@ -215,15 +205,9 @@ test("bob, carl, then bob joining no tracks yet", () => { a: daveMembership, }); expectObservable(item.connection$).toBe("a", { - a: expect.toSatisfy((connection) => { - expect( - areLivekitTransportsEqual( - connection.transport, - daveMembership.transports[0]! as UnstableLivekitTransport, - ), - ).toBe(true); - return true; - }), + a: expect.toSatisfy((connection) => + deepCompare(connection.transport, daveMembership.transports[0]), + ), }); expectObservable(item.participant.value$).toBe("a", { a: null, diff --git a/src/state/SessionBehaviors.ts b/src/state/SessionBehaviors.ts index 435e77021..6620350cf 100644 --- a/src/state/SessionBehaviors.ts +++ b/src/state/SessionBehaviors.ts @@ -7,7 +7,6 @@ Please see LICENSE in the repository root for full details. import { type CallMembership, - type UnstableLivekitTransport, type MatrixRTCSession, MatrixRTCSessionEvent, isUnstableLivekitTransport, @@ -21,6 +20,7 @@ import { type ObservableScope, } from "./ObservableScope"; import { type Behavior } from "./Behavior"; +import { type TransportLocator } from "../livekit/auth"; /** * Tracks the transports used by ourselves, plus all other MatrixRTC session @@ -32,17 +32,14 @@ export const membershipsAndTransports$ = ( ): { membershipsWithTransport$: Behavior< Epoch< - { membership: CallMembership; transport?: UnstableLivekitTransport }[] + { membership: CallMembership; transport: TransportLocator | undefined }[] > >; - transports$: Behavior>; + transports$: Behavior>; } => { const membershipsWithTransport$: Behavior< Epoch< - { - membership: CallMembership; - transport: UnstableLivekitTransport | undefined; - }[] + { membership: CallMembership; transport: TransportLocator | undefined }[] > > = scope.behavior( memberships$.pipe( @@ -52,7 +49,10 @@ export const membershipsAndTransports$ = ( return { membership, transport: isUnstableLivekitTransport(transport) - ? transport + ? { + transport, + serverName: membership.userId.replace(/^.*?:/, ""), + } : undefined, }; }); @@ -60,12 +60,11 @@ export const membershipsAndTransports$ = ( ), ); - const transports$: Behavior> = - scope.behavior( - membershipsWithTransport$.pipe( - mapEpoch((mts) => mts.flatMap(({ transport: t }) => (t ? [t] : []))), - ), - ); + const transports$: Behavior> = scope.behavior( + membershipsWithTransport$.pipe( + mapEpoch((mts) => mts.flatMap(({ transport: t }) => (t ? [t] : []))), + ), + ); return { membershipsWithTransport$, diff --git a/src/utils/test-viewmodel.ts b/src/utils/test-viewmodel.ts index 9903b99f8..df4f21bb1 100644 --- a/src/utils/test-viewmodel.ts +++ b/src/utils/test-viewmodel.ts @@ -69,7 +69,7 @@ export function getBasicRTCSession( getUserId: () => localRtcMember.userId, getDeviceId: () => localRtcMember.deviceId, getSyncState: () => SyncState.Syncing, - getDomain: () => null, + getDomain: () => "example.com", getAccessToken: () => "fake-token", sendEvent: vitest.fn().mockResolvedValue({ event_id: "$fake:event" }), redactEvent: vitest.fn().mockResolvedValue({ event_id: "$fake:event" }), diff --git a/src/utils/test.ts b/src/utils/test.ts index c3efc6b89..e40e56fca 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -78,6 +78,7 @@ import { type RemoteScreenShareViewModel, } from "../state/media/RemoteScreenShareViewModel"; import { Connection } from "../state/CallViewModel/remoteMembers/Connection"; +import { type TransportLocator } from "../livekit/auth"; export function withFakeTimers(continuation: () => void): void { vi.useFakeTimers(); @@ -213,6 +214,11 @@ export const exampleTransport: UnstableLivekitTransport = { livekit_service_url: "https://lk.example.org", }; +export const exampleTransportLocator: TransportLocator = { + transport: exampleTransport, + serverName: "example.org", +}; + export function mockRtcMembership( user: string | RoomMember, deviceId: string, From 17c3a7cb5d5ea13ce6c6179917f88e198afa734d Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 12 Oct 2026 10:32:13 +0200 Subject: [PATCH 13/25] Pin matrix-js-sdk to the member_id branch Temporary, until matrix-org/matrix-js-sdk#5588 merges and this can go back to a develop bump. The branch only changes the get_token request body from a member object to a member_id, so this is kept separate from the code that uses it and can be dropped without touching that code. --- package.json | 2 +- pnpm-lock.yaml | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/package.json b/package.json index b30f57ce3..0a76e0c01 100644 --- a/package.json +++ b/package.json @@ -103,7 +103,7 @@ "livekit-client": "^2.18.1", "lodash-es": "^4.17.21", "loglevel": "^1.9.1", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#develop", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#robin/livekit-updates", "matrix-widget-api": "^1.18.0", "node-stdlib-browser": "^1.3.1", "normalize.css": "^8.0.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 69805cfcd..5866d9f8c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -191,11 +191,11 @@ importers: specifier: ^1.9.1 version: 1.9.2 matrix-js-sdk: - specifier: github:matrix-org/matrix-js-sdk#develop - version: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/fa4772a4ddd335a5a8bfbf8bb21df49b70f7a999 + specifier: github:matrix-org/matrix-js-sdk#robin/livekit-updates + version: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/29099d884f08c90a1bc61b31bae0a937b0dc94ff matrix-widget-api: specifier: ^1.18.0 - version: 1.19.0 + version: 1.20.0 node-stdlib-browser: specifier: ^1.3.1 version: 1.3.1 @@ -4673,13 +4673,13 @@ packages: matrix-events-sdk@0.0.1: resolution: {integrity: sha512-1QEOsXO+bhyCroIe2/A5OwaxHvBm7EsSQ46DEDn8RBIfQwN5HWBpFvyWWR4QY0KHPPnnJdI99wgRiAl7Ad5qaA==} - matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/fa4772a4ddd335a5a8bfbf8bb21df49b70f7a999: - resolution: {gitHosted: true, integrity: sha512-qF8++74MX36KS1KEw6Lc8HHXgZ6nHD1oUU4yBZUj5vVPvZTIspk1lOGztkFmizE5vmLAHd5tdadL9Xu1Asy4bA==, tarball: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/fa4772a4ddd335a5a8bfbf8bb21df49b70f7a999} - version: 42.3.0 + matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/29099d884f08c90a1bc61b31bae0a937b0dc94ff: + resolution: {gitHosted: true, integrity: sha512-3614DKOUwhaTbGbLbWWMTrdQvRE0J7oucUTbrfr95fbAn/Zb1TZvwhVs3B94o4i9HrBf2hq1m1VwusAslg2WTQ==, tarball: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/29099d884f08c90a1bc61b31bae0a937b0dc94ff} + version: 43.0.0 engines: {node: '>=22.0.0'} - matrix-widget-api@1.19.0: - resolution: {integrity: sha512-Qe7L6G1mQJphBC47/Bzp5ljPIK1UARgLiuGgS+/KmEs/eaNHkQ3+ArD0/0kvMV+d6kCGAZ9OKRqsnuPjizI+tw==} + matrix-widget-api@1.20.0: + resolution: {integrity: sha512-qwFP37rgns7mPesM5/vvZgknzb+kvyRB5BISv5sCuBxsGMceMr+Kogf7OkjRxBXdMCPBFuf6gIuYRe5S+lpzQw==} md5.js@1.3.5: resolution: {integrity: sha512-xitP+WxNPcTTOgnTJcrhM0xvdPepipPSf3I8EIpGKeFLjt3PlJLIDG3u8EX53ZIubkb+5U2+3rELYpEhHhzdkg==} @@ -10227,7 +10227,7 @@ snapshots: matrix-events-sdk@0.0.1: {} - matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/fa4772a4ddd335a5a8bfbf8bb21df49b70f7a999: + matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/29099d884f08c90a1bc61b31bae0a937b0dc94ff: dependencies: '@babel/runtime': 8.0.0 '@matrix-org/matrix-sdk-crypto-wasm': 18.9.0 @@ -10236,12 +10236,12 @@ snapshots: content-type: 3.1.1 loglevel: 1.9.2 matrix-events-sdk: 0.0.1 - matrix-widget-api: 1.19.0 + matrix-widget-api: 1.20.0 p-retry: 8.0.1 sdp-transform: 3.0.0 unhomoglyph: 1.0.6 - matrix-widget-api@1.19.0: + matrix-widget-api@1.20.0: dependencies: '@types/events': 3.0.3 events: 3.3.0 From e336ee5cfad56d8f3b7ea1ebf48d7e70cd500470 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 12 Oct 2026 10:33:00 +0200 Subject: [PATCH 14/25] Get SFU tokens and delegate delayed leaves through the MSC4195 CS API src/livekit/auth now fronts two flows. For a transport with a LiveKit SFU `url`, tokens come from our own homeserver's MSC4195 get_token endpoint and delayed leaves are delegated through delegate_delayed_leave, both via the js-sdk's unstable client methods. Transports with only a livekit_service_url keep using the OpenID flow against the JWT service. A transport advertising both shapes lets a subscriber fall back to the OpenID flow when the homeserver answers get_token with a 404 (Synapse reports M_UNRECOGNIZED for unknown routes rather than M_NOT_FOUND), so a client can still join calls hosted on servers that have not deployed MSC4195 yet. Publishers never fall back: a homeserver that advertises a `url` but does not implement the endpoints is misconfigured and the admin should hear about it, so they get NoMatrix2AuthorizationService. Delegation uses the same flow as the token so that the participant identity the service computes matches. The local member only probes a transport's delegate_delayed_leave endpoint when it has a service URL. The developer settings tab validates a custom service URL through the same facade. --- src/livekit/auth/csApi.ts | 124 +++++++++ src/livekit/auth/index.test.ts | 252 ++++++++++++++++++ src/livekit/auth/index.ts | 201 +++++++++++++- src/settings/DeveloperSettingsTab.test.tsx | 6 +- src/settings/DeveloperSettingsTab.tsx | 6 +- .../CallViewModel/CallViewModelTestUtils.ts | 2 + .../localMember/LocalMember.test.ts | 35 ++- .../CallViewModel/localMember/LocalMember.ts | 22 +- .../remoteMembers/Connection.test.ts | 94 ++++--- .../CallViewModel/remoteMembers/Connection.ts | 10 +- .../remoteMembers/ConnectionFactory.ts | 7 +- .../remoteMembers/ECConnectionFactory.test.ts | 9 +- .../remoteMembers/integration.test.ts | 14 +- src/utils/test-fixtures.ts | 31 ++- 14 files changed, 709 insertions(+), 104 deletions(-) create mode 100644 src/livekit/auth/csApi.ts create mode 100644 src/livekit/auth/index.test.ts diff --git a/src/livekit/auth/csApi.ts b/src/livekit/auth/csApi.ts new file mode 100644 index 000000000..63b9d0cb9 --- /dev/null +++ b/src/livekit/auth/csApi.ts @@ -0,0 +1,124 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type MatrixClient } from "matrix-js-sdk"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; + +import { extractFullConfigFromToken, type SFUConfig } from "./types.ts"; +import { doNetworkOperationWithRetry } from "../../utils/matrix.ts"; + +export type ClientGetTokenParts = Pick< + MatrixClient, + "_unstable_getLivekitToken" +>; + +export interface GetSFUConfigParams { + /** + * The Matrix client. + */ + client: ClientGetTokenParts; + /** + * Data identifying the local user's session membership. + */ + membership: CallMembershipIdentityParts; + /** + * The WebSocket URL of the SFU for which we wish to get an access token. + */ + url: string; + /** + * The name of the homeserver to which the SFU belongs. + */ + serverName: string; + /** + * The ID of the Matrix room in which the session takes place. + */ + roomId: string; + /** + * The ID of the MatrixRTC slot in which the session takes place. + */ + slotId: string; +} + +/** + * Gets an {@link SFUConfig} appropriate for connecting to a given SFU, using + * the MSC4195 LiveKit client-server API endpoints. + */ +export async function getSFUConfig({ + client, + membership, + url, + serverName, + roomId, + slotId, +}: GetSFUConfigParams): Promise { + const res = await doNetworkOperationWithRetry(async () => + client._unstable_getLivekitToken({ + url, + server_name: serverName, + room_id: roomId, + slot_id: slotId, + member_id: membership.memberId, + }), + ); + return extractFullConfigFromToken({ url, jwt: res.jwt }); +} + +export type ClientDelegationParts = Pick< + MatrixClient, + "_unstable_delegateDelayedLeave" | "baseUrl" +>; + +export interface DelegateDelayedLeaveParams { + /** + * The Matrix client. + */ + client: ClientDelegationParts; + /** + * Data identifying the local user's session membership. + */ + membership: CallMembershipIdentityParts; + /** + * The WebSocket URL of the SFU to which we are publishing and wish to + * delegate the delayed leave event. + */ + url: string; + /** + * The ID of the Matrix room in which the session takes place. + */ + roomId: string; + /** + * The ID of the MatrixRTC slot in which the session takes place. + */ + slotId: string; + /** + * The delay ID of the leave event to be delegated. + */ + delayId: string; +} + +/** + * Delegates a delayed leave event to a given SFU, so that the event will + * be sent on our behalf whenever we disconnect from the SFU. + */ +export async function delegateDelayedLeave({ + client, + membership, + url, + roomId, + slotId, + delayId, +}: DelegateDelayedLeaveParams): Promise { + await doNetworkOperationWithRetry(async () => + client._unstable_delegateDelayedLeave({ + url, + room_id: roomId, + slot_id: slotId, + member_id: membership.memberId, + delay_id: delayId, + }), + ); +} diff --git a/src/livekit/auth/index.test.ts b/src/livekit/auth/index.test.ts new file mode 100644 index 000000000..015e0b036 --- /dev/null +++ b/src/livekit/auth/index.test.ts @@ -0,0 +1,252 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { afterEach, expect, type Mocked, vi, describe, it } from "vitest"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { logger } from "matrix-js-sdk/lib/logger"; + +import { + aliceDeviceId, + aliceId, + aliceUserId, + createJWTToken, +} from "../../utils/test-fixtures"; +import type * as csApi from "./csApi"; +import * as openID from "./openID"; +import { delegateDelayedLeave, getSFUConfig } from "."; +import { mocked } from "storybook/test"; +import { ConnectionError, MatrixError } from "matrix-js-sdk"; +import { NoMatrix2AuthorizationService } from "../../utils/errors"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; + +vi.mock("./openID", () => ({ getSFUConfig: vi.fn() })); +vi.useFakeTimers(); + +const csApiJWTToken = createJWTToken(aliceId, "via CS API"); +const openIDJWTToken = createJWTToken(aliceId, "via OpenID"); + +mocked(openID).getSFUConfig.mockResolvedValue({ + url: "mock-openID-url", + jwt: openIDJWTToken, + livekitAlias: "via OpenID", + livekitIdentity: aliceId, +}); + +const client: Mocked< + csApi.ClientGetTokenParts & + csApi.ClientDelegationParts & + openID.ClientOpenIDParts +> = { + baseUrl: "https://matrix.example.org", + _unstable_getLivekitToken: vi.fn().mockResolvedValue({ jwt: csApiJWTToken }), + _unstable_delegateDelayedLeave: vi.fn().mockResolvedValue({}), + getOpenIdToken: vi + .fn() + .mockResolvedValue({ access_token: "mock-openID-token" }), +}; + +const membership: CallMembershipIdentityParts = { + userId: aliceUserId, + deviceId: aliceDeviceId, + memberId: "123", +}; + +const roomId = "!room:example.org"; +const slotId = "m.call#ROOM"; + +const unsupportedError = new MatrixError({ errcode: "M_UNSUPPORTED" }, 404); + +const urlTransport: UnstableLivekitTransport = { + type: "livekit", + url: "wss://livekit.example.org", +}; + +const serviceUrlTransport: UnstableLivekitTransport = { + type: "livekit", + livekit_service_url: "https://livekit-jwt.example.org", +}; + +const urlAndServiceUrlTransport: UnstableLivekitTransport = { + ...urlTransport, + ...serviceUrlTransport, +}; + +afterEach(() => vi.clearAllMocks()); + +describe("getSFUConfig", () => { + const defaultParams = { + client, + membership, + serverName: "example.org", + role: "subscriber" as const, + roomId, + slotId, + logger, + }; + + it("uses CS API with url transport", async () => { + expect( + await getSFUConfig({ ...defaultParams, transport: urlTransport }), + ).toEqual( + expect.objectContaining({ + jwt: csApiJWTToken, + livekitAlias: "via CS API", + }), + ); + expect(client._unstable_getLivekitToken).toHaveBeenCalled(); + expect(openID.getSFUConfig).not.toHaveBeenCalled(); + }); + + it("does not fall back to OpenID with url transport", async () => { + client._unstable_getLivekitToken.mockRejectedValueOnce(unsupportedError); + await expect( + getSFUConfig({ ...defaultParams, transport: urlTransport }), + ).rejects.toThrow(expect.any(NoMatrix2AuthorizationService)); + expect(client._unstable_getLivekitToken).toHaveBeenCalled(); + expect(openID.getSFUConfig).not.toHaveBeenCalled(); + }); + + it("uses OpenID with livekit_service_url transport", async () => { + expect( + await getSFUConfig({ ...defaultParams, transport: serviceUrlTransport }), + ).toEqual( + expect.objectContaining({ + jwt: openIDJWTToken, + livekitAlias: "via OpenID", + }), + ); + expect(client._unstable_getLivekitToken).not.toHaveBeenCalled(); + expect(openID.getSFUConfig).toHaveBeenCalled(); + }); + + it("uses CS API with url + livekit_service_url transport", async () => { + expect( + await getSFUConfig({ + ...defaultParams, + transport: urlAndServiceUrlTransport, + }), + ).toEqual( + expect.objectContaining({ + jwt: csApiJWTToken, + livekitAlias: "via CS API", + }), + ); + expect(client._unstable_getLivekitToken).toHaveBeenCalled(); + expect(openID.getSFUConfig).not.toHaveBeenCalled(); + }); + + it("falls back to OpenID with url + livekit_service_url transport", async () => { + client._unstable_getLivekitToken.mockRejectedValueOnce(unsupportedError); + expect( + await getSFUConfig({ + ...defaultParams, + transport: urlAndServiceUrlTransport, + }), + ).toEqual( + expect.objectContaining({ + jwt: openIDJWTToken, + livekitAlias: "via OpenID", + }), + ); + expect(client._unstable_getLivekitToken).toHaveBeenCalled(); + expect(openID.getSFUConfig).toHaveBeenCalled(); + }); + + it("does not fall back to OpenID when publishing", async () => { + client._unstable_getLivekitToken.mockRejectedValueOnce(unsupportedError); + await expect( + getSFUConfig({ + ...defaultParams, + transport: urlAndServiceUrlTransport, + role: "publisher", + }), + ).rejects.toThrow(expect.any(NoMatrix2AuthorizationService)); + expect(client._unstable_getLivekitToken).toHaveBeenCalled(); + expect(openID.getSFUConfig).not.toHaveBeenCalled(); + }); + + it("does not fall back to OpenID on connection error", async () => { + const client_ = { + ...client, + _unstable_getLivekitToken: vi + .fn() + .mockRejectedValue(new ConnectionError("No connection")), + }; + await Promise.all([ + expect( + getSFUConfig({ + ...defaultParams, + client: client_, + transport: urlAndServiceUrlTransport, + }), + ).rejects.toThrow(expect.any(ConnectionError)), + vi.runAllTimersAsync(), // Allow request to be retried + ]); + expect(client_._unstable_getLivekitToken).toHaveBeenCalled(); + expect(openID.getSFUConfig).not.toHaveBeenCalled(); + }); +}); + +describe("delegateDelayedLeave", () => { + const defaultParams = { + client, + membership, + roomId, + slotId, + delayId: "delay123", + logger, + }; + + it("uses CS API with url transport", async () => { + await delegateDelayedLeave({ ...defaultParams, transport: urlTransport }); + expect(client._unstable_delegateDelayedLeave).toHaveBeenCalled(); + expect(openID.getSFUConfig).not.toHaveBeenCalled(); + }); + + it("does not fall back to OpenID with url transport", async () => { + client._unstable_delegateDelayedLeave.mockRejectedValueOnce( + unsupportedError, + ); + await expect( + delegateDelayedLeave({ ...defaultParams, transport: urlTransport }), + ).rejects.toThrow(); + expect(client._unstable_delegateDelayedLeave).toHaveBeenCalled(); + expect(openID.getSFUConfig).not.toHaveBeenCalled(); + }); + + it("uses OpenID with livekit_service_url transport", async () => { + await delegateDelayedLeave({ + ...defaultParams, + transport: serviceUrlTransport, + }); + expect(client._unstable_delegateDelayedLeave).not.toHaveBeenCalled(); + expect(openID.getSFUConfig).toHaveBeenCalled(); + }); + + it("uses CS API with url + livekit_service_url transport", async () => { + await delegateDelayedLeave({ + ...defaultParams, + transport: urlAndServiceUrlTransport, + }); + expect(client._unstable_delegateDelayedLeave).toHaveBeenCalled(); + expect(openID.getSFUConfig).not.toHaveBeenCalled(); + }); + + it("does not fall back to OpenID with url + livekit_service_url transport", async () => { + client._unstable_delegateDelayedLeave.mockRejectedValueOnce( + unsupportedError, + ); + await expect( + delegateDelayedLeave({ + ...defaultParams, + transport: urlAndServiceUrlTransport, + }), + ).rejects.toThrow(); + expect(client._unstable_delegateDelayedLeave).toHaveBeenCalled(); + expect(openID.getSFUConfig).not.toHaveBeenCalled(); + }); +}); diff --git a/src/livekit/auth/index.ts b/src/livekit/auth/index.ts index 6c11768f7..c535e72f5 100644 --- a/src/livekit/auth/index.ts +++ b/src/livekit/auth/index.ts @@ -5,5 +5,204 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -export * from "./openID"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; +import { type Logger } from "matrix-js-sdk/lib/logger"; + +import { MatrixError } from "matrix-js-sdk"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import * as openID from "./openID"; +import * as csApi from "./csApi"; +import { type SFUConfig } from "./types"; +import { NoMatrix2AuthorizationService } from "../../utils/errors"; + +export type { ClientOpenIDParts } from "./openID"; +export type { ClientGetTokenParts, ClientDelegationParts } from "./csApi"; export type * from "./types"; + +/** + * Checks whether an error means "this homeserver does not implement the + * (unstable) MSC4195 endpoints". + * + * MSC4195 documents `M_NOT_FOUND` for this, but a homeserver without support + * answers with whatever it uses for unknown endpoints (Synapse: 404 with + * `M_UNRECOGNIZED`), so we accept any 404 as "unsupported". + * @param e The error to check. + */ +function isEndpointUnsupported(e: unknown): boolean { + return ( + e instanceof MatrixError && + (e.httpStatus === 404 || + e.errcode === "M_NOT_FOUND" || + e.errcode === "M_UNRECOGNIZED") + ); +} + +export interface GetSFUConfigParams { + /** + * The Matrix client. + */ + client: csApi.ClientGetTokenParts & openID.ClientOpenIDParts; + /** + * Data identifying the local user's session membership. + */ + membership: CallMembershipIdentityParts; + /** + * The transport for which we wish to get an access token. + */ + transport: UnstableLivekitTransport; + /** + * The name of the homeserver to which the {@link transport} belongs. + */ + serverName: string; + /** + * The ID of the Matrix room in which the session takes place. + */ + roomId: string; + /** + * The ID of the MatrixRTC slot in which the session takes place. + */ + slotId: string; + /** + * Whether we want to publish or only subscribe on the {@link transport}. + */ + role: "publisher" | "subscriber"; + logger: Logger; +} + +/** + * Gets an {@link SFUConfig} appropriate for connecting to a given transport. + */ +export async function getSFUConfig({ + client, + membership, + transport, + serverName, + roomId, + slotId, + role, + logger, +}: GetSFUConfigParams): Promise { + if ("url" in transport) { + try { + return await csApi.getSFUConfig({ + client, + membership, + url: transport.url, + serverName, + roomId, + slotId, + }); + } catch (e) { + if (isEndpointUnsupported(e)) { + // Publishers should never fall back. If a homeserver advertises a `url` + // but doesn't support MSC4195 itself, that's a misconfiguration which + // the admin should know about so they can prepare for the future + // deprecation of `livekit_service_url`. + const mayFallBack = + role === "subscriber" && "livekit_service_url" in transport; + if (mayFallBack) { + logger.warn( + `Homeserver does not support the MSC4195 get_token endpoint, falling back to the OpenID flow with service ${transport.livekit_service_url}`, + ); + // Fall through + } else { + logger.error( + `Homeserver does not support the MSC4195 get_token endpoint, cannot get token for transport ${transport.url}`, + ); + throw new NoMatrix2AuthorizationService(e as Error); + } + } else { + throw e; + } + } + } + + return await openID.getSFUConfig({ + client, + membership, + serviceUrl: transport.livekit_service_url, + roomId, + slotId, + role, + logger, + }); +} + +export interface DelegateDelayedLeaveParams { + /** + * The Matrix client. + */ + client: csApi.ClientDelegationParts & openID.ClientOpenIDParts; + /** + * Data identifying the local user's session membership. + */ + membership: CallMembershipIdentityParts; + /** + * The transport to which we are publishing and wish to delegate the delayed + * leave event. + */ + transport: UnstableLivekitTransport; + /** + * The ID of the Matrix room in which the session takes place. + */ + roomId: string; + /** + * The ID of the MatrixRTC slot in which the session takes place. + */ + slotId: string; + /** + * The delay ID of the leave event to be delegated. + */ + delayId: string; + logger: Logger; +} + +/** + * Delegates a delayed leave event to a given transport, so that the event will + * be sent on our behalf whenever we disconnect from the transport's SFU. + */ +export async function delegateDelayedLeave({ + client, + membership, + transport, + roomId, + slotId, + delayId, + logger, +}: DelegateDelayedLeaveParams): Promise { + // It's important that we use the same flow (CS API or OpenID) as whatever we + // would've used to get a JWT token for this transport, since the participant + // identity computed by the service needs to match. + if ("url" in transport) { + try { + await csApi.delegateDelayedLeave({ + client, + membership, + url: transport.url, + roomId, + slotId, + delayId, + }); + } catch (e) { + if (isEndpointUnsupported(e)) + logger.error( + "Server misconfigured: transport contains a 'url' field but server does not support the MSC4195 delegate_delayed_leave endpoint", + ); + throw e; + } + } else { + // This will technically cause the service to issue a new JWT token, but + // it's safe to discard. We're only interested in triggering delegation. + await openID.getSFUConfig({ + client, + membership, + serviceUrl: transport.livekit_service_url, + roomId, + slotId, + role: "publisher", + delayEndpointBaseUrl: client.baseUrl, + delayId, + logger, + }); + } +} diff --git a/src/settings/DeveloperSettingsTab.test.tsx b/src/settings/DeveloperSettingsTab.test.tsx index bb0808761..034249ced 100644 --- a/src/settings/DeveloperSettingsTab.test.tsx +++ b/src/settings/DeveloperSettingsTab.test.tsx @@ -133,6 +133,7 @@ describe("DeveloperSettingsTab", () => { getCrypto: () => ({ getVersion: (): string => "x" }), getUserId: () => "@u:hs", getDeviceId: () => "DEVICE", + getDomain: () => "hs", } as unknown as MatrixClient; it("will not update custom livekit url without roomId", async () => { const user = userEvent.setup(); @@ -228,7 +229,10 @@ describe("DeveloperSettingsTab", () => { await user.click(saveButton); expect(getSFUConfig).toHaveBeenCalledWith( expect.objectContaining({ - serviceUrl: "wss://example.livekit.valid", + transport: { + type: "livekit", + livekit_service_url: "wss://example.livekit.valid", + }, roomId: "#testRoom", }), ); diff --git a/src/settings/DeveloperSettingsTab.tsx b/src/settings/DeveloperSettingsTab.tsx index c8cc2a524..d7a9164c5 100644 --- a/src/settings/DeveloperSettingsTab.tsx +++ b/src/settings/DeveloperSettingsTab.tsx @@ -521,7 +521,11 @@ export const DeveloperSettingsTab: FC = ({ await getSFUConfig({ client, membership: { userId, deviceId, memberId: "" }, - serviceUrl: customLivekitUrlTextBuffer, + transport: { + type: "livekit", + livekit_service_url: customLivekitUrlTextBuffer, + }, + serverName: client.getDomain()!, roomId, slotId: "org.example.fake-slot#room", role: "publisher", diff --git a/src/state/CallViewModel/CallViewModelTestUtils.ts b/src/state/CallViewModel/CallViewModelTestUtils.ts index 809dc75a5..93c190ace 100644 --- a/src/state/CallViewModel/CallViewModelTestUtils.ts +++ b/src/state/CallViewModel/CallViewModelTestUtils.ts @@ -122,6 +122,8 @@ export function withCallViewModel(mode: MatrixRTCMode) { }; const room = mockMatrixRoom({ client: new (class extends EventEmitter { + public readonly baseUrl = "https://matrix.example.com"; + public getUserId(): string | undefined { return localRtcMember.userId; } diff --git a/src/state/CallViewModel/localMember/LocalMember.test.ts b/src/state/CallViewModel/localMember/LocalMember.test.ts index f1c95d1cb..2d721f7d6 100644 --- a/src/state/CallViewModel/localMember/LocalMember.test.ts +++ b/src/state/CallViewModel/localMember/LocalMember.test.ts @@ -116,6 +116,7 @@ const delegatedTimings: ResolvedDelayedLeaveTimings = { }; const mockedClient = { + baseUrl: "https://matrix.example.org", getDomain: vi.fn().mockReturnValue("example.org"), getDeviceId: vi.fn().mockReturnValue("AAAA"), getOpenIdToken: vi.fn().mockResolvedValue({ @@ -124,6 +125,7 @@ const mockedClient = { matrix_server_name: "localhost", expires_in: 10000, }), + _unstable_delegateDelayedLeave: vi.fn().mockResolvedValue({}), }; describe("enterRTCSession", () => { @@ -239,6 +241,7 @@ describe("LocalMembership", () => { ownMembershipIdentity: ownMemberMock, client: mockedClient, delayId$: constant(null), + matrixRTCMode: MATRIX_RTC_MODE, }; beforeEach(() => { @@ -304,9 +307,11 @@ describe("LocalMembership", () => { expect(internalLogger).toHaveBeenCalledWith(expextedLog); }); - const mockTransport = { - livekit_service_url: "a", - } as UnstableLivekitTransport; + const mockTransport: UnstableLivekitTransport = { + type: "livekit", + url: "https://sfu.example.org", + livekit_service_url: "https://jwt.example.org", + }; const mockTransportLocator: livekitAuth.TransportLocator = { transport: mockTransport, @@ -336,10 +341,10 @@ describe("LocalMembership", () => { livekitRoom: mockLivekitRoom({}), } as unknown as Connection; - const authCallSpy = vi - .spyOn(livekitAuth, "getSFUConfig") - .mockImplementation(() => mockedClient.getOpenIdToken()); - afterEach(() => authCallSpy.mockClear()); + const delegationSpy = vi + .spyOn(livekitAuth, "delegateDelayedLeave") + .mockResolvedValue(); + afterEach(() => delegationSpy.mockClear()); it.each([ ["no", null, timings], @@ -348,7 +353,11 @@ describe("LocalMembership", () => { "https://matrix.example.org/_matrix/client/unstable/io.element.msc4195/rtc/livekit/delegate_delayed_leave", delegatedTimings, ], - ["transport", "/a/delegate_delayed_leave", delegatedTimings], + [ + "transport", + "https://jwt.example.org/delegate_delayed_leave", + delegatedTimings, + ], ])( "joins session with %s delegation support", async (_serviceName, delegationUrl, delayedLeaveTimings) => { @@ -379,22 +388,20 @@ describe("LocalMembership", () => { delayedLeaveTimings, ); - expect(authCallSpy).not.toHaveBeenCalled(); + expect(delegationSpy).not.toHaveBeenCalled(); delayId$.next("leave1"); await flushPromises(); if (delegationUrl === null) { - expect(authCallSpy).not.toHaveBeenCalled(); + expect(delegationSpy).not.toHaveBeenCalled(); } else { // Delegation is supported in this test case, so go on to check that // LocalMember actually performs delegation const expectDelegation = (delayId: string) => - expect(authCallSpy).toHaveBeenLastCalledWith({ + expect(delegationSpy).toHaveBeenLastCalledWith({ client: mockedClient, membership: ownMemberMock, - serviceUrl: "a", + transport: mockTransport, roomId: "!test-room-id:example.org", - role: "publisher", - delayEndpointBaseUrl: "https://matrix.example.org", delayId, logger: expect.anything(), }); diff --git a/src/state/CallViewModel/localMember/LocalMember.ts b/src/state/CallViewModel/localMember/LocalMember.ts index 14f853403..244c728dc 100644 --- a/src/state/CallViewModel/localMember/LocalMember.ts +++ b/src/state/CallViewModel/localMember/LocalMember.ts @@ -79,8 +79,9 @@ import { } from "../remoteMembers/Connection.ts"; import { type HomeserverConnected } from "./HomeserverConnected.ts"; import { + type ClientDelegationParts, + delegateDelayedLeave, type ClientOpenIDParts, - getSFUConfig, type TransportLocator, } from "../../../livekit/auth"; @@ -152,7 +153,7 @@ interface Props { roomId: string; ownMembershipIdentity: CallMembershipIdentityParts; localTransport$: Observable; - client: ClientOpenIDParts; + client: ClientDelegationParts & ClientOpenIDParts; matrixRTCSession: Pick< MatrixRTCSession, "slotId" | "updateCallIntent" | "leave" @@ -720,22 +721,19 @@ export const createLocalMembership$ = ({ async ([joinParams, delayId]) => { if (joinParams?.delegationSupported && delayId !== null) { try { - if (!("livekit_service_url" in joinParams.transport)) - throw new Error("Transport has no livekit_service_url"); - // This will technically cause the service to issue a new JWT token, - // but it's safe to discard. We're only interested in triggering - // delegation. - await getSFUConfig({ + logger.info( + `Delegating delayed leave to ${JSON.stringify(joinParams.transport)}…`, + ); + await delegateDelayedLeave({ client, membership: ownMembershipIdentity, - serviceUrl: joinParams.transport.livekit_service_url, - roomId, + transport: joinParams.transport, slotId: matrixRTCSession.slotId, - role: "publisher", - delayEndpointBaseUrl: baseUrl, + roomId, delayId, logger, }); + logger.info("Delayed leave successfully delegated"); } catch (e) { // TODO: Surface this to the user as a service interruption? logger.error(`Failed to delegate leave`, e); diff --git a/src/state/CallViewModel/remoteMembers/Connection.test.ts b/src/state/CallViewModel/remoteMembers/Connection.test.ts index f0fc6bfd5..bdab8e7af 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.test.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.test.ts @@ -8,6 +8,7 @@ Please see LICENSE in the repository root for full details. import { afterEach, + beforeEach, describe, expect, it, @@ -27,9 +28,12 @@ import { } from "livekit-client"; import fetchMock from "fetch-mock"; import EventEmitter from "events"; -import { type IOpenIDToken } from "matrix-js-sdk"; +import { MatrixError } from "matrix-js-sdk"; import { logger, type Logger } from "matrix-js-sdk/lib/logger"; -import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { + type LivekitGetTokenResponse, + type UnstableLivekitTransport, +} from "matrix-js-sdk/lib/matrixrtc"; import { Connection, @@ -40,11 +44,14 @@ import { ObservableScope } from "../../ObservableScope.ts"; import { FailToGetOpenIdToken } from "../../../utils/errors.ts"; import { testJWTToken } from "../../../utils/test-fixtures.ts"; import { mockRemoteParticipant, ownMemberMock } from "../../../utils/test.ts"; -import { type ClientOpenIDParts } from "../../../livekit/auth"; +import { + type ClientGetTokenParts, + type ClientOpenIDParts, +} from "../../../livekit/auth"; let testScope: ObservableScope; -let client: MockedObject; +let client: MockedObject; let fakeLivekitRoom: MockedObject; @@ -54,21 +61,21 @@ let fakeLocalParticipant: MockedObject; const ROOM_ID = "!roomID:example.org"; const SLOT_ID = "m.call#ROOM"; -const livekitServiceUrl = "https://matrix-rtc.example.org/livekit/jwt"; -const livekitFocus: UnstableLivekitTransport = { - livekit_service_url: livekitServiceUrl, +const transport: UnstableLivekitTransport = { type: "livekit", + url: "wss://matrix-rtc.m.localhost/livekit/sfu", }; function setupTest(): void { testScope = new ObservableScope(); - client = vi.mocked({ + client = vi.mocked({ getOpenIdToken: vi.fn().mockResolvedValue({ access_token: "rYsmGUEwNjKgJYyeNUkZseJN", token_type: "Bearer", matrix_server_name: "example.org", expires_in: 3600, }), + _unstable_getLivekitToken: vi.fn().mockResolvedValue({ jwt: testJWTToken }), }); localParticipantEventEmiter = new EventEmitter(); @@ -117,23 +124,13 @@ function setupRemoteConnection(): Connection { roomId: ROOM_ID, slotId: SLOT_ID, role: "subscriber", - transport: livekitFocus, + transport, serverName: "m.localhost", scope: testScope, ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, }; - fetchMock.post(`${livekitServiceUrl}/get_token`, () => { - return { - status: 200, - body: { - url: "wss://matrix-rtc.m.localhost/livekit/sfu", - jwt: testJWTToken, - }, - }; - }); - fakeLivekitRoom.connect.mockImplementation(async (): Promise => { const changeEv = RoomEvent.ConnectionStateChanged; @@ -155,6 +152,19 @@ afterEach(() => { }); describe("Start connection states", () => { + beforeEach(() => { + fetchMock.post( + `https://matrix-rtc.example.org/livekit/jwt/get_token`, + () => { + return { + // Return a non-retryable error, if not, the retry logic will + // wait and fail the test with a timeout. + status: 404, + }; + }, + ); + }); + it("start in initialized state", () => { setupTest(); @@ -163,7 +173,7 @@ describe("Start connection states", () => { roomId: ROOM_ID, slotId: SLOT_ID, role: "subscriber", - transport: livekitFocus, + transport, serverName: "m.localhost", scope: testScope, ownMembershipIdentity: ownMemberMock, @@ -174,7 +184,7 @@ describe("Start connection states", () => { expect(connection.state$.getValue()).toEqual({ state: "initialized" }); }); - it("fail to getOpenId token then error state", async () => { + it("fail to get SFU token then error state", async () => { setupTest(); vi.useFakeTimers(); @@ -183,7 +193,7 @@ describe("Start connection states", () => { roomId: ROOM_ID, slotId: SLOT_ID, role: "subscriber", - transport: livekitFocus, + transport, serverName: "m.localhost", scope: testScope, ownMembershipIdentity: ownMemberMock, @@ -198,10 +208,10 @@ describe("Start connection states", () => { }); onTestFinished(() => s.unsubscribe()); - const deferred = Promise.withResolvers(); + const deferred = Promise.withResolvers(); - client.getOpenIdToken.mockImplementation( - async (): Promise => { + client._unstable_getLivekitToken.mockImplementation( + async (): Promise => { return await deferred.promise; }, ); @@ -221,7 +231,7 @@ describe("Start connection states", () => { capturedState = capturedStates.pop(); if (capturedState?.state === "error") { expect(capturedState.error.message).toEqual("Something went wrong"); - expect(connection.transport).toEqual(livekitFocus); + expect(connection.transport).toEqual(transport); } else { expect.fail( "Expected FailedToStart state but got " + JSON.stringify(capturedState), @@ -238,7 +248,7 @@ describe("Start connection states", () => { roomId: ROOM_ID, slotId: SLOT_ID, role: "subscriber", - transport: livekitFocus, + transport, serverName: "m.localhost", scope: testScope, ownMembershipIdentity: ownMemberMock, @@ -254,16 +264,15 @@ describe("Start connection states", () => { onTestFinished(() => s.unsubscribe()); const deferredSFU = Promise.withResolvers(); - // mock the /get_token call - fetchMock.post(`${livekitServiceUrl}/get_token`, async () => { + client._unstable_getLivekitToken.mockImplementation(async () => { await deferredSFU.promise; - return { - status: 500, - body: { + throw new MatrixError( + { errcode: "M_LOOKUP_FAILED", error: "Failed to look up user info from homeserver", }, - }; + 500, + ); }); connection.start().catch(() => { @@ -286,7 +295,7 @@ describe("Start connection states", () => { expect(capturedState.error.cause.message).toContain( "Failed to look up user info from homeserver", ); - expect(connection.transport).toEqual(livekitFocus); + expect(connection.transport).toEqual(transport); } else { expect.fail( "Expected FailedToStart state but got " + JSON.stringify(capturedState), @@ -303,7 +312,7 @@ describe("Start connection states", () => { roomId: ROOM_ID, slotId: SLOT_ID, role: "subscriber", - transport: livekitFocus, + transport, serverName: "m.localhost", scope: testScope, ownMembershipIdentity: ownMemberMock, @@ -319,17 +328,6 @@ describe("Start connection states", () => { onTestFinished(() => s.unsubscribe()); const deferredSFU = Promise.withResolvers(); - // mock the /get_token call - fetchMock.post(`${livekitServiceUrl}/get_token`, () => { - return { - status: 200, - body: { - url: "wss://matrix-rtc.m.localhost/livekit/sfu", - jwt: testJWTToken, - }, - }; - }); - fakeLivekitRoom.connect.mockImplementation(async () => { await deferredSFU.promise; throw new Error("Failed to connect to livekit"); @@ -356,7 +354,7 @@ describe("Start connection states", () => { expect(capturedState.error.cause.message).toContain( "Failed to connect to livekit", ); - expect(connection.transport).toEqual(livekitFocus); + expect(connection.transport).toEqual(transport); } else { expect.fail( "Expected FailedToStart state but got " + JSON.stringify(capturedState), @@ -564,7 +562,7 @@ describe("remote track logging", () => { roomId: ROOM_ID, slotId: SLOT_ID, role: "subscriber", - transport: livekitFocus, + transport, serverName: "m.localhost", scope: testScope, ownMembershipIdentity: ownMemberMock, diff --git a/src/state/CallViewModel/remoteMembers/Connection.ts b/src/state/CallViewModel/remoteMembers/Connection.ts index c0fe4fd9a..7e94b39f1 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.ts @@ -39,6 +39,7 @@ import { UnknownCallError, } from "../../../utils/errors.ts"; import { + type ClientGetTokenParts, getSFUConfig, type ClientOpenIDParts, type SFUConfig, @@ -54,7 +55,7 @@ export interface ConnectionOpts { /** The name of the homeserver to which the {@link transport} belongs. */ serverName: string; /** The Matrix client to use for OpenID and SFU config requests. */ - client: ClientOpenIDParts; + client: ClientGetTokenParts & ClientOpenIDParts; /** The ID of the Matrix room in which the session takes place. */ roomId: string; /** The ID of the MatrixRTC slot in which the session takes place. */ @@ -134,7 +135,7 @@ export class Connection { protected stopped = false; // TODO: can we just keep the ConnectionOpts object instead of spreading? - private readonly client: ClientOpenIDParts; + private readonly client: ClientGetTokenParts & ClientOpenIDParts; private readonly roomId: string; private readonly slotId: string; private readonly logger: Logger; @@ -385,13 +386,12 @@ export class Connection { } protected async getSFUConfig(): Promise { - if (!("livekit_service_url" in this.transport)) - throw new Error("Transport has no livekit_service_url"); return await getSFUConfig({ client: this.client, membership: this.ownMembershipIdentity, role: this.role, - serviceUrl: this.transport.livekit_service_url, + transport: this.transport, + serverName: this.serverName, roomId: this.roomId, slotId: this.slotId, logger: this.logger, diff --git a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts index 86cc01b51..e5240f096 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts @@ -35,7 +35,10 @@ import { noiseSuppressionSetting, autoGainControlSetting, } from "../../../settings/settings.ts"; -import { type ClientOpenIDParts } from "../../../livekit/auth"; +import { + type ClientGetTokenParts, + type ClientOpenIDParts, +} from "../../../livekit/auth"; // TODO evaluate if this should be done like the Publisher Factory export interface ConnectionFactory { @@ -65,7 +68,7 @@ export class ECConnectionFactory implements ConnectionFactory { * @param livekitRoomFactory - Optional factory function (for testing) to create LivekitRoom instances. If not provided, a default factory is used. */ public constructor( - private client: ClientOpenIDParts, + private client: ClientGetTokenParts & ClientOpenIDParts, private readonly roomId: string, private readonly slotId: string, private devices: MediaDevices, diff --git a/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts b/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts index 5d9abd0fe..be6c149f4 100644 --- a/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts +++ b/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts @@ -14,7 +14,10 @@ import EventEmitter from "events"; import { ObservableScope } from "../../ObservableScope.ts"; import { ECConnectionFactory } from "./ConnectionFactory.ts"; -import { type ClientOpenIDParts } from "../../../livekit/auth"; +import { + type ClientGetTokenParts, + type ClientOpenIDParts, +} from "../../../livekit/auth"; import { exampleTransport, mockMediaDevices, @@ -32,6 +35,7 @@ import { cameraBitrate, cameraCodec, } from "../../../settings/settings.ts"; +import { testJWTToken } from "../../../utils/test-fixtures.ts"; // At the top of your test file, after imports vi.mock("livekit-client", async (importOriginal) => { @@ -51,12 +55,13 @@ vi.mock("livekit-client", async (importOriginal) => { }); let testScope: ObservableScope; -let mockClient: ClientOpenIDParts; +let mockClient: ClientGetTokenParts & ClientOpenIDParts; beforeEach(() => { testScope = new ObservableScope(); mockClient = { getOpenIdToken: vi.fn().mockReturnValue(""), + _unstable_getLivekitToken: vi.fn().mockResolvedValue({ jwt: testJWTToken }), }; }); diff --git a/src/state/CallViewModel/remoteMembers/integration.test.ts b/src/state/CallViewModel/remoteMembers/integration.test.ts index 39737ae6a..bf90e9eb4 100644 --- a/src/state/CallViewModel/remoteMembers/integration.test.ts +++ b/src/state/CallViewModel/remoteMembers/integration.test.ts @@ -18,7 +18,10 @@ import { trackEpoch, } from "../../ObservableScope.ts"; import { ECConnectionFactory } from "./ConnectionFactory.ts"; -import { type ClientOpenIDParts } from "../../../livekit/auth"; +import { + type ClientGetTokenParts, + type ClientOpenIDParts, +} from "../../../livekit/auth"; import { exampleTransportLocator, mockMediaDevices, @@ -40,7 +43,7 @@ import { deepCompare } from "matrix-js-sdk/lib/utils"; let testScope: ObservableScope; let ecConnectionFactory: ECConnectionFactory; -let mockClient: ClientOpenIDParts; +let mockClient: ClientGetTokenParts & ClientOpenIDParts; let lkRoomFactory: () => LivekitRoom; const createdMockLivekitRooms: Map = new Map(); @@ -49,6 +52,7 @@ beforeEach(() => { testScope = new ObservableScope(); mockClient = { getOpenIdToken: vi.fn().mockReturnValue(""), + _unstable_getLivekitToken: vi.fn().mockResolvedValue({ jwt: testJWTToken }), }; lkRoomFactory = vi.fn().mockImplementation(() => { @@ -82,7 +86,7 @@ beforeEach(() => { ); //TODO a bit annoying to have to do a http mock? - const sfuConfigResponse = (url: string): { status: number; body: object } => { + fetchMock.post(`path:/sfu/get`, (url) => { const domain = new URL(url).hostname; // Extract the domain from the URL return { status: 200, @@ -91,9 +95,7 @@ beforeEach(() => { jwt: testJWTToken, }, }; - }; - fetchMock.post(`path:/sfu/get`, sfuConfigResponse); - fetchMock.post(`path:/get_token`, sfuConfigResponse); + }); }); afterEach(() => { diff --git a/src/utils/test-fixtures.ts b/src/utils/test-fixtures.ts index dcdb9f9c9..dac0c6972 100644 --- a/src/utils/test-fixtures.ts +++ b/src/utils/test-fixtures.ts @@ -64,16 +64,23 @@ export const daveRTL = mockMatrixRoomMember(daveRTLRtcMember, { rawDisplayName: "\u202eevaD", }); -export const testJWTToken = [ - {}, // header - { - // payload - sub: "@me:example.org:ABCDEF", - video: { - room: "!example_room_id", +export function createJWTToken(sub: string, room: string): string { + return [ + {}, // header + { + // payload + sub, + video: { + room, + }, }, - }, - {}, // signature -] - .map((d) => global.btoa(JSON.stringify(d))) - .join("."); + {}, // signature + ] + .map((d) => global.btoa(JSON.stringify(d))) + .join("."); +} + +export const testJWTToken = createJWTToken( + "@me:example.org:ABCDEF", + "!example_room_id", +); From f366e7cbe28c647992586c27718030222e729eea Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 12 Oct 2026 10:33:19 +0200 Subject: [PATCH 15/25] Let developers override the whole transport The developer setting that overrode the LiveKit service URL becomes a JSON transport, so that a `url`-only or combined transport can be tested against a homeserver that does not advertise one yet. The settings modal shows each LiveKit room's role instead of a never-populated isLocal flag. --- locales/en/app.json | 6 +- src/room/InCallView.tsx | 25 +++--- src/settings/DeveloperSettingsTab.test.tsx | 83 ++++++++++++------- src/settings/DeveloperSettingsTab.tsx | 69 +++++++-------- src/settings/SettingsModal.tsx | 2 +- .../DeveloperSettingsTab.test.tsx.snap | 4 +- src/settings/settings.ts | 5 +- .../localMember/LocalTransport.test.ts | 9 +- .../localMember/LocalTransport.ts | 12 ++- 9 files changed, 120 insertions(+), 95 deletions(-) diff --git a/locales/en/app.json b/locales/en/app.json index e31c7d50a..5be1bfff6 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -59,10 +59,10 @@ "developer_mode": { "always_show_iphone_earpiece": "Show iPhone earpiece option on all platforms", "crypto_version": "Crypto version: {{version}}", - "custom_livekit_url": { - "current_url": "Currently set to: ", + "custom_transport": { + "current": "Currently set to: ", "from_config": "Currently, no overwrite is set. Url from config is used.", - "label": "Custom Livekit-url", + "label": "Custom transport JSON", "reset": "Reset overwrite", "save": "Save", "saving": "Saving..." diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 7f35c3887..bc0d68c4e 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -694,18 +694,19 @@ export const InCallView: FC = ({ tab={settingsTab} onTabChange={setSettingsTab} developerSettingsVm={developerSettingsVm} - livekitRooms={allConnections.map((connectionItem) => ({ - room: connectionItem.livekitRoom, - livekitAlias: connectionItem.livekitAlias, - // TODO compute is local or tag it in the livekit room items already - isLocal: undefined, - url: - // It should be relatively clear to the user which kind of URL - // this is, based on the scheme (ws:// vs. http://) - "url" in connectionItem.transport - ? connectionItem.transport.url - : connectionItem.transport.livekit_service_url, - }))} + livekitRooms={allConnections.map( + ({ livekitRoom: room, livekitAlias, transport, role }) => ({ + room, + livekitAlias, + role, + url: + // It should be relatively clear to the user which kind of URL + // this is, based on the scheme (ws:// vs. http://) + "url" in transport + ? transport.url + : transport.livekit_service_url, + }), + )} /> )} diff --git a/src/settings/DeveloperSettingsTab.test.tsx b/src/settings/DeveloperSettingsTab.test.tsx index 034249ced..82764d8ec 100644 --- a/src/settings/DeveloperSettingsTab.test.tsx +++ b/src/settings/DeveloperSettingsTab.test.tsx @@ -7,7 +7,7 @@ Please see LICENSE in the repository root for full details. import { afterEach, describe, expect, it, type Mock, vi } from "vitest"; import { render, waitFor, screen } from "@testing-library/react"; -import userEvent from "@testing-library/user-event"; +import userEvent, { type UserEvent } from "@testing-library/user-event"; import { TooltipProvider } from "@vector-im/compound-web"; import type { MatrixClient } from "matrix-js-sdk"; @@ -19,12 +19,12 @@ import { import { outOfCallDeveloperSettingsTabViewModel } from "./DeveloperSettingsTabViewModel"; import { createStaticViewModel } from "../state/ViewModel"; import { - customLivekitUrl as customLivekitUrlSetting, + customTransport as customTransportSetting, enableExtendedLivekitLogs as enableExtendedLivekitLogsSetting, matrixRTCMode as matrixRTCModeSetting, } from "./settings"; import { MatrixRTCMode } from "../config/ConfigOptions"; -import { mockConfig } from "../utils/test"; +import { exampleTransport, mockConfig } from "../utils/test"; import { getSFUConfig } from "../livekit/auth"; // Mock url params hook to avoid environment-dependent snapshot churn. @@ -77,6 +77,21 @@ function createMockMatrixClient(): MatrixClient { } as unknown as MatrixClient; } +/** + * Types the JSON representation of an object into the given element. + */ +async function typeJson( + user: UserEvent, + element: Element, + object: object, +): Promise { + // `user.type` requires us to escape brackets by doubling them + await user.type( + element, + JSON.stringify(object).replace(/([{[\]])/g, (c) => c + c), + ); +} + describe("DeveloperSettingsTab", () => { it("renders and matches snapshot", async () => { const client = createMockMatrixClient(); @@ -124,9 +139,9 @@ describe("DeveloperSettingsTab", () => { expect(container).toMatchSnapshot(); }); - describe("custom livekit url", () => { + describe("custom transport", () => { afterEach(() => { - customLivekitUrlSetting.setValue(null); + customTransportSetting.setValue(null); }); const client = { doesServerSupportUnstableFeature: vi.fn().mockResolvedValue(true), @@ -135,7 +150,7 @@ describe("DeveloperSettingsTab", () => { getDeviceId: () => "DEVICE", getDomain: () => "hs", } as unknown as MatrixClient; - it("will not update custom livekit url without roomId", async () => { + it("will not update custom transport without roomId", async () => { const user = userEvent.setup(); render( @@ -148,17 +163,17 @@ describe("DeveloperSettingsTab", () => { , ); - const input = screen.getByLabelText("Custom Livekit-url"); + const input = screen.getByLabelText("Custom transport JSON"); await user.clear(input); - await user.type(input, "wss://example.livekit.invalid"); + await typeJson(user, input, exampleTransport); const saveButton = screen.getByRole("button", { name: "Save" }); await user.click(saveButton); expect(getSFUConfig).not.toHaveBeenCalled(); - expect(customLivekitUrlSetting.getValue()).toBe(null); + expect(customTransportSetting.getValue()).toBe(null); }); - it("will not update custom livekit url without text in input", async () => { + it("will not update custom transport without text in input", async () => { const user = userEvent.setup(); render( @@ -172,16 +187,16 @@ describe("DeveloperSettingsTab", () => { , ); - const input = screen.getByLabelText("Custom Livekit-url"); + const input = screen.getByLabelText("Custom transport JSON"); await user.clear(input); const saveButton = screen.getByRole("button", { name: "Save" }); await user.click(saveButton); expect(getSFUConfig).not.toHaveBeenCalled(); - expect(customLivekitUrlSetting.getValue()).toBe(null); + expect(customTransportSetting.getValue()).toBe(null); }); - it("will not update custom livekit url when pressing cancel", async () => { + it("will not update custom transport when pressing cancel", async () => { const user = userEvent.setup(); render( @@ -195,7 +210,7 @@ describe("DeveloperSettingsTab", () => { , ); - const input = screen.getByLabelText("Custom Livekit-url"); + const input = screen.getByLabelText("Custom transport JSON"); await user.clear(input); await user.type(input, "wss://example.livekit.invalid"); @@ -205,9 +220,9 @@ describe("DeveloperSettingsTab", () => { await user.click(cancelButton); expect(getSFUConfig).not.toHaveBeenCalled(); - expect(customLivekitUrlSetting.getValue()).toBe(null); + expect(customTransportSetting.getValue()).toBe(null); }); - it("will update custom livekit url", async () => { + it("will update custom transport", async () => { const user = userEvent.setup(); render( @@ -221,27 +236,28 @@ describe("DeveloperSettingsTab", () => { , ); - const input = screen.getByLabelText("Custom Livekit-url"); + const input = screen.getByLabelText("Custom transport JSON"); await user.clear(input); - await user.type(input, "wss://example.livekit.valid"); + await typeJson(user, input, { + type: "livekit", + url: "wss://example.livekit.valid", + }); const saveButton = screen.getByRole("button", { name: "Save" }); await user.click(saveButton); expect(getSFUConfig).toHaveBeenCalledWith( expect.objectContaining({ - transport: { - type: "livekit", - livekit_service_url: "wss://example.livekit.valid", - }, + transport: { type: "livekit", url: "wss://example.livekit.valid" }, roomId: "#testRoom", }), ); - expect(customLivekitUrlSetting.getValue()).toBe( - "wss://example.livekit.valid", - ); + expect(customTransportSetting.getValue()).toEqual({ + type: "livekit", + url: "wss://example.livekit.valid", + }); }); - it("will show error on invalid url", async () => { + it("will show error on unreachable transport", async () => { const user = userEvent.setup(); render( @@ -255,24 +271,27 @@ describe("DeveloperSettingsTab", () => { , ); - const input = screen.getByLabelText("Custom Livekit-url"); + const input = screen.getByLabelText("Custom transport JSON"); await user.clear(input); - await user.type(input, "wss://example.livekit.valid"); + await typeJson(user, input, { + type: "livekit", + url: "wss://example.livekit.valid", + }); const saveButton = screen.getByRole("button", { name: "Save" }); (getSFUConfig as Mock).mockImplementation(() => { - throw new Error("Invalid URL"); + throw new Error("Failed to get SFU config"); }); await user.click(saveButton); expect( - screen.getByText("invalid URL (did not update)"), + screen.getByText("invalid transport (did not update)"), ).toBeInTheDocument(); - expect(customLivekitUrlSetting.getValue()).toBe(null); + expect(customTransportSetting.getValue()).toBe(null); }); }); // Add this test inside the describe("DeveloperSettingsTab", () => { block, - // after the custom livekit url tests: + // after the custom transport tests: describe("enable extended livekit logs", () => { afterEach(() => { diff --git a/src/settings/DeveloperSettingsTab.tsx b/src/settings/DeveloperSettingsTab.tsx index d7a9164c5..3ef943ddf 100644 --- a/src/settings/DeveloperSettingsTab.tsx +++ b/src/settings/DeveloperSettingsTab.tsx @@ -45,7 +45,7 @@ import { muteAllAudio as muteAllAudioSetting, alwaysShowIphoneEarpiece as alwaysShowIphoneEarpieceSetting, matrixRTCMode as matrixRTCModeSetting, - customLivekitUrl as customLivekitUrlSetting, + customTransport as customTransportSetting, advancedScreenShare as advancedScreenShareSetting, screenShareResolution as screenShareResolutionSetting, screenShareFramerate as screenShareFramerateSetting, @@ -167,16 +167,18 @@ export const DeveloperSettingsTab: FC = ({ enableExtendedLivekitLogsSetting, ); - const [customLivekitUrlUpdateError, setCustomLivekitUrlUpdateError] = - useState(null); - const [customLivekitUrl, setCustomLivekitUrl] = useSetting( - customLivekitUrlSetting, + const [customTransportUpdateError, setCustomLivekitUrlUpdateError] = useState< + string | null + >(null); + const [customTransport, setCustomTransport] = useSetting( + customTransportSetting, + ); + const [customTransportTextBuffer, setCustomLivekitUrlTextBuffer] = useState( + () => JSON.stringify(customTransport), ); - const [customLivekitUrlTextBuffer, setCustomLivekitUrlTextBuffer] = - useState(customLivekitUrl); useEffect(() => { - setCustomLivekitUrlTextBuffer(customLivekitUrl); - }, [customLivekitUrl]); + setCustomLivekitUrlTextBuffer(JSON.stringify(customTransport)); + }, [customTransport]); const [muteAllAudio, setMuteAllAudio] = useSetting(muteAllAudioSetting); @@ -491,23 +493,23 @@ export const DeveloperSettingsTab: FC = ({ e.preventDefault()} helpLabel={ - customLivekitUrl === null - ? t("developer_mode.custom_livekit_url.from_config") - : t("developer_mode.custom_livekit_url.current_url") + - customLivekitUrl + customTransport === null + ? t("developer_mode.custom_transport.from_config") + : t("developer_mode.custom_transport.current") + + JSON.stringify(customTransport) } - label={t("developer_mode.custom_livekit_url.label")} - saveButtonLabel={t("developer_mode.custom_livekit_url.save")} - savingLabel={t("developer_mode.custom_livekit_url.saving")} - cancelButtonLabel={t("developer_mode.custom_livekit_url.reset")} + label={t("developer_mode.custom_transport.label")} + saveButtonLabel={t("developer_mode.custom_transport.save")} + savingLabel={t("developer_mode.custom_transport.saving")} + cancelButtonLabel={t("developer_mode.custom_transport.reset")} onSave={useCallback( async (e: React.FormEvent): Promise => { if ( roomId === undefined || - customLivekitUrlTextBuffer === "" || - customLivekitUrlTextBuffer === null + customTransportTextBuffer === "" || + customTransportTextBuffer === null ) { - setCustomLivekitUrl(null); + setCustomTransport(null); return; } @@ -518,13 +520,12 @@ export const DeveloperSettingsTab: FC = ({ if (userId === null || deviceId === null) { throw new Error("Invalid user or device ID"); } + const transport = JSON.parse(customTransportTextBuffer); + // Validate the transport by attempting to get a token for it await getSFUConfig({ client, membership: { userId, deviceId, memberId: "" }, - transport: { - type: "livekit", - livekit_service_url: customLivekitUrlTextBuffer, - }, + transport, serverName: client.getDomain()!, roomId, slotId: "org.example.fake-slot#room", @@ -532,14 +533,16 @@ export const DeveloperSettingsTab: FC = ({ logger, }); setCustomLivekitUrlUpdateError(null); - setCustomLivekitUrl(customLivekitUrlTextBuffer); + setCustomTransport(transport); } catch { - setCustomLivekitUrlUpdateError("invalid URL (did not update)"); + setCustomLivekitUrlUpdateError( + "invalid transport (did not update)", + ); } }, - [customLivekitUrlTextBuffer, setCustomLivekitUrl, client, roomId], + [customTransportTextBuffer, setCustomTransport, client, roomId], )} - value={customLivekitUrlTextBuffer ?? ""} + value={customTransportTextBuffer ?? ""} onChange={useCallback( (event: ChangeEvent): void => { setCustomLivekitUrlTextBuffer(event.target.value); @@ -548,14 +551,14 @@ export const DeveloperSettingsTab: FC = ({ )} onCancel={useCallback( (e: React.FormEvent) => { - setCustomLivekitUrl(null); + setCustomTransport(null); }, - [setCustomLivekitUrl], + [setCustomTransport], )} - serverInvalid={customLivekitUrlUpdateError !== null} + serverInvalid={customTransportUpdateError !== null} > - {customLivekitUrlUpdateError !== null && ( - {customLivekitUrlUpdateError} + {customTransportUpdateError !== null && ( + {customTransportUpdateError} )} diff --git a/src/settings/SettingsModal.tsx b/src/settings/SettingsModal.tsx index 933ac66f5..733068acc 100644 --- a/src/settings/SettingsModal.tsx +++ b/src/settings/SettingsModal.tsx @@ -59,7 +59,7 @@ interface Props { livekitRooms?: { room: LivekitRoom; url: string; - isLocal?: boolean; + role: "publisher" | "subscriber"; }[]; /** Only available while in a call. Used by the developer tab. */ developerSettingsVm?: ViewModel; diff --git a/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap b/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap index 63a68ac4b..6bfc9502f 100644 --- a/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap +++ b/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap @@ -239,7 +239,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = ` class="_label_1o4d9_60" for="radix-_r_7_" > - Custom Livekit-url + Custom transport JSON
renders and matches snapshot 1`] = ` id="radix-_r_7_" name="input" title="" - value="" + value="null" />
{ public constructor( @@ -153,8 +154,8 @@ export const matrixRTCMode = new Setting( MatrixRTCMode.Compatibility, ); -export const customLivekitUrl = new Setting( - "custom-livekit-url", +export const customTransport = new Setting( + "custom-transport", null, ); diff --git a/src/state/CallViewModel/localMember/LocalTransport.test.ts b/src/state/CallViewModel/localMember/LocalTransport.test.ts index 0b3caa42e..ed6654e2f 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.test.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.test.ts @@ -19,7 +19,7 @@ import fetchMock from "fetch-mock"; import { mockConfig } from "../../../utils/test"; import { getLocalTransport } from "./LocalTransport"; import { MatrixRTCTransportMissingError } from "../../../utils/errors"; -import { customLivekitUrl } from "../../../settings/settings"; +import { customTransport } from "../../../settings/settings"; import { type MatrixClient } from "matrix-js-sdk"; describe("LocalTransport", () => { @@ -60,7 +60,7 @@ describe("LocalTransport", () => { >; beforeEach(() => { mockConfig({}); - customLivekitUrl.setValue(customLivekitUrl.defaultValue); + customTransport.setValue(customTransport.defaultValue); client = { getDomain: vi.fn().mockReturnValue("example.org"), _unstable_getRTCTransports: vi.fn().mockResolvedValue([]), @@ -86,7 +86,10 @@ describe("LocalTransport", () => { }); it("supports getting transport via user settings", async () => { - customLivekitUrl.setValue("https://lk.example.org"); + customTransport.setValue({ + type: "livekit", + livekit_service_url: "https://lk.example.org", + }); expect(await getLocalTransport(client)).toStrictEqual({ transport: { diff --git a/src/state/CallViewModel/localMember/LocalTransport.ts b/src/state/CallViewModel/localMember/LocalTransport.ts index feb8aee6f..3b43f8112 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.ts @@ -5,13 +5,12 @@ SPDX-License-IdFentifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { type MatrixClient } from "matrix-js-sdk"; import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { Config } from "../../../config/Config.ts"; import { MatrixRTCTransportMissingError } from "../../../utils/errors.ts"; -import { customLivekitUrl } from "../../../settings/settings.ts"; +import { customTransport as customTransportSetting } from "../../../settings/settings.ts"; import { RtcTransportAutoDiscovery } from "./RtcTransportAutoDiscovery.ts"; import { type TransportLocator } from "../../../livekit/auth/types.ts"; @@ -30,12 +29,11 @@ export async function getLocalTransport( resolvedConfig: Config.get(), logger: rootLogger.getChild("[LocalTransport]"), }); - const customUrl = customLivekitUrl.value$.value; - // Respect the user's custom URL, if set - const transport: UnstableLivekitTransport | null = customUrl - ? { type: "livekit", livekit_service_url: customUrl } - : await discovery.discoverPreferredTransport(); + // Respect the user's custom transport, if set + const transport = + customTransportSetting.value$.value ?? + (await discovery.discoverPreferredTransport()); if (transport === null) throw new MatrixRTCTransportMissingError(serverName); From c5044119ce2f296c29ef5d3756b25d5bab2b5c42 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 12 Oct 2026 10:33:38 +0200 Subject: [PATCH 16/25] Deprecate the livekit.livekit_service_url config option Homeservers should advertise their transports over the MSC4143 rtc/transports endpoint instead of relying on a fallback in the app config. --- src/config/ConfigOptions.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/config/ConfigOptions.ts b/src/config/ConfigOptions.ts index 5d18695a9..5f9910662 100644 --- a/src/config/ConfigOptions.ts +++ b/src/config/ConfigOptions.ts @@ -97,11 +97,19 @@ export interface ConfigOptions { }; }; - // Describes the LiveKit configuration to be used. + /** + * Describes the LiveKit configuration to be used. + */ livekit?: { - // The link to the service that returns a livekit url and token to use it. - // This is a fallback link in case the homeserver in use does not advertise - // a livekit service url over the transports endpoint. + /** + * The link to the service that returns a livekit url and token to use it. + * This is a fallback link in case the homeserver in use does not advertise + * a livekit service url over the transports endpoint. + * + * @deprecated Homeservers should advertise available transports over the + * /_matrix/client/unstable/org.matrix.msc4143/rtc/transports endpoint + * instead. + */ livekit_service_url: string; }; From 49223e720b73002f4f0d476e6501f69ed3cd21cd Mon Sep 17 00:00:00 2001 From: Johannes Marbach Date: Mon, 12 Oct 2026 10:33:58 +0200 Subject: [PATCH 17/25] Cover url transports end to end The specs replace the transports the e2e homeserver advertises with each shape a migrating deployment may produce and record which token endpoints each side used: a url-only or combined transport on both sides goes through the CS API alone, a legacy guest joining a url host uses the legacy endpoint to publish and the CS API to subscribe, a subscriber falls back to the JWT service when the homeserver lacks MSC4195, a publisher gets an error instead, a delayed leave delegated through the CS API fires on disconnect, and subscribers get tokens for a remote homeserver's SFU over federation. The JWT service's app service registration has to cover users of the e2e Synapse's real server name for the get_token flow, and lk-jwt-service 0.8.0 implements the endpoints. --- backend/app-service-othersite.yaml | 2 +- backend/app-service.yaml | 2 +- docker-compose-dev.yml | 4 +- playwright/url-transport.spec.ts | 405 +++++++++++++++++++++++++++++ 4 files changed, 409 insertions(+), 4 deletions(-) create mode 100644 playwright/url-transport.spec.ts diff --git a/backend/app-service-othersite.yaml b/backend/app-service-othersite.yaml index ef2a7ada7..a80322baf 100644 --- a/backend/app-service-othersite.yaml +++ b/backend/app-service-othersite.yaml @@ -5,7 +5,7 @@ sender_localpart: "_lk_jwt_service" namespaces: users: - exclusive: false - regex: "@.*:homeserver-1" # Cover all local users + regex: "@.*:synapse\\.othersite\\.m\\.localhost" # Cover all local users # Required for Synapse to call POST /_matrix/app/v1/ping on this service. url: "http://auth-server-1:16080" # Enable membership look-ups via /is_joined. diff --git a/backend/app-service.yaml b/backend/app-service.yaml index b480df64e..75716ca59 100644 --- a/backend/app-service.yaml +++ b/backend/app-service.yaml @@ -5,7 +5,7 @@ sender_localpart: "_lk_jwt_service" namespaces: users: - exclusive: false - regex: "@.*:homeserver" # Cover all local users + regex: "@.*:synapse\\.m\\.localhost" # Cover all local users # Required for Synapse to call POST /_matrix/app/v1/ping on this service. url: "http://auth-server:6080" # Enable membership look-ups via /is_joined. diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 3de89ec38..2db9a63a0 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -3,7 +3,7 @@ networks: services: auth-service: - image: ghcr.io/element-hq/lk-jwt-service:0.7.0 + image: ghcr.io/element-hq/lk-jwt-service:0.8.0 pull_policy: always hostname: auth-server environment: @@ -33,7 +33,7 @@ services: - ecbackend auth-service-1: - image: ghcr.io/element-hq/lk-jwt-service:0.7.0 + image: ghcr.io/element-hq/lk-jwt-service:0.8.0 pull_policy: always hostname: auth-server-1 environment: diff --git a/playwright/url-transport.spec.ts b/playwright/url-transport.spec.ts new file mode 100644 index 000000000..994c0f5cc --- /dev/null +++ b/playwright/url-transport.spec.ts @@ -0,0 +1,405 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { expect, type Page, test } from "@playwright/test"; + +import { SpaHelpers } from "./spa-helpers"; + +const sfuUrl = "wss://matrix-rtc.m.localhost/livekit/sfu"; +const serviceUrl = "https://matrix-rtc.m.localhost/livekit/jwt"; + +const otherHomeserver = { + base_url: "https://synapse.othersite.m.localhost", + server_name: "synapse.othersite.m.localhost", +}; +const otherSfuUrl = "wss://matrix-rtc.othersite.m.localhost/livekit/sfu"; + +interface Transport { + type: "livekit"; + url?: string; + livekit_service_url?: string; +} + +/** + * The transport shapes a homeserver may advertise while migrating to the + * latest MSC4195. `legacy` is what the e2e Synapse advertises on its own. + */ +const shapes = { + url: { type: "livekit", url: sfuUrl }, + both: { type: "livekit", url: sfuUrl, livekit_service_url: serviceUrl }, + legacy: null, +} satisfies Record; +type Shape = keyof typeof shapes; + +/** Which endpoint a page asks for its SFU tokens. */ +type Flow = "cs-api" | "service-legacy" | "service-default"; + +const cases: { + hostShape: Shape; + guestShape: Shape; + hostFlows: Flow[]; + guestFlows: Flow[]; +}[] = [ + { + hostShape: "url", + guestShape: "url", + hostFlows: ["cs-api"], + guestFlows: ["cs-api"], + }, + { + hostShape: "both", + guestShape: "both", + hostFlows: ["cs-api"], + guestFlows: ["cs-api"], + }, + { + hostShape: "url", + guestShape: "legacy", + hostFlows: ["cs-api", "service-default"], + guestFlows: ["service-legacy", "cs-api"], + }, +]; + +for (const { hostShape, guestShape, hostFlows, guestFlows } of cases) { + test(`One to one call with ${hostShape} host and ${guestShape} guest transport`, async ({ + browser, + page, + browserName, + }) => { + skipOnFirefox(browserName); + + // The host starts the call. + await advertiseTransport(page, shapes[hostShape]); + const hostRequests = recordAuthRequests(page); + await page.goto("/"); + await SpaHelpers.createCall(page, "Androl", "HelloCall", true); + const inviteLink = await SpaHelpers.getCallInviteLink(page); + + // The guest joins. + const guestContext = await browser.newContext({ reducedMotion: "reduce" }); + const guestPage = await guestContext.newPage(); + await advertiseTransport(guestPage, shapes[guestShape]); + const guestRequests = recordAuthRequests(guestPage); + await SpaHelpers.joinCallFromInviteLink(guestPage, inviteLink, "Pevara"); + + // Wait for the call to connect and render. + await SpaHelpers.expectVideoTilesCount(page, 2); + await SpaHelpers.expectVideoTilesCount(guestPage, 2); + + // The host and guest ran through the expected flows. + expect(flowsOf(hostRequests)).toEqual(new Set(hostFlows)); + expect(flowsOf(guestRequests)).toEqual(new Set(guestFlows)); + + // All requests succeeded and the SFU URL was set on C-S requests. + for (const request of [...hostRequests, ...guestRequests]) { + expect(request.status).toBe(200); + if (request.flow === "cs-api") expect(request.sfuUrl).toBe(sfuUrl); + } + }); +} + +test("Subscriber falls back to the JWT service when the homeserver lacks MSC4195", async ({ + browser, + page, + browserName, +}) => { + skipOnFirefox(browserName); + + // The host starts the call with both transport shapes. + await advertiseTransport(page, shapes.both); + await page.goto("/"); + await SpaHelpers.createCall(page, "Androl", "HelloCall", true); + const inviteLink = await SpaHelpers.getCallInviteLink(page); + + // The guest joins but their C-S requests fail. + const guestContext = await browser.newContext({ reducedMotion: "reduce" }); + const guestPage = await guestContext.newPage(); + await rejectCsApiTokens(guestPage); + const guestRequests = recordAuthRequests(guestPage); + await SpaHelpers.joinCallFromInviteLink(guestPage, inviteLink, "Pevara"); + + // Wait for the call to connect and render. + await SpaHelpers.expectVideoTilesCount(page, 2); + await SpaHelpers.expectVideoTilesCount(guestPage, 2); + + // The guest ran through all flows. + expect(flowsOf(guestRequests)).toEqual( + new Set(["cs-api", "service-legacy", "service-default"]), + ); + + // The C-S token requests failed. + for (const request of guestRequests) { + expect(request.status).toBe(request.flow === "cs-api" ? 404 : 200); + } + + // The C-S token requests preceeded the fallback request. + const flows = guestRequests.map((r) => r.flow); + expect(flows.indexOf("service-default")).toBeGreaterThan( + flows.indexOf("cs-api"), + ); +}); + +test("Publisher shows an error when the homeserver lacks MSC4195", async ({ + page, + browserName, +}) => { + skipOnFirefox(browserName); + + // The host tries to start a call with only the C-S transport shape available but + // their C-S requests fail. + await advertiseTransport(page, shapes.url); + await rejectCsApiTokens(page); + await page.goto("/"); + await SpaHelpers.createCall(page, "Androl", "HelloCall", true); + + // We land on an error page. + await expect( + page.getByRole("heading", { name: "Something went wrong" }), + ).toBeVisible(); + await expect( + page.getByText( + "The authorization service for your media server (SFU) is out of date.", + ), + ).toBeVisible(); +}); + +test("Delayed leave event delegated to a url transport gets sent on disconnect", async ({ + browser, + page, + browserName, +}) => { + skipOnFirefox(browserName); + + // The host starts the call. + await advertiseTransport(page, shapes.url); + await page.goto("/"); + await SpaHelpers.createCall(page, "Androl", "HelloCall", true); + const inviteLink = await SpaHelpers.getCallInviteLink(page); + + // The guest joins. + const guestContext = await browser.newContext({ reducedMotion: "reduce" }); + const guestPage = await guestContext.newPage(); + await advertiseTransport(guestPage, shapes.url); + const delayedLeave = recordDelayedLeave(guestPage); + const delegations = recordDelegations(guestPage); + await SpaHelpers.joinCallFromInviteLink(guestPage, inviteLink, "Pevara"); + + // Wait for the call to connect and render. + await SpaHelpers.expectVideoTilesCount(page, 2); + await SpaHelpers.expectVideoTilesCount(guestPage, 2); + + // The guest handed its delayed leave to the homeserver, after probing it for + // delegation support with bodiless requests. + await expect + .poll(() => delegations.at(-1)) + .toEqual({ status: 200, sfuUrl, delayId: delayedLeave.delayId }); + const probes = delegations.slice(0, -1); + expect(probes.length).toBeGreaterThan(0); + for (const probe of probes) expect(probe).toEqual({ status: 401 }); + + // The delay is long enough that a leave within the window below cannot come + // from the delayed event expiring on its own. + expect(delayedLeave.delayMs).toBeGreaterThan(60_000); + + // The guest drops without leaving. + await guestPage.reload(); + + // The host eventually sees the guest go once the delayed event fires. + await expect(page.getByTestId("videoTile")).toHaveCount(1, { + timeout: 30_000, + }); +}); + +test("Subscribers get tokens for a remote homeserver's SFU over federation", async ({ + browser, + page, + browserName, +}) => { + skipOnFirefox(browserName); + + // The host lives on the other homeserver and publishes on its SFU. + await useHomeserver(page, otherHomeserver); + await advertiseTransport(page, { type: "livekit", url: otherSfuUrl }); + const hostRequests = recordAuthRequests(page); + await page.goto("/"); + await SpaHelpers.createCall(page, "Androl", "HelloCall", true); + const inviteLink = await SpaHelpers.getCallInviteLink(page); + + // The guest lives on the default homeserver and joins over federation. + const guestContext = await browser.newContext({ reducedMotion: "reduce" }); + const guestPage = await guestContext.newPage(); + await advertiseTransport(guestPage, shapes.url); + const guestRequests = recordAuthRequests(guestPage); + await SpaHelpers.joinCallFromInviteLink( + guestPage, + `${inviteLink}&viaServers=${otherHomeserver.server_name}`, + "Pevara", + ); + + // Wait for the call to connect and render. + await SpaHelpers.expectVideoTilesCount(page, 2); + await SpaHelpers.expectVideoTilesCount(guestPage, 2); + + // Each side got a token for its own SFU and for the other side's SFU, all + // through its own homeserver. + for (const requests of [hostRequests, guestRequests]) { + expect(flowsOf(requests)).toEqual(new Set(["cs-api"])); + expect(new Set(requests.map((r) => r.sfuUrl))).toEqual( + new Set([sfuUrl, otherSfuUrl]), + ); + for (const request of requests) expect(request.status).toBe(200); + } +}); + +/** Skips the calling test if we're running on Firefox. */ +function skipOnFirefox(browserName: string): void { + test.skip( + browserName === "firefox", + "The is test is not working on firefox CI environment. No mic/audio device inputs so cam/mic are disabled", + ); +} + +/** Points this page's Element Call at another homeserver. */ +async function useHomeserver( + page: Page, + homeserver: typeof otherHomeserver, +): Promise { + await page.route("**/config.json", async (route) => { + const response = await route.fetch(); + const config = (await response.json()) as Record; + await route.fulfill({ + response, + json: { + ...config, + default_server_config: { "m.homeserver": homeserver }, + }, + }); + }); +} + +/** + * Maps recorded requests to a set of flows. Publisher and subscriber connections + * start concurrently so using arrays for comparison isn't convenient. + */ +function flowsOf(requests: AuthRequest[]): Set { + return new Set(requests.map((r) => r.flow)); +} + +/** + * Replaces the transports the homeserver advertises to this page. The real + * response is fetched first so its CORS headers carry over. + */ +async function advertiseTransport( + page: Page, + transport: Transport | null, +): Promise { + if (transport === null) return; + await page.route("**/org.matrix.msc4143/rtc/transports", async (route) => { + if (route.request().method() !== "GET") return route.continue(); + const response = await route.fetch(); + await route.fulfill({ response, json: { rtc_transports: [transport] } }); + }); +} + +/** + * Makes the homeserver look like one without MSC4195: its `get_token` + * endpoint answers as Synapse does for an unknown route. + */ +async function rejectCsApiTokens(page: Page): Promise { + await page.route("**/rtc/livekit/get_token", async (route) => { + if (route.request().method() !== "POST") return route.continue(); + await route.fulfill({ + status: 404, + json: { errcode: "M_UNRECOGNIZED", error: "Unrecognized request" }, + headers: { "Access-Control-Allow-Origin": "*" }, + }); + }); +} + +/** A recorded delayed leave event. */ +interface DelayedLeave { + delayMs?: number; + delayId?: string; +} + +/** Records the delayed leave event the page schedules on joining. */ +function recordDelayedLeave(page: Page): DelayedLeave { + const delayedLeave: DelayedLeave = {}; + page.on("response", (response) => { + const delay = new URL(response.request().url()).searchParams.get( + "org.matrix.msc4140.delay", + ); + if (delay === null || response.status() !== 200) return; + delayedLeave.delayMs = Number(delay); + response + .json() + .then((body: { delay_id: string }) => { + delayedLeave.delayId = body.delay_id; + }) + .catch(() => {}); + }); + return delayedLeave; +} + +/** A recorded delegation request for a delayed leave event. */ +interface Delegation { + status: number; + sfuUrl?: string; + delayId?: string; +} + +/** Records requests to the MSC4195 `delegate_delayed_leave` endpoint. */ +function recordDelegations(page: Page): Delegation[] { + const delegations: Delegation[] = []; + page.on("response", (response) => { + const request = response.request(); + if ( + request.method() !== "POST" || + !request.url().endsWith("/rtc/livekit/delegate_delayed_leave") + ) + return; + const body = request.postDataJSON() as { + url: string; + delay_id: string; + } | null; + delegations.push({ + status: response.status(), + ...(body && { sfuUrl: body.url, delayId: body.delay_id }), + }); + }); + return delegations; +} + +/** A recorded token request. */ +interface AuthRequest { + flow: Flow; + status: number; + /** The SFU a request to the MSC4195 `get_token` endpoint asked for. */ + sfuUrl?: string; +} + +/** Records token requests. */ +function recordAuthRequests(page: Page): AuthRequest[] { + const requests: AuthRequest[] = []; + page.on("response", (response) => { + const request = response.request(); + if (request.method() !== "POST") return; + const url = request.url(); + if (url.endsWith("/rtc/livekit/get_token")) { + requests.push({ + flow: "cs-api", + status: response.status(), + sfuUrl: (request.postDataJSON() as { url: string }).url, + }); + } else if (url.endsWith("/livekit/jwt/sfu/get")) { + requests.push({ flow: "service-legacy", status: response.status() }); + } else if (url.endsWith("/livekit/jwt/get_token")) { + requests.push({ flow: "service-default", status: response.status() }); + } + }); + return requests; +} From 68107d119ef5bcd5c42422658cf9db99c4f58ec9 Mon Sep 17 00:00:00 2001 From: Johannes Marbach Date: Mon, 12 Oct 2026 15:40:27 +0200 Subject: [PATCH 18/25] Retain compatiblity in compatiblity mode --- playwright/url-transport.spec.ts | 27 +++- src/livekit/auth/index.test.ts | 67 +++++++++ src/livekit/auth/index.ts | 140 ++++++++++++------ src/settings/DeveloperSettingsTab.tsx | 9 +- src/state/CallViewModel/CallViewModel.ts | 2 + .../CallViewModel/CallViewModelTestUtils.ts | 1 + .../localMember/LocalMember.test.ts | 14 +- .../CallViewModel/localMember/LocalMember.ts | 27 ++-- .../remoteMembers/Connection.test.ts | 7 + .../CallViewModel/remoteMembers/Connection.ts | 6 + .../remoteMembers/ConnectionFactory.ts | 4 + .../remoteMembers/ECConnectionFactory.test.ts | 6 + .../remoteMembers/integration.test.ts | 2 + 13 files changed, 247 insertions(+), 65 deletions(-) diff --git a/playwright/url-transport.spec.ts b/playwright/url-transport.spec.ts index 994c0f5cc..6400657fe 100644 --- a/playwright/url-transport.spec.ts +++ b/playwright/url-transport.spec.ts @@ -8,6 +8,7 @@ Please see LICENSE in the repository root for full details. import { expect, type Page, test } from "@playwright/test"; import { SpaHelpers } from "./spa-helpers"; +import { type RtcMode } from "./widget/test-helpers.ts"; const sfuUrl = "wss://matrix-rtc.m.localhost/livekit/sfu"; const serviceUrl = "https://matrix-rtc.m.localhost/livekit/jwt"; @@ -41,6 +42,8 @@ type Flow = "cs-api" | "service-legacy" | "service-default"; const cases: { hostShape: Shape; guestShape: Shape; + /** The MatrixRTC mode both sides use; the default is compatibility mode. */ + mode?: RtcMode; hostFlows: Flow[]; guestFlows: Flow[]; }[] = [ @@ -51,8 +54,19 @@ const cases: { guestFlows: ["cs-api"], }, { + // In compatibility mode both publish a membership state event, so both + // must keep the legacy participant identity that clients predating MSC4195 + // expect, which only the JWT service assigns. hostShape: "both", guestShape: "both", + hostFlows: ["service-legacy"], + guestFlows: ["service-legacy"], + }, + { + // In Matrix 2.0 mode nobody expects a legacy identity, so the CS API wins. + hostShape: "both", + guestShape: "both", + mode: "2_0", hostFlows: ["cs-api"], guestFlows: ["cs-api"], }, @@ -64,8 +78,8 @@ const cases: { }, ]; -for (const { hostShape, guestShape, hostFlows, guestFlows } of cases) { - test(`One to one call with ${hostShape} host and ${guestShape} guest transport`, async ({ +for (const { hostShape, guestShape, mode, hostFlows, guestFlows } of cases) { + test(`One to one call with ${hostShape} host and ${guestShape} guest transport${mode === "2_0" ? " in Matrix 2.0 mode" : ""}`, async ({ browser, page, browserName, @@ -76,7 +90,7 @@ for (const { hostShape, guestShape, hostFlows, guestFlows } of cases) { await advertiseTransport(page, shapes[hostShape]); const hostRequests = recordAuthRequests(page); await page.goto("/"); - await SpaHelpers.createCall(page, "Androl", "HelloCall", true); + await SpaHelpers.createCall(page, "Androl", "HelloCall", true, mode); const inviteLink = await SpaHelpers.getCallInviteLink(page); // The guest joins. @@ -84,7 +98,12 @@ for (const { hostShape, guestShape, hostFlows, guestFlows } of cases) { const guestPage = await guestContext.newPage(); await advertiseTransport(guestPage, shapes[guestShape]); const guestRequests = recordAuthRequests(guestPage); - await SpaHelpers.joinCallFromInviteLink(guestPage, inviteLink, "Pevara"); + await SpaHelpers.joinCallFromInviteLink( + guestPage, + inviteLink, + "Pevara", + mode, + ); // Wait for the call to connect and render. await SpaHelpers.expectVideoTilesCount(page, 2); diff --git a/src/livekit/auth/index.test.ts b/src/livekit/auth/index.test.ts index 015e0b036..6112e4caf 100644 --- a/src/livekit/auth/index.test.ts +++ b/src/livekit/auth/index.test.ts @@ -21,6 +21,7 @@ import { delegateDelayedLeave, getSFUConfig } from "."; import { mocked } from "storybook/test"; import { ConnectionError, MatrixError } from "matrix-js-sdk"; import { NoMatrix2AuthorizationService } from "../../utils/errors"; +import { MatrixRTCMode } from "../../config/ConfigOptions"; import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; vi.mock("./openID", () => ({ getSFUConfig: vi.fn() })); @@ -83,6 +84,7 @@ describe("getSFUConfig", () => { membership, serverName: "example.org", role: "subscriber" as const, + matrixRTCMode: MatrixRTCMode.Matrix_2_0, roomId, slotId, logger, @@ -169,6 +171,53 @@ describe("getSFUConfig", () => { expect(openID.getSFUConfig).not.toHaveBeenCalled(); }); + it("uses OpenID when publishing in compatibility mode with url + livekit_service_url transport", async () => { + // Subscribers predating MSC4195 expect a legacy participant identity from + // anyone advertising a state event, which only the JWT service assigns + expect( + await getSFUConfig({ + ...defaultParams, + transport: urlAndServiceUrlTransport, + role: "publisher", + matrixRTCMode: MatrixRTCMode.Compatibility, + }), + ).toEqual( + expect.objectContaining({ + jwt: openIDJWTToken, + livekitAlias: "via OpenID", + }), + ); + expect(client._unstable_getLivekitToken).not.toHaveBeenCalled(); + expect(openID.getSFUConfig).toHaveBeenCalledWith( + expect.objectContaining({ role: "publisher" }), + ); + }); + + it("uses CS API when publishing in compatibility mode with url transport", async () => { + expect( + await getSFUConfig({ + ...defaultParams, + transport: urlTransport, + role: "publisher", + matrixRTCMode: MatrixRTCMode.Compatibility, + }), + ).toEqual(expect.objectContaining({ jwt: csApiJWTToken })); + expect(client._unstable_getLivekitToken).toHaveBeenCalled(); + expect(openID.getSFUConfig).not.toHaveBeenCalled(); + }); + + it("uses CS API when subscribing in compatibility mode with url + livekit_service_url transport", async () => { + expect( + await getSFUConfig({ + ...defaultParams, + transport: urlAndServiceUrlTransport, + matrixRTCMode: MatrixRTCMode.Compatibility, + }), + ).toEqual(expect.objectContaining({ jwt: csApiJWTToken })); + expect(client._unstable_getLivekitToken).toHaveBeenCalled(); + expect(openID.getSFUConfig).not.toHaveBeenCalled(); + }); + it("does not fall back to OpenID on connection error", async () => { const client_ = { ...client, @@ -198,6 +247,7 @@ describe("delegateDelayedLeave", () => { roomId, slotId, delayId: "delay123", + matrixRTCMode: MatrixRTCMode.Matrix_2_0, logger, }; @@ -236,6 +286,23 @@ describe("delegateDelayedLeave", () => { expect(openID.getSFUConfig).not.toHaveBeenCalled(); }); + it("uses OpenID in compatibility mode with url + livekit_service_url transport", async () => { + // Must match the flow that got the token, so the identities line up + await delegateDelayedLeave({ + ...defaultParams, + transport: urlAndServiceUrlTransport, + matrixRTCMode: MatrixRTCMode.Compatibility, + }); + expect(client._unstable_delegateDelayedLeave).not.toHaveBeenCalled(); + expect(openID.getSFUConfig).toHaveBeenCalledWith( + expect.objectContaining({ + role: "publisher", + delayId: "delay123", + delayEndpointBaseUrl: client.baseUrl, + }), + ); + }); + it("does not fall back to OpenID with url + livekit_service_url transport", async () => { client._unstable_delegateDelayedLeave.mockRejectedValueOnce( unsupportedError, diff --git a/src/livekit/auth/index.ts b/src/livekit/auth/index.ts index c535e72f5..20258c81d 100644 --- a/src/livekit/auth/index.ts +++ b/src/livekit/auth/index.ts @@ -14,6 +14,7 @@ import * as openID from "./openID"; import * as csApi from "./csApi"; import { type SFUConfig } from "./types"; import { NoMatrix2AuthorizationService } from "../../utils/errors"; +import { MatrixRTCMode } from "../../config/ConfigOptions"; export type { ClientOpenIDParts } from "./openID"; export type { ClientGetTokenParts, ClientDelegationParts } from "./csApi"; @@ -66,6 +67,11 @@ export interface GetSFUConfigParams { * Whether we want to publish or only subscribe on the {@link transport}. */ role: "publisher" | "subscriber"; + /** + * The MatrixRTC mode, which decides the format of our membership event and + * so the participant identity a publisher must have. + */ + matrixRTCMode: MatrixRTCMode; logger: Logger; } @@ -80,52 +86,51 @@ export async function getSFUConfig({ roomId, slotId, role, + matrixRTCMode, logger, }: GetSFUConfigParams): Promise { - if ("url" in transport) { - try { - return await csApi.getSFUConfig({ - client, - membership, - url: transport.url, - serverName, - roomId, - slotId, - }); - } catch (e) { - if (isEndpointUnsupported(e)) { - // Publishers should never fall back. If a homeserver advertises a `url` - // but doesn't support MSC4195 itself, that's a misconfiguration which - // the admin should know about so they can prepare for the future - // deprecation of `livekit_service_url`. - const mayFallBack = - role === "subscriber" && "livekit_service_url" in transport; - if (mayFallBack) { - logger.warn( - `Homeserver does not support the MSC4195 get_token endpoint, falling back to the OpenID flow with service ${transport.livekit_service_url}`, - ); - // Fall through - } else { - logger.error( - `Homeserver does not support the MSC4195 get_token endpoint, cannot get token for transport ${transport.url}`, - ); - throw new NoMatrix2AuthorizationService(e as Error); - } - } else { - throw e; - } - } - } + const viaOpenID = async (serviceUrl: string): Promise => + await openID.getSFUConfig({ + client, + membership, + serviceUrl, + roomId, + slotId, + role, + logger, + }); - return await openID.getSFUConfig({ - client, - membership, - serviceUrl: transport.livekit_service_url, - roomId, - slotId, - role, - logger, - }); + const flow = chooseAuthFlow(transport, role, matrixRTCMode); + if (flow.kind === "openID") return await viaOpenID(flow.serviceUrl); + + try { + return await csApi.getSFUConfig({ + client, + membership, + url: flow.url, + serverName, + roomId, + slotId, + }); + } catch (e) { + if (!isEndpointUnsupported(e)) throw e; + // Publishers should never fall back. If a homeserver advertises a `url` + // but doesn't support MSC4195 itself, that's a misconfiguration which + // the admin should know about so they can prepare for the future + // deprecation of `livekit_service_url`. + const mayFallBack = + role === "subscriber" && "livekit_service_url" in transport; + if (!mayFallBack) { + logger.error( + `Homeserver does not support the MSC4195 get_token endpoint, cannot get token for transport ${flow.url}`, + ); + throw new NoMatrix2AuthorizationService(e as Error); + } + logger.warn( + `Homeserver does not support the MSC4195 get_token endpoint, falling back to the OpenID flow with service ${transport.livekit_service_url}`, + ); + return await viaOpenID(transport.livekit_service_url); + } } export interface DelegateDelayedLeaveParams { @@ -154,6 +159,10 @@ export interface DelegateDelayedLeaveParams { * The delay ID of the leave event to be delegated. */ delayId: string; + /** + * The MatrixRTC mode, which decides the format of our membership event. + */ + matrixRTCMode: MatrixRTCMode; logger: Logger; } @@ -168,17 +177,19 @@ export async function delegateDelayedLeave({ roomId, slotId, delayId, + matrixRTCMode, logger, }: DelegateDelayedLeaveParams): Promise { // It's important that we use the same flow (CS API or OpenID) as whatever we // would've used to get a JWT token for this transport, since the participant // identity computed by the service needs to match. - if ("url" in transport) { + const flow = chooseAuthFlow(transport, "publisher", matrixRTCMode); + if (flow.kind === "csApi") { try { await csApi.delegateDelayedLeave({ client, membership, - url: transport.url, + url: flow.url, roomId, slotId, delayId, @@ -196,7 +207,7 @@ export async function delegateDelayedLeave({ await openID.getSFUConfig({ client, membership, - serviceUrl: transport.livekit_service_url, + serviceUrl: flow.serviceUrl, roomId, slotId, role: "publisher", @@ -206,3 +217,40 @@ export async function delegateDelayedLeave({ }); } } + +/** + * The flow with which the local member authenticates against a transport. + */ +export type AuthFlow = + | { kind: "csApi"; url: string } + | { kind: "openID"; serviceUrl: string }; + +/** + * Chooses between the MSC4195 CS API and the legacy OpenID flow against the + * transport's JWT service. + * + * Only the JWT service's `/sfu/get` endpoint assigns the legacy participant + * identity that clients predating MSC4195 expect from anyone advertising a + * membership state event, which is what compatibility mode sends. So a + * publisher in compatibility mode keeps using the JWT service wherever the + * transport still offers one, even if it also has a `url`. Subscribers may use + * the CS API regardless, as nobody matches their identity. + * @param transport The transport to authenticate against. + * @param role Whether we want to publish or only subscribe on the transport. + * @param matrixRTCMode The MatrixRTC mode, which decides the format of our + * membership event. + */ +export function chooseAuthFlow( + transport: UnstableLivekitTransport, + role: "publisher" | "subscriber", + matrixRTCMode: MatrixRTCMode, +): AuthFlow { + if ("url" in transport) { + const useCsApi = + !("livekit_service_url" in transport) || + role === "subscriber" || + matrixRTCMode === MatrixRTCMode.Matrix_2_0; + if (useCsApi) return { kind: "csApi", url: transport.url }; + } + return { kind: "openID", serviceUrl: transport.livekit_service_url }; +} diff --git a/src/settings/DeveloperSettingsTab.tsx b/src/settings/DeveloperSettingsTab.tsx index 3ef943ddf..42cc608c7 100644 --- a/src/settings/DeveloperSettingsTab.tsx +++ b/src/settings/DeveloperSettingsTab.tsx @@ -530,6 +530,7 @@ export const DeveloperSettingsTab: FC = ({ roomId, slotId: "org.example.fake-slot#room", role: "publisher", + matrixRTCMode: effectiveMatrixRTCMode, logger, }); setCustomLivekitUrlUpdateError(null); @@ -540,7 +541,13 @@ export const DeveloperSettingsTab: FC = ({ ); } }, - [customTransportTextBuffer, setCustomTransport, client, roomId], + [ + customTransportTextBuffer, + setCustomTransport, + client, + roomId, + effectiveMatrixRTCMode, + ], )} value={customTransportTextBuffer ?? ""} onChange={useCallback( diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 169abb98c..5821204a1 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -575,6 +575,7 @@ export function createCallViewModel$( client, matrixRoom.roomId, matrixRTCSession.slotId, + matrixRTCMode, mediaDevices, trackProcessorState$, livekitKeyProvider, @@ -665,6 +666,7 @@ export function createCallViewModel$( ).pipe(map(([delayId]) => delayId ?? null)), matrixRTCSession.delayId ?? null, ), + matrixRTCMode, logger: logger.getChild(`[${Date.now()}]`), }); diff --git a/src/state/CallViewModel/CallViewModelTestUtils.ts b/src/state/CallViewModel/CallViewModelTestUtils.ts index 93c190ace..6ef3e9f8d 100644 --- a/src/state/CallViewModel/CallViewModelTestUtils.ts +++ b/src/state/CallViewModel/CallViewModelTestUtils.ts @@ -230,6 +230,7 @@ export function withCallViewModel(mode: MatrixRTCMode) { client: room.client, roomId: room.roomId, slotId: "m.call#ROOM", + matrixRTCMode: mode, livekitRoomFactory, }, logger, diff --git a/src/state/CallViewModel/localMember/LocalMember.test.ts b/src/state/CallViewModel/localMember/LocalMember.test.ts index 2d721f7d6..273b6c8f3 100644 --- a/src/state/CallViewModel/localMember/LocalMember.test.ts +++ b/src/state/CallViewModel/localMember/LocalMember.test.ts @@ -346,21 +346,27 @@ describe("LocalMembership", () => { .mockResolvedValue(); afterEach(() => delegationSpy.mockClear()); + // The transport has both a `url` and a `livekit_service_url`. In Matrix 2.0 + // mode we publish through the CS API, so the homeserver delegates; in + // compatibility mode we publish through the JWT service, so that delegates. it.each([ - ["no", null, timings], + ["no", MatrixRTCMode.Compatibility, null, timings], + ["no", MatrixRTCMode.Matrix_2_0, null, timings], [ "homeserver", + MatrixRTCMode.Matrix_2_0, "https://matrix.example.org/_matrix/client/unstable/io.element.msc4195/rtc/livekit/delegate_delayed_leave", delegatedTimings, ], [ "transport", + MatrixRTCMode.Compatibility, "https://jwt.example.org/delegate_delayed_leave", delegatedTimings, ], ])( - "joins session with %s delegation support", - async (_serviceName, delegationUrl, delayedLeaveTimings) => { + "joins session with %s delegation support in %s mode", + async (_serviceName, matrixRTCMode, delegationUrl, delayedLeaveTimings) => { const scope = testScope(); const joinMatrixRTC = vi.fn(); const delayId$ = new BehaviorSubject(null); @@ -377,6 +383,7 @@ describe("LocalMembership", () => { joinMatrixRTC, localTransport$: constant(mockTransportLocator), delayId$, + matrixRTCMode, }); localMembership.requestJoinAndPublish(); @@ -403,6 +410,7 @@ describe("LocalMembership", () => { transport: mockTransport, roomId: "!test-room-id:example.org", delayId, + matrixRTCMode, logger: expect.anything(), }); diff --git a/src/state/CallViewModel/localMember/LocalMember.ts b/src/state/CallViewModel/localMember/LocalMember.ts index 244c728dc..b4af36b58 100644 --- a/src/state/CallViewModel/localMember/LocalMember.ts +++ b/src/state/CallViewModel/localMember/LocalMember.ts @@ -82,6 +82,7 @@ import { type ClientDelegationParts, delegateDelayedLeave, type ClientOpenIDParts, + chooseAuthFlow, type TransportLocator, } from "../../../livekit/auth"; @@ -164,6 +165,8 @@ interface Props { hostBridge: HostBridge; baseUrl: string; delayId$: Behavior; + /** The MatrixRTC mode, which decides the format of our membership event. */ + matrixRTCMode: MatrixRTCMode; logger: Logger; } @@ -211,6 +214,7 @@ export const createLocalMembership$ = ({ hostBridge, ownMembershipIdentity, delayId$, + matrixRTCMode, }: Props): { /** * This request to start audio and video tracks. @@ -312,17 +316,17 @@ export const createLocalMembership$ = ({ const joinParams$ = scope.behavior( localTransport$.pipe( switchMap(async ({ transport }) => { - const transportSupportsDelegation = - "livekit_service_url" in transport && - (await checkDelegationSupport( - transport.livekit_service_url + "/delegate_delayed_leave", - `transport ${transport.livekit_service_url}`, - )); - return { - transport, - delegationSupported: - transportSupportsDelegation || (await homeserverSupportsDelegation), - }; + // Delegation goes through whichever service authenticates us as a + // publisher, so only that service's support counts + const flow = chooseAuthFlow(transport, "publisher", matrixRTCMode); + const delegationSupported = + flow.kind === "openID" + ? await checkDelegationSupport( + flow.serviceUrl + "/delegate_delayed_leave", + `transport ${flow.serviceUrl}`, + ) + : await homeserverSupportsDelegation; + return { transport, delegationSupported }; }), ), null, @@ -731,6 +735,7 @@ export const createLocalMembership$ = ({ slotId: matrixRTCSession.slotId, roomId, delayId, + matrixRTCMode, logger, }); logger.info("Delayed leave successfully delegated"); diff --git a/src/state/CallViewModel/remoteMembers/Connection.test.ts b/src/state/CallViewModel/remoteMembers/Connection.test.ts index bdab8e7af..632122a3f 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.test.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.test.ts @@ -42,6 +42,7 @@ import { } from "./Connection.ts"; import { ObservableScope } from "../../ObservableScope.ts"; import { FailToGetOpenIdToken } from "../../../utils/errors.ts"; +import { MatrixRTCMode } from "../../../config/ConfigOptions.ts"; import { testJWTToken } from "../../../utils/test-fixtures.ts"; import { mockRemoteParticipant, ownMemberMock } from "../../../utils/test.ts"; import { @@ -123,6 +124,7 @@ function setupRemoteConnection(): Connection { client, roomId: ROOM_ID, slotId: SLOT_ID, + matrixRTCMode: MatrixRTCMode.Matrix_2_0, role: "subscriber", transport, serverName: "m.localhost", @@ -172,6 +174,7 @@ describe("Start connection states", () => { client, roomId: ROOM_ID, slotId: SLOT_ID, + matrixRTCMode: MatrixRTCMode.Matrix_2_0, role: "subscriber", transport, serverName: "m.localhost", @@ -192,6 +195,7 @@ describe("Start connection states", () => { client, roomId: ROOM_ID, slotId: SLOT_ID, + matrixRTCMode: MatrixRTCMode.Matrix_2_0, role: "subscriber", transport, serverName: "m.localhost", @@ -247,6 +251,7 @@ describe("Start connection states", () => { client, roomId: ROOM_ID, slotId: SLOT_ID, + matrixRTCMode: MatrixRTCMode.Matrix_2_0, role: "subscriber", transport, serverName: "m.localhost", @@ -311,6 +316,7 @@ describe("Start connection states", () => { client, roomId: ROOM_ID, slotId: SLOT_ID, + matrixRTCMode: MatrixRTCMode.Matrix_2_0, role: "subscriber", transport, serverName: "m.localhost", @@ -561,6 +567,7 @@ describe("remote track logging", () => { client, roomId: ROOM_ID, slotId: SLOT_ID, + matrixRTCMode: MatrixRTCMode.Matrix_2_0, role: "subscriber", transport, serverName: "m.localhost", diff --git a/src/state/CallViewModel/remoteMembers/Connection.ts b/src/state/CallViewModel/remoteMembers/Connection.ts index 7e94b39f1..cbdffcbde 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.ts @@ -44,12 +44,15 @@ import { type ClientOpenIDParts, type SFUConfig, } from "../../../livekit/auth"; +import { type MatrixRTCMode } from "../../../config/ConfigOptions.ts"; export interface ConnectionOpts { /** The identity parts to use on this connection */ ownMembershipIdentity: CallMembershipIdentityParts; /** Whether we want to publish or only subscribe on this connection. */ role: "publisher" | "subscriber"; + /** The MatrixRTC mode, which decides the format of our membership event. */ + matrixRTCMode: MatrixRTCMode; /** The media transport to connect to. */ transport: UnstableLivekitTransport; /** The name of the homeserver to which the {@link transport} belongs. */ @@ -138,6 +141,7 @@ export class Connection { private readonly client: ClientGetTokenParts & ClientOpenIDParts; private readonly roomId: string; private readonly slotId: string; + private readonly matrixRTCMode: MatrixRTCMode; private readonly logger: Logger; private readonly ownMembershipIdentity: CallMembershipIdentityParts; /** @@ -151,6 +155,7 @@ export class Connection { this.ownMembershipIdentity = opts.ownMembershipIdentity; this.roomId = opts.roomId; this.slotId = opts.slotId; + this.matrixRTCMode = opts.matrixRTCMode; this.logger = logger.getChild( `[Connection ${JSON.stringify(opts.transport)}]`, ); @@ -394,6 +399,7 @@ export class Connection { serverName: this.serverName, roomId: this.roomId, slotId: this.slotId, + matrixRTCMode: this.matrixRTCMode, logger: this.logger, }); } diff --git a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts index e5240f096..39ac35706 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts @@ -39,6 +39,7 @@ import { type ClientGetTokenParts, type ClientOpenIDParts, } from "../../../livekit/auth"; +import { type MatrixRTCMode } from "../../../config/ConfigOptions.ts"; // TODO evaluate if this should be done like the Publisher Factory export interface ConnectionFactory { @@ -61,6 +62,7 @@ export class ECConnectionFactory implements ConnectionFactory { * @param client - The OpenID client parts for authentication, needed to get openID and JWT tokens. * @param roomId - The ID of the Matrix room in which the session takes place. * @param slotId - The ID of the MatrixRTC slot in which the session takes place. + * @param matrixRTCMode - The MatrixRTC mode, which decides the format of our membership event. * @param devices - Used for video/audio out/in capture options. * @param processorState$ - Effects like background blur (only for publishing connection?) * @param livekitKeyProvider - Optional key provider for end-to-end encryption. @@ -71,6 +73,7 @@ export class ECConnectionFactory implements ConnectionFactory { private client: ClientGetTokenParts & ClientOpenIDParts, private readonly roomId: string, private readonly slotId: string, + private readonly matrixRTCMode: MatrixRTCMode, private devices: MediaDevices, private processorState$: Behavior, livekitKeyProvider: BaseKeyProvider | undefined, @@ -117,6 +120,7 @@ export class ECConnectionFactory implements ConnectionFactory { { roomId: this.roomId, slotId: this.slotId, + matrixRTCMode: this.matrixRTCMode, role, transport, serverName, diff --git a/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts b/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts index be6c149f4..0a9310f82 100644 --- a/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts +++ b/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts @@ -36,6 +36,7 @@ import { cameraCodec, } from "../../../settings/settings.ts"; import { testJWTToken } from "../../../utils/test-fixtures.ts"; +import { MatrixRTCMode } from "../../../config/ConfigOptions.ts"; // At the top of your test file, after imports vi.mock("livekit-client", async (importOriginal) => { @@ -84,6 +85,7 @@ describe("ECConnectionFactory - Audio inputs options", () => { mockClient, "!roomid:example.org", "m.call#ROOM", + MatrixRTCMode.Matrix_2_0, mockMediaDevices({}), new BehaviorSubject({ supported: true, @@ -129,6 +131,7 @@ describe("ECConnectionFactory - ControlledAudioDevice", () => { mockClient, "!roomid:example.org", "m.call#ROOM", + MatrixRTCMode.Matrix_2_0, mockMediaDevices({ audioOutput: { available$: constant(new Map()), @@ -173,6 +176,7 @@ describe("ECConnectionFactory - Camera quality settings", () => { mockClient, "!roomid:example.org", "m.call#ROOM", + MatrixRTCMode.Matrix_2_0, mockMediaDevices({}), new BehaviorSubject({ supported: true, @@ -213,6 +217,7 @@ describe("ECConnectionFactory - Camera quality settings", () => { mockClient, "!roomid:example.org", "m.call#ROOM", + MatrixRTCMode.Matrix_2_0, mockMediaDevices({}), new BehaviorSubject({ supported: true, @@ -254,6 +259,7 @@ describe("ECConnectionFactory - Camera quality settings", () => { mockClient, "!roomid:example.org", "m.call#ROOM", + MatrixRTCMode.Matrix_2_0, mockMediaDevices({}), new BehaviorSubject({ supported: true, diff --git a/src/state/CallViewModel/remoteMembers/integration.test.ts b/src/state/CallViewModel/remoteMembers/integration.test.ts index bf90e9eb4..b610447d9 100644 --- a/src/state/CallViewModel/remoteMembers/integration.test.ts +++ b/src/state/CallViewModel/remoteMembers/integration.test.ts @@ -38,6 +38,7 @@ import { createConnectionMap$ } from "./ConnectionMap.ts"; import { membershipsAndTransports$ } from "../../SessionBehaviors.ts"; import { localRtcMember, testJWTToken } from "../../../utils/test-fixtures.ts"; import { deepCompare } from "matrix-js-sdk/lib/utils"; +import { MatrixRTCMode } from "../../../config/ConfigOptions.ts"; // Test the integration of ConnectionMap and MatrixLivekitMembers @@ -75,6 +76,7 @@ beforeEach(() => { mockClient, "!roomid:example.org", "m.call#ROOM", + MatrixRTCMode.Matrix_2_0, mockMediaDevices({}), new BehaviorSubject({ supported: true, From f8258f6a1d5091650b2b935517053b993b522341 Mon Sep 17 00:00:00 2001 From: Johannes Marbach Date: Mon, 12 Oct 2026 15:43:44 +0200 Subject: [PATCH 19/25] Switch JS SDK back to develop --- package.json | 2 +- pnpm-lock.yaml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 0a76e0c01..b30f57ce3 100644 --- a/package.json +++ b/package.json @@ -103,7 +103,7 @@ "livekit-client": "^2.18.1", "lodash-es": "^4.17.21", "loglevel": "^1.9.1", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#robin/livekit-updates", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#develop", "matrix-widget-api": "^1.18.0", "node-stdlib-browser": "^1.3.1", "normalize.css": "^8.0.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5866d9f8c..c55527011 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -191,8 +191,8 @@ importers: specifier: ^1.9.1 version: 1.9.2 matrix-js-sdk: - specifier: github:matrix-org/matrix-js-sdk#robin/livekit-updates - version: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/29099d884f08c90a1bc61b31bae0a937b0dc94ff + specifier: github:matrix-org/matrix-js-sdk#develop + version: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/4df2f2bfb5826c25f05d9535a8556acecaf2e570 matrix-widget-api: specifier: ^1.18.0 version: 1.20.0 @@ -4673,8 +4673,8 @@ packages: matrix-events-sdk@0.0.1: resolution: {integrity: sha512-1QEOsXO+bhyCroIe2/A5OwaxHvBm7EsSQ46DEDn8RBIfQwN5HWBpFvyWWR4QY0KHPPnnJdI99wgRiAl7Ad5qaA==} - matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/29099d884f08c90a1bc61b31bae0a937b0dc94ff: - resolution: {gitHosted: true, integrity: sha512-3614DKOUwhaTbGbLbWWMTrdQvRE0J7oucUTbrfr95fbAn/Zb1TZvwhVs3B94o4i9HrBf2hq1m1VwusAslg2WTQ==, tarball: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/29099d884f08c90a1bc61b31bae0a937b0dc94ff} + matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/4df2f2bfb5826c25f05d9535a8556acecaf2e570: + resolution: {gitHosted: true, integrity: sha512-b1o6ywaPY9jakj+3GIgx3samM9AbqohzPQ5nZKbyOGYB16IHQtNMyjEo/PY0fW/s0uwBVApEHNb/MMtN40c/rQ==, tarball: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/4df2f2bfb5826c25f05d9535a8556acecaf2e570} version: 43.0.0 engines: {node: '>=22.0.0'} @@ -10227,7 +10227,7 @@ snapshots: matrix-events-sdk@0.0.1: {} - matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/29099d884f08c90a1bc61b31bae0a937b0dc94ff: + matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/4df2f2bfb5826c25f05d9535a8556acecaf2e570: dependencies: '@babel/runtime': 8.0.0 '@matrix-org/matrix-sdk-crypto-wasm': 18.9.0 From 97b5e61ad8e5e890a10578d7134b8c8a6c4d8c26 Mon Sep 17 00:00:00 2001 From: Johannes Marbach Date: Mon, 12 Oct 2026 16:03:00 +0200 Subject: [PATCH 20/25] Fall back to get_token as subscriber on widget API errors --- src/livekit/auth/csApi.ts | 47 ++++++++++++++++++++++------------ src/livekit/auth/index.test.ts | 28 ++++++++++++++++++++ src/livekit/auth/index.ts | 29 ++++++++++++++++----- src/utils/matrix.ts | 8 +++++- 4 files changed, 87 insertions(+), 25 deletions(-) diff --git a/src/livekit/auth/csApi.ts b/src/livekit/auth/csApi.ts index 63b9d0cb9..d62111f02 100644 --- a/src/livekit/auth/csApi.ts +++ b/src/livekit/auth/csApi.ts @@ -5,12 +5,21 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type MatrixClient } from "matrix-js-sdk"; +import { ConnectionError, HTTPError, type MatrixClient } from "matrix-js-sdk"; import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; import { extractFullConfigFromToken, type SFUConfig } from "./types.ts"; import { doNetworkOperationWithRetry } from "../../utils/matrix.ts"; +/** + * In widget mode the host may refuse an action before any request is made + * which surfaces as a plain error. Asking again cannot change its answer, + * so only errors from the network are worth retrying. + */ +function isNetworkError(e: unknown): boolean { + return e instanceof HTTPError || e instanceof ConnectionError; +} + export type ClientGetTokenParts = Pick< MatrixClient, "_unstable_getLivekitToken" @@ -55,14 +64,16 @@ export async function getSFUConfig({ roomId, slotId, }: GetSFUConfigParams): Promise { - const res = await doNetworkOperationWithRetry(async () => - client._unstable_getLivekitToken({ - url, - server_name: serverName, - room_id: roomId, - slot_id: slotId, - member_id: membership.memberId, - }), + const res = await doNetworkOperationWithRetry( + async () => + client._unstable_getLivekitToken({ + url, + server_name: serverName, + room_id: roomId, + slot_id: slotId, + member_id: membership.memberId, + }), + isNetworkError, ); return extractFullConfigFromToken({ url, jwt: res.jwt }); } @@ -112,13 +123,15 @@ export async function delegateDelayedLeave({ slotId, delayId, }: DelegateDelayedLeaveParams): Promise { - await doNetworkOperationWithRetry(async () => - client._unstable_delegateDelayedLeave({ - url, - room_id: roomId, - slot_id: slotId, - member_id: membership.memberId, - delay_id: delayId, - }), + await doNetworkOperationWithRetry( + async () => + client._unstable_delegateDelayedLeave({ + url, + room_id: roomId, + slot_id: slotId, + member_id: membership.memberId, + delay_id: delayId, + }), + isNetworkError, ); } diff --git a/src/livekit/auth/index.test.ts b/src/livekit/auth/index.test.ts index 6112e4caf..f16987a34 100644 --- a/src/livekit/auth/index.test.ts +++ b/src/livekit/auth/index.test.ts @@ -158,6 +158,34 @@ describe("getSFUConfig", () => { expect(openID.getSFUConfig).toHaveBeenCalled(); }); + it("falls back to OpenID when the widget host refuses the CS API", async () => { + // In widget mode a host without MSC4533 rejects the request before it + // reaches the homeserver, with a plain error rather than a Matrix error + client._unstable_getLivekitToken.mockRejectedValueOnce( + new Error( + "The rtc_livekit_get_token action is not supported by the client.", + ), + ); + expect( + await getSFUConfig({ + ...defaultParams, + transport: urlAndServiceUrlTransport, + }), + ).toEqual(expect.objectContaining({ jwt: openIDJWTToken })); + expect(openID.getSFUConfig).toHaveBeenCalled(); + }); + + it("surfaces the widget host's refusal with url transport", async () => { + const error = new Error( + "The rtc_livekit_get_token action is not supported by the client.", + ); + client._unstable_getLivekitToken.mockRejectedValueOnce(error); + await expect( + getSFUConfig({ ...defaultParams, transport: urlTransport }), + ).rejects.toThrow(error); + expect(openID.getSFUConfig).not.toHaveBeenCalled(); + }); + it("does not fall back to OpenID when publishing", async () => { client._unstable_getLivekitToken.mockRejectedValueOnce(unsupportedError); await expect( diff --git a/src/livekit/auth/index.ts b/src/livekit/auth/index.ts index 20258c81d..018347cb3 100644 --- a/src/livekit/auth/index.ts +++ b/src/livekit/auth/index.ts @@ -8,7 +8,7 @@ Please see LICENSE in the repository root for full details. import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; import { type Logger } from "matrix-js-sdk/lib/logger"; -import { MatrixError } from "matrix-js-sdk"; +import { ConnectionError, MatrixError } from "matrix-js-sdk"; import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import * as openID from "./openID"; import * as csApi from "./csApi"; @@ -38,6 +38,19 @@ function isEndpointUnsupported(e: unknown): boolean { ); } +/** + * Checks whether an error means the MSC4195 CS API cannot be used from here: + * the homeserver does not implement it, or the request never reached the + * homeserver because in widget mode the host does not expose the endpoint + * (MSC4533), which the widget API reports as a plain error rather than a + * Matrix error. A connection error is neither, as the endpoint may well exist. + * @param e The error to check. + */ +function isCsApiUnavailable(e: unknown): boolean { + if (isEndpointUnsupported(e)) return true; + return !(e instanceof MatrixError) && !(e instanceof ConnectionError); +} + export interface GetSFUConfigParams { /** * The Matrix client. @@ -113,23 +126,25 @@ export async function getSFUConfig({ slotId, }); } catch (e) { - if (!isEndpointUnsupported(e)) throw e; // Publishers should never fall back. If a homeserver advertises a `url` // but doesn't support MSC4195 itself, that's a misconfiguration which // the admin should know about so they can prepare for the future // deprecation of `livekit_service_url`. const mayFallBack = role === "subscriber" && "livekit_service_url" in transport; - if (!mayFallBack) { + if (mayFallBack && isCsApiUnavailable(e)) { + logger.warn( + `Cannot use the MSC4195 get_token endpoint (${e}), falling back to the OpenID flow with service ${transport.livekit_service_url}`, + ); + return await viaOpenID(transport.livekit_service_url); + } + if (isEndpointUnsupported(e)) { logger.error( `Homeserver does not support the MSC4195 get_token endpoint, cannot get token for transport ${flow.url}`, ); throw new NoMatrix2AuthorizationService(e as Error); } - logger.warn( - `Homeserver does not support the MSC4195 get_token endpoint, falling back to the OpenID flow with service ${transport.livekit_service_url}`, - ); - return await viaOpenID(transport.livekit_service_url); + throw e; } } diff --git a/src/utils/matrix.ts b/src/utils/matrix.ts index 4e3ae3c32..823de6880 100644 --- a/src/utils/matrix.ts +++ b/src/utils/matrix.ts @@ -341,9 +341,13 @@ export function getRelativeRoomUrl( * Perform a network operation with retries on ConnectionError. * If the error is not retryable, or the max number of retries is reached, the error is rethrown. * Supports handling of matrix quotas. + * @param operation The operation to perform. + * @param shouldRetry Rules out errors that retrying cannot fix, on top of the + * standard backoff rule. Retries everything by default. */ export async function doNetworkOperationWithRetry( operation: () => Promise, + shouldRetry: (e: unknown) => boolean = () => true, ): Promise { let currentRetryCount = 0; @@ -353,7 +357,9 @@ export async function doNetworkOperationWithRetry( return await operation(); } catch (e) { currentRetryCount++; - const backoff = calculateRetryBackoff(e, currentRetryCount, true); + const backoff = shouldRetry(e) + ? calculateRetryBackoff(e, currentRetryCount, true) + : -1; if (backoff < 0) { // Max number of retries reached, or error is not retryable. rethrow the error throw e; From a1575cb55411b2a6bfe210faeaeb6bfc6f645375 Mon Sep 17 00:00:00 2001 From: Johannes Marbach Date: Mon, 12 Oct 2026 16:08:56 +0200 Subject: [PATCH 21/25] Share a connection between homeservers using the same JWT service --- .../remoteMembers/ConnectionMap.test.ts | 78 ++++++++++++++++++- .../remoteMembers/ConnectionMap.ts | 46 +++++------ 2 files changed, 96 insertions(+), 28 deletions(-) diff --git a/src/state/CallViewModel/remoteMembers/ConnectionMap.test.ts b/src/state/CallViewModel/remoteMembers/ConnectionMap.test.ts index 90c0e4c67..2344c1e55 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionMap.test.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionMap.test.ts @@ -127,6 +127,82 @@ describe("connections$ stream", () => { }); }); + test("Shares one connection between homeservers advertising the same JWT service", () => { + withTestScheduler(({ behavior, expectObservable }) => { + const sameServiceElsewhere: TransportLocator = { + transport: TRANSPORT_1.transport, + serverName: "elsewhere.org", + }; + const connectionMap$ = createConnectionMap$({ + scope: testScope, + connectionFactory: fakeConnectionFactory, + localTransport: TRANSPORT_1, + remoteTransports$: behavior("a", { + a: new Epoch([sameServiceElsewhere], 0), + }), + logger: logger, + ownMembershipIdentity: ownMemberMock, + }); + + expectObservable( + connectionMap$.pipe(mapEpoch((d) => d.getConnections())), + ).toBe("a", { + a: expect.toSatisfy((e: Epoch) => { + expect(e.value.length).toBe(1); + // The JWT service is reached directly, so the server name plays no + // part and the local locator, which came first, is the one used + expect( + vi.mocked(fakeConnectionFactory).createConnection, + ).toHaveBeenCalledExactlyOnceWith( + expect.anything(), + "publisher", + TRANSPORT_1.transport, + "example.org", + expect.anything(), + expect.anything(), + ); + return true; + }), + }); + }); + }); + + test("Keeps one connection per homeserver for transports with a url", () => { + withTestScheduler(({ behavior, expectObservable }) => { + const urlTransport: TransportLocator = { + transport: { type: "livekit", url: "wss://sfu.example.org" }, + serverName: "example.org", + }; + const sameUrlElsewhere: TransportLocator = { + transport: urlTransport.transport, + serverName: "elsewhere.org", + }; + const connectionMap$ = createConnectionMap$({ + scope: testScope, + connectionFactory: fakeConnectionFactory, + localTransport: urlTransport, + remoteTransports$: behavior("a", { + a: new Epoch([sameUrlElsewhere], 0), + }), + logger: logger, + ownMembershipIdentity: ownMemberMock, + }); + + expectObservable( + connectionMap$.pipe(mapEpoch((d) => d.getConnections())), + ).toBe("a", { + a: expect.toSatisfy((e: Epoch) => { + // Our homeserver asks the named server for the token, so each + // server name needs its own connection + expect(e.value.length).toBe(2); + expect(findConnection(urlTransport, e.value)).toBeDefined(); + expect(findConnection(sameUrlElsewhere, e.value)).toBeDefined(); + return true; + }), + }); + }); + }); + test("Should start connection only once", () => { withTestScheduler(({ behavior, expectObservable }) => { const connectionMap$ = createConnectionMap$({ @@ -228,7 +304,7 @@ describe("ConnectionMap", () => { data.add({ ...connection } as Connection, [p("b")]); expect(warn).toHaveBeenCalledWith( expect.stringContaining( - 'second connection to [null,"https://lk.example.org","example.org"]: existing [a], adding [b]', + 'second connection to [null,"https://lk.example.org",null]: existing [a], adding [b]', ), ); expect(data.getParticipantsForTransport(TRANSPORT_1)).toHaveLength(2); diff --git a/src/state/CallViewModel/remoteMembers/ConnectionMap.ts b/src/state/CallViewModel/remoteMembers/ConnectionMap.ts index 4adc92b04..586e8d49b 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionMap.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionMap.ts @@ -6,7 +6,6 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { combineLatest, map, of, switchMap, tap } from "rxjs"; import { type Logger } from "matrix-js-sdk/lib/logger"; import { type RemoteParticipant } from "livekit-client"; @@ -23,46 +22,40 @@ import { generateItemsWithEpoch } from "../../../utils/observable.ts"; import { type ConnectionFactory } from "./ConnectionFactory.ts"; import { type TransportLocator } from "../../../livekit/auth/types.ts"; -// Branded string type, to enforce that `fromKey` must only be called with -// values returned by `keyFor` +// Branded string type, so that only values returned by `keyFor` can be used +// as keys export type TransportKey = string & { __transportKey: true }; /** * Converts a {@link TransportLocator} to a key suitable for identifying it in a - * {@link Map} or {@link Set}. + * {@link Map} or {@link Set}. Locators with the same key share a connection. */ export function keyFor({ transport, serverName, }: TransportLocator): TransportKey { + // The server name only takes part in authentication for a transport with a + // `url`, where our homeserver asks that server for the token. A JWT service + // is reached directly, so members of several homeservers advertising the + // same one must share a connection rather than each get their own. + const hasUrl = "url" in transport; // Serialize the fields in a consistent order return JSON.stringify([ - "url" in transport ? transport.url : null, + hasUrl ? transport.url : null, "livekit_service_url" in transport ? transport.livekit_service_url : null, - serverName, + hasUrl ? serverName : null, ]) as TransportKey; } -/** - * Inverse of {@link keyFor}: turns a key back into a {@link TransportLocator}. - */ -export function fromKey(key: TransportKey): TransportLocator { - const [url, serviceUrl, serverName] = JSON.parse(key); - return { - transport: { - type: "livekit", - ...(url === null ? {} : { url }), - ...(serviceUrl === null ? {} : { livekit_service_url: serviceUrl }), - } as UnstableLivekitTransport, - serverName, - }; -} - function removeDuplicateTransports( transports: TransportLocator[], ): TransportLocator[] { + // The first locator wins, so the local one stays first in the list const map = new Map(); - for (const transport of transports) map.set(keyFor(transport), transport); + for (const transport of transports) { + const key = keyFor(transport); + if (!map.has(key)) map.set(key, transport); + } return [...map.values()]; } @@ -173,15 +166,14 @@ export function createConnectionMap$({ ? ("publisher" as const) : ("subscriber" as const); yield { - // In order to maintain a static set of map keys, we must - // temporarily serialize `transport` into a stable key format + // The locator is an object, so a stable key stands in for it keys: [role, keyFor(transport)], - data: undefined, + data: transport, }; } }, - (scope, _data$, role, transportKey) => { - const { transport, serverName } = fromKey(transportKey); + (scope, locator$, role, _key) => { + const { transport, serverName } = locator$.value; const connection = connectionFactory.createConnection( scope, role, From 022415e98e4f3a727c4e6bf3aa932f777a106aa5 Mon Sep 17 00:00:00 2001 From: Johannes Marbach Date: Mon, 12 Oct 2026 16:11:03 +0200 Subject: [PATCH 22/25] Avoid null in transport dev setting --- src/settings/DeveloperSettingsTab.tsx | 12 ++++++++++-- .../__snapshots__/DeveloperSettingsTab.test.tsx.snap | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/settings/DeveloperSettingsTab.tsx b/src/settings/DeveloperSettingsTab.tsx index 42cc608c7..cd21320e1 100644 --- a/src/settings/DeveloperSettingsTab.tsx +++ b/src/settings/DeveloperSettingsTab.tsx @@ -70,6 +70,7 @@ import { useUrlParams } from "../UrlParams"; import { useBehavior } from "../useBehavior"; import { type ViewModel } from "../state/ViewModel.ts"; import { getSFUConfig } from "../livekit/auth/index.ts"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; /** * The state of MatrixRTC's media key rotation. @@ -174,10 +175,10 @@ export const DeveloperSettingsTab: FC = ({ customTransportSetting, ); const [customTransportTextBuffer, setCustomLivekitUrlTextBuffer] = useState( - () => JSON.stringify(customTransport), + () => transportText(customTransport), ); useEffect(() => { - setCustomLivekitUrlTextBuffer(JSON.stringify(customTransport)); + setCustomLivekitUrlTextBuffer(transportText(customTransport)); }, [customTransport]); const [muteAllAudio, setMuteAllAudio] = useSetting(muteAllAudioSetting); @@ -703,3 +704,10 @@ export const DeveloperSettingsTab: FC = ({ ); }; + +/** + * The text shown in the custom transport field for a given setting value. + */ +function transportText(transport: UnstableLivekitTransport | null): string { + return transport === null ? "" : JSON.stringify(transport); +} diff --git a/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap b/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap index 6bfc9502f..54d1ea478 100644 --- a/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap +++ b/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap @@ -250,7 +250,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = ` id="radix-_r_7_" name="input" title="" - value="null" + value="" /> Date: Mon, 12 Oct 2026 20:37:14 +0200 Subject: [PATCH 23/25] Make the auth choice mode independent again --- playwright/url-transport.spec.ts | 11 ++- src/livekit/auth/index.test.ts | 71 ++++--------------- src/livekit/auth/index.ts | 30 ++------ src/settings/DeveloperSettingsTab.tsx | 9 +-- src/state/CallViewModel/CallViewModel.ts | 2 - .../CallViewModel/CallViewModelTestUtils.ts | 1 - .../localMember/LocalMember.test.ts | 35 +++++---- .../CallViewModel/localMember/LocalMember.ts | 6 +- .../remoteMembers/Connection.test.ts | 7 -- .../CallViewModel/remoteMembers/Connection.ts | 6 -- .../remoteMembers/ConnectionFactory.ts | 4 -- .../remoteMembers/ECConnectionFactory.test.ts | 6 -- .../remoteMembers/integration.test.ts | 2 - 13 files changed, 48 insertions(+), 142 deletions(-) diff --git a/playwright/url-transport.spec.ts b/playwright/url-transport.spec.ts index 6400657fe..917d29e77 100644 --- a/playwright/url-transport.spec.ts +++ b/playwright/url-transport.spec.ts @@ -54,21 +54,20 @@ const cases: { guestFlows: ["cs-api"], }, { - // In compatibility mode both publish a membership state event, so both - // must keep the legacy participant identity that clients predating MSC4195 - // expect, which only the JWT service assigns. + // Publishers keep the legacy participant identity that clients predating + // MSC4195 expect, which only the JWT service assigns. hostShape: "both", guestShape: "both", hostFlows: ["service-legacy"], guestFlows: ["service-legacy"], }, { - // In Matrix 2.0 mode nobody expects a legacy identity, so the CS API wins. + // The MatrixRTC mode has no bearing on the choice. hostShape: "both", guestShape: "both", mode: "2_0", - hostFlows: ["cs-api"], - guestFlows: ["cs-api"], + hostFlows: ["service-legacy"], + guestFlows: ["service-legacy"], }, { hostShape: "url", diff --git a/src/livekit/auth/index.test.ts b/src/livekit/auth/index.test.ts index f16987a34..6c48c5032 100644 --- a/src/livekit/auth/index.test.ts +++ b/src/livekit/auth/index.test.ts @@ -21,7 +21,6 @@ import { delegateDelayedLeave, getSFUConfig } from "."; import { mocked } from "storybook/test"; import { ConnectionError, MatrixError } from "matrix-js-sdk"; import { NoMatrix2AuthorizationService } from "../../utils/errors"; -import { MatrixRTCMode } from "../../config/ConfigOptions"; import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; vi.mock("./openID", () => ({ getSFUConfig: vi.fn() })); @@ -84,7 +83,6 @@ describe("getSFUConfig", () => { membership, serverName: "example.org", role: "subscriber" as const, - matrixRTCMode: MatrixRTCMode.Matrix_2_0, roomId, slotId, logger, @@ -186,12 +184,24 @@ describe("getSFUConfig", () => { expect(openID.getSFUConfig).not.toHaveBeenCalled(); }); + it("uses CS API when publishing with url transport", async () => { + expect( + await getSFUConfig({ + ...defaultParams, + transport: urlTransport, + role: "publisher", + }), + ).toEqual(expect.objectContaining({ jwt: csApiJWTToken })); + expect(client._unstable_getLivekitToken).toHaveBeenCalled(); + expect(openID.getSFUConfig).not.toHaveBeenCalled(); + }); + it("does not fall back to OpenID when publishing", async () => { client._unstable_getLivekitToken.mockRejectedValueOnce(unsupportedError); await expect( getSFUConfig({ ...defaultParams, - transport: urlAndServiceUrlTransport, + transport: urlTransport, role: "publisher", }), ).rejects.toThrow(expect.any(NoMatrix2AuthorizationService)); @@ -199,7 +209,7 @@ describe("getSFUConfig", () => { expect(openID.getSFUConfig).not.toHaveBeenCalled(); }); - it("uses OpenID when publishing in compatibility mode with url + livekit_service_url transport", async () => { + it("uses OpenID when publishing with url + livekit_service_url transport", async () => { // Subscribers predating MSC4195 expect a legacy participant identity from // anyone advertising a state event, which only the JWT service assigns expect( @@ -207,7 +217,6 @@ describe("getSFUConfig", () => { ...defaultParams, transport: urlAndServiceUrlTransport, role: "publisher", - matrixRTCMode: MatrixRTCMode.Compatibility, }), ).toEqual( expect.objectContaining({ @@ -221,31 +230,6 @@ describe("getSFUConfig", () => { ); }); - it("uses CS API when publishing in compatibility mode with url transport", async () => { - expect( - await getSFUConfig({ - ...defaultParams, - transport: urlTransport, - role: "publisher", - matrixRTCMode: MatrixRTCMode.Compatibility, - }), - ).toEqual(expect.objectContaining({ jwt: csApiJWTToken })); - expect(client._unstable_getLivekitToken).toHaveBeenCalled(); - expect(openID.getSFUConfig).not.toHaveBeenCalled(); - }); - - it("uses CS API when subscribing in compatibility mode with url + livekit_service_url transport", async () => { - expect( - await getSFUConfig({ - ...defaultParams, - transport: urlAndServiceUrlTransport, - matrixRTCMode: MatrixRTCMode.Compatibility, - }), - ).toEqual(expect.objectContaining({ jwt: csApiJWTToken })); - expect(client._unstable_getLivekitToken).toHaveBeenCalled(); - expect(openID.getSFUConfig).not.toHaveBeenCalled(); - }); - it("does not fall back to OpenID on connection error", async () => { const client_ = { ...client, @@ -275,7 +259,6 @@ describe("delegateDelayedLeave", () => { roomId, slotId, delayId: "delay123", - matrixRTCMode: MatrixRTCMode.Matrix_2_0, logger, }; @@ -305,21 +288,11 @@ describe("delegateDelayedLeave", () => { expect(openID.getSFUConfig).toHaveBeenCalled(); }); - it("uses CS API with url + livekit_service_url transport", async () => { - await delegateDelayedLeave({ - ...defaultParams, - transport: urlAndServiceUrlTransport, - }); - expect(client._unstable_delegateDelayedLeave).toHaveBeenCalled(); - expect(openID.getSFUConfig).not.toHaveBeenCalled(); - }); - - it("uses OpenID in compatibility mode with url + livekit_service_url transport", async () => { + it("uses OpenID with url + livekit_service_url transport", async () => { // Must match the flow that got the token, so the identities line up await delegateDelayedLeave({ ...defaultParams, transport: urlAndServiceUrlTransport, - matrixRTCMode: MatrixRTCMode.Compatibility, }); expect(client._unstable_delegateDelayedLeave).not.toHaveBeenCalled(); expect(openID.getSFUConfig).toHaveBeenCalledWith( @@ -330,18 +303,4 @@ describe("delegateDelayedLeave", () => { }), ); }); - - it("does not fall back to OpenID with url + livekit_service_url transport", async () => { - client._unstable_delegateDelayedLeave.mockRejectedValueOnce( - unsupportedError, - ); - await expect( - delegateDelayedLeave({ - ...defaultParams, - transport: urlAndServiceUrlTransport, - }), - ).rejects.toThrow(); - expect(client._unstable_delegateDelayedLeave).toHaveBeenCalled(); - expect(openID.getSFUConfig).not.toHaveBeenCalled(); - }); }); diff --git a/src/livekit/auth/index.ts b/src/livekit/auth/index.ts index 018347cb3..292dac4d4 100644 --- a/src/livekit/auth/index.ts +++ b/src/livekit/auth/index.ts @@ -14,7 +14,6 @@ import * as openID from "./openID"; import * as csApi from "./csApi"; import { type SFUConfig } from "./types"; import { NoMatrix2AuthorizationService } from "../../utils/errors"; -import { MatrixRTCMode } from "../../config/ConfigOptions"; export type { ClientOpenIDParts } from "./openID"; export type { ClientGetTokenParts, ClientDelegationParts } from "./csApi"; @@ -80,11 +79,6 @@ export interface GetSFUConfigParams { * Whether we want to publish or only subscribe on the {@link transport}. */ role: "publisher" | "subscriber"; - /** - * The MatrixRTC mode, which decides the format of our membership event and - * so the participant identity a publisher must have. - */ - matrixRTCMode: MatrixRTCMode; logger: Logger; } @@ -99,7 +93,6 @@ export async function getSFUConfig({ roomId, slotId, role, - matrixRTCMode, logger, }: GetSFUConfigParams): Promise { const viaOpenID = async (serviceUrl: string): Promise => @@ -113,7 +106,7 @@ export async function getSFUConfig({ logger, }); - const flow = chooseAuthFlow(transport, role, matrixRTCMode); + const flow = chooseAuthFlow(transport, role); if (flow.kind === "openID") return await viaOpenID(flow.serviceUrl); try { @@ -174,10 +167,6 @@ export interface DelegateDelayedLeaveParams { * The delay ID of the leave event to be delegated. */ delayId: string; - /** - * The MatrixRTC mode, which decides the format of our membership event. - */ - matrixRTCMode: MatrixRTCMode; logger: Logger; } @@ -192,13 +181,12 @@ export async function delegateDelayedLeave({ roomId, slotId, delayId, - matrixRTCMode, logger, }: DelegateDelayedLeaveParams): Promise { // It's important that we use the same flow (CS API or OpenID) as whatever we // would've used to get a JWT token for this transport, since the participant // identity computed by the service needs to match. - const flow = chooseAuthFlow(transport, "publisher", matrixRTCMode); + const flow = chooseAuthFlow(transport, "publisher"); if (flow.kind === "csApi") { try { await csApi.delegateDelayedLeave({ @@ -246,25 +234,19 @@ export type AuthFlow = * * Only the JWT service's `/sfu/get` endpoint assigns the legacy participant * identity that clients predating MSC4195 expect from anyone advertising a - * membership state event, which is what compatibility mode sends. So a - * publisher in compatibility mode keeps using the JWT service wherever the - * transport still offers one, even if it also has a `url`. Subscribers may use - * the CS API regardless, as nobody matches their identity. + * membership state event. So a publisher keeps using the JWT service wherever + * the transport still offers one, even if it also has a `url`. Subscribers may + * use the CS API regardless, as nobody matches their identity. * @param transport The transport to authenticate against. * @param role Whether we want to publish or only subscribe on the transport. - * @param matrixRTCMode The MatrixRTC mode, which decides the format of our - * membership event. */ export function chooseAuthFlow( transport: UnstableLivekitTransport, role: "publisher" | "subscriber", - matrixRTCMode: MatrixRTCMode, ): AuthFlow { if ("url" in transport) { const useCsApi = - !("livekit_service_url" in transport) || - role === "subscriber" || - matrixRTCMode === MatrixRTCMode.Matrix_2_0; + !("livekit_service_url" in transport) || role === "subscriber"; if (useCsApi) return { kind: "csApi", url: transport.url }; } return { kind: "openID", serviceUrl: transport.livekit_service_url }; diff --git a/src/settings/DeveloperSettingsTab.tsx b/src/settings/DeveloperSettingsTab.tsx index cd21320e1..e31282e81 100644 --- a/src/settings/DeveloperSettingsTab.tsx +++ b/src/settings/DeveloperSettingsTab.tsx @@ -531,7 +531,6 @@ export const DeveloperSettingsTab: FC = ({ roomId, slotId: "org.example.fake-slot#room", role: "publisher", - matrixRTCMode: effectiveMatrixRTCMode, logger, }); setCustomLivekitUrlUpdateError(null); @@ -542,13 +541,7 @@ export const DeveloperSettingsTab: FC = ({ ); } }, - [ - customTransportTextBuffer, - setCustomTransport, - client, - roomId, - effectiveMatrixRTCMode, - ], + [customTransportTextBuffer, setCustomTransport, client, roomId], )} value={customTransportTextBuffer ?? ""} onChange={useCallback( diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 5821204a1..169abb98c 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -575,7 +575,6 @@ export function createCallViewModel$( client, matrixRoom.roomId, matrixRTCSession.slotId, - matrixRTCMode, mediaDevices, trackProcessorState$, livekitKeyProvider, @@ -666,7 +665,6 @@ export function createCallViewModel$( ).pipe(map(([delayId]) => delayId ?? null)), matrixRTCSession.delayId ?? null, ), - matrixRTCMode, logger: logger.getChild(`[${Date.now()}]`), }); diff --git a/src/state/CallViewModel/CallViewModelTestUtils.ts b/src/state/CallViewModel/CallViewModelTestUtils.ts index 6ef3e9f8d..93c190ace 100644 --- a/src/state/CallViewModel/CallViewModelTestUtils.ts +++ b/src/state/CallViewModel/CallViewModelTestUtils.ts @@ -230,7 +230,6 @@ export function withCallViewModel(mode: MatrixRTCMode) { client: room.client, roomId: room.roomId, slotId: "m.call#ROOM", - matrixRTCMode: mode, livekitRoomFactory, }, logger, diff --git a/src/state/CallViewModel/localMember/LocalMember.test.ts b/src/state/CallViewModel/localMember/LocalMember.test.ts index 273b6c8f3..f8b4e6b6b 100644 --- a/src/state/CallViewModel/localMember/LocalMember.test.ts +++ b/src/state/CallViewModel/localMember/LocalMember.test.ts @@ -346,27 +346,31 @@ describe("LocalMembership", () => { .mockResolvedValue(); afterEach(() => delegationSpy.mockClear()); - // The transport has both a `url` and a `livekit_service_url`. In Matrix 2.0 - // mode we publish through the CS API, so the homeserver delegates; in - // compatibility mode we publish through the JWT service, so that delegates. + const urlOnlyTransport: UnstableLivekitTransport = { + type: "livekit", + url: "https://sfu.example.org", + }; + + // With a `url` only we publish through the CS API, so the homeserver + // delegates. With a `livekit_service_url` we publish through the JWT + // service, so that delegates. it.each([ - ["no", MatrixRTCMode.Compatibility, null, timings], - ["no", MatrixRTCMode.Matrix_2_0, null, timings], + ["no", mockTransport, null, timings], [ "homeserver", - MatrixRTCMode.Matrix_2_0, + urlOnlyTransport, "https://matrix.example.org/_matrix/client/unstable/io.element.msc4195/rtc/livekit/delegate_delayed_leave", delegatedTimings, ], [ "transport", - MatrixRTCMode.Compatibility, + mockTransport, "https://jwt.example.org/delegate_delayed_leave", delegatedTimings, ], ])( - "joins session with %s delegation support in %s mode", - async (_serviceName, matrixRTCMode, delegationUrl, delayedLeaveTimings) => { + "joins session with %s delegation support", + async (_serviceName, transport, delegationUrl, delayedLeaveTimings) => { const scope = testScope(); const joinMatrixRTC = vi.fn(); const delayId$ = new BehaviorSubject(null); @@ -375,15 +379,17 @@ describe("LocalMembership", () => { fetchMock.post(delegationUrl, () => ({ status: 401, body: {} })); const connectionMap = new ConnectionMap(); - connectionMap.add(connectionTransportAConnected, []); + connectionMap.add( + { ...connectionTransportAConnected, transport } as Connection, + [], + ); const localMembership = createLocalMembership$({ scope, ...defaultCreateLocalMemberValues, connectionMap$: constant(new Epoch(connectionMap)), joinMatrixRTC, - localTransport$: constant(mockTransportLocator), + localTransport$: constant({ transport, serverName: "example.org" }), delayId$, - matrixRTCMode, }); localMembership.requestJoinAndPublish(); @@ -391,7 +397,7 @@ describe("LocalMembership", () => { await flushPromises(); // Joins with timings appropriate for the level of delegation support expect(joinMatrixRTC).toHaveBeenCalledWith( - mockTransport, + transport, delayedLeaveTimings, ); @@ -407,10 +413,9 @@ describe("LocalMembership", () => { expect(delegationSpy).toHaveBeenLastCalledWith({ client: mockedClient, membership: ownMemberMock, - transport: mockTransport, + transport, roomId: "!test-room-id:example.org", delayId, - matrixRTCMode, logger: expect.anything(), }); diff --git a/src/state/CallViewModel/localMember/LocalMember.ts b/src/state/CallViewModel/localMember/LocalMember.ts index b4af36b58..baa6b473d 100644 --- a/src/state/CallViewModel/localMember/LocalMember.ts +++ b/src/state/CallViewModel/localMember/LocalMember.ts @@ -165,8 +165,6 @@ interface Props { hostBridge: HostBridge; baseUrl: string; delayId$: Behavior; - /** The MatrixRTC mode, which decides the format of our membership event. */ - matrixRTCMode: MatrixRTCMode; logger: Logger; } @@ -214,7 +212,6 @@ export const createLocalMembership$ = ({ hostBridge, ownMembershipIdentity, delayId$, - matrixRTCMode, }: Props): { /** * This request to start audio and video tracks. @@ -318,7 +315,7 @@ export const createLocalMembership$ = ({ switchMap(async ({ transport }) => { // Delegation goes through whichever service authenticates us as a // publisher, so only that service's support counts - const flow = chooseAuthFlow(transport, "publisher", matrixRTCMode); + const flow = chooseAuthFlow(transport, "publisher"); const delegationSupported = flow.kind === "openID" ? await checkDelegationSupport( @@ -735,7 +732,6 @@ export const createLocalMembership$ = ({ slotId: matrixRTCSession.slotId, roomId, delayId, - matrixRTCMode, logger, }); logger.info("Delayed leave successfully delegated"); diff --git a/src/state/CallViewModel/remoteMembers/Connection.test.ts b/src/state/CallViewModel/remoteMembers/Connection.test.ts index 632122a3f..bdab8e7af 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.test.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.test.ts @@ -42,7 +42,6 @@ import { } from "./Connection.ts"; import { ObservableScope } from "../../ObservableScope.ts"; import { FailToGetOpenIdToken } from "../../../utils/errors.ts"; -import { MatrixRTCMode } from "../../../config/ConfigOptions.ts"; import { testJWTToken } from "../../../utils/test-fixtures.ts"; import { mockRemoteParticipant, ownMemberMock } from "../../../utils/test.ts"; import { @@ -124,7 +123,6 @@ function setupRemoteConnection(): Connection { client, roomId: ROOM_ID, slotId: SLOT_ID, - matrixRTCMode: MatrixRTCMode.Matrix_2_0, role: "subscriber", transport, serverName: "m.localhost", @@ -174,7 +172,6 @@ describe("Start connection states", () => { client, roomId: ROOM_ID, slotId: SLOT_ID, - matrixRTCMode: MatrixRTCMode.Matrix_2_0, role: "subscriber", transport, serverName: "m.localhost", @@ -195,7 +192,6 @@ describe("Start connection states", () => { client, roomId: ROOM_ID, slotId: SLOT_ID, - matrixRTCMode: MatrixRTCMode.Matrix_2_0, role: "subscriber", transport, serverName: "m.localhost", @@ -251,7 +247,6 @@ describe("Start connection states", () => { client, roomId: ROOM_ID, slotId: SLOT_ID, - matrixRTCMode: MatrixRTCMode.Matrix_2_0, role: "subscriber", transport, serverName: "m.localhost", @@ -316,7 +311,6 @@ describe("Start connection states", () => { client, roomId: ROOM_ID, slotId: SLOT_ID, - matrixRTCMode: MatrixRTCMode.Matrix_2_0, role: "subscriber", transport, serverName: "m.localhost", @@ -567,7 +561,6 @@ describe("remote track logging", () => { client, roomId: ROOM_ID, slotId: SLOT_ID, - matrixRTCMode: MatrixRTCMode.Matrix_2_0, role: "subscriber", transport, serverName: "m.localhost", diff --git a/src/state/CallViewModel/remoteMembers/Connection.ts b/src/state/CallViewModel/remoteMembers/Connection.ts index cbdffcbde..7e94b39f1 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.ts @@ -44,15 +44,12 @@ import { type ClientOpenIDParts, type SFUConfig, } from "../../../livekit/auth"; -import { type MatrixRTCMode } from "../../../config/ConfigOptions.ts"; export interface ConnectionOpts { /** The identity parts to use on this connection */ ownMembershipIdentity: CallMembershipIdentityParts; /** Whether we want to publish or only subscribe on this connection. */ role: "publisher" | "subscriber"; - /** The MatrixRTC mode, which decides the format of our membership event. */ - matrixRTCMode: MatrixRTCMode; /** The media transport to connect to. */ transport: UnstableLivekitTransport; /** The name of the homeserver to which the {@link transport} belongs. */ @@ -141,7 +138,6 @@ export class Connection { private readonly client: ClientGetTokenParts & ClientOpenIDParts; private readonly roomId: string; private readonly slotId: string; - private readonly matrixRTCMode: MatrixRTCMode; private readonly logger: Logger; private readonly ownMembershipIdentity: CallMembershipIdentityParts; /** @@ -155,7 +151,6 @@ export class Connection { this.ownMembershipIdentity = opts.ownMembershipIdentity; this.roomId = opts.roomId; this.slotId = opts.slotId; - this.matrixRTCMode = opts.matrixRTCMode; this.logger = logger.getChild( `[Connection ${JSON.stringify(opts.transport)}]`, ); @@ -399,7 +394,6 @@ export class Connection { serverName: this.serverName, roomId: this.roomId, slotId: this.slotId, - matrixRTCMode: this.matrixRTCMode, logger: this.logger, }); } diff --git a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts index 39ac35706..e5240f096 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts @@ -39,7 +39,6 @@ import { type ClientGetTokenParts, type ClientOpenIDParts, } from "../../../livekit/auth"; -import { type MatrixRTCMode } from "../../../config/ConfigOptions.ts"; // TODO evaluate if this should be done like the Publisher Factory export interface ConnectionFactory { @@ -62,7 +61,6 @@ export class ECConnectionFactory implements ConnectionFactory { * @param client - The OpenID client parts for authentication, needed to get openID and JWT tokens. * @param roomId - The ID of the Matrix room in which the session takes place. * @param slotId - The ID of the MatrixRTC slot in which the session takes place. - * @param matrixRTCMode - The MatrixRTC mode, which decides the format of our membership event. * @param devices - Used for video/audio out/in capture options. * @param processorState$ - Effects like background blur (only for publishing connection?) * @param livekitKeyProvider - Optional key provider for end-to-end encryption. @@ -73,7 +71,6 @@ export class ECConnectionFactory implements ConnectionFactory { private client: ClientGetTokenParts & ClientOpenIDParts, private readonly roomId: string, private readonly slotId: string, - private readonly matrixRTCMode: MatrixRTCMode, private devices: MediaDevices, private processorState$: Behavior, livekitKeyProvider: BaseKeyProvider | undefined, @@ -120,7 +117,6 @@ export class ECConnectionFactory implements ConnectionFactory { { roomId: this.roomId, slotId: this.slotId, - matrixRTCMode: this.matrixRTCMode, role, transport, serverName, diff --git a/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts b/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts index 0a9310f82..be6c149f4 100644 --- a/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts +++ b/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts @@ -36,7 +36,6 @@ import { cameraCodec, } from "../../../settings/settings.ts"; import { testJWTToken } from "../../../utils/test-fixtures.ts"; -import { MatrixRTCMode } from "../../../config/ConfigOptions.ts"; // At the top of your test file, after imports vi.mock("livekit-client", async (importOriginal) => { @@ -85,7 +84,6 @@ describe("ECConnectionFactory - Audio inputs options", () => { mockClient, "!roomid:example.org", "m.call#ROOM", - MatrixRTCMode.Matrix_2_0, mockMediaDevices({}), new BehaviorSubject({ supported: true, @@ -131,7 +129,6 @@ describe("ECConnectionFactory - ControlledAudioDevice", () => { mockClient, "!roomid:example.org", "m.call#ROOM", - MatrixRTCMode.Matrix_2_0, mockMediaDevices({ audioOutput: { available$: constant(new Map()), @@ -176,7 +173,6 @@ describe("ECConnectionFactory - Camera quality settings", () => { mockClient, "!roomid:example.org", "m.call#ROOM", - MatrixRTCMode.Matrix_2_0, mockMediaDevices({}), new BehaviorSubject({ supported: true, @@ -217,7 +213,6 @@ describe("ECConnectionFactory - Camera quality settings", () => { mockClient, "!roomid:example.org", "m.call#ROOM", - MatrixRTCMode.Matrix_2_0, mockMediaDevices({}), new BehaviorSubject({ supported: true, @@ -259,7 +254,6 @@ describe("ECConnectionFactory - Camera quality settings", () => { mockClient, "!roomid:example.org", "m.call#ROOM", - MatrixRTCMode.Matrix_2_0, mockMediaDevices({}), new BehaviorSubject({ supported: true, diff --git a/src/state/CallViewModel/remoteMembers/integration.test.ts b/src/state/CallViewModel/remoteMembers/integration.test.ts index b610447d9..bf90e9eb4 100644 --- a/src/state/CallViewModel/remoteMembers/integration.test.ts +++ b/src/state/CallViewModel/remoteMembers/integration.test.ts @@ -38,7 +38,6 @@ import { createConnectionMap$ } from "./ConnectionMap.ts"; import { membershipsAndTransports$ } from "../../SessionBehaviors.ts"; import { localRtcMember, testJWTToken } from "../../../utils/test-fixtures.ts"; import { deepCompare } from "matrix-js-sdk/lib/utils"; -import { MatrixRTCMode } from "../../../config/ConfigOptions.ts"; // Test the integration of ConnectionMap and MatrixLivekitMembers @@ -76,7 +75,6 @@ beforeEach(() => { mockClient, "!roomid:example.org", "m.call#ROOM", - MatrixRTCMode.Matrix_2_0, mockMediaDevices({}), new BehaviorSubject({ supported: true, From 8d1ed31b0bb6c4aeb2b6fd53a38011d8e05a85ae Mon Sep 17 00:00:00 2001 From: Johannes Marbach Date: Mon, 12 Oct 2026 21:09:15 +0200 Subject: [PATCH 24/25] Cover identity matching and delegation across transport shapes --- playwright/url-transport.spec.ts | 180 ++++++++++++------ .../MatrixLivekitMembers.test.ts | 84 ++++++++ src/utils/test.ts | 4 +- 3 files changed, 212 insertions(+), 56 deletions(-) diff --git a/playwright/url-transport.spec.ts b/playwright/url-transport.spec.ts index 917d29e77..77fe00827 100644 --- a/playwright/url-transport.spec.ts +++ b/playwright/url-transport.spec.ts @@ -116,6 +116,12 @@ for (const { hostShape, guestShape, mode, hostFlows, guestFlows } of cases) { for (const request of [...hostRequests, ...guestRequests]) { expect(request.status).toBe(200); if (request.flow === "cs-api") expect(request.sfuUrl).toBe(sfuUrl); + // Clients predating MSC4195 match a publisher by the legacy identity, + // which only the JWT service's legacy endpoint assigns. The other flows + // hash it. + expect(request.identity).toMatch( + request.flow === "service-legacy" ? legacyIdentity : hashedIdentity, + ); } }); } @@ -185,52 +191,74 @@ test("Publisher shows an error when the homeserver lacks MSC4195", async ({ ).toBeVisible(); }); -test("Delayed leave event delegated to a url transport gets sent on disconnect", async ({ - browser, - page, - browserName, -}) => { - skipOnFirefox(browserName); +/** + * Delegation follows the flow that authenticated the publisher: the homeserver + * for a `url`-only transport, the JWT service wherever there is one. + */ +const delegationCases: { shape: Shape; via: Delegation["via"] }[] = [ + { shape: "url", via: "homeserver" }, + { shape: "legacy", via: "service" }, + { shape: "both", via: "service" }, +]; - // The host starts the call. - await advertiseTransport(page, shapes.url); - await page.goto("/"); - await SpaHelpers.createCall(page, "Androl", "HelloCall", true); - const inviteLink = await SpaHelpers.getCallInviteLink(page); +for (const { shape, via } of delegationCases) { + test(`Delayed leave event delegated through the ${via} for a ${shape} transport gets sent on disconnect`, async ({ + browser, + page, + browserName, + }) => { + skipOnFirefox(browserName); - // The guest joins. - const guestContext = await browser.newContext({ reducedMotion: "reduce" }); - const guestPage = await guestContext.newPage(); - await advertiseTransport(guestPage, shapes.url); - const delayedLeave = recordDelayedLeave(guestPage); - const delegations = recordDelegations(guestPage); - await SpaHelpers.joinCallFromInviteLink(guestPage, inviteLink, "Pevara"); + // The host starts the call. + await advertiseTransport(page, shapes[shape]); + await page.goto("/"); + await SpaHelpers.createCall(page, "Androl", "HelloCall", true); + const inviteLink = await SpaHelpers.getCallInviteLink(page); - // Wait for the call to connect and render. - await SpaHelpers.expectVideoTilesCount(page, 2); - await SpaHelpers.expectVideoTilesCount(guestPage, 2); + // The guest joins. + const guestContext = await browser.newContext({ reducedMotion: "reduce" }); + const guestPage = await guestContext.newPage(); + await advertiseTransport(guestPage, shapes[shape]); + const delayedLeave = recordDelayedLeave(guestPage); + const delegations = recordDelegations(guestPage); + await SpaHelpers.joinCallFromInviteLink(guestPage, inviteLink, "Pevara"); - // The guest handed its delayed leave to the homeserver, after probing it for - // delegation support with bodiless requests. - await expect - .poll(() => delegations.at(-1)) - .toEqual({ status: 200, sfuUrl, delayId: delayedLeave.delayId }); - const probes = delegations.slice(0, -1); - expect(probes.length).toBeGreaterThan(0); - for (const probe of probes) expect(probe).toEqual({ status: 401 }); + // Wait for the call to connect and render. + await SpaHelpers.expectVideoTilesCount(page, 2); + await SpaHelpers.expectVideoTilesCount(guestPage, 2); - // The delay is long enough that a leave within the window below cannot come - // from the delayed event expiring on its own. - expect(delayedLeave.delayMs).toBeGreaterThan(60_000); + // The guest handed its delayed leave over, after probing for delegation + // support with bodiless requests, which count as supported unless they + // 404. The homeserver is always probed; the JWT service only when there + // is one. + await expect + .poll(() => delegations.at(-1)) + .toEqual({ + via, + status: 200, + delayId: delayedLeave.delayId, + ...(via === "homeserver" && { sfuUrl }), + }); + const probes = delegations.slice(0, -1); + expect(probes.some((probe) => probe.via === via)).toBe(true); + for (const probe of probes) { + expect(probe.status).toBeGreaterThanOrEqual(400); + expect(probe.status).not.toBe(404); + } - // The guest drops without leaving. - await guestPage.reload(); + // The delay is long enough that a leave within the window below cannot + // come from the delayed event expiring on its own. + expect(delayedLeave.delayMs).toBeGreaterThan(60_000); - // The host eventually sees the guest go once the delayed event fires. - await expect(page.getByTestId("videoTile")).toHaveCount(1, { - timeout: 30_000, + // The guest drops without leaving. + await guestPage.reload(); + + // The host eventually sees the guest go once the delayed event fires. + await expect(page.getByTestId("videoTile")).toHaveCount(1, { + timeout: 30_000, + }); }); -}); +} test("Subscribers get tokens for a remote homeserver's SFU over federation", async ({ browser, @@ -365,29 +393,43 @@ function recordDelayedLeave(page: Page): DelayedLeave { /** A recorded delegation request for a delayed leave event. */ interface Delegation { + /** Who was asked to send the leave event on the guest's behalf. */ + via: "homeserver" | "service"; status: number; sfuUrl?: string; delayId?: string; } -/** Records requests to the MSC4195 `delegate_delayed_leave` endpoint. */ +/** + * Records requests that delegate a delayed leave event, or probe for support: + * the MSC4195 `delegate_delayed_leave` endpoint of the homeserver, and the JWT + * service's `delegate_delayed_leave` probe and `/sfu/get` with delay fields. + */ function recordDelegations(page: Page): Delegation[] { const delegations: Delegation[] = []; page.on("response", (response) => { const request = response.request(); - if ( - request.method() !== "POST" || - !request.url().endsWith("/rtc/livekit/delegate_delayed_leave") - ) - return; + if (request.method() !== "POST") return; + const url = request.url(); const body = request.postDataJSON() as { - url: string; - delay_id: string; + url?: string; + delay_id?: string; } | null; - delegations.push({ - status: response.status(), - ...(body && { sfuUrl: body.url, delayId: body.delay_id }), - }); + if (url.endsWith("/rtc/livekit/delegate_delayed_leave")) { + delegations.push({ + via: "homeserver", + status: response.status(), + ...(body && { sfuUrl: body.url, delayId: body.delay_id }), + }); + } else if (url.endsWith("/livekit/jwt/delegate_delayed_leave")) { + delegations.push({ via: "service", status: response.status() }); + } else if (url.endsWith("/livekit/jwt/sfu/get") && body?.delay_id) { + delegations.push({ + via: "service", + status: response.status(), + delayId: body.delay_id, + }); + } }); return delegations; } @@ -398,26 +440,54 @@ interface AuthRequest { status: number; /** The SFU a request to the MSC4195 `get_token` endpoint asked for. */ sfuUrl?: string; + /** The participant identity the issued token carries. */ + identity?: string; } -/** Records token requests. */ +/** The legacy `:` participant identity. */ +const legacyIdentity = /^@[^:]+:[^:]+:[A-Z]+$/; +/** A hashed participant identity: unpadded base64 of a SHA-256. */ +const hashedIdentity = /^[A-Za-z0-9+/]{43}$/; + +/** Reads the subject of a JWT without verifying it. */ +function jwtSubject(jwt: string): string { + const [, payload] = jwt.split("."); + return ( + JSON.parse(Buffer.from(payload, "base64url").toString()) as { + sub: string; + } + ).sub; +} + +/** Records token requests, including the identity each token carries. */ function recordAuthRequests(page: Page): AuthRequest[] { const requests: AuthRequest[] = []; page.on("response", (response) => { const request = response.request(); if (request.method() !== "POST") return; const url = request.url(); + let entry: AuthRequest; if (url.endsWith("/rtc/livekit/get_token")) { - requests.push({ + entry = { flow: "cs-api", status: response.status(), sfuUrl: (request.postDataJSON() as { url: string }).url, - }); + }; } else if (url.endsWith("/livekit/jwt/sfu/get")) { - requests.push({ flow: "service-legacy", status: response.status() }); + entry = { flow: "service-legacy", status: response.status() }; } else if (url.endsWith("/livekit/jwt/get_token")) { - requests.push({ flow: "service-default", status: response.status() }); + entry = { flow: "service-default", status: response.status() }; + } else { + return; } + requests.push(entry); + if (entry.status !== 200) return; + response + .json() + .then((body: { jwt: string }) => { + entry.identity = jwtSubject(body.jwt); + }) + .catch(() => {}); }); return requests; } diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts index 85f21b478..1c6a32790 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts @@ -170,6 +170,90 @@ test("should signal participant on a connection that is publishing", async () => ); }); +// A member advertising both a `url` and a `livekit_service_url` may have +// authenticated through either flow, so it has two candidate identities. +const bothTransport: UnstableLivekitTransport = { + type: "livekit", + url: "wss://sfu.example.org", + livekit_service_url: "https://lk.example.org", +}; +const bobLegacyId = fallbackMemberId("@bob:example.org", "DEV000"); +const bobHashedId = "2qkXfWz1S7jQmU0bJ6yN8aVtR3pLcDgHe9oZxKwMiA4"; +const bobWithBothIdentities = mockRtcMembership("@bob:example.org", "DEV000", { + fociPreferred: [bothTransport], + backendIdentities: [bobHashedId, bobLegacyId], +}); + +test.each([ + ["legacy", bobLegacyId], + ["hashed", bobHashedId], +])( + "matches a member with two candidate identities publishing under the %s one", + async (_name, identity) => { + const { memberships$, membershipsWithTransport$ } = + createEpochedMemberships$(constant([bobWithBothIdentities])); + + const connection = { + transport: bothTransport, + serverName: "example.org", + } as unknown as Connection; + const dataWithPublisher = new ConnectionMap(); + dataWithPublisher.add(connection, [mockRemoteParticipant({ identity })]); + + const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({ + scope: testScope, + membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), + connectionMap$: testScope.behavior( + epochMeWith$(memberships$, constant(dataWithPublisher)), + ), + localUser: localRtcMember, + }); + + await flushPromises(); + expect(remoteMatrixLivekitMembers$.value.value).toSatisfy( + (data: RemoteMatrixLivekitMember[]) => { + expect(data.length).toEqual(1); + expect(data[0].participant.value$.value?.identity).toEqual(identity); + expect(data[0].connection$.value).toBe(connection); + return true; + }, + ); + }, +); + +test("does not match a member with two candidate identities to anyone else", async () => { + const { memberships$, membershipsWithTransport$ } = createEpochedMemberships$( + constant([bobWithBothIdentities]), + ); + + const connection = { + transport: bothTransport, + serverName: "example.org", + } as unknown as Connection; + const dataWithStranger = new ConnectionMap(); + dataWithStranger.add(connection, [ + mockRemoteParticipant({ identity: "@stranger:example.org:DEV999" }), + ]); + + const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({ + scope: testScope, + membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), + connectionMap$: testScope.behavior( + epochMeWith$(memberships$, constant(dataWithStranger)), + ), + localUser: localRtcMember, + }); + + await flushPromises(); + expect(remoteMatrixLivekitMembers$.value.value).toSatisfy( + (data: RemoteMatrixLivekitMember[]) => { + expect(data.length).toEqual(1); + expect(data[0].participant.value$.value).toBe(null); + return true; + }, + ); +}); + test("should signal participant on a connection that is not publishing", async () => { const { memberships$, membershipsWithTransport$ } = createEpochedMemberships$( constant([bobMembership]), diff --git a/src/utils/test.ts b/src/utils/test.ts index e40e56fca..5ab16f2e5 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -224,6 +224,8 @@ export function mockRtcMembership( deviceId: string, customOverwrites?: { rtcBackendIdentity?: string; + /** Every identity the member may have in LiveKit, when more than one. */ + backendIdentities?: string[]; callId?: string; fociPreferred?: Transport[]; focusActive?: LivekitFocusSelection; @@ -263,7 +265,7 @@ export function mockRtcMembership( const cms = new CallMembership( event, CallMembership.membershipDataFromMatrixEvent(event), - [backendIdentity], + customOverwrites?.backendIdentities ?? [backendIdentity], backendIdentity, ); vi.mocked(cms).getTransport = vi.fn().mockReturnValue(fociPreferred[0]); From ee7dcdee7e7b47cbaf6a655d9bd2fe1678c21694 Mon Sep 17 00:00:00 2001 From: Johannes Marbach Date: Mon, 12 Oct 2026 21:28:04 +0200 Subject: [PATCH 25/25] Cover the subscriber token path, widget fallback and server names --- playwright/widget/url-transport.test.ts | 108 ++++++++++++++++++++++++ src/livekit/auth/openID.test.ts | 30 +++++++ src/state/SessionBehaviors.test.ts | 67 ++++++++++++++- 3 files changed, 201 insertions(+), 4 deletions(-) create mode 100644 playwright/widget/url-transport.test.ts diff --git a/playwright/widget/url-transport.test.ts b/playwright/widget/url-transport.test.ts new file mode 100644 index 000000000..c4e39cfdd --- /dev/null +++ b/playwright/widget/url-transport.test.ts @@ -0,0 +1,108 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { expect, type Page, test } from "@playwright/test"; + +import { widgetTest } from "../fixtures/widget-user.ts"; +import { HOST1, TestHelpers } from "./test-helpers.ts"; + +const sfuUrl = "wss://matrix-rtc.m.localhost/livekit/sfu"; +const serviceUrl = "https://matrix-rtc.m.localhost/livekit/jwt"; + +/** Which endpoint a page asks for its SFU tokens. */ +type Flow = "cs-api" | "service-legacy" | "service-default"; + +widgetTest( + "Call between a url + livekit_service_url transport and a legacy one", + async ({ addUser, browserName }) => { + test.skip( + browserName === "firefox", + "The is test is not working on firefox CI environment. No mic/audio device inputs so cam/mic are disabled", + ); + test.slow(); + + const [host, guest] = await Promise.all([ + addUser("Androl", HOST1), + addUser("Pevara", HOST1), + ]); + + // The host's homeserver has started advertising a `url` next to its JWT + // service; the guest's still advertises the JWT service alone. + await advertiseTransport(host.page, { + type: "livekit", + url: sfuUrl, + livekit_service_url: serviceUrl, + }); + // Element Web answers the widget from a cache it filled at login, so make + // it fetch the transports again now that they are rerouted + await host.clientHandle.evaluate(async (client) => { + await client.cachedRtcTransports.refresh(); + }); + const hostFlows = recordFlows(host.page); + const guestFlows = recordFlows(guest.page); + + const roomName = "Transport Room"; + await TestHelpers.createRoom(roomName, host.page, [guest.mxId]); + await TestHelpers.acceptRoomInvite(roomName, guest.page); + + await TestHelpers.startCallInCurrentRoom(host.page, false); + await TestHelpers.joinCallFromLobby(host.page); + await TestHelpers.joinCallInCurrentRoom(guest.page); + + for (const user of [host, guest]) { + const frame = user.page + .locator('iframe[title="Element Call"]') + .contentFrame(); + await expect(frame.getByTestId("videoTile")).toHaveCount(2, { + timeout: 10000, + }); + await expect(frame.getByText("Waiting for media...")).not.toBeVisible({ + timeout: 10000, + }); + await TestHelpers.expectVisibleVideoCount(frame, 2); + } + + // Element Web does not expose the MSC4195 endpoints to widgets yet, so + // neither side reached the CS API. The host published through the JWT + // service's legacy endpoint and subscribed to the guest through its + // default one; the guest published the same way and fell back to the + // default endpoint for the host's transport. + expect(hostFlows).toEqual(new Set(["service-legacy", "service-default"])); + expect(guestFlows).toEqual(new Set(["service-legacy", "service-default"])); + }, +); + +/** + * Replaces the transports the homeserver advertises to this page, including + * to the widget host, which fetches them on the widget's behalf. The real + * response is fetched first so its CORS headers carry over. + */ +async function advertiseTransport( + page: Page, + transport: { type: "livekit"; url?: string; livekit_service_url?: string }, +): Promise { + await page.route("**/org.matrix.msc4143/rtc/transports", async (route) => { + if (route.request().method() !== "GET") return route.continue(); + const response = await route.fetch(); + await route.fulfill({ response, json: { rtc_transports: [transport] } }); + }); +} + +/** Records which token endpoints the page's widget asks. */ +function recordFlows(page: Page): Set { + const flows = new Set(); + page.on("response", (response) => { + const request = response.request(); + if (request.method() !== "POST") return; + const url = request.url(); + if (url.endsWith("/rtc/livekit/get_token")) flows.add("cs-api"); + else if (url.endsWith("/livekit/jwt/sfu/get")) flows.add("service-legacy"); + else if (url.endsWith("/livekit/jwt/get_token")) + flows.add("service-default"); + }); + return flows; +} diff --git a/src/livekit/auth/openID.test.ts b/src/livekit/auth/openID.test.ts index a722dcd0a..7498b8fe1 100644 --- a/src/livekit/auth/openID.test.ts +++ b/src/livekit/auth/openID.test.ts @@ -169,6 +169,36 @@ describe("getSFUConfig", () => { }); }); + it("should use the default endpoint when subscribing", async () => { + fetchMock.post("https://sfu.example.org/get_token", (url, opts) => { + const body = JSON.parse(opts.body as string); + expect(body).toMatchObject({ + room_id: "!example_room_id", + slot_id: "m.call#ROOM", + member: { + id: ownMemberMock.memberId, + claimed_user_id: ownMemberMock.userId, + claimed_device_id: ownMemberMock.deviceId, + }, + }); + return { status: 200, body: { url: serviceUrl, jwt: testJWTToken } }; + }); + const config = await getSFUConfig({ ...params, role: "subscriber" }); + expect(config.jwt).toBe(testJWTToken); + expect(fetchMock.called("https://sfu.example.org/sfu/get")).toBe(false); + }); + + it("should fail when subscribing if the default endpoint is missing", async () => { + // Nothing but the catch-all 404 answers the default endpoint. Subscribers + // must not fall back to the legacy endpoint, which would hand them a + // legacy identity. + await expect( + getSFUConfig({ ...params, role: "subscriber" }), + ).rejects.toThrow(FailToGetOpenIdToken); + expect(fetchMock.called("https://sfu.example.org/get_token")).toBe(true); + expect(fetchMock.called("https://sfu.example.org/sfu/get")).toBe(false); + }); + it("should retry fetching the openid token", async () => { let count = 0; matrixClient.getOpenIdToken.mockImplementation(async () => { diff --git a/src/state/SessionBehaviors.test.ts b/src/state/SessionBehaviors.test.ts index fca24767c..d9da6f830 100644 --- a/src/state/SessionBehaviors.test.ts +++ b/src/state/SessionBehaviors.test.ts @@ -1,16 +1,25 @@ /* -Copyright 2025 Element Creations Ltd. +Copyright 2025, 2026 Element Creations Ltd. SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ import { describe, expect, it, vi } from "vitest"; -import { MatrixRTCSessionEvent } from "matrix-js-sdk/lib/matrixrtc"; +import { + MatrixRTCSessionEvent, + type Transport, + type UnstableLivekitTransport, +} from "matrix-js-sdk/lib/matrixrtc"; import { EventEmitter } from "events"; -import { createKeyRotationSuppressed$ } from "./SessionBehaviors"; -import { testScope } from "../utils/test"; +import { + createKeyRotationSuppressed$, + membershipsAndTransports$, +} from "./SessionBehaviors"; +import { Epoch } from "./ObservableScope"; +import { constant } from "./Behavior"; +import { mockRtcMembership, testScope } from "../utils/test"; describe("SessionBehaviors", () => { describe("createKeyRotationSuppressed$", () => { @@ -38,4 +47,54 @@ describe("SessionBehaviors", () => { expect(keyRotationSuppressed$.value).toBe(false); }); }); + + describe("membershipsAndTransports$", () => { + const serviceTransport: UnstableLivekitTransport = { + type: "livekit", + livekit_service_url: "https://lk.example.org", + }; + const urlTransport: UnstableLivekitTransport = { + type: "livekit", + url: "wss://sfu.sample.com", + }; + + it("locates each member's transport on the member's homeserver", () => { + const bob = mockRtcMembership("@bob:example.org", "DEV000", { + fociPreferred: [serviceTransport], + }); + const carl = mockRtcMembership("@carl:sample.com", "DEV111", { + fociPreferred: [urlTransport], + }); + + const { membershipsWithTransport$, transports$ } = + membershipsAndTransports$( + testScope(), + constant(new Epoch([bob, carl])), + ); + + expect(transports$.value.value).toEqual([ + { transport: serviceTransport, serverName: "example.org" }, + { transport: urlTransport, serverName: "sample.com" }, + ]); + expect( + membershipsWithTransport$.value.value.map( + (m) => m.transport?.serverName, + ), + ).toEqual(["example.org", "sample.com"]); + }); + + it("leaves out members without a LiveKit transport", () => { + const dave = mockRtcMembership("@dave:example.org", "DEV222", { + fociPreferred: [{ type: "something-else" } as Transport], + }); + + const { membershipsWithTransport$, transports$ } = + membershipsAndTransports$(testScope(), constant(new Epoch([dave]))); + + expect(transports$.value.value).toEqual([]); + expect(membershipsWithTransport$.value.value).toEqual([ + { membership: dave, transport: undefined }, + ]); + }); + }); });