From b4bd1f81a02a733a08541069e74910a4836b6415 Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 13 Oct 2026 12:59:13 +0200 Subject: [PATCH] [SDK] Move SFU authentication into livekit/auth and choose the endpoint by role The OpenID flow moves to src/livekit/auth/openID.ts with the SFU config types in a sibling file, takes an options object, and replaces the MatrixRTC mode with the role we take on the transport plus the session's slot ID instead of a hard-coded m.call#ROOM. This changes behaviour: publishers on a livekit_service_url transport always use the legacy /sfu/get endpoint, so that subscribers can predict their user:device participant identity, and subscribers always use the default /get_token endpoint. The MatrixRTC mode no longer influences which endpoint is used. On the SDK this lands as sdk/src/media-backend/livekit/auth/ with the connection map, the discriminated connection state and the client-server API flow of the following commits of element-hq/element-call#4331, which rewrite the same files: - Model ConnectionState as a discriminated union - Show the whole transport in debug UI and logs (SDK part) - Let the publishing Connection authenticate itself - Reduce ConnectionManager to a ConnectionMap - Use the unstable LiveKit transport type - Identify transports by server name as well as URLs - Get SFU tokens and delegate delayed leaves through the MSC4195 CS API - Fall back to get_token as subscriber on widget API errors - Share a connection between homeservers using the same JWT service - Make the auth choice mode independent again Co-authored-by: Johannes Marbach (cherry picked from commit adb380cd9ed108856e42d992fc03ced8c44f1e56) --- sdk/index.ts | 8 +- sdk/src/RTCParticipation.ts | 11 +- sdk/src/config.ts | 9 +- sdk/src/matrixrtc/Transports.ts | 14 +- sdk/src/media-backend/api.ts | 2 + .../media-backend/livekit/Connection.test.ts | 257 +++++------ sdk/src/media-backend/livekit/Connection.ts | 174 ++++---- .../livekit/ConnectionFactory.ts | 51 ++- .../livekit/ConnectionManager.ts | 297 ------------- ...nManager.test.ts => ConnectionMap.test.ts} | 257 ++++++++--- .../media-backend/livekit/ConnectionMap.ts | 194 +++++++++ .../media-backend/livekit/DataChannel.test.ts | 33 +- sdk/src/media-backend/livekit/DataChannel.ts | 23 +- .../livekit/LivekitBackend.test.ts | 397 +++++++++++++++-- .../media-backend/livekit/LivekitBackend.ts | 298 +++++++------ sdk/src/media-backend/livekit/LocalMedia.ts | 36 +- sdk/src/media-backend/livekit/auth/csApi.ts | 102 +++++ .../media-backend/livekit/auth/index.test.ts | 407 ++++++++++++++++++ sdk/src/media-backend/livekit/auth/index.ts | 212 +++++++++ .../media-backend/livekit/auth/openID.test.ts | 229 ++++++++++ sdk/src/media-backend/livekit/auth/openID.ts | 203 +++++++++ sdk/src/media-backend/livekit/auth/types.ts | 58 +++ .../media-backend/livekit/connectionState.ts | 36 +- .../media-backend/livekit/openIDSFU.test.ts | 332 -------------- sdk/src/media-backend/livekit/openIDSFU.ts | 321 -------------- sdk/src/media-backend/livekit/test.ts | 8 +- sdk/src/utils/network.ts | 8 +- sdk/src/utils/serverName.ts | 11 + sdk/src/utils/test-fixtures.ts | 32 +- 29 files changed, 2495 insertions(+), 1525 deletions(-) delete mode 100644 sdk/src/media-backend/livekit/ConnectionManager.ts rename sdk/src/media-backend/livekit/{ConnectionManager.test.ts => ConnectionMap.test.ts} (58%) create mode 100644 sdk/src/media-backend/livekit/ConnectionMap.ts create mode 100644 sdk/src/media-backend/livekit/auth/csApi.ts create mode 100644 sdk/src/media-backend/livekit/auth/index.test.ts create mode 100644 sdk/src/media-backend/livekit/auth/index.ts create mode 100644 sdk/src/media-backend/livekit/auth/openID.test.ts create mode 100644 sdk/src/media-backend/livekit/auth/openID.ts create mode 100644 sdk/src/media-backend/livekit/auth/types.ts delete mode 100644 sdk/src/media-backend/livekit/openIDSFU.test.ts delete mode 100644 sdk/src/media-backend/livekit/openIDSFU.ts create mode 100644 sdk/src/utils/serverName.ts diff --git a/sdk/index.ts b/sdk/index.ts index e6e417cf1..471d3862f 100644 --- a/sdk/index.ts +++ b/sdk/index.ts @@ -63,8 +63,8 @@ export { } from "./src/media-backend/livekit/LivekitBackend"; // For a host that creates LiveKit tracks itself, such as a preview before joining export { convertToLivekitProcessor } from "./src/media-backend/livekit/videoProcessor"; -// For a developer panel to check a transport url before using it +// For a developer panel to check a transport before using it export { - getSFUConfigWithOpenID as authenticateWithTransport, - type TokenEndpoint, -} from "./src/media-backend/livekit/openIDSFU"; + getSFUConfig as authenticateWithTransport, + type GetSFUConfigParams as AuthenticateWithTransportParams, +} from "./src/media-backend/livekit/auth"; diff --git a/sdk/src/RTCParticipation.ts b/sdk/src/RTCParticipation.ts index 740857b89..adbafda38 100644 --- a/sdk/src/RTCParticipation.ts +++ b/sdk/src/RTCParticipation.ts @@ -119,8 +119,9 @@ export function createRTCParticipation( : `${userId}:${deviceId}`, }; - const backend = createBackend(scope, client, options, matrixRTCMode, { + const backend = createBackend(scope, client, options, { roomId: room.roomId, + slotId: jsSdkSession.slotId, ownMembershipIdentity, publish: options.publish, encryptionSystem, @@ -285,16 +286,10 @@ function createBackend( scope: ObservableScope, client: MatrixClient, { backend, mediaQuality }: RTCParticipationOptions, - matrixRTCMode: MatrixRTCMode, context: MediaBackendContext, ): MediaBackend { if (backend) return backend(scope, context); - return createLivekitBackend(scope, context, { - client, - mediaQuality, - tokenEndpoint: - matrixRTCMode === MatrixRTCMode.Matrix_2_0 ? "msc4195" : "legacy", - }); + return createLivekitBackend(scope, context, { client, mediaQuality }); } /** The slot's per-participant keys: the ones it already holds on subscribe, then each new one. */ diff --git a/sdk/src/config.ts b/sdk/src/config.ts index be234f8cb..eedc01b2b 100644 --- a/sdk/src/config.ts +++ b/sdk/src/config.ts @@ -10,12 +10,13 @@ Please see LICENSE in the repository root for full details. * backend and other participants. Resolved by the host. */ export enum MatrixRTCMode { - /** Multi-SFU transport, legacy JWT endpoint, state events. */ + /** Multi-SFU transport, membership state events. */ Compatibility = "compatibility", /** - * Multi-SFU transport with sticky events, a hashed RTC backend identity and - * the new JWT endpoint for the local membership. Remote memberships always - * try the new endpoint first, then the legacy one. + * Multi-SFU transport with sticky membership events and a uuid member id. + * The mode decides how the membership is written, not how the media + * transport is authenticated with: that follows the transport's shape and + * our role on it. */ Matrix_2_0 = "matrix_2_0", } diff --git a/sdk/src/matrixrtc/Transports.ts b/sdk/src/matrixrtc/Transports.ts index fbeb839d9..d9d12aaa2 100644 --- a/sdk/src/matrixrtc/Transports.ts +++ b/sdk/src/matrixrtc/Transports.ts @@ -36,13 +36,17 @@ export function createTransportRegistry( return transport; }; + // Deduplicated: a backend may hold more than one connection to a transport, + // one per homeserver it has to authenticate through const connected$ = scope.behavior( connections$.pipe( - map((connections) => - connections - .filter(({ state }) => state === MediaConnectionState.Connected) - .map(({ transport }) => get(transport)), - ), + map((connections) => [ + ...new Set( + connections + .filter(({ state }) => state === MediaConnectionState.Connected) + .map(({ transport }) => get(transport)), + ), + ]), ), ); diff --git a/sdk/src/media-backend/api.ts b/sdk/src/media-backend/api.ts index 36252e795..4d5f5562f 100644 --- a/sdk/src/media-backend/api.ts +++ b/sdk/src/media-backend/api.ts @@ -53,6 +53,8 @@ export interface MediaKey { export interface MediaBackendContext { roomId: string; + /** The slot the session is in, `application#id`; a token is issued for it. */ + slotId: string; ownMembershipIdentity: CallMembershipIdentityParts; /** What the local member publishes at the join. */ publish: PublishRequest[]; diff --git a/sdk/src/media-backend/livekit/Connection.test.ts b/sdk/src/media-backend/livekit/Connection.test.ts index 41dbde4bd..7224cf7c9 100644 --- a/sdk/src/media-backend/livekit/Connection.test.ts +++ b/sdk/src/media-backend/livekit/Connection.test.ts @@ -8,7 +8,6 @@ Please see LICENSE in the repository root for full details. import { afterEach, - beforeEach, describe, expect, it, @@ -28,25 +27,28 @@ import { } from "livekit-client"; import fetchMock from "fetch-mock"; import EventEmitter from "events"; -import { type IOpenIDToken } from "matrix-js-sdk"; +import { MatrixError } from "matrix-js-sdk"; import { logger, type Logger } from "matrix-js-sdk/lib/logger"; -import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { + type LivekitGetTokenResponse, + type UnstableLivekitTransport, +} from "matrix-js-sdk/lib/matrixrtc"; import { Connection, - ConnectionState, + type ConnectionState, type ConnectionOpts, } from "./Connection.ts"; import { ObservableScope } from "../../reactive/ObservableScope"; -import { type OpenIDClientParts } from "./openIDSFU"; -import { MatrixRTCError, FailToGetOpenIdToken } from "../../errors"; +import { type ClientGetTokenParts, type ClientOpenIDParts } from "./auth"; +import { MatrixRTCError } from "../../errors"; import { testJWTToken } from "../../utils/test-fixtures"; import { ownMemberMock } from "../../utils/test"; import { mockRemoteParticipant } from "./test"; let testScope: ObservableScope; -let client: MockedObject; +let client: MockedObject; let fakeLivekitRoom: MockedObject; @@ -54,23 +56,39 @@ let localParticipantEventEmiter: EventEmitter; let fakeLocalParticipant: MockedObject; const ROOM_ID = "!roomID:example.org"; +const SLOT_ID = "m.call#ROOM"; -const livekitFocus: LivekitTransport = { - livekit_service_url: "https://matrix-rtc.example.org/livekit/jwt", +const transport: UnstableLivekitTransport = { type: "livekit", + url: "wss://matrix-rtc.m.localhost/livekit/sfu", }; +/** The options of a subscriber connection to `transport`. */ +function connectionOpts(): ConnectionOpts { + return { + client, + roomId: ROOM_ID, + slotId: SLOT_ID, + role: "subscriber", + transport, + serverName: "m.localhost", + scope: testScope, + ownMembershipIdentity: ownMemberMock, + livekitRoomFactory: () => fakeLivekitRoom, + }; +} + function setupTest(): void { testScope = new ObservableScope(); - client = vi.mocked({ + client = vi.mocked({ getOpenIdToken: vi.fn().mockResolvedValue({ access_token: "rYsmGUEwNjKgJYyeNUkZseJN", token_type: "Bearer", matrix_server_name: "example.org", expires_in: 3600, }), - getDeviceId: vi.fn().mockReturnValue("ABCDEF"), - } as unknown as OpenIDClientParts); + _unstable_getLivekitToken: vi.fn().mockResolvedValue({ jwt: testJWTToken }), + }); localParticipantEventEmiter = new EventEmitter(); @@ -113,24 +131,7 @@ function setupTest(): void { } function setupRemoteConnection(): Connection { - const opts: ConnectionOpts = { - client: client, - roomId: ROOM_ID, - transport: livekitFocus, - scope: testScope, - ownMembershipIdentity: ownMemberMock, - livekitRoomFactory: () => fakeLivekitRoom, - }; - - fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, () => { - return { - status: 200, - body: { - url: "wss://matrix-rtc.m.localhost/livekit/sfu", - jwt: testJWTToken, - }, - }; - }); + const opts = connectionOpts(); fakeLivekitRoom.connect.mockImplementation(async (): Promise => { const changeEv = RoomEvent.ConnectionStateChanged; @@ -153,60 +154,33 @@ afterEach(() => { }); describe("Start connection states", () => { - beforeEach(() => { - fetchMock.post( - `https://matrix-rtc.example.org/livekit/jwt/get_token`, - () => { - return { - // Return a non-retryable error, if not, the retry logic will - // wait and fail the test with a timeout. - status: 404, - }; - }, - ); - }); - it("start in initialized state", () => { setupTest(); - const opts: ConnectionOpts = { - client: client, - roomId: ROOM_ID, - transport: livekitFocus, - scope: testScope, - ownMembershipIdentity: ownMemberMock, - livekitRoomFactory: () => fakeLivekitRoom, - }; + const opts = connectionOpts(); const connection = new Connection(opts, logger); - expect(connection.state$.getValue()).toEqual("Initialized"); + expect(connection.state$.getValue()).toEqual({ state: "initialized" }); }); - it("fail to getOpenId token then error state", async () => { + it("fail to get SFU token then error state", async () => { setupTest(); vi.useFakeTimers(); - const opts: ConnectionOpts = { - client: client, - roomId: ROOM_ID, - transport: livekitFocus, - scope: testScope, - ownMembershipIdentity: ownMemberMock, - livekitRoomFactory: () => fakeLivekitRoom, - }; + const opts = connectionOpts(); const connection = new Connection(opts, logger); - const capturedStates: (ConnectionState | Error)[] = []; + const capturedStates: ConnectionState[] = []; const s = connection.state$.subscribe((value) => { capturedStates.push(value); }); onTestFinished(() => s.unsubscribe()); - const deferred = Promise.withResolvers(); + const deferred = Promise.withResolvers(); - client.getOpenIdToken.mockImplementation( - async (): Promise => { + client._unstable_getLivekitToken.mockImplementation( + async (): Promise => { return await deferred.promise; }, ); @@ -217,18 +191,20 @@ describe("Start connection states", () => { let capturedState = capturedStates.pop(); expect(capturedState).toBeDefined(); - expect(capturedState!).toEqual("FetchingConfig"); + expect(capturedState).toEqual({ state: "authenticating" }); - deferred.reject(new FailToGetOpenIdToken(new Error("Failed to get token"))); + deferred.reject(new MatrixRTCError("Failed to get token")); await vi.runAllTimersAsync(); capturedState = capturedStates.pop(); - if (capturedState instanceof Error) { - expect(capturedState).toBeInstanceOf(FailToGetOpenIdToken); - expect(connection.transport).toEqual(livekitFocus); + if (capturedState?.state === "error") { + expect(capturedState.error.message).toEqual("Failed to get token"); + expect(connection.transport).toEqual(transport); } else { - expect.fail("Expected FailedToStart state but got " + capturedState); + expect.fail( + "Expected error state but got " + JSON.stringify(capturedState), + ); } }); @@ -236,34 +212,26 @@ describe("Start connection states", () => { setupTest(); vi.useFakeTimers(); - const opts: ConnectionOpts = { - client: client, - roomId: ROOM_ID, - transport: livekitFocus, - scope: testScope, - ownMembershipIdentity: ownMemberMock, - livekitRoomFactory: () => fakeLivekitRoom, - }; + const opts = connectionOpts(); const connection = new Connection(opts, logger); - const capturedStates: (ConnectionState | Error)[] = []; + const capturedStates: ConnectionState[] = []; const s = connection.state$.subscribe((value) => { capturedStates.push(value); }); onTestFinished(() => s.unsubscribe()); const deferredSFU = Promise.withResolvers(); - // mock the /sfu/get call - fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, async () => { + client._unstable_getLivekitToken.mockImplementation(async () => { await deferredSFU.promise; - return { - status: 500, - body: { + throw new MatrixError( + { errcode: "M_LOOKUP_FAILED", error: "Failed to look up user info from homeserver", }, - }; + 500, + ); }); connection.start().catch(() => { @@ -272,7 +240,7 @@ describe("Start connection states", () => { let capturedState = capturedStates.pop(); expect(capturedState).toBeDefined(); - expect(capturedState).toEqual(ConnectionState.FetchingConfig); + expect(capturedState).toEqual({ state: "authenticating" }); deferredSFU.resolve(); await vi.runAllTimersAsync(); @@ -280,15 +248,17 @@ describe("Start connection states", () => { capturedState = capturedStates.pop(); if ( - capturedState instanceof MatrixRTCError && - capturedState.cause instanceof Error + capturedState?.state === "error" && + capturedState.error.cause instanceof Error ) { - expect(capturedState.cause.message).toContain( + expect(capturedState.error.cause.message).toContain( "Failed to look up user info from homeserver", ); - expect(connection.transport).toEqual(livekitFocus); + expect(connection.transport).toEqual(transport); } else { - expect.fail("Expected FailedToStart state but got " + capturedState); + expect.fail( + "Expected error state but got " + JSON.stringify(capturedState), + ); } }); @@ -296,35 +266,17 @@ describe("Start connection states", () => { setupTest(); vi.useFakeTimers(); - const opts: ConnectionOpts = { - client: client, - roomId: ROOM_ID, - transport: livekitFocus, - scope: testScope, - ownMembershipIdentity: ownMemberMock, - livekitRoomFactory: () => fakeLivekitRoom, - }; + const opts = connectionOpts(); const connection = new Connection(opts, logger); - const capturedStates: (ConnectionState | Error)[] = []; + const capturedStates: ConnectionState[] = []; const s = connection.state$.subscribe((value) => { capturedStates.push(value); }); onTestFinished(() => s.unsubscribe()); const deferredSFU = Promise.withResolvers(); - // mock the /sfu/get call - fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, () => { - return { - status: 200, - body: { - url: "wss://matrix-rtc.m.localhost/livekit/sfu", - jwt: testJWTToken, - }, - }; - }); - fakeLivekitRoom.connect.mockImplementation(async () => { await deferredSFU.promise; throw new Error("Failed to connect to livekit"); @@ -337,7 +289,7 @@ describe("Start connection states", () => { let capturedState = capturedStates.pop(); expect(capturedState).toBeDefined(); - expect(capturedState).toEqual(ConnectionState.FetchingConfig); + expect(capturedState).toEqual({ state: "authenticating" }); deferredSFU.resolve(); await vi.runAllTimersAsync(); @@ -345,13 +297,13 @@ describe("Start connection states", () => { capturedState = capturedStates.pop(); if ( - capturedState instanceof MatrixRTCError && - capturedState.cause instanceof Error + capturedState?.state === "error" && + capturedState.error.cause instanceof Error ) { - expect(capturedState.cause.message).toContain( + expect(capturedState.error.cause.message).toContain( "Failed to connect to livekit", ); - expect(connection.transport).toEqual(livekitFocus); + expect(connection.transport).toEqual(transport); } else { expect.fail( "Expected FailedToStart state but got " + JSON.stringify(capturedState), @@ -365,7 +317,7 @@ describe("Start connection states", () => { const connection = setupRemoteConnection(); - const capturedStates: (ConnectionState | Error)[] = []; + const capturedStates: ConnectionState[] = []; const s = connection.state$.subscribe((value) => { capturedStates.push(value); }); @@ -375,22 +327,59 @@ describe("Start connection states", () => { await vi.runAllTimersAsync(); const initialState = capturedStates.shift(); - expect(initialState).toEqual(ConnectionState.Initialized); + expect(initialState).toEqual({ state: "initialized" }); const fetchingState = capturedStates.shift(); - expect(fetchingState).toEqual(ConnectionState.FetchingConfig); + expect(fetchingState).toEqual({ state: "authenticating" }); const disconnectedState = capturedStates.shift(); - expect(disconnectedState).toEqual(ConnectionState.LivekitDisconnected); + expect(disconnectedState).toEqual({ + state: "authenticated", + livekitState: LivekitConnectionState.Disconnected, + }); const connectingState = capturedStates.shift(); - expect(connectingState).toEqual(ConnectionState.LivekitConnecting); + expect(connectingState).toEqual({ + state: "authenticated", + livekitState: LivekitConnectionState.Connecting, + }); const connectedState = capturedStates.shift(); - expect(connectedState).toEqual(ConnectionState.LivekitConnected); + expect(connectedState).toEqual({ + state: "authenticated", + livekitState: LivekitConnectionState.Connected, + }); + }); + + it("authenticates a publisher on a livekit_service_url transport through the JWT service", async () => { + setupTest(); + const serviceUrl = "https://matrix-rtc.example.org/livekit/jwt"; + fetchMock.post(`${serviceUrl}/sfu/get`, { + status: 200, + body: { url: "wss://sfu", jwt: testJWTToken }, + }); + fakeLivekitRoom.connect.mockResolvedValue(undefined); + const connection = new Connection( + { + ...connectionOpts(), + role: "publisher", + transport: { type: "livekit", livekit_service_url: serviceUrl }, + }, + logger, + ); + + await connection.start(); + + expect(client._unstable_getLivekitToken).not.toHaveBeenCalled(); + expect(fetchMock.called(`${serviceUrl}/sfu/get`)).toBe(true); + expect(fakeLivekitRoom.connect).toHaveBeenCalledWith( + "wss://sfu", + testJWTToken, + ); + expect(connection.livekitAlias).toBe("!example_room_id"); }); it("stopping while connecting does not report an error", async () => { setupTest(); const connection = setupRemoteConnection(); - const capturedStates: (ConnectionState | Error)[] = []; + const capturedStates: ConnectionState[] = []; const s = connection.state$.subscribe((value) => { capturedStates.push(value); }); @@ -407,11 +396,11 @@ describe("Start connection states", () => { pendingConnect.reject(new Error("Client initiated disconnect")); await stopping; - // start() resolves rather than rejecting (it is not awaited by the - // ConnectionManager, so a rejection would be unhandled). + // start() resolves rather than rejecting (it is not awaited by the caller, + // so a rejection would be unhandled). await expect(started).resolves.toBeUndefined(); - expect(capturedStates.at(-1)).toEqual(ConnectionState.Stopped); - expect(capturedStates.some((st) => st instanceof Error)).toBe(false); + expect(capturedStates.at(-1)).toEqual({ state: "stopped" }); + expect(capturedStates.some((st) => st.state === "error")).toBe(false); }); it("shutting down the scope should stop the connection", async () => { @@ -544,17 +533,7 @@ describe("remote track logging", () => { warn, error: vi.fn(), } as unknown as Logger; - new Connection( - { - client, - roomId: ROOM_ID, - transport: livekitFocus, - scope: testScope, - ownMembershipIdentity: ownMemberMock, - livekitRoomFactory: () => fakeLivekitRoom, - }, - testLogger, - ); + new Connection(connectionOpts(), testLogger); info.mockClear(); // drop the constructor log line const bob = mockRemoteParticipant({ diff --git a/sdk/src/media-backend/livekit/Connection.ts b/sdk/src/media-backend/livekit/Connection.ts index 95dcb1541..1af4427c8 100644 --- a/sdk/src/media-backend/livekit/Connection.ts +++ b/sdk/src/media-backend/livekit/Connection.ts @@ -13,6 +13,7 @@ import { import { ConnectionError, ConnectionErrorReason, + type ConnectionState as LivekitConnectionState, type Participant, type RemoteParticipant, type RemoteTrackPublication, @@ -22,16 +23,18 @@ import { type Track, type TrackPublication, } from "livekit-client"; -import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; -import { BehaviorSubject, map } from "rxjs"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { BehaviorSubject } from "rxjs"; import { type Logger } from "matrix-js-sdk/lib/logger"; import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; import { - getSFUConfigWithOpenID, - type OpenIDClientParts, + type ClientGetTokenParts, + type ClientOpenIDParts, + type ConnectionRole, + getSFUConfig, type SFUConfig, -} from "./openIDSFU"; +} from "./auth"; import { type Behavior } from "../../reactive/Behavior"; import { type ObservableScope } from "../../reactive/ObservableScope"; import { @@ -44,19 +47,20 @@ import { } from "../../errors"; export interface ConnectionOpts { - /** - * For the local transport we already do know the jwt token and url. We can reuse it. - * On top the local transport will send additional data to the jwt server to use delayed event delegation. - */ - existingSFUConfig?: SFUConfig; /** The identity parts to use on this connection */ ownMembershipIdentity: CallMembershipIdentityParts; + /** Whether we publish on this connection or only subscribe. */ + role: ConnectionRole; /** The media transport to connect to. */ - transport: LivekitTransport; - /** The Matrix client to use for OpenID and SFU config requests. */ - client: OpenIDClientParts; - /** The room ID this connection is associated with. */ + transport: UnstableLivekitTransport; + /** The homeserver the transport belongs to. */ + serverName: string; + /** The Matrix client, for the token exchange. */ + client: ClientGetTokenParts & ClientOpenIDParts; + /** The Matrix room the session is in. */ roomId: string; + /** The MatrixRTC slot the session is in. */ + slotId: string; /** The observable scope to use for this connection. */ scope: ObservableScope; @@ -64,23 +68,15 @@ export interface ConnectionOpts { livekitRoomFactory: () => LivekitRoom; } -export enum ConnectionState { - /** The start state of a connection. It has been created but nothing has loaded yet. */ - Initialized = "Initialized", - /** `start` has been called on the connection. It aquires the jwt info to conenct to the LK Room */ - FetchingConfig = "FetchingConfig", - Stopped = "Stopped", - /** The same as ConnectionState.Disconnected from `livekit-client` */ - LivekitDisconnected = "disconnected", - /** The same as ConnectionState.Connecting from `livekit-client` */ - LivekitConnecting = "connecting", - /** The same as ConnectionState.Connected from `livekit-client` */ - LivekitConnected = "connected", - /** The same as ConnectionState.Reconnecting from `livekit-client` */ - LivekitReconnecting = "reconnecting", - /** The same as ConnectionState.SignalReconnecting from `livekit-client` */ - LivekitSignalReconnecting = "signalReconnecting", -} +export type ConnectionState = + /** Created, nothing started yet. */ + | { state: "initialized" } + /** `start` was called; the SFU token is being fetched. */ + | { state: "authenticating" } + /** The token is in hand; the LiveKit room's own state from here. */ + | { state: "authenticated"; livekitState: LivekitConnectionState } + | { state: "error"; error: MatrixRTCError } + | { state: "stopped" }; /** * A connection to a Matrix RTC LiveKit backend. @@ -89,19 +85,25 @@ export enum ConnectionState { */ export class Connection { // Private Behavior - private readonly _state$ = new BehaviorSubject< - ConnectionState | MatrixRTCError - >(ConnectionState.Initialized); + private readonly _state$ = new BehaviorSubject({ + state: "initialized", + }); /** * The current state of the connection to the media transport. */ - public readonly state$: Behavior = this._state$; + public readonly state$: Behavior = this._state$; + + /** Whether we publish on this connection or only subscribe. */ + public readonly role: ConnectionRole; /** * The media transport to connect to. */ - public readonly transport: LivekitTransport; + public readonly transport: UnstableLivekitTransport; + + /** The homeserver the transport belongs to. */ + public readonly serverName: string; public readonly livekitRoom: LivekitRoom; @@ -130,12 +132,11 @@ export class Connection { * */ protected stopped = false; - // TODO: can we just keep the ConnectionOpts object instead of spreading? - private readonly client: OpenIDClientParts; + private readonly client: ClientGetTokenParts & ClientOpenIDParts; private readonly roomId: string; + private readonly slotId: string; private readonly logger: Logger; private readonly ownMembershipIdentity: CallMembershipIdentityParts; - private readonly existingSFUConfig?: SFUConfig; /** * Creates a new connection to a matrix RTC LiveKit backend. * @@ -145,19 +146,19 @@ export class Connection { */ public constructor(opts: ConnectionOpts, logger: Logger) { this.ownMembershipIdentity = opts.ownMembershipIdentity; - this.existingSFUConfig = opts.existingSFUConfig; this.roomId = opts.roomId; + this.slotId = opts.slotId; this.logger = logger.getChild( - "[Connection " + opts.transport.livekit_service_url + "]", + `[Connection ${JSON.stringify(opts.transport)}]`, ); - this.logger.info( - `constructor: ${opts.transport.livekit_service_url} roomId: ${this.roomId} withSfuConfig?: ${opts.existingSFUConfig ? JSON.stringify(opts.existingSFUConfig) : "undefined"}`, - ); - const { transport, client, scope } = opts; + this.logger.debug(`constructor called`); + const { role, transport, serverName, client, scope } = opts; this.scope = scope; this.livekitRoom = opts.livekitRoomFactory(); + this.role = role; this.transport = transport; + this.serverName = serverName; this.client = client; this.remoteParticipants$ = scope.behavior( @@ -284,9 +285,9 @@ export class Connection { * Starts the connection. * * This will: - * 1. Request an OpenId token `request_token` (allows matrix users to verify their identity with a third-party service.) - * 2. Use this token to request the SFU config to the MatrixRtc authentication service. - * 3. Connect to the configured LiveKit room. + * 1. Authenticate with the transport for our role, through the homeserver + * or the JWT service. + * 2. Connect to the LiveKit room the token is for. * * The errors are also represented as a state in the `state$` observable. * It is safe to ignore those errors and handle them accordingly via the `state$` observable. @@ -298,16 +299,10 @@ export class Connection { this.logger.debug("Starting Connection"); this.stopped = false; try { - this._state$.next(ConnectionState.FetchingConfig); - // We should already have this information after creating the localTransport. - // only call getSFUConfigWithOpenID for connections where we do not have a token yet. (existingJwtTokenData === undefined) - const { url, jwt, livekitAlias } = - this.existingSFUConfig ?? - (await this.getSFUConfigForRemoteConnection()); + this._state$.next({ state: "authenticating" }); + const { url, jwt, livekitAlias } = await this.getSFUConfig(); this.logger.debug( - "Starting Connection to: ", - this.transport.livekit_service_url, - "jwt: ", + "Starting Connection - jwt: ", jwt, "wss: ", url, @@ -318,16 +313,12 @@ export class Connection { // If we were stopped while fetching the config, don't proceed to connect if (this.stopped) return; - // Setup observer once we are done with getSFUConfigWithOpenID + // Setup observer once we are done with getSFUConfig connectionStateObserver(this.livekitRoom) - .pipe( - this.scope.bind(), - map((s) => s as unknown as ConnectionState), - ) - .subscribe((lkState) => { - // It is save to cast lkState to ConnectionState as they are fully overlapping. - this._state$.next(lkState); - }); + .pipe(this.scope.bind()) + .subscribe((livekitState) => + this._state$.next({ state: "authenticated", livekitState }), + ); try { this.logger.info(`livekitRoom.connect ${url}`); @@ -371,36 +362,37 @@ export class Connection { // stop() was called while we were connecting, which makes the pending // connect reject. That is the abort we asked for, not a failure, so // don't record an error state on a stopped connection or rethrow it - // (start() is not awaited by the ConnectionManager, so a throw here + // (start() is not necessarily awaited by the caller, so a throw here // becomes an unhandled promise rejection). this.logger.debug(`Connect aborted because the connection was stopped`); return; } this.logger.debug(`Failed to connect to LiveKit room: ${error}`); - this._state$.next( - error instanceof MatrixRTCError - ? error - : error instanceof Error - ? new UnknownRTCError(error) - : new UnknownRTCError(new Error(`${error}`)), - ); + this._state$.next({ + state: "error", + error: + error instanceof MatrixRTCError + ? error + : error instanceof Error + ? new UnknownRTCError(error) + : new UnknownRTCError(new Error(`${error}`)), + }); // Its okay to ignore the throw. The error is part of the state. throw error; } } - protected async getSFUConfigForRemoteConnection(): Promise { - // This will only be called for sfu's where we do not publish ourselves. - // For the local connection we will use the existingJwtTokenData - return await getSFUConfigWithOpenID( - this.client, - this.ownMembershipIdentity, - this.transport.livekit_service_url, - this.roomId, - // dont pass any custom opts for the subscribe only connections - {}, - this.logger, - ); + protected async getSFUConfig(): Promise { + return await getSFUConfig({ + client: this.client, + membership: this.ownMembershipIdentity, + role: this.role, + transport: this.transport, + serverName: this.serverName, + roomId: this.roomId, + slotId: this.slotId, + logger: this.logger, + }); } /** @@ -410,17 +402,13 @@ export class Connection { * If the connection is already stopped, this is a no-op. */ public async stop(): Promise { - this.logger.debug( - `stop: disconnecing from lk room ${this.transport.livekit_service_url}`, - ); + this.logger.debug("stop: disconnecting from lk room"); if (this.stopped) return; // Mark as stopped before disconnecting so that a connect() aborted by the // disconnect sees the flag and does not report the abort as an error. this.stopped = true; await this.livekitRoom.disconnect(); - this._state$.next(ConnectionState.Stopped); - this.logger.debug( - `stop: DONE disconnecing from lk room ${this.transport.livekit_service_url}`, - ); + this._state$.next({ state: "stopped" }); + this.logger.debug("stop: DONE disconnecting from lk room"); } } diff --git a/sdk/src/media-backend/livekit/ConnectionFactory.ts b/sdk/src/media-backend/livekit/ConnectionFactory.ts index cc01c2a55..eb8230060 100644 --- a/sdk/src/media-backend/livekit/ConnectionFactory.ts +++ b/sdk/src/media-backend/livekit/ConnectionFactory.ts @@ -14,18 +14,23 @@ import { // origin than the page import E2EEWorker from "livekit-client/e2ee-worker?worker&inline"; import { type Logger } from "matrix-js-sdk/lib/logger"; -import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; import { BehaviorSubject, combineLatest, map } from "rxjs"; -import { buildLiveKitOptions } from "./livekitOptions"; -import { type OpenIDClientParts, type SFUConfig } from "./openIDSFU"; -import { type Behavior } from "../../reactive/Behavior"; -import { Connection, type ConnectionOpts, ConnectionState } from "./Connection"; -import { type ObservableScope } from "../../reactive/ObservableScope"; import { type ResolvedTransport } from "../../api"; -import { type PublishRequest } from "../../media-api"; import { type MediaQuality } from "../../config"; +import { type PublishRequest } from "../../media-api"; +import { type Behavior } from "../../reactive/Behavior"; +import { type ObservableScope } from "../../reactive/ObservableScope"; +import { + type ClientGetTokenParts, + type ClientOpenIDParts, + type ConnectionRole, + type SFUConfig, +} from "./auth"; +import { Connection, type ConnectionOpts } from "./Connection"; +import { buildLiveKitOptions } from "./livekitOptions"; import { audioCaptureOptions, type PublishRequestFor, @@ -36,25 +41,27 @@ import { export interface ConnectionFactory { createConnection( scope: ObservableScope, - transport: LivekitTransport, + role: ConnectionRole, + transport: UnstableLivekitTransport, + serverName: string, ownMembershipIdentity: CallMembershipIdentityParts, logger: Logger, - sfuConfig?: SFUConfig, ): Connection; } /** - * A connection that keeps what it fetched from the JWT service, so that the - * transport can report it as `resolved$`. Nothing while the config is still + * A connection that keeps what it fetched when authenticating, so that the + * transport can report it as `resolved$`. Nothing while the token is still * being fetched, and nothing again once the connection has stopped or failed. */ export class ResolvedConnection extends Connection { - private readonly sfuConfig$: BehaviorSubject; + private readonly sfuConfig$ = new BehaviorSubject( + undefined, + ); public readonly resolved$: Behavior; public constructor(opts: ConnectionOpts, logger: Logger) { super(opts, logger); - this.sfuConfig$ = new BehaviorSubject(opts.existingSFUConfig); // One object per fetch, so that a change of state does not read as a // change of what was fetched const resolved$ = this.sfuConfig$.pipe( @@ -63,7 +70,7 @@ export class ResolvedConnection extends Connection { this.resolved$ = opts.scope.behavior( combineLatest([resolved$, this.state$]).pipe( map(([resolved, state]) => - state === ConnectionState.Stopped || state instanceof Error + state.state === "stopped" || state.state === "error" ? undefined : resolved, ), @@ -71,8 +78,8 @@ export class ResolvedConnection extends Connection { ); } - protected override async getSFUConfigForRemoteConnection(): Promise { - const config = await super.getSFUConfigForRemoteConnection(); + protected override async getSFUConfig(): Promise { + const config = await super.getSFUConfig(); this.sfuConfig$.next(config); return config; } @@ -94,8 +101,9 @@ function resolvedTransport(config: SFUConfig): ResolvedTransport { */ export class LivekitConnectionFactory implements ConnectionFactory { public constructor( - private readonly client: OpenIDClientParts, + private readonly client: ClientGetTokenParts & ClientOpenIDParts, private readonly roomId: string, + private readonly slotId: string, private readonly publish: PublishRequest[], private readonly audioOutputDeviceId$: Behavior, private readonly keyProvider: BaseKeyProvider | undefined, @@ -104,16 +112,19 @@ export class LivekitConnectionFactory implements ConnectionFactory { public createConnection( scope: ObservableScope, - transport: LivekitTransport, + role: ConnectionRole, + transport: UnstableLivekitTransport, + serverName: string, ownMembershipIdentity: CallMembershipIdentityParts, logger: Logger, - sfuConfig?: SFUConfig, ): Connection { return new ResolvedConnection( { - existingSFUConfig: sfuConfig, roomId: this.roomId, + slotId: this.slotId, + role, transport, + serverName, client: this.client, scope, livekitRoomFactory: () => diff --git a/sdk/src/media-backend/livekit/ConnectionManager.ts b/sdk/src/media-backend/livekit/ConnectionManager.ts deleted file mode 100644 index e3cc20015..000000000 --- a/sdk/src/media-backend/livekit/ConnectionManager.ts +++ /dev/null @@ -1,297 +0,0 @@ -/* -Copyright 2025 Element Creations Ltd. -Copyright 2025 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; -import { - combineLatest, - map, - type Observable, - of, - switchMap, - startWith, - catchError, - NEVER, -} from "rxjs"; -import { type Logger } from "matrix-js-sdk/lib/logger"; -import { type RemoteParticipant } from "livekit-client"; -import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; - -import { type Behavior } from "../../reactive/Behavior"; -import { type Connection } from "./Connection.ts"; -import { Epoch, type ObservableScope } from "../../reactive/ObservableScope"; -import { generateItemsWithEpoch } from "../../reactive/observable"; -import { type ConnectionFactory } from "./ConnectionFactory.ts"; -import { type SFUConfig } from "./openIDSFU"; - -export class ConnectionManagerData { - private readonly store: Map< - string, - { connection: Connection; participants: RemoteParticipant[] } - > = new Map(); - - public constructor(private readonly logger?: Logger) {} - - public add(connection: Connection, participants: RemoteParticipant[]): void { - const key = this.getKey(connection.transport); - const existing = this.store.get(key); - if (!existing) { - this.store.set(key, { connection, participants }); - } else { - // Transports are deduplicated by URL upstream, so this should never - // happen; if it does, members may be matched against the wrong room. - this.logger?.warn( - `Merging participants from a second connection to ${key}: existing [${existing.participants.map((p) => p.identity).join(", ")}], adding [${participants.map((p) => p.identity).join(", ")}]`, - ); - existing.participants.push(...participants); - } - } - - private getKey(transport: LivekitTransport): string { - // This is enough as a key because the ConnectionManager is already scoped by room. - // We also do not need to consider the slotId at this point since each `MatrixRTCSession` is already scoped by `slotDescription: {id, application}`. - return transport.livekit_service_url; - } - - public getConnections(): Connection[] { - return Array.from(this.store.values()).map(({ connection }) => connection); - } - - public getConnectionForTransport( - transport: LivekitTransport, - ): Connection | null { - return this.store.get(this.getKey(transport))?.connection ?? null; - } - - public getParticipantsForTransport( - transport: LivekitTransport, - ): RemoteParticipant[] { - const key = this.getKey(transport); - const existing = this.store.get(key); - if (existing) { - return existing.participants; - } - return []; - } -} - -interface Props { - scope: ObservableScope; - connectionFactory: ConnectionFactory; - localTransport$: Observable; - remoteTransports$: Behavior>; - - logger: Logger; - ownMembershipIdentity: CallMembershipIdentityParts; -} - -// TODO - write test for scopes (do we really need to bind scope) -export interface IConnectionManager { - connectionManagerData$: Behavior>; -} - -/** - * Crete a `ConnectionManager` - * @param props - Configuration object - * @param props.scope - The observable scope used by this object - * @param props.connectionFactory - Used to create new connections - * @param props.localTransport$ - The transport to publish local media on. (deduplicated with remoteTransports$) - * @param props.remoteTransports$ - All other transports. The connection manager will create connections for each transport. (deduplicated with localTransport$) - * @param props.ownMembershipIdentity - The own membership identity to use. - * @param props.logger - The logger to use. - - * - * Each of these behaviors can be interpreted as subscribed list of transports. - * - * Using `registerTransports` independent external modules can control what connections - * are created by the ConnectionManager. - * - * The connection manager will remove all duplicate transports in each subscibed list. - * - * See `unregisterAllTransports` and `unregisterTransport` for details on how to unsubscribe. - */ -export function createConnectionManager$({ - scope, - connectionFactory, - localTransport$, - remoteTransports$, - logger: parentLogger, - ownMembershipIdentity, -}: Props): IConnectionManager { - const logger = parentLogger.getChild("[ConnectionManager]"); - // TODO logger: only construct one logger from the client and make it compatible via a EC specific sing - - const localTransportAsArray$ = localTransport$.pipe( - // LocalMembership already surfaces local transport errors properly in the UI, - // here we can just swallow them - catchError(() => NEVER), - map((transport) => [transport]), - startWith([]), - ); - - /** - * All transports currently managed by the ConnectionManager. - * - * This list does not include duplicate transports. - * - * It is build based on the list of subscribed transports (`transportsSubscriptions$`). - * externally this is modified via `registerTransports()`. - */ - const localAndRemoteTransports$: Behavior< - Epoch<(LivekitTransport | LocalTransport)[]> - > = scope.behavior( - combineLatest([localTransportAsArray$, remoteTransports$]).pipe( - // Combine local and remote transports into one transport array - // and set the forceOldJwtEndpoint property on the local transport - map(([localTransportAsArray, remoteTransports]) => { - const dedupedRemote = removeDuplicateTransports(remoteTransports.value); - const remoteWithoutLocal = dedupedRemote.filter( - (transport) => - !localTransportAsArray.find((l) => - areLivekitTransportsEqual(l.transport, transport), - ), - ); - logger.debug( - "remoteWithoutLocal", - remoteWithoutLocal, - "localTransportAsArray", - localTransportAsArray, - ); - return new Epoch( - [...localTransportAsArray, ...remoteWithoutLocal], - remoteTransports.epoch, - ); - }), - ), - ); - - /** - * Connections for each transport in use by one or more session members. - */ - const connections$ = scope.behavior( - localAndRemoteTransports$.pipe( - generateItemsWithEpoch( - "ConnectionManager connections$", - function* (transports) { - for (const transport of transports) { - if (isLocalTransport(transport)) { - // This is the local transport; only the `LocalTransport` - // interface has a `sfuConfig` field. - yield { - keys: [ - transport.transport.livekit_service_url, - transport.sfuConfig, - ], - data: undefined, - }; - } else { - yield { - keys: [ - transport.livekit_service_url, - undefined as SFUConfig | undefined, - ], - data: undefined, - }; - } - } - }, - (scope, _data$, serviceUrl, sfuConfig) => { - const connection = connectionFactory.createConnection( - scope, - { - type: "livekit", - livekit_service_url: serviceUrl, - }, - ownMembershipIdentity, - logger, - // TODO: This whole optional SFUConfig parameter is not particularly elegant. - // I would like it if connections always fetched the SFUConfig by themselves. - sfuConfig, - ); - // Start the connection immediately - // Use connection state to track connection progress - void connection.start(); - // TODO subscribe to connection state to retry or log issues? - return connection; - }, - ), - ), - ); - - const connectionManagerData$ = scope.behavior( - connections$.pipe( - switchMap((connections) => { - const epoch = connections.epoch; - - // Map the connections to list of {connection, participants}[] - const listOfConnectionsWithRemoteParticipants = connections.value.map( - (connection) => { - return connection.remoteParticipants$.pipe( - map((participants) => ({ - connection, - participants, - })), - ); - }, - ); - - // probably not required - - if (listOfConnectionsWithRemoteParticipants.length === 0) { - return of(new Epoch(new ConnectionManagerData(), epoch)); - } - - // combineLatest the several streams into a single stream with the ConnectionManagerData - return combineLatest(listOfConnectionsWithRemoteParticipants).pipe( - map( - (lists) => - new Epoch( - lists.reduce((data, { connection, participants }) => { - data.add(connection, participants); - return data; - }, new ConnectionManagerData(logger)), - epoch, - ), - ), - ); - }), - ), - new Epoch(new ConnectionManagerData(), -1), - ); - - return { connectionManagerData$ }; -} - -function removeDuplicateTransports( - transports: T[], -): T[] { - return transports.reduce((acc, transport) => { - if (!acc.some((t) => areLivekitTransportsEqual(t, transport))) - acc.push(transport); - return acc; - }, [] as T[]); -} - -/** The transport the local member publishes on, authenticated with. */ -export interface LocalTransport { - transport: LivekitTransport; - sfuConfig: SFUConfig; -} - -export function isLocalTransport( - obj: LivekitTransport | LocalTransport, -): obj is LocalTransport { - return "transport" in obj && "sfuConfig" in obj; -} - -export function areLivekitTransportsEqual( - t1: T | null, - t2: T | null, -): boolean { - if (t1 && t2) return t1.livekit_service_url === t2.livekit_service_url; - return !t1 && !t2; -} diff --git a/sdk/src/media-backend/livekit/ConnectionManager.test.ts b/sdk/src/media-backend/livekit/ConnectionMap.test.ts similarity index 58% rename from sdk/src/media-backend/livekit/ConnectionManager.test.ts rename to sdk/src/media-backend/livekit/ConnectionMap.test.ts index 3b289d982..a3acae247 100644 --- a/sdk/src/media-backend/livekit/ConnectionManager.test.ts +++ b/sdk/src/media-backend/livekit/ConnectionMap.test.ts @@ -6,8 +6,8 @@ Please see LICENSE in the repository root for full details. */ import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; -import { BehaviorSubject, NEVER } from "rxjs"; -import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { BehaviorSubject } from "rxjs"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { type RemoteParticipant } from "livekit-client"; import { logger } from "matrix-js-sdk/lib/logger"; @@ -17,27 +17,40 @@ import { ObservableScope, } from "../../reactive/ObservableScope"; import { - createConnectionManager$, - ConnectionManagerData, - areLivekitTransportsEqual, -} from "./ConnectionManager.ts"; + createConnectionMap$, + ConnectionMap, + keyFor, +} from "./ConnectionMap.ts"; import { type ConnectionFactory } from "./ConnectionFactory.ts"; import { type Connection } from "./Connection.ts"; import { ownMemberMock, withTestScheduler } from "../../utils/test"; import { type Behavior } from "../../reactive/Behavior"; +import { type TransportLocator } from "./auth"; -// Some test constants - -const TRANSPORT_1: LivekitTransport = { - type: "livekit", - livekit_service_url: "https://lk.example.org", +const TRANSPORT_1: TransportLocator = { + transport: { + type: "livekit", + livekit_service_url: "https://lk.example.org", + }, + serverName: "example.org", }; -const TRANSPORT_2: LivekitTransport = { - type: "livekit", - livekit_service_url: "https://lk.sample.com", +const TRANSPORT_2: TransportLocator = { + transport: { + type: "livekit", + livekit_service_url: "https://lk.sample.com", + }, + serverName: "sample.com", }; +function findConnection( + transport: TransportLocator, + connections: Connection[], +): Connection | undefined { + const key = keyFor(transport); + return connections.find((c) => keyFor(c) === key); +} + let fakeConnectionFactory: ConnectionFactory; let testScope: ObservableScope; @@ -51,9 +64,15 @@ beforeEach(() => { vi.mocked(fakeConnectionFactory).createConnection = vi .fn() .mockImplementation( - (scope: ObservableScope, transport: LivekitTransport) => { + ( + scope: ObservableScope, + _role: unknown, + transport: UnstableLivekitTransport, + serverName: string, + ) => { const mockConnection = { transport, + serverName, remoteParticipants$: new BehaviorSubject([]), } as unknown as Connection; vi.mocked(mockConnection).start = vi.fn(); @@ -75,10 +94,10 @@ afterEach(() => { describe("connections$ stream", () => { test("Should create and start new connections for each transports", () => { withTestScheduler(({ behavior, expectObservable }) => { - const { connectionManagerData$ } = createConnectionManager$({ + const connectionMap$ = createConnectionMap$({ scope: testScope, connectionFactory: fakeConnectionFactory, - localTransport$: NEVER, + localTransport: TRANSPORT_1, remoteTransports$: behavior("a", { a: new Epoch([TRANSPORT_1, TRANSPORT_2], 0), }), @@ -87,7 +106,7 @@ describe("connections$ stream", () => { }); expectObservable( - connectionManagerData$.pipe(mapEpoch((d) => d.getConnections())), + connectionMap$.pipe(mapEpoch((d) => d.getConnections())), ).toBe("a", { a: expect.toSatisfy((e: Epoch) => { const connections = e.value; @@ -97,15 +116,11 @@ describe("connections$ stream", () => { vi.mocked(fakeConnectionFactory).createConnection, ).toHaveBeenCalledTimes(2); - const conn1 = connections.find((c) => - areLivekitTransportsEqual(c.transport, TRANSPORT_1), - ); + const conn1 = findConnection(TRANSPORT_1, connections); expect(conn1).toBeDefined(); expect(conn1!.start).toHaveBeenCalled(); - const conn2 = connections.find((c) => - areLivekitTransportsEqual(c.transport, TRANSPORT_2), - ); + const conn2 = findConnection(TRANSPORT_2, connections); expect(conn2).toBeDefined(); expect(conn2!.start).toHaveBeenCalled(); return true; @@ -114,12 +129,126 @@ describe("connections$ stream", () => { }); }); - test("Should start connection only once", () => { + test("Tags the local transport as the publisher and the rest as subscribers", () => { withTestScheduler(({ behavior, expectObservable }) => { - const { connectionManagerData$ } = createConnectionManager$({ + const connectionMap$ = createConnectionMap$({ scope: testScope, connectionFactory: fakeConnectionFactory, - localTransport$: NEVER, + localTransport: TRANSPORT_1, + remoteTransports$: behavior("a", { + a: new Epoch([TRANSPORT_2], 0), + }), + logger: logger, + ownMembershipIdentity: ownMemberMock, + }); + + expectObservable(connectionMap$).toBe("a", { + a: expect.toSatisfy(() => { + const { createConnection } = vi.mocked(fakeConnectionFactory); + expect(createConnection).toHaveBeenCalledWith( + expect.anything(), + "publisher", + TRANSPORT_1.transport, + TRANSPORT_1.serverName, + ownMemberMock, + expect.anything(), + ); + expect(createConnection).toHaveBeenCalledWith( + expect.anything(), + "subscriber", + TRANSPORT_2.transport, + TRANSPORT_2.serverName, + ownMemberMock, + expect.anything(), + ); + return true; + }), + }); + }); + }); + + test("Shares one connection between homeservers advertising the same JWT service", () => { + withTestScheduler(({ behavior, expectObservable }) => { + const sameServiceElsewhere: TransportLocator = { + transport: TRANSPORT_1.transport, + serverName: "elsewhere.org", + }; + const connectionMap$ = createConnectionMap$({ + scope: testScope, + connectionFactory: fakeConnectionFactory, + localTransport: TRANSPORT_1, + remoteTransports$: behavior("a", { + a: new Epoch([sameServiceElsewhere], 0), + }), + logger: logger, + ownMembershipIdentity: ownMemberMock, + }); + + expectObservable( + connectionMap$.pipe(mapEpoch((d) => d.getConnections())), + ).toBe("a", { + a: expect.toSatisfy((e: Epoch) => { + expect(e.value.length).toBe(1); + // The JWT service is reached directly, so the server name plays no + // part and the local locator, which came first, is the one used + expect( + vi.mocked(fakeConnectionFactory).createConnection, + ).toHaveBeenCalledExactlyOnceWith( + expect.anything(), + "publisher", + TRANSPORT_1.transport, + "example.org", + expect.anything(), + expect.anything(), + ); + return true; + }), + }); + }); + }); + + test("Keeps one connection per homeserver for transports with a url", () => { + withTestScheduler(({ behavior, expectObservable }) => { + const urlTransport: TransportLocator = { + transport: { type: "livekit", url: "wss://sfu.example.org" }, + serverName: "example.org", + }; + const sameUrlElsewhere: TransportLocator = { + transport: urlTransport.transport, + serverName: "elsewhere.org", + }; + const connectionMap$ = createConnectionMap$({ + scope: testScope, + connectionFactory: fakeConnectionFactory, + localTransport: urlTransport, + remoteTransports$: behavior("a", { + a: new Epoch([sameUrlElsewhere], 0), + }), + logger: logger, + ownMembershipIdentity: ownMemberMock, + }); + + expectObservable( + connectionMap$.pipe(mapEpoch((d) => d.getConnections())), + ).toBe("a", { + a: expect.toSatisfy((e: Epoch) => { + // Our homeserver asks the named server for the token, so each + // server name needs its own connection + expect(e.value.length).toBe(2); + expect(findConnection(urlTransport, e.value)).toBeDefined(); + expect(findConnection(sameUrlElsewhere, e.value)).toBeDefined(); + return true; + }), + }); + }); + }); + + test("Should start connection only once", () => { + withTestScheduler(({ behavior, expectObservable }) => { + const connectionMap$ = createConnectionMap$({ + scope: testScope, + connectionFactory: fakeConnectionFactory, + localTransport: TRANSPORT_1, remoteTransports$: behavior("abcdef", { a: new Epoch([TRANSPORT_1], 0), b: new Epoch([TRANSPORT_1], 1), @@ -133,7 +262,7 @@ describe("connections$ stream", () => { }); expectObservable( - connectionManagerData$.pipe(mapEpoch((d) => d.getConnections())), + connectionMap$.pipe(mapEpoch((d) => d.getConnections())), ).toBe("xxxxxa", { x: expect.anything(), a: expect.toSatisfy((e: Epoch) => { @@ -144,14 +273,10 @@ describe("connections$ stream", () => { vi.mocked(fakeConnectionFactory).createConnection, ).toHaveBeenCalledTimes(2); - const conn2 = connections.find((c) => - areLivekitTransportsEqual(c.transport, TRANSPORT_2), - ); + const conn2 = findConnection(TRANSPORT_2, connections); expect(conn2).toBeDefined(); - const conn1 = connections.find((c) => - areLivekitTransportsEqual(c.transport, TRANSPORT_1), - ); + const conn1 = findConnection(TRANSPORT_1, connections); expect(conn1).toBeDefined(); expect(conn1!.start).toHaveBeenCalledOnce(); @@ -163,10 +288,10 @@ describe("connections$ stream", () => { test("Should cleanup connections when not needed anymore", () => { withTestScheduler(({ behavior, expectObservable }) => { - const { connectionManagerData$ } = createConnectionManager$({ + const connectionMap$ = createConnectionMap$({ scope: testScope, connectionFactory: fakeConnectionFactory, - localTransport$: NEVER, + localTransport: TRANSPORT_1, remoteTransports$: behavior("abc", { a: new Epoch([TRANSPORT_1], 0), b: new Epoch([TRANSPORT_1, TRANSPORT_2], 1), @@ -177,7 +302,7 @@ describe("connections$ stream", () => { }); expectObservable( - connectionManagerData$.pipe(mapEpoch((d) => d.getConnections())), + connectionMap$.pipe(mapEpoch((d) => d.getConnections())), ).toBe("xab", { x: expect.anything(), a: expect.toSatisfy((e: Epoch) => { @@ -190,8 +315,9 @@ describe("connections$ stream", () => { expect(connections.length).toBe(1); // The second connection should have been stopped has it is no longer needed. - const connection2 = allCreatedConnections.find((c) => - areLivekitTransportsEqual(c.transport, TRANSPORT_2), + const connection2 = findConnection( + TRANSPORT_2, + allCreatedConnections, ); expect(connection2).toBeDefined(); expect(connection2!.stop).toHaveBeenCalled(); @@ -207,40 +333,36 @@ describe("connections$ stream", () => { }); }); -describe("ConnectionManagerData", () => { +describe("ConnectionMap", () => { test("warns when a second connection to the same URL is merged", () => { const warn = vi.spyOn(logger, "warn").mockImplementation(() => {}); - const data = new ConnectionManagerData(logger); - const connection = { transport: TRANSPORT_1 } as unknown as Connection; + const data = new ConnectionMap(logger); + const connection = TRANSPORT_1 as unknown as Connection; const p = (identity: string): RemoteParticipant => ({ identity }) as unknown as RemoteParticipant; data.add(connection, [p("a")]); data.add({ ...connection } as Connection, [p("b")]); expect(warn).toHaveBeenCalledWith( expect.stringContaining( - "second connection to https://lk.example.org: existing [a], adding [b]", + 'second connection to [null,"https://lk.example.org",null]: existing [a], adding [b]', ), ); expect(data.getParticipantsForTransport(TRANSPORT_1)).toHaveLength(2); }); }); -describe("connectionManagerData$ stream", () => { +describe("connectionMap$", () => { // Used in test to control fake connections' remoteParticipants$ streams let fakeRemoteParticipantsStreams: Map>; - function keyForTransport(transport: LivekitTransport): string { - return `${transport.livekit_service_url}`; - } - beforeEach(() => { fakeRemoteParticipantsStreams = new Map(); function getRemoteParticipantsFor( - transport: LivekitTransport, + transport: TransportLocator, ): Behavior { return ( - fakeRemoteParticipantsStreams.get(keyForTransport(transport)) ?? + fakeRemoteParticipantsStreams.get(keyFor(transport)) ?? new BehaviorSubject([]) ); } @@ -249,13 +371,22 @@ describe("connectionManagerData$ stream", () => { vi.mocked(fakeConnectionFactory).createConnection = vi .fn() .mockImplementation( - (scope: ObservableScope, transport: LivekitTransport) => { + ( + scope: ObservableScope, + _role: unknown, + transport: UnstableLivekitTransport, + serverName: string, + ) => { const fakeRemoteParticipants$ = new BehaviorSubject< RemoteParticipant[] >([]); const mockConnection = { transport, - remoteParticipants$: getRemoteParticipantsFor(transport), + serverName, + remoteParticipants$: getRemoteParticipantsFor({ + transport, + serverName, + }), } as unknown as Connection; vi.mocked(mockConnection).start = vi.fn(); vi.mocked(mockConnection).stop = vi.fn(); @@ -265,7 +396,7 @@ describe("connectionManagerData$ stream", () => { }); fakeRemoteParticipantsStreams.set( - keyForTransport(transport), + keyFor({ transport, serverName }), fakeRemoteParticipants$, ); return mockConnection; @@ -274,11 +405,11 @@ describe("connectionManagerData$ stream", () => { }); test("Should report connections with the remote participants", () => { - withTestScheduler(({ expectObservable, schedule, behavior }) => { + withTestScheduler(({ expectObservable, behavior }) => { // Setup the fake participants streams behavior // ============================== fakeRemoteParticipantsStreams.set( - keyForTransport(TRANSPORT_1), + keyFor(TRANSPORT_1), behavior("oa-b", { o: [], a: [{ identity: "user1A" } as RemoteParticipant], @@ -290,7 +421,7 @@ describe("connectionManagerData$ stream", () => { ); fakeRemoteParticipantsStreams.set( - keyForTransport(TRANSPORT_2), + keyFor(TRANSPORT_2), behavior("o-a", { o: [], a: [{ identity: "user2A" } as RemoteParticipant], @@ -298,10 +429,10 @@ describe("connectionManagerData$ stream", () => { ); // ============================== - const { connectionManagerData$ } = createConnectionManager$({ + const connectionMap$ = createConnectionMap$({ scope: testScope, connectionFactory: fakeConnectionFactory, - localTransport$: NEVER, + localTransport: TRANSPORT_1, remoteTransports$: behavior("a", { a: new Epoch([TRANSPORT_1, TRANSPORT_2], 0), }), @@ -309,16 +440,16 @@ describe("connectionManagerData$ stream", () => { ownMembershipIdentity: ownMemberMock, }); - expectObservable(connectionManagerData$).toBe("abcd", { + expectObservable(connectionMap$).toBe("abcd", { a: expect.toSatisfy((e) => { - const data: ConnectionManagerData = e.value; + const data: ConnectionMap = e.value; expect(data.getConnections().length).toBe(2); expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(0); expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(0); return true; }), b: expect.toSatisfy((e) => { - const data: ConnectionManagerData = e.value; + const data: ConnectionMap = e.value; expect(data.getConnections().length).toBe(2); expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(1); expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(0); @@ -328,7 +459,7 @@ describe("connectionManagerData$ stream", () => { return true; }), c: expect.toSatisfy((e) => { - const data: ConnectionManagerData = e.value; + const data: ConnectionMap = e.value; expect(data.getConnections().length).toBe(2); expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(1); expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(1); @@ -341,7 +472,7 @@ describe("connectionManagerData$ stream", () => { return true; }), d: expect.toSatisfy((e) => { - const data: ConnectionManagerData = e.value; + const data: ConnectionMap = e.value; expect(data.getConnections().length).toBe(2); expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(2); expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(1); diff --git a/sdk/src/media-backend/livekit/ConnectionMap.ts b/sdk/src/media-backend/livekit/ConnectionMap.ts new file mode 100644 index 000000000..a0fe4871f --- /dev/null +++ b/sdk/src/media-backend/livekit/ConnectionMap.ts @@ -0,0 +1,194 @@ +/* +Copyright 2025-2026 Element Creations Ltd. +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type RemoteParticipant } from "livekit-client"; +import { type Logger } from "matrix-js-sdk/lib/logger"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; +import { combineLatest, map, of, switchMap, tap } from "rxjs"; + +import { type Behavior } from "../../reactive/Behavior"; +import { + Epoch, + mapEpoch, + type ObservableScope, +} from "../../reactive/ObservableScope"; +import { generateItemsWithEpoch } from "../../reactive/observable"; +import { type ConnectionRole, type TransportLocator } from "./auth"; +import { type Connection } from "./Connection"; +import { type ConnectionFactory } from "./ConnectionFactory"; + +// Branded, so that only values returned by `keyFor` can be used as keys +export type TransportKey = string & { __transportKey: true }; + +/** + * The key a locator is identified by in a map or set. Locators with the same + * key share a connection: the server name only takes part in authentication + * for a transport with a `url`, where our homeserver asks that server for the + * token. A JWT service is reached directly, so members of several homeservers + * advertising the same one share a connection rather than each get their own. + */ +export function keyFor({ + transport, + serverName, +}: TransportLocator): TransportKey { + const hasUrl = "url" in transport; + // Serialize the fields in a consistent order + return JSON.stringify([ + hasUrl ? transport.url : null, + "livekit_service_url" in transport ? transport.livekit_service_url : null, + hasUrl ? serverName : null, + ]) as TransportKey; +} + +function removeDuplicateTransports( + transports: TransportLocator[], +): TransportLocator[] { + // The first locator wins, so the local one stays first in the list + const map = new Map(); + for (const transport of transports) { + const key = keyFor(transport); + if (!map.has(key)) map.set(key, transport); + } + return [...map.values()]; +} + +/** The connections at one moment, with the remote participants seen on each. */ +export class ConnectionMap { + private readonly map = new Map< + TransportKey, + { connection: Connection; participants: RemoteParticipant[] } + >(); + + public constructor(private readonly logger?: Logger) {} + + public add(connection: Connection, participants: RemoteParticipant[]): void { + const key = keyFor(connection); + const existing = this.map.get(key); + if (!existing) { + this.map.set(key, { connection, participants }); + } else { + // Transports are deduplicated by key upstream, so this should never + // happen; if it does, members may be matched against the wrong room. + this.logger?.warn( + `Merging participants from a second connection to ${key}: existing [${existing.participants.map((p) => p.identity).join(", ")}], adding [${participants.map((p) => p.identity).join(", ")}]`, + ); + existing.participants.push(...participants); + } + } + + public getConnections(): Connection[] { + return Array.from(this.map.values()).map(({ connection }) => connection); + } + + public getConnectionForTransport( + transport: TransportLocator, + ): Connection | null { + return this.map.get(keyFor(transport))?.connection ?? null; + } + + public getParticipantsForTransport( + transport: TransportLocator, + ): RemoteParticipant[] { + return this.map.get(keyFor(transport))?.participants ?? []; + } +} + +interface Props { + scope: ObservableScope; + connectionFactory: ConnectionFactory; + /** The transport we publish on; the one connection with the publisher role. */ + localTransport: TransportLocator; + /** Every transport some member is on, deduplicated against the local one. */ + remoteTransports$: Behavior>; + logger: Logger; + ownMembershipIdentity: CallMembershipIdentityParts; +} + +/** + * One connection per transport in use by one or more session members, with + * the remote participants each one sees. The connections start as they are + * created and stop as their transport drops out of the list. + */ +export function createConnectionMap$({ + scope, + connectionFactory, + localTransport, + remoteTransports$, + logger: parentLogger, + ownMembershipIdentity, +}: Props): Behavior> { + const logger = parentLogger.getChild("[ConnectionMap]"); + + const localAndRemoteTransports$: Behavior> = + scope.behavior( + remoteTransports$.pipe( + mapEpoch((remoteTransports) => + removeDuplicateTransports([localTransport, ...remoteTransports]), + ), + tap((transports) => + logger.debug("localAndRemoteTransports$ = ", transports), + ), + ), + ); + + const localTransportKey = keyFor(localTransport); + + const connections$ = scope.behavior( + localAndRemoteTransports$.pipe( + generateItemsWithEpoch( + "ConnectionMap connections$", + function* (transports) { + for (const transport of transports) { + const role: ConnectionRole = + keyFor(transport) === localTransportKey + ? "publisher" + : "subscriber"; + // The locator is an object, so a stable key stands in for it + yield { keys: [role, keyFor(transport)], data: transport }; + } + }, + (scope, locator$, role, _key) => { + const { transport, serverName } = locator$.value; + const connection = connectionFactory.createConnection( + scope, + role, + transport, + serverName, + ownMembershipIdentity, + logger, + ); + // The state carries the progress, errors included + void connection.start(); + return connection; + }, + ), + ), + ); + + return scope.behavior( + connections$.pipe( + switchMap(({ value: connections, epoch }) => { + if (connections.length === 0) + return of(new Epoch(new ConnectionMap(), epoch)); + return combineLatest( + connections.map((connection) => + connection.remoteParticipants$.pipe( + map((participants) => ({ connection, participants })), + ), + ), + (...list) => { + const map = new ConnectionMap(logger); + for (const item of list) + map.add(item.connection, item.participants); + return new Epoch(map, epoch); + }, + ); + }), + ), + ); +} diff --git a/sdk/src/media-backend/livekit/DataChannel.test.ts b/sdk/src/media-backend/livekit/DataChannel.test.ts index 6e0de4f65..9dff925b2 100644 --- a/sdk/src/media-backend/livekit/DataChannel.test.ts +++ b/sdk/src/media-backend/livekit/DataChannel.test.ts @@ -6,24 +6,33 @@ Please see LICENSE in the repository root for full details. */ import { EventEmitter } from "events"; -import { type RemoteParticipant, RoomEvent } from "livekit-client"; +import { + ConnectionState as LivekitConnectionState, + type RemoteParticipant, + RoomEvent, +} from "livekit-client"; import { BehaviorSubject } from "rxjs"; import { describe, expect, it, vi } from "vitest"; import { Epoch } from "../../reactive/ObservableScope"; import { constant } from "../../reactive/Behavior"; import { type DataPacket } from "../api"; -import { type Connection, ConnectionState } from "./Connection"; -import { ConnectionManagerData } from "./ConnectionManager"; +import { type Connection, type ConnectionState } from "./Connection"; +import { ConnectionMap } from "./ConnectionMap"; import { createDataChannel$ } from "./DataChannel"; import { testScope } from "../../utils/test"; import { exampleTransport } from "./test"; const encode = (text: string): Uint8Array => new TextEncoder().encode(text); +const connected: ConnectionState = { + state: "authenticated", + livekitState: LivekitConnectionState.Connected, +}; + function fakeConnection( url: string, - state = ConnectionState.LivekitConnected, + state: ConnectionState = connected, ): { connection: Connection; room: EventEmitter; @@ -33,6 +42,7 @@ function fakeConnection( const publishData = vi.fn(async () => Promise.resolve()); const connection = { transport: { ...exampleTransport, livekit_service_url: url }, + serverName: "example.org", state$: constant(state), livekitRoom: Object.assign(room, { localParticipant: { publishData } }), } as unknown as Connection; @@ -41,15 +51,15 @@ function fakeConnection( function setup(connections: Connection[]): { received: DataPacket[]; - connections$: BehaviorSubject>; + connections$: BehaviorSubject>; sendData: (topic: string, text: string) => Promise; } { - const data = new ConnectionManagerData(); + const data = new ConnectionMap(); for (const connection of connections) data.add(connection, []); const connections$ = new BehaviorSubject(new Epoch(data)); const { data$, sendData } = createDataChannel$({ scope: testScope(), - connectionManager: { connectionManagerData$: connections$ }, + connectionMap$: connections$, connection$: constant(connections[0] ?? null), }); const received: DataPacket[] = []; @@ -89,7 +99,7 @@ describe("data$", () => { const second = fakeConnection("https://sfu-2"); const { received, connections$ } = setup([first.connection]); - const data = new ConnectionManagerData(); + const data = new ConnectionMap(); data.add(first.connection, []); data.add(second.connection, []); connections$.next(new Epoch(data, 1)); @@ -122,10 +132,9 @@ describe("sendData", () => { }); it("rejects while the local transport is not connected", async () => { - const { connection, publishData } = fakeConnection( - "https://sfu", - ConnectionState.Initialized, - ); + const { connection, publishData } = fakeConnection("https://sfu", { + state: "initialized", + }); const { sendData } = setup([connection]); await expect(sendData("chat", "hello")).rejects.toThrow("not connected"); diff --git a/sdk/src/media-backend/livekit/DataChannel.ts b/sdk/src/media-backend/livekit/DataChannel.ts index 69d494ac7..4849ab96b 100644 --- a/sdk/src/media-backend/livekit/DataChannel.ts +++ b/sdk/src/media-backend/livekit/DataChannel.ts @@ -6,6 +6,7 @@ Please see LICENSE in the repository root for full details. */ import { + ConnectionState as LivekitConnectionState, type RemoteParticipant, type Room as LivekitRoom, RoomEvent, @@ -22,14 +23,17 @@ import { } from "rxjs"; import { type Behavior } from "../../reactive/Behavior"; -import { type ObservableScope } from "../../reactive/ObservableScope"; +import { + type Epoch, + type ObservableScope, +} from "../../reactive/ObservableScope"; import { type DataPacket } from "../api"; -import { type Connection, ConnectionState } from "./Connection"; -import { type IConnectionManager } from "./ConnectionManager"; +import { type Connection } from "./Connection"; +import { type ConnectionMap } from "./ConnectionMap"; interface Props { scope: ObservableScope; - connectionManager: IConnectionManager; + connectionMap$: Behavior>; /** The local member's connection, which carries what we send. */ connection$: Behavior; } @@ -47,13 +51,13 @@ export interface DataChannel { */ export function createDataChannel$({ scope, - connectionManager, + connectionMap$, connection$, }: Props): DataChannel { const decoder = new TextDecoder(); const encoder = new TextEncoder(); - const rooms$ = connectionManager.connectionManagerData$.pipe( + const rooms$ = connectionMap$.pipe( map(({ value }) => value.getConnections().map((c) => c.livekitRoom)), distinctUntilChanged( (a, b) => a.length === b.length && a.every((room, i) => room === b[i]), @@ -79,7 +83,12 @@ export function createDataChannel$({ const sendData = async (topic: string, text: string): Promise => { const connection = connection$.value; - if (connection?.state$.value !== ConnectionState.LivekitConnected) + const state = connection?.state$.value; + if ( + connection === null || + state?.state !== "authenticated" || + state.livekitState !== LivekitConnectionState.Connected + ) throw new Error("Cannot send data: the local transport is not connected"); await connection.livekitRoom.localParticipant.publishData( encoder.encode(text), diff --git a/sdk/src/media-backend/livekit/LivekitBackend.test.ts b/sdk/src/media-backend/livekit/LivekitBackend.test.ts index b743c387f..ae8759b1a 100644 --- a/sdk/src/media-backend/livekit/LivekitBackend.test.ts +++ b/sdk/src/media-backend/livekit/LivekitBackend.test.ts @@ -5,21 +5,52 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ +import { + ConnectionState as LivekitConnectionState, + type Room as LivekitRoom, + type RemoteParticipant, +} from "livekit-client"; import fetchMock from "fetch-mock"; import { logger } from "matrix-js-sdk/lib/logger"; -import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; -import { BehaviorSubject } from "rxjs"; -import { afterEach, describe, expect, it } from "vitest"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { BehaviorSubject, NEVER } from "rxjs"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { E2eeType } from "../../encryption"; +import { MatrixRTCError, NoMatrix2AuthorizationService } from "../../errors"; +import { constant } from "../../reactive/Behavior"; import { ObservableScope } from "../../reactive/ObservableScope"; -import { testScope } from "../../utils/test"; +import { type RTCMember } from "../../api"; +import { defaultSessionTimings } from "../../config"; +import { + flushPromises, + mockEmitter, + ownMemberMock, + testScope, +} from "../../utils/test"; import { MediaConnectionState } from "../api"; -import { ConnectionState } from "./Connection"; +import * as auth from "./auth"; +import { type TransportLocator } from "./auth"; +import { type Connection, type ConnectionState } from "./Connection"; +import * as connectionFactory from "./ConnectionFactory"; import { toMediaConnectionState } from "./connectionState"; -import { createTransportRegistry, supportsDelegation } from "./LivekitBackend"; -import { exampleTransport } from "./test"; +import { + createLivekitBackend, + createTransportRegistry, + supportsDelegation, +} from "./LivekitBackend"; +import { + exampleServiceUrl, + exampleTransport, + mockRemoteParticipant, +} from "./test"; -const otherTransport: LivekitTransport = { +const locator = ( + transport: UnstableLivekitTransport, + serverName = "example.org", +): TransportLocator => ({ transport, serverName }); + +const otherTransport: UnstableLivekitTransport = { type: "livekit", livekit_service_url: "https://other.example.org", }; @@ -27,31 +58,31 @@ const otherTransport: LivekitTransport = { describe("createTransportRegistry", () => { it("connects to a transport once however many members are on it, and drops it with the last", () => { const registry = createTransportRegistry(testScope()); - const urls = (): string[] => - registry.transports$.value.value.map((t) => t.livekit_service_url); + const transports = (): UnstableLivekitTransport[] => + registry.transports$.value.value.map((t) => t.transport); const alice = new ObservableScope(); const bob = new ObservableScope(); - registry.follow(alice, new BehaviorSubject(exampleTransport)); - registry.follow(bob, new BehaviorSubject(exampleTransport)); - expect(urls()).toEqual([exampleTransport.livekit_service_url]); + registry.follow(alice, new BehaviorSubject(locator(exampleTransport))); + registry.follow(bob, new BehaviorSubject(locator(exampleTransport))); + expect(transports()).toEqual([exampleTransport]); alice.end(); - expect(urls()).toEqual([exampleTransport.livekit_service_url]); + expect(transports()).toEqual([exampleTransport]); bob.end(); - expect(urls()).toEqual([]); + expect(transports()).toEqual([]); }); it("follows a member that changes transport", () => { const registry = createTransportRegistry(testScope()); - const transport$ = new BehaviorSubject( - exampleTransport, + const transport$ = new BehaviorSubject( + locator(exampleTransport), ); registry.follow(testScope(), transport$); - transport$.next(otherTransport); - expect(registry.transports$.value.value).toEqual([otherTransport]); + transport$.next(locator(otherTransport)); + expect(registry.transports$.value.value).toEqual([locator(otherTransport)]); transport$.next(undefined); expect(registry.transports$.value.value).toEqual([]); @@ -61,7 +92,7 @@ describe("createTransportRegistry", () => { const scope = new ObservableScope(); const registry = createTransportRegistry(scope); const member = new ObservableScope(); - registry.follow(member, new BehaviorSubject(exampleTransport)); + registry.follow(member, new BehaviorSubject(locator(exampleTransport))); scope.end(); expect(() => member.end()).not.toThrow(); @@ -70,43 +101,327 @@ describe("createTransportRegistry", () => { describe("supportsDelegation", () => { afterEach(() => fetchMock.reset()); - const homeserver = "https://matrix.example.org"; - const homeserverEndpoint = `${homeserver}/_matrix/client/unstable/io.element.msc4195/rtc/livekit/delegate_delayed_leave`; - const transportEndpoint = `${exampleTransport.livekit_service_url}/delegate_delayed_leave`; + const endpoint = `${exampleServiceUrl}/delegate_delayed_leave`; it.each([ - ["the homeserver supports it", 400, 404, true], - ["only the transport supports it", 404, 400, true], - ["neither supports it", 404, 404, false], - ])("%s", async (_name, homeserverStatus, transportStatus, expected) => { - fetchMock.post(homeserverEndpoint, homeserverStatus); - fetchMock.post(transportEndpoint, transportStatus); + ["anything but a 404", 401, true], + ["a 404", 404, false], + ])("reads %s as the answer", async (_name, status, expected) => { + fetchMock.post(endpoint, status); await expect( - supportsDelegation(homeserver, exampleTransport, logger), + supportsDelegation(endpoint, "transport", logger), ).resolves.toBe(expected); }); it("assumes no support where the probe itself fails", async () => { - fetchMock.post(homeserverEndpoint, { throws: new TypeError("CORS") }); - fetchMock.post(transportEndpoint, { throws: new TypeError("CORS") }); + fetchMock.post(endpoint, { throws: new TypeError("CORS") }); await expect( - supportsDelegation(homeserver, exampleTransport, logger), + supportsDelegation(endpoint, "transport", logger), ).resolves.toBe(false); }); }); describe("toMediaConnectionState", () => { - it("maps every LiveKit state and passes an error through", () => { - expect(toMediaConnectionState(ConnectionState.FetchingConfig)).toBe( + it("maps every state and passes an error through", () => { + expect(toMediaConnectionState({ state: "authenticating" })).toBe( MediaConnectionState.Preparing, ); - expect(toMediaConnectionState(ConnectionState.LivekitConnected)).toBe( - MediaConnectionState.Connected, - ); expect( - toMediaConnectionState(ConnectionState.LivekitSignalReconnecting), + toMediaConnectionState({ + state: "authenticated", + livekitState: LivekitConnectionState.Connected, + }), + ).toBe(MediaConnectionState.Connected); + expect( + toMediaConnectionState({ + state: "authenticated", + livekitState: LivekitConnectionState.SignalReconnecting, + }), ).toBe(MediaConnectionState.Reconnecting); - const error = new Error("gone"); - expect(toMediaConnectionState(error)).toBe(error); + expect(toMediaConnectionState({ state: "stopped" })).toBe( + MediaConnectionState.Stopped, + ); + const error = new MatrixRTCError("gone"); + expect(toMediaConnectionState({ state: "error", error })).toBe(error); }); }); + +describe("createLivekitBackend", () => { + const homeserver = "https://matrix.example.org"; + const homeserverEndpoint = `${homeserver}/_matrix/client/unstable/io.element.msc4195/rtc/livekit/delegate_delayed_leave`; + const urlTransport: UnstableLivekitTransport = { + type: "livekit", + url: "wss://sfu.example.org", + }; + const bothTransport: UnstableLivekitTransport = { + ...urlTransport, + ...exampleTransport, + }; + const client = { + baseUrl: homeserver, + getOpenIdToken: vi.fn(), + _unstable_getLivekitToken: vi.fn(), + _unstable_delegateDelayedLeave: vi.fn(), + }; + /** The connections the stubbed factory handed out, with their participants to drive. */ + let connections: { + connection: Connection; + participants$: BehaviorSubject; + state$: BehaviorSubject; + }[]; + const connected: ConnectionState = { + state: "authenticated", + livekitState: LivekitConnectionState.Connected, + }; + + /** Stands in for the factory: connections whose state and participants the test drives. */ + function stubFactory(initialState: ConnectionState): void { + vi.spyOn(connectionFactory, "LivekitConnectionFactory").mockImplementation( + function (this: unknown) { + return { + createConnection: ( + scope: ObservableScope, + role: auth.ConnectionRole, + transport: UnstableLivekitTransport, + serverName: string, + ): Connection => { + const participants$ = new BehaviorSubject([]); + const state$ = new BehaviorSubject(initialState); + const connection = { + role, + transport, + serverName, + state$, + remoteParticipants$: scope.behavior(participants$), + livekitRoom: fakeLivekitRoom(), + start: vi.fn(async () => Promise.resolve()), + stop: vi.fn(async () => Promise.resolve()), + } as unknown as Connection; + connections.push({ connection, participants$, state$ }); + return connection; + }, + } as unknown as connectionFactory.LivekitConnectionFactory; + }, + ); + } + + beforeEach(() => { + connections = []; + stubFactory(connected); + }); + afterEach(() => { + vi.restoreAllMocks(); + fetchMock.reset(); + }); + + function backend(): ReturnType { + return createLivekitBackend( + testScope(), + { + roomId: "!room:example.org", + slotId: "m.call#ROOM", + ownMembershipIdentity: ownMemberMock, + publish: [], + encryptionSystem: { kind: E2eeType.NONE }, + mediaKeys$: NEVER, + timings: defaultSessionTimings, + logger, + }, + { client }, + ); + } + + // Delegation goes through whichever service authenticates us as a + // publisher: the JWT service wherever the transport names one, the + // homeserver for a `url` alone + it.each([ + ["the JWT service", exampleTransport, 404, 401, true], + ["the JWT service, and only it", bothTransport, 401, 404, false], + ["the homeserver", urlTransport, 401, 404, true], + ["neither", exampleTransport, 404, 404, false], + ])( + "prepares the local transport and asks %s about delegation", + async (_name, transport, homeserverStatus, serviceStatus, expected) => { + fetchMock.post(homeserverEndpoint, homeserverStatus); + fetchMock.post( + `${exampleServiceUrl}/delegate_delayed_leave`, + serviceStatus, + ); + + const capabilities = await backend().prepareLocalTransport(transport); + + expect(capabilities).toEqual({ canDelegateDelayedLeave: expected }); + expect(connections).toHaveLength(1); + expect(connections[0].connection.role).toBe("publisher"); + expect(connections[0].connection.transport).toEqual(transport); + expect(connections[0].connection.serverName).toBe("example.org"); + }, + ); + + it("waits for the publisher connection to hold its token before the join", async () => { + fetchMock.post(homeserverEndpoint, 404); + fetchMock.post(`${exampleServiceUrl}/delegate_delayed_leave`, 404); + stubFactory({ state: "authenticating" }); + + let prepared = false; + const preparing = backend() + .prepareLocalTransport(exampleTransport) + .then(() => (prepared = true)); + await flushPromises(); + expect(prepared).toBe(false); + + connections[0].state$.next(connected); + await preparing; + expect(prepared).toBe(true); + }); + + it("fails the preparation with the publisher connection's error", async () => { + fetchMock.post(homeserverEndpoint, 404); + fetchMock.post(`${exampleServiceUrl}/delegate_delayed_leave`, 404); + stubFactory({ state: "authenticating" }); + const preparing = backend().prepareLocalTransport(exampleTransport); + await flushPromises(); + const error = new NoMatrix2AuthorizationService(new Error("404")); + connections[0].state$.next({ state: "error", error }); + await expect(preparing).rejects.toBe(error); + }); + + it("delegates the leave through the flow that authenticated the publisher", async () => { + fetchMock.post(homeserverEndpoint, 401); + const delegate = vi.spyOn(auth, "delegateDelayedLeave").mockResolvedValue(); + const b = backend(); + await expect(b.delegateDelayedLeave("delay-1")).rejects.toThrow( + "No local transport", + ); + + await b.prepareLocalTransport(urlTransport); + await b.delegateDelayedLeave("delay-1"); + + expect(delegate).toHaveBeenCalledWith( + expect.objectContaining({ + client, + membership: ownMemberMock, + transport: urlTransport, + roomId: "!room:example.org", + slotId: "m.call#ROOM", + delayId: "delay-1", + delayTimeoutMs: defaultSessionTimings.delegatedDelayedLeave.delay_ms, + }), + ); + }); + + describe("mediaFor", () => { + // A member advertising both a `url` and a `livekit_service_url` may have + // authenticated through either flow, so it has two candidate identities + const legacyId = "@bob:example.org:DEV000"; + const hashedId = "2qkXfWz1S7jQmU0bJ6yN8aVtR3pLcDgHe9oZxKwMiA4"; + + function bob( + transport: UnstableLivekitTransport, + backendIdentities: string[], + ): RTCMember { + return { + local: false, + rtcBackendIdentity: legacyId, + backendIdentities, + userId: "@bob:example.org", + deviceId: "DEV000", + memberId: legacyId, + transport$: constant({ + type: "livekit", + id: JSON.stringify(transport), + raw: transport, + resolved$: constant(undefined), + }), + applicationData$: constant({}), + displayName$: constant("Bob"), + avatarUrl$: constant(undefined), + }; + } + + async function prepared(): Promise< + ReturnType + > { + fetchMock.post(homeserverEndpoint, 404); + fetchMock.post(`${exampleServiceUrl}/delegate_delayed_leave`, 404); + const b = backend(); + await b.prepareLocalTransport(bothTransport); + return b; + } + + it.each([ + ["legacy", legacyId], + ["hashed", hashedId], + ])( + "matches a member with two candidate identities publishing under the %s one", + async (_name, identity) => { + const b = await prepared(); + const media = b.mediaFor( + testScope(), + bob(bothTransport, [hashedId, legacyId]), + ); + await flushPromises(); + // Bob is on our transport, so his media arrives on the publisher + // connection + expect(connections).toHaveLength(1); + expect(media.tracks$.value).toBeNull(); + + connections[0].participants$.next([ + mockRemoteParticipant({ identity, trackPublications: new Map() }), + ]); + await flushPromises(); + expect(media.tracks$.value).toEqual([]); + }, + ); + + it("does not match a member with two candidate identities to anyone else", async () => { + const b = await prepared(); + const media = b.mediaFor( + testScope(), + bob(bothTransport, [hashedId, legacyId]), + ); + connections[0].participants$.next([ + mockRemoteParticipant({ + identity: "@stranger:example.org:DEV999", + trackPublications: new Map(), + }), + ]); + await flushPromises(); + expect(media.tracks$.value).toBeNull(); + }); + + it("connects to a remote member's transport on the member's homeserver", async () => { + const b = await prepared(); + const carlsTransport: UnstableLivekitTransport = { + type: "livekit", + url: "wss://sfu.sample.com", + }; + b.mediaFor(testScope(), { + ...bob(carlsTransport, ["carl-hashed"]), + userId: "@carl:sample.com", + }); + await flushPromises(); + + const remote = connections.find( + (c) => c.connection.role === "subscriber", + ); + expect(remote?.connection.transport).toEqual(carlsTransport); + expect(remote?.connection.serverName).toBe("sample.com"); + }); + }); +}); + +/** The parts of a LiveKit room the local media and the member media read. */ +function fakeLivekitRoom(): LivekitRoom { + return { + options: {}, + setE2EEEnabled: vi.fn(), + localParticipant: { + ...mockEmitter(), + trackPublications: new Map(), + getTrackPublication: () => undefined, + }, + remoteParticipants: new Map(), + ...mockEmitter(), + } as unknown as LivekitRoom; +} diff --git a/sdk/src/media-backend/livekit/LivekitBackend.ts b/sdk/src/media-backend/livekit/LivekitBackend.ts index 0ba67bf37..db7be819c 100644 --- a/sdk/src/media-backend/livekit/LivekitBackend.ts +++ b/sdk/src/media-backend/livekit/LivekitBackend.ts @@ -8,71 +8,86 @@ Please see LICENSE in the repository root for full details. import { type MatrixClient } from "matrix-js-sdk"; import { type Logger } from "matrix-js-sdk/lib/logger"; import { - isLivekitTransport, - type LivekitTransport, + isUnstableLivekitTransport, type Transport, } from "matrix-js-sdk/lib/matrixrtc"; import { BehaviorSubject, combineLatest, distinctUntilChanged, + filter, + firstValueFrom, map, type Observable, of, - ReplaySubject, switchMap, } from "rxjs"; import { type ResolvedTransport, type RTCMember } from "../../api"; - import { type MediaQuality } from "../../config"; import { MatrixRTCError } from "../../errors"; -import { type Behavior } from "../../reactive/Behavior"; import { type MemberMedia } from "../../media-api"; +import { type Behavior } from "../../reactive/Behavior"; import { - type Epoch, + Epoch, type ObservableScope, trackEpoch, } from "../../reactive/ObservableScope"; +import { mapScoped } from "../../utils/mapScoped"; +import { serverNameOf } from "../../utils/serverName"; import { type BackendConnection, type MediaBackend, type MediaBackendContext, type TransportCapabilities, } from "../api"; +import { + chooseAuthFlow, + delegateDelayedLeave, + type TransportLocator, +} from "./auth"; import { type Connection } from "./Connection"; +import { + ConnectionMap, + createConnectionMap$, + keyFor, + type TransportKey, +} from "./ConnectionMap"; import { LivekitConnectionFactory, ResolvedConnection, } from "./ConnectionFactory"; -import { - createConnectionManager$, - type LocalTransport, -} from "./ConnectionManager"; import { toMediaConnectionState } from "./connectionState"; import { createDataChannel$ } from "./DataChannel"; import { createKeyProvider } from "./KeyProvider"; import { createLivekitMemberMedia } from "./LivekitMemberMedia"; import { createLivekitLocalMedia } from "./LocalMedia"; -import { getSFUConfigWithOpenID, type TokenEndpoint } from "./openIDSFU"; export interface LivekitBackendOptions { - /** `getOpenIdToken` for the token exchange, `baseUrl` for the delegation probe and handover. */ - client: Pick; + /** + * `getOpenIdToken` for the JWT service flow, the MSC4195 methods for the + * client-server API flow, `baseUrl` for the delegation probe and handover. + */ + client: Pick< + MatrixClient, + | "getOpenIdToken" + | "_unstable_getLivekitToken" + | "_unstable_delegateDelayedLeave" + | "baseUrl" + >; mediaQuality?: MediaQuality; - /** Which token service endpoint the local transport uses; remote ones try both. */ - tokenEndpoint: TokenEndpoint; } /** * MatrixRTC media over LiveKit SFUs: one room per transport, the local one - * published on, every remote one subscribed to, with the token service - * protocol in front of each. + * published on, every remote one subscribed to, each authenticated for its + * role through the homeserver or the transport's JWT service. */ export function createLivekitBackend( scope: ObservableScope, { roomId, + slotId, ownMembershipIdentity, publish, encryptionSystem, @@ -80,102 +95,133 @@ export function createLivekitBackend( timings, logger: parentLogger, }: MediaBackendContext, - { client, mediaQuality, tokenEndpoint }: LivekitBackendOptions, + { client, mediaQuality }: LivekitBackendOptions, ): MediaBackend { const logger = parentLogger.getChild("[LivekitBackend]"); const keyProvider = createKeyProvider(encryptionSystem, mediaKeys$, logger); const transports = createTransportRegistry(scope); - const localTransport$ = new ReplaySubject(1); + const localTransport$ = new BehaviorSubject(null); const audioOutputDeviceId$ = new BehaviorSubject( undefined, ); + const connectionFactory = new LivekitConnectionFactory( + client, + roomId, + slotId, + publish, + audioOutputDeviceId$, + keyProvider, + mediaQuality, + ); - const connectionManager = createConnectionManager$({ + // A new map, in a scope of its own, whenever the local transport changes: + // the publisher role sits on one connection, and which one is fixed per map + const connectionMap$ = scope.behavior>( + mapScoped(scope, localTransport$, (mapScope, localTransport) => + createConnectionMap$({ + scope: mapScope, + connectionFactory, + localTransport, + remoteTransports$: transports.transports$, + logger, + ownMembershipIdentity, + }), + ).pipe(switchMap((map$) => map$ ?? of(new Epoch(new ConnectionMap())))), + ); + + const localMedia_ = createLivekitLocalMedia({ scope, - connectionFactory: new LivekitConnectionFactory( - client, - roomId, - publish, - audioOutputDeviceId$, - keyProvider, - mediaQuality, - ), + connectionMap$, localTransport$, - remoteTransports$: transports.transports$, + publish, + audioOutputDeviceId$, + encryptionSystem, logger, - ownMembershipIdentity, }); - let local: LivekitTransport | undefined; + // Probed once: the answer cannot change within a session + const homeserverSupportsDelegation = supportsDelegation( + `${client.baseUrl}/_matrix/client/unstable/io.element.msc4195/rtc/livekit/delegate_delayed_leave`, + "homeserver", + logger, + ); + const prepareLocalTransport = async ( transport: Transport, ): Promise => { - if (!isLivekitTransport(transport)) + if (!isUnstableLivekitTransport(transport)) throw new MatrixRTCError(`Not a LiveKit transport: ${transport.type}`); - const [sfuConfig, canDelegateDelayedLeave] = await Promise.all([ - getSFUConfigWithOpenID( - client, - ownMembershipIdentity, - transport.livekit_service_url, - roomId, - { tokenEndpoint }, - logger, - ), - supportsDelegation(client.baseUrl, transport, logger), + const locator: TransportLocator = { + transport, + serverName: serverNameOf(ownMembershipIdentity.userId), + }; + // Delegation goes through whichever service authenticates us as a + // publisher, so only that service's support counts + const flow = chooseAuthFlow(transport, "publisher"); + const canDelegate = + flow.kind === "openID" + ? supportsDelegation( + `${flow.serviceUrl}/delegate_delayed_leave`, + `transport ${flow.serviceUrl}`, + logger, + ) + : homeserverSupportsDelegation; + localTransport$.next(locator); + // The membership is only sent once the publisher connection holds its + // token, so that the LiveKit room exists before anyone subscribes to it: + // a subscriber lacks the permission to create it and would fail without + // retrying + const [, canDelegateDelayedLeave] = await Promise.all([ + authenticated(localMedia_.connection$), + canDelegate, ]); - local = transport; - localTransport$.next({ transport, sfuConfig }); return { canDelegateDelayedLeave }; }; - // The token this issues is discarded; the request is what hands the leave over - const delegateDelayedLeave = async (delayId: string): Promise => { - if (local === undefined) + const delegateLeave = async (delayId: string): Promise => { + const local = localTransport$.value; + if (local === null) throw new MatrixRTCError("No local transport to delegate the leave to"); - await getSFUConfigWithOpenID( + await delegateDelayedLeave({ client, - ownMembershipIdentity, - local.livekit_service_url, + membership: ownMembershipIdentity, + transport: local.transport, roomId, - { - tokenEndpoint, - delayEndpointBaseUrl: client.baseUrl, - delayId, - delayTimeoutMs: timings.delegatedDelayedLeave.delay_ms, - }, + slotId, + delayId, + delayTimeoutMs: timings.delegatedDelayedLeave.delay_ms, logger, - ); + }); }; const mediaFor = ( memberScope: ObservableScope, member: RTCMember, ): MemberMedia => { - const identity = member.rtcBackendIdentity; - const transport$ = memberScope.behavior( + const { backendIdentities } = member; + const locator$ = memberScope.behavior( member.transport$.pipe( - map((transport) => transport?.raw), - map((transport) => - isLivekitTransport(transport) ? transport : undefined, - ), - distinctUntilChanged( - (a, b) => a?.livekit_service_url === b?.livekit_service_url, - ), + map((transport): TransportLocator | undefined => { + const raw = transport?.raw; + return isUnstableLivekitTransport(raw) + ? { transport: raw, serverName: serverNameOf(member.userId) } + : undefined; + }), + distinctUntilChanged((a, b) => (a && keyFor(a)) === (b && keyFor(b))), ), ); - transports.follow(memberScope, transport$); + transports.follow(memberScope, locator$); const source$ = memberScope.behavior( - combineLatest([ - transport$, - connectionManager.connectionManagerData$, - ]).pipe( - map(([transport, { value: connections }]) => { - if (transport === undefined) return null; - const connection = connections.getConnectionForTransport(transport); + combineLatest([locator$, connectionMap$]).pipe( + map(([locator, { value: connections }]) => { + if (locator === undefined) return null; + const connection = connections.getConnectionForTransport(locator); + // A member advertising both a `url` and a `livekit_service_url` may + // have authenticated through either flow, under either identity const participant = connections - .getParticipantsForTransport(transport) - .find((p) => p.identity === identity); + .getParticipantsForTransport(locator) + .find((p) => backendIdentities.includes(p.identity)); return connection && participant ? { participant, room: connection.livekitRoom } : null; @@ -187,14 +233,14 @@ export function createLivekitBackend( ); source$.pipe(memberScope.bind()).subscribe((source) => { logger.info( - `${identity}: LiveKit participant ${source ? `matched (${source.participant.sid})` : "missing"}`, + `${member.rtcBackendIdentity}: LiveKit participant ${source ? `matched (${source.participant.sid})` : "missing"} on ${locator$.value ? JSON.stringify(locator$.value.transport) : "no transport"}`, ); }); return createLivekitMemberMedia(memberScope, source$, encryptionSystem); }; const connections$ = scope.behavior( - connectionManager.connectionManagerData$.pipe( + connectionMap$.pipe( switchMap(({ value }) => { const connections = value.getConnections(); if (connections.length === 0) return of([]); @@ -203,26 +249,16 @@ export function createLivekitBackend( ), ); - const localMedia_ = createLivekitLocalMedia({ - scope, - connectionManager, - localTransport$, - publish, - audioOutputDeviceId$, - encryptionSystem, - logger, - }); - const { data$, sendData } = createDataChannel$({ scope, - connectionManager, + connectionMap$, connection$: localMedia_.connection$, }); return { transportType: "livekit", prepareLocalTransport, - delegateDelayedLeave, + delegateDelayedLeave: delegateLeave, local: localMedia_, mediaFor, connections$, @@ -232,6 +268,29 @@ export function createLivekitBackend( }; } +/** Resolves once the connection holds its token; rejects with its error otherwise. */ +async function authenticated( + connection$: Behavior, +): Promise { + const state = await firstValueFrom( + connection$.pipe( + switchMap((connection) => connection?.state$ ?? of(null)), + filter( + (state) => + state !== null && + (state.state === "authenticated" || + state.state === "error" || + state.state === "stopped"), + ), + ), + ); + if (state?.state === "error") throw state.error; + if (state?.state === "stopped") + throw new MatrixRTCError( + "The local connection stopped before it was authenticated", + ); +} + function backendConnection$( connection: Connection, ): Observable { @@ -252,10 +311,10 @@ export interface TransportRegistry { /** Keeps a member's transport registered for as long as its scope lives. */ follow( scope: ObservableScope, - transport$: Behavior, + locator$: Behavior, ): void; - /** Every transport some member is on, for the connection manager. */ - transports$: Behavior>; + /** Every transport some member is on, for the connection map. */ + transports$: Behavior>; } /** @@ -267,34 +326,34 @@ export function createTransportRegistry( scope: ObservableScope, ): TransportRegistry { const counts = new Map< - string, - { transport: LivekitTransport; count: number } + TransportKey, + { locator: TransportLocator; count: number } >(); - const transports = new BehaviorSubject([]); + const transports = new BehaviorSubject([]); const publish = (): void => - transports.next([...counts.values()].map(({ transport }) => transport)); + transports.next([...counts.values()].map(({ locator }) => locator)); - const register = (transport: LivekitTransport): (() => void) => { - const url = transport.livekit_service_url; - const entry = counts.get(url) ?? { transport, count: 0 }; + const register = (locator: TransportLocator): (() => void) => { + const key = keyFor(locator); + const entry = counts.get(key) ?? { locator, count: 0 }; entry.count++; if (entry.count === 1) { - counts.set(url, entry); + counts.set(key, entry); publish(); } return () => { if (--entry.count > 0) return; - counts.delete(url); + counts.delete(key); publish(); }; }; return { - follow: (memberScope, transport$) => { + follow: (memberScope, locator$) => { let release: (() => void) | undefined; - transport$.pipe(memberScope.bind()).subscribe((transport) => { + locator$.pipe(memberScope.bind()).subscribe((locator) => { release?.(); - release = transport && register(transport); + release = locator && register(locator); }); memberScope.onEnd(() => release?.()); }, @@ -303,33 +362,12 @@ export function createTransportRegistry( } /** - * Whether the SFU can take over restarting the delayed leave. Either the - * homeserver or the transport has to support it. Each endpoint is hit without - * credentials and read for a 404, not retried: many servers predate the - * endpoint altogether and answer with a CORS failure that a retry loop would - * only repeat. + * Whether a service can take over restarting the delayed leave. The endpoint + * is hit without credentials and read for a 404, not retried: many servers + * predate the endpoint altogether and answer with a CORS failure that a retry + * loop would only repeat. */ export async function supportsDelegation( - homeserverUrl: string, - transport: LivekitTransport, - logger: Logger, -): Promise { - const [homeserver, sfu] = await Promise.all([ - endpointExists( - `${homeserverUrl}/_matrix/client/unstable/io.element.msc4195/rtc/livekit/delegate_delayed_leave`, - "homeserver", - logger, - ), - endpointExists( - `${transport.livekit_service_url}/delegate_delayed_leave`, - `transport ${transport.livekit_service_url}`, - logger, - ), - ]); - return homeserver || sfu; -} - -async function endpointExists( url: string, serviceName: string, logger: Logger, diff --git a/sdk/src/media-backend/livekit/LocalMedia.ts b/sdk/src/media-backend/livekit/LocalMedia.ts index 472cc4f54..bb5a85249 100644 --- a/sdk/src/media-backend/livekit/LocalMedia.ts +++ b/sdk/src/media-backend/livekit/LocalMedia.ts @@ -35,13 +35,14 @@ import { import { type EncryptionSystem } from "../../encryption"; import { FailToStartLivekitConnection, toMatrixRTCError } from "../../errors"; import { type Behavior } from "../../reactive/Behavior"; -import { type ObservableScope } from "../../reactive/ObservableScope"; -import { type LocalMediaBackend, MediaConnectionState } from "../api"; -import { type Connection } from "./Connection"; import { - type IConnectionManager, - type LocalTransport, -} from "./ConnectionManager"; + type Epoch, + type ObservableScope, +} from "../../reactive/ObservableScope"; +import { type LocalMediaBackend, MediaConnectionState } from "../api"; +import { type TransportLocator } from "./auth"; +import { type Connection } from "./Connection"; +import { type ConnectionMap } from "./ConnectionMap"; import { createLocalLivekitMemberMedia } from "./LivekitMemberMedia"; import { livekitSources } from "./LivekitMediaTrack"; import { type DesiredMedia, Publisher } from "./Publisher"; @@ -49,8 +50,9 @@ import { toMediaConnectionState } from "./connectionState"; interface Props { scope: ObservableScope; - connectionManager: IConnectionManager; - localTransport$: Observable; + connectionMap$: Behavior>; + /** The transport we publish on; null until the session has found it. */ + localTransport$: Behavior; publish: PublishRequest[]; /** Read by the connection factory for the rooms it creates from now on. */ audioOutputDeviceId$: BehaviorSubject; @@ -71,7 +73,7 @@ export interface LivekitLocalMedia extends LocalMediaBackend { */ export function createLivekitLocalMedia({ scope, - connectionManager, + connectionMap$, localTransport$, publish: initialRequests, audioOutputDeviceId$, @@ -81,15 +83,11 @@ export function createLivekitLocalMedia({ const logger = parentLogger.getChild("[LocalMedia]"); const connection$ = scope.behavior( - combineLatest([ - connectionManager.connectionManagerData$, - localTransport$, - ]).pipe( - map(([{ value: connections }, { transport }]) => - connections.getConnectionForTransport(transport), - ), + combineLatest( + [connectionMap$, localTransport$], + ({ value: connections }, transport) => + transport && connections.getConnectionForTransport(transport), ), - null, ); const connectionState$ = scope.behavior( @@ -117,9 +115,7 @@ export function createLivekitLocalMedia({ const publisher = new Publisher( connection.livekitRoom, desired, - logger.getChild( - `[Publisher ${connection.transport.livekit_service_url}]`, - ), + logger.getChild(`[Publisher ${JSON.stringify(connection.transport)}]`), ); publisher$.next(publisher); return Promise.resolve(async (): Promise => { diff --git a/sdk/src/media-backend/livekit/auth/csApi.ts b/sdk/src/media-backend/livekit/auth/csApi.ts new file mode 100644 index 000000000..fd38f202a --- /dev/null +++ b/sdk/src/media-backend/livekit/auth/csApi.ts @@ -0,0 +1,102 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { ConnectionError, HTTPError, type MatrixClient } from "matrix-js-sdk"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; + +import { doNetworkOperationWithRetry } from "../../../utils/network"; +import { extractFullConfigFromToken, type SFUConfig } from "./types"; + +/** + * In widget mode the host may refuse an action before any request is made, + * which surfaces as a plain error. Asking again cannot change its answer, so + * only errors from the network are worth retrying. + */ +function isNetworkError(e: unknown): boolean { + return e instanceof HTTPError || e instanceof ConnectionError; +} + +export type ClientGetTokenParts = Pick< + MatrixClient, + "_unstable_getLivekitToken" +>; + +export interface GetSFUConfigParams { + client: ClientGetTokenParts; + /** Our own membership identity; the member id is what the token is issued for. */ + membership: CallMembershipIdentityParts; + /** The WebSocket URL of the SFU to get a token for. */ + url: string; + /** The homeserver the SFU belongs to. */ + serverName: string; + roomId: string; + slotId: string; +} + +/** Gets an SFU token through the homeserver's MSC4195 `get_token` endpoint. */ +export async function getSFUConfig({ + client, + membership, + url, + serverName, + roomId, + slotId, +}: GetSFUConfigParams): Promise { + const res = await doNetworkOperationWithRetry( + async () => + client._unstable_getLivekitToken({ + url, + server_name: serverName, + room_id: roomId, + slot_id: slotId, + member_id: membership.memberId, + }), + isNetworkError, + ); + return extractFullConfigFromToken({ url, jwt: res.jwt }); +} + +export type ClientDelegationParts = Pick< + MatrixClient, + "_unstable_delegateDelayedLeave" | "baseUrl" +>; + +export interface DelegateDelayedLeaveParams { + client: ClientDelegationParts; + membership: CallMembershipIdentityParts; + /** The WebSocket URL of the SFU we publish on. */ + url: string; + roomId: string; + slotId: string; + /** The delay ID of the leave event to hand over. */ + delayId: string; +} + +/** + * Hands the delayed leave to the homeserver through the MSC4195 + * `delegate_delayed_leave` endpoint, which sends it once we drop off the SFU. + */ +export async function delegateDelayedLeave({ + client, + membership, + url, + roomId, + slotId, + delayId, +}: DelegateDelayedLeaveParams): Promise { + await doNetworkOperationWithRetry( + async () => + client._unstable_delegateDelayedLeave({ + url, + room_id: roomId, + slot_id: slotId, + member_id: membership.memberId, + delay_id: delayId, + }), + isNetworkError, + ); +} diff --git a/sdk/src/media-backend/livekit/auth/index.test.ts b/sdk/src/media-backend/livekit/auth/index.test.ts new file mode 100644 index 000000000..bce375312 --- /dev/null +++ b/sdk/src/media-backend/livekit/auth/index.test.ts @@ -0,0 +1,407 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { afterEach, describe, expect, it, type Mocked, vi } from "vitest"; +import fetchMock from "fetch-mock"; +import { ConnectionError, MatrixError } from "matrix-js-sdk"; +import { logger } from "matrix-js-sdk/lib/logger"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; + +import { NoMatrix2AuthorizationService } from "../../../errors"; +import { + aliceDeviceId, + aliceUserId, + createJWTToken, +} from "../../../utils/test-fixtures"; +import type * as csApi from "./csApi"; +import type * as openID from "./openID"; +import { chooseAuthFlow, delegateDelayedLeave, getSFUConfig } from "."; + +const aliceId = `${aliceUserId}:${aliceDeviceId}`; +const csApiJWTToken = createJWTToken(aliceId, "via CS API"); +const legacyJWTToken = createJWTToken(aliceId, "via legacy endpoint"); +const defaultJWTToken = createJWTToken(aliceId, "via default endpoint"); + +const client: Mocked< + csApi.ClientGetTokenParts & + csApi.ClientDelegationParts & + openID.ClientOpenIDParts +> = { + baseUrl: "https://matrix.example.org", + _unstable_getLivekitToken: vi.fn().mockResolvedValue({ jwt: csApiJWTToken }), + _unstable_delegateDelayedLeave: vi.fn().mockResolvedValue({}), + getOpenIdToken: vi + .fn() + .mockResolvedValue({ access_token: "mock-openID-token" }), +}; + +const membership: CallMembershipIdentityParts = { + userId: aliceUserId, + deviceId: aliceDeviceId, + memberId: "123", +}; + +const roomId = "!room:example.org"; +const slotId = "m.call#ROOM"; + +const unsupportedError = new MatrixError({ errcode: "M_UNSUPPORTED" }, 404); + +const urlTransport: UnstableLivekitTransport = { + type: "livekit", + url: "wss://livekit.example.org", +}; + +const serviceUrl = "https://livekit-jwt.example.org"; +const serviceUrlTransport: UnstableLivekitTransport = { + type: "livekit", + livekit_service_url: serviceUrl, +}; + +const urlAndServiceUrlTransport: UnstableLivekitTransport = { + ...urlTransport, + ...serviceUrlTransport, +}; + +/** The JWT service, answering each endpoint with a token that names it. */ +function mockJWTService(): void { + fetchMock.post(`${serviceUrl}/sfu/get`, { + status: 200, + body: { url: "wss://sfu", jwt: legacyJWTToken }, + }); + fetchMock.post(`${serviceUrl}/get_token`, { + status: 200, + body: { url: "wss://sfu", jwt: defaultJWTToken }, + }); +} + +/** Whether the JWT service was asked at all, and which endpoint. */ +function jwtServiceCalls(): string[] { + return fetchMock.calls().map(([url]) => url.replace(serviceUrl, "")); +} + +afterEach(() => { + vi.clearAllMocks(); + fetchMock.reset(); +}); + +describe("getSFUConfig", () => { + const defaultParams = { + client, + membership, + serverName: "example.org", + role: "subscriber" as const, + roomId, + slotId, + logger, + }; + + it("uses CS API with url transport", async () => { + mockJWTService(); + expect( + await getSFUConfig({ ...defaultParams, transport: urlTransport }), + ).toEqual( + expect.objectContaining({ + jwt: csApiJWTToken, + livekitAlias: "via CS API", + }), + ); + expect(client._unstable_getLivekitToken).toHaveBeenCalledWith({ + url: urlTransport.url, + server_name: "example.org", + room_id: roomId, + slot_id: slotId, + member_id: "123", + }); + expect(jwtServiceCalls()).toEqual([]); + }); + + it("does not fall back to OpenID with url transport", async () => { + mockJWTService(); + client._unstable_getLivekitToken.mockRejectedValueOnce(unsupportedError); + await expect( + getSFUConfig({ ...defaultParams, transport: urlTransport }), + ).rejects.toThrow(expect.any(NoMatrix2AuthorizationService)); + expect(client._unstable_getLivekitToken).toHaveBeenCalled(); + expect(jwtServiceCalls()).toEqual([]); + }); + + it("uses OpenID with livekit_service_url transport", async () => { + mockJWTService(); + expect( + await getSFUConfig({ ...defaultParams, transport: serviceUrlTransport }), + ).toEqual( + expect.objectContaining({ + jwt: defaultJWTToken, + livekitAlias: "via default endpoint", + }), + ); + expect(client._unstable_getLivekitToken).not.toHaveBeenCalled(); + expect(jwtServiceCalls()).toEqual(["/get_token"]); + }); + + it("uses CS API when subscribing with url + livekit_service_url transport", async () => { + mockJWTService(); + expect( + await getSFUConfig({ + ...defaultParams, + transport: urlAndServiceUrlTransport, + }), + ).toEqual( + expect.objectContaining({ + jwt: csApiJWTToken, + livekitAlias: "via CS API", + }), + ); + expect(client._unstable_getLivekitToken).toHaveBeenCalled(); + expect(jwtServiceCalls()).toEqual([]); + }); + + it("falls back to OpenID with url + livekit_service_url transport", async () => { + mockJWTService(); + client._unstable_getLivekitToken.mockRejectedValueOnce(unsupportedError); + expect( + await getSFUConfig({ + ...defaultParams, + transport: urlAndServiceUrlTransport, + }), + ).toEqual( + expect.objectContaining({ + jwt: defaultJWTToken, + livekitAlias: "via default endpoint", + }), + ); + expect(client._unstable_getLivekitToken).toHaveBeenCalled(); + expect(jwtServiceCalls()).toEqual(["/get_token"]); + }); + + it("falls back to OpenID when the widget host refuses the CS API", async () => { + mockJWTService(); + // In widget mode a host without MSC4533 rejects the request before it + // reaches the homeserver, with a plain error rather than a Matrix error + client._unstable_getLivekitToken.mockRejectedValueOnce( + new Error( + "The rtc_livekit_get_token action is not supported by the client.", + ), + ); + expect( + await getSFUConfig({ + ...defaultParams, + transport: urlAndServiceUrlTransport, + }), + ).toEqual(expect.objectContaining({ jwt: defaultJWTToken })); + expect(jwtServiceCalls()).toEqual(["/get_token"]); + }); + + it("surfaces the widget host's refusal with url transport", async () => { + mockJWTService(); + const error = new Error( + "The rtc_livekit_get_token action is not supported by the client.", + ); + client._unstable_getLivekitToken.mockRejectedValueOnce(error); + await expect( + getSFUConfig({ ...defaultParams, transport: urlTransport }), + ).rejects.toThrow(error); + expect(jwtServiceCalls()).toEqual([]); + }); + + it("uses CS API when publishing with url transport", async () => { + mockJWTService(); + expect( + await getSFUConfig({ + ...defaultParams, + transport: urlTransport, + role: "publisher", + }), + ).toEqual(expect.objectContaining({ jwt: csApiJWTToken })); + expect(client._unstable_getLivekitToken).toHaveBeenCalled(); + expect(jwtServiceCalls()).toEqual([]); + }); + + it("does not fall back to OpenID when publishing", async () => { + mockJWTService(); + client._unstable_getLivekitToken.mockRejectedValueOnce(unsupportedError); + await expect( + getSFUConfig({ + ...defaultParams, + transport: urlTransport, + role: "publisher", + }), + ).rejects.toThrow(expect.any(NoMatrix2AuthorizationService)); + expect(client._unstable_getLivekitToken).toHaveBeenCalled(); + expect(jwtServiceCalls()).toEqual([]); + }); + + it("uses the legacy OpenID endpoint when publishing with url + livekit_service_url transport", async () => { + mockJWTService(); + // Subscribers predating MSC4195 expect a legacy participant identity from + // anyone advertising a membership, which only the JWT service assigns + expect( + await getSFUConfig({ + ...defaultParams, + transport: urlAndServiceUrlTransport, + role: "publisher", + }), + ).toEqual( + expect.objectContaining({ + jwt: legacyJWTToken, + livekitAlias: "via legacy endpoint", + }), + ); + expect(client._unstable_getLivekitToken).not.toHaveBeenCalled(); + expect(jwtServiceCalls()).toEqual(["/sfu/get"]); + }); + + it("does not fall back to OpenID on connection error", async () => { + vi.useFakeTimers(); + try { + mockJWTService(); + const client_ = { + ...client, + _unstable_getLivekitToken: vi + .fn() + .mockRejectedValue(new ConnectionError("No connection")), + }; + await Promise.all([ + expect( + getSFUConfig({ + ...defaultParams, + client: client_, + transport: urlAndServiceUrlTransport, + }), + ).rejects.toThrow(expect.any(ConnectionError)), + vi.runAllTimersAsync(), // Allow request to be retried + ]); + expect(client_._unstable_getLivekitToken).toHaveBeenCalled(); + expect(jwtServiceCalls()).toEqual([]); + } finally { + vi.useRealTimers(); + } + }); +}); + +describe("delegateDelayedLeave", () => { + const defaultParams = { + client, + membership, + roomId, + slotId, + delayId: "delay123", + delayTimeoutMs: 3_600_000, + logger, + }; + + /** The delay fields the JWT service was handed, if any. */ + function delegationViaJWTService(): object | undefined { + const call = fetchMock.calls().find(([url]) => url.endsWith("/sfu/get")); + if (call === undefined) return undefined; + const { delay_id, delay_timeout, delay_cs_api_url } = JSON.parse( + call[1]?.body as string, + ); + return { delay_id, delay_timeout, delay_cs_api_url }; + } + + it("uses CS API with url transport", async () => { + mockJWTService(); + await delegateDelayedLeave({ ...defaultParams, transport: urlTransport }); + expect(client._unstable_delegateDelayedLeave).toHaveBeenCalledWith({ + url: urlTransport.url, + room_id: roomId, + slot_id: slotId, + member_id: "123", + delay_id: "delay123", + }); + expect(jwtServiceCalls()).toEqual([]); + }); + + it("does not fall back to OpenID with url transport", async () => { + mockJWTService(); + client._unstable_delegateDelayedLeave.mockRejectedValueOnce( + unsupportedError, + ); + await expect( + delegateDelayedLeave({ ...defaultParams, transport: urlTransport }), + ).rejects.toThrow(); + expect(client._unstable_delegateDelayedLeave).toHaveBeenCalled(); + expect(jwtServiceCalls()).toEqual([]); + }); + + it("uses OpenID with livekit_service_url transport", async () => { + mockJWTService(); + await delegateDelayedLeave({ + ...defaultParams, + transport: serviceUrlTransport, + }); + expect(client._unstable_delegateDelayedLeave).not.toHaveBeenCalled(); + expect(delegationViaJWTService()).toEqual({ + delay_id: "delay123", + delay_timeout: 3_600_000, + delay_cs_api_url: client.baseUrl, + }); + }); + + it("uses OpenID with url + livekit_service_url transport", async () => { + mockJWTService(); + // Must match the flow that got the token, so the identities line up + await delegateDelayedLeave({ + ...defaultParams, + transport: urlAndServiceUrlTransport, + }); + expect(client._unstable_delegateDelayedLeave).not.toHaveBeenCalled(); + expect(delegationViaJWTService()).toEqual({ + delay_id: "delay123", + delay_timeout: 3_600_000, + delay_cs_api_url: client.baseUrl, + }); + }); +}); + +describe("chooseAuthFlow", () => { + it.each([ + [ + "url", + "publisher", + urlTransport, + { kind: "csApi", url: urlTransport.url }, + ], + [ + "url", + "subscriber", + urlTransport, + { kind: "csApi", url: urlTransport.url }, + ], + [ + "livekit_service_url", + "publisher", + serviceUrlTransport, + { kind: "openID", serviceUrl }, + ], + [ + "livekit_service_url", + "subscriber", + serviceUrlTransport, + { kind: "openID", serviceUrl }, + ], + [ + "url + livekit_service_url", + "publisher", + urlAndServiceUrlTransport, + { kind: "openID", serviceUrl }, + ], + [ + "url + livekit_service_url", + "subscriber", + urlAndServiceUrlTransport, + { kind: "csApi", url: urlTransport.url }, + ], + ] as const)( + "chooses the flow for a %s transport as %s", + (_shape, role, transport, flow) => { + expect(chooseAuthFlow(transport, role)).toEqual(flow); + }, + ); +}); diff --git a/sdk/src/media-backend/livekit/auth/index.ts b/sdk/src/media-backend/livekit/auth/index.ts new file mode 100644 index 000000000..a9756cf5e --- /dev/null +++ b/sdk/src/media-backend/livekit/auth/index.ts @@ -0,0 +1,212 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +/** + * Authentication with a LiveKit transport: the MSC4195 client-server API + * through our homeserver for a transport with an SFU `url`, the OpenID flow + * against the JWT service for one with a `livekit_service_url`, and the + * choice between them where a transport carries both. + */ + +import { ConnectionError, MatrixError } from "matrix-js-sdk"; +import { type Logger } from "matrix-js-sdk/lib/logger"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; + +import { NoMatrix2AuthorizationService } from "../../../errors"; +import * as csApi from "./csApi"; +import * as openID from "./openID"; +import { type ConnectionRole, type SFUConfig } from "./types"; + +export type { ClientOpenIDParts } from "./openID"; +export type { ClientGetTokenParts, ClientDelegationParts } from "./csApi"; +export type * from "./types"; + +/** + * Whether an error means the homeserver does not implement the MSC4195 + * endpoints. The MSC says `M_NOT_FOUND`; a homeserver without support answers + * with whatever it uses for unknown routes (Synapse: 404 with + * `M_UNRECOGNIZED`), so any 404 counts. + */ +function isEndpointUnsupported(e: unknown): boolean { + return ( + e instanceof MatrixError && + (e.httpStatus === 404 || + e.errcode === "M_NOT_FOUND" || + e.errcode === "M_UNRECOGNIZED") + ); +} + +/** + * Whether an error means the MSC4195 client-server API cannot be used from + * here: the homeserver does not implement it, or the request never reached + * the homeserver because a widget host does not expose the endpoint + * (MSC4533), which the widget API reports as a plain error. A connection + * error is neither: the endpoint may well exist. + */ +function isCsApiUnavailable(e: unknown): boolean { + if (isEndpointUnsupported(e)) return true; + return !(e instanceof MatrixError) && !(e instanceof ConnectionError); +} + +export interface GetSFUConfigParams { + client: csApi.ClientGetTokenParts & openID.ClientOpenIDParts; + membership: CallMembershipIdentityParts; + transport: UnstableLivekitTransport; + /** The homeserver the transport belongs to. */ + serverName: string; + roomId: string; + slotId: string; + role: ConnectionRole; + logger: Logger; +} + +/** Authenticates with a transport, through whichever flow suits it and our role. */ +export async function getSFUConfig({ + client, + membership, + transport, + serverName, + roomId, + slotId, + role, + logger, +}: GetSFUConfigParams): Promise { + const viaOpenID = async (serviceUrl: string): Promise => + await openID.getSFUConfig({ + client, + membership, + serviceUrl, + roomId, + slotId, + role, + logger, + }); + + const flow = chooseAuthFlow(transport, role); + if (flow.kind === "openID") return await viaOpenID(flow.serviceUrl); + + try { + return await csApi.getSFUConfig({ + client, + membership, + url: flow.url, + serverName, + roomId, + slotId, + }); + } catch (e) { + // A publisher never falls back: a homeserver that advertises a `url` but + // does not implement MSC4195 is misconfigured, and the admin should hear + // about it before `livekit_service_url` is deprecated + const mayFallBack = + role === "subscriber" && "livekit_service_url" in transport; + if (mayFallBack && isCsApiUnavailable(e)) { + logger.warn( + `Cannot use the MSC4195 get_token endpoint (${e}), falling back to the OpenID flow with service ${transport.livekit_service_url}`, + ); + return await viaOpenID(transport.livekit_service_url); + } + if (isEndpointUnsupported(e)) { + logger.error( + `Homeserver does not support the MSC4195 get_token endpoint, cannot get token for transport ${flow.url}`, + ); + throw new NoMatrix2AuthorizationService(e as Error); + } + throw e; + } +} + +export interface DelegateDelayedLeaveParams { + client: csApi.ClientDelegationParts & openID.ClientOpenIDParts; + membership: CallMembershipIdentityParts; + /** The transport we publish on. */ + transport: UnstableLivekitTransport; + roomId: string; + slotId: string; + delayId: string; + /** How long the service waits before sending the leave, for the JWT service flow. */ + delayTimeoutMs: number; + logger: Logger; +} + +/** + * Hands the delayed leave to the service that authenticated us as a + * publisher, so that the identity it watches for on the SFU is the one we + * publish under. + */ +export async function delegateDelayedLeave({ + client, + membership, + transport, + roomId, + slotId, + delayId, + delayTimeoutMs, + logger, +}: DelegateDelayedLeaveParams): Promise { + const flow = chooseAuthFlow(transport, "publisher"); + if (flow.kind === "csApi") { + try { + await csApi.delegateDelayedLeave({ + client, + membership, + url: flow.url, + roomId, + slotId, + delayId, + }); + } catch (e) { + if (isEndpointUnsupported(e)) + logger.error( + "Server misconfigured: transport contains a 'url' field but server does not support the MSC4195 delegate_delayed_leave endpoint", + ); + throw e; + } + } else { + // The token this issues is discarded; the request is what hands the leave over + await openID.getSFUConfig({ + client, + membership, + serviceUrl: flow.serviceUrl, + roomId, + slotId, + role: "publisher", + delayEndpointBaseUrl: client.baseUrl, + delayId, + delayTimeoutMs, + logger, + }); + } +} + +/** The flow with which the local member authenticates against a transport. */ +export type AuthFlow = + | { kind: "csApi"; url: string } + | { kind: "openID"; serviceUrl: string }; + +/** + * Chooses between the MSC4195 client-server API and the OpenID flow against + * the transport's JWT service. + * + * Only the JWT service's `/sfu/get` endpoint assigns the legacy participant + * identity that clients predating MSC4195 expect from anyone advertising a + * membership, so a publisher keeps using the JWT service wherever the + * transport still offers one, even next to a `url`. A subscriber may use the + * client-server API regardless, as nobody matches its identity. + */ +export function chooseAuthFlow( + transport: UnstableLivekitTransport, + role: ConnectionRole, +): AuthFlow { + if ("url" in transport) { + const useCsApi = + !("livekit_service_url" in transport) || role === "subscriber"; + if (useCsApi) return { kind: "csApi", url: transport.url }; + } + return { kind: "openID", serviceUrl: transport.livekit_service_url }; +} diff --git a/sdk/src/media-backend/livekit/auth/openID.test.ts b/sdk/src/media-backend/livekit/auth/openID.test.ts new file mode 100644 index 000000000..26cde69c4 --- /dev/null +++ b/sdk/src/media-backend/livekit/auth/openID.test.ts @@ -0,0 +1,229 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + beforeEach, + afterEach, + describe, + expect, + it, + type MockedObject, + vitest, +} from "vitest"; +import fetchMock from "fetch-mock"; +import { MatrixError } from "matrix-js-sdk"; +import { logger } from "matrix-js-sdk/lib/logger"; + +import { + getSFUConfig, + type GetSFUConfigParams, + type ClientOpenIDParts, +} from "./openID"; +import { testJWTToken } from "../../../utils/test-fixtures"; +import { ownMemberMock } from "../../../utils/test"; +import { FailToGetOpenIdToken } from "../../../errors"; + +const serviceUrl = "https://sfu.example.org"; + +const delayParams = { + delayEndpointBaseUrl: "https://matrix.homeserverserver.org", + delayId: "mock_delay_id", + delayTimeoutMs: 3_600_000, +}; + +describe("getSFUConfig", () => { + let matrixClient: MockedObject; + let params: GetSFUConfigParams; + beforeEach(() => { + fetchMock.catch(404); + matrixClient = { getOpenIdToken: vitest.fn() }; + params = { + client: matrixClient, + membership: ownMemberMock, + serviceUrl: "https://sfu.example.org", + roomId: "!example_room_id", + slotId: "m.call#ROOM", + role: "publisher", + logger, + }; + }); + afterEach(async () => { + void (await fetchMock.flush()); + vitest.clearAllMocks(); + fetchMock.reset(); + }); + + it("should handle fetching a token", async () => { + fetchMock.post("https://sfu.example.org/sfu/get", () => { + return { + status: 200, + body: { url: serviceUrl, jwt: testJWTToken }, + }; + }); + const config = await getSFUConfig(params); + expect(config).toEqual({ + jwt: testJWTToken, + url: serviceUrl, + livekitIdentity: "@me:example.org:ABCDEF", + livekitAlias: "!example_room_id", + }); + }); + + it("should fail if the SFU errors", async () => { + fetchMock.post("https://sfu.example.org/sfu/get", () => { + return { + status: 500, + body: { + errcode: "M_LOOKUP_FAILED", + error: "Failed to look up user info from homeserver", + }, + }; + }); + try { + await getSFUConfig(params); + } catch (ex: unknown) { + expect(ex).toBeInstanceOf(FailToGetOpenIdToken); + expect((ex as FailToGetOpenIdToken).cause).toBeInstanceOf(MatrixError); + const mxError = (ex as Error).cause as MatrixError; + expect(mxError.message).toEqual( + "MatrixError: [500] Failed to look up user info from homeserver", + ); + + return; + } + expect.fail("Expected test to throw;"); + }); + + it("should retry without delay params if the JWT service legacy endpoint returns M_BAD_JSON 400", async () => { + let callCount = 0; + + fetchMock.post( + "https://sfu.example.org/sfu/get", + (url, opts) => { + callCount++; + const body = JSON.parse(opts.body as string); + + // First call: check if it has delay parts and return 400 + if (callCount === 1) { + expect(body).toHaveProperty("delay_id", "mock_delay_id"); + return { + status: 400, + body: { errcode: "M_BAD_JSON", error: "Unsupported parameters" }, + }; + } + + // Second call: check if delay parts were stripped and return success + expect(body).not.toHaveProperty("delay_id"); + expect(body).not.toHaveProperty("delay_timeout"); + expect(body).not.toHaveProperty("delay_cs_api_url"); + + return { + status: 200, + body: { url: serviceUrl, jwt: testJWTToken }, + }; + }, + { overwriteRoutes: true }, + ); + + const config = await getSFUConfig({ ...params, ...delayParams }); + + expect(config.jwt).toBe(testJWTToken); + expect(callCount).toBe(2); + }); + + it("should successfully send delay parameters to the JWT service legacy endpoint", async () => { + fetchMock.post( + "https://sfu.example.org/sfu/get", + (url, opts) => { + const body = JSON.parse(opts.body as string); + + // Verify, that the request contains the expected delay parameters + if ( + body.delay_id === "mock_delay_id" && + body.delay_timeout === 3600000 && + body.delay_cs_api_url === "https://matrix.homeserverserver.org" + ) { + return { + status: 200, + body: { url: serviceUrl, jwt: testJWTToken }, + }; + } + return { + status: 400, + body: { error: "Missing expected delay params" }, + }; + }, + { overwriteRoutes: true }, + ); + + const config = await getSFUConfig({ ...params, ...delayParams }); + + expect(config).toMatchObject({ + jwt: testJWTToken, + url: serviceUrl, + }); + }); + + it("should use the default endpoint when subscribing", async () => { + fetchMock.post("https://sfu.example.org/get_token", (url, opts) => { + const body = JSON.parse(opts.body as string); + expect(body).toMatchObject({ + room_id: "!example_room_id", + slot_id: "m.call#ROOM", + member: { + id: ownMemberMock.memberId, + claimed_user_id: ownMemberMock.userId, + claimed_device_id: ownMemberMock.deviceId, + }, + }); + return { status: 200, body: { url: serviceUrl, jwt: testJWTToken } }; + }); + const config = await getSFUConfig({ ...params, role: "subscriber" }); + expect(config.jwt).toBe(testJWTToken); + expect(fetchMock.called("https://sfu.example.org/sfu/get")).toBe(false); + }); + + it("should fail when subscribing if the default endpoint is missing", async () => { + // Nothing but the catch-all 404 answers the default endpoint. Subscribers + // must not fall back to the legacy endpoint, which would hand them a + // legacy identity. + await expect( + getSFUConfig({ ...params, role: "subscriber" }), + ).rejects.toThrow(FailToGetOpenIdToken); + expect(fetchMock.called("https://sfu.example.org/get_token")).toBe(true); + expect(fetchMock.called("https://sfu.example.org/sfu/get")).toBe(false); + }); + + it("should retry fetching the openid token", async () => { + let count = 0; + matrixClient.getOpenIdToken.mockImplementation(async () => { + count++; + if (count < 2) { + throw Error("Test failure"); + } + return Promise.resolve({ + token_type: "Bearer", + access_token: "foobar", + matrix_server_name: "example.org", + expires_in: 30, + }); + }); + fetchMock.post("https://sfu.example.org/sfu/get", () => { + return { + status: 200, + body: { url: serviceUrl, jwt: testJWTToken }, + }; + }); + const config = await getSFUConfig({ ...params, ...delayParams }); + expect(config).toEqual({ + jwt: testJWTToken, + url: serviceUrl, + livekitIdentity: "@me:example.org:ABCDEF", + livekitAlias: "!example_room_id", + }); + }); +}); diff --git a/sdk/src/media-backend/livekit/auth/openID.ts b/sdk/src/media-backend/livekit/auth/openID.ts new file mode 100644 index 000000000..8b767662b --- /dev/null +++ b/sdk/src/media-backend/livekit/auth/openID.ts @@ -0,0 +1,203 @@ +/* +Copyright 2023, 2024 New Vector Ltd. +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + type IOpenIDToken, + type MatrixClient, + parseErrorResponse, +} from "matrix-js-sdk"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; +import { type Logger } from "matrix-js-sdk/lib/logger"; + +import { FailToGetOpenIdToken } from "../../../errors"; +import { doNetworkOperationWithRetry } from "../../../utils/network"; +import { + type ConnectionRole, + extractFullConfigFromToken, + type SFUConfig, +} from "./types"; + +export type ClientOpenIDParts = Pick; + +export interface GetSFUConfigParams { + client: ClientOpenIDParts; + /** Our own membership identity, as the JWT service wants it. */ + membership: CallMembershipIdentityParts; + /** The base URL of the LiveKit JWT service. */ + serviceUrl: string; + /** The Matrix room the session is in; the JWT service maps it to a LiveKit room. */ + roomId: string; + /** The MatrixRTC slot the session is in, e.g. `m.call#ROOM`. */ + slotId: string; + role: ConnectionRole; + /** The base URL of the Matrix homeserver, for delayed leave delegation. */ + delayEndpointBaseUrl?: string; + /** The delay ID of the leave event to delegate to the JWT service. */ + delayId?: string; + /** How long the SFU waits before sending the delegated leave. Needed with `delayId`. */ + delayTimeoutMs?: number; + logger: Logger; +} + +/** + * Authenticates with a LiveKit JWT service through the OpenID flow: an OpenID + * token from the homeserver, exchanged at the service for an SFU token. + * + * A publisher uses the legacy `/sfu/get` endpoint, the only one that assigns + * the `:` identity that clients predating MSC4195 match a + * membership against. A subscriber uses the `/get_token` endpoint, which + * hashes the identity: nobody matches a subscriber. + * @throws {@link FailToGetOpenIdToken} + */ +export async function getSFUConfig({ + client, + membership, + serviceUrl, + roomId, + slotId, + role, + delayEndpointBaseUrl, + delayId, + delayTimeoutMs, + logger, +}: GetSFUConfigParams): Promise { + let openIdToken: IOpenIDToken; + try { + openIdToken = await doNetworkOperationWithRetry(async () => + client.getOpenIdToken(), + ); + } catch (error) { + throw new FailToGetOpenIdToken( + error instanceof Error ? error : new Error("Unknown error"), + ); + } + logger.debug("Got openID token", openIdToken); + + const delay: DelayParams = + // Also check for empty string + delayId && delayEndpointBaseUrl + ? { + delay_id: delayId, + delay_timeout: delayTimeoutMs, + delay_cs_api_url: delayEndpointBaseUrl, + } + : {}; + const endpoint = role === "publisher" ? "legacy" : "default"; + try { + logger.info( + `Trying to get JWT with ${endpoint} endpoint for focus ${serviceUrl}...`, + ); + const sfuConfig = await ( + endpoint === "legacy" ? getLiveKitJWTLegacy : getLiveKitJWT + )(membership, serviceUrl, roomId, slotId, openIdToken, delay); + logger.info(`Got JWT from call's active focus URL.`); + return extractFullConfigFromToken(sfuConfig); + } catch (e) { + logger.error(`Failed fetching jwt with ${endpoint} endpoint:`, e); + throw new FailToGetOpenIdToken( + e instanceof Error ? e : new Error(`Unknown error ${e}`), + ); + } +} + +interface DelayParams { + delay_id?: string; + delay_timeout?: number; + delay_cs_api_url?: string; +} + +/** The legacy `/sfu/get` endpoint, which assigns the `:` identity. */ +async function getLiveKitJWTLegacy( + membership: CallMembershipIdentityParts, + livekitServiceURL: string, + matrixRoomId: string, + _slotId: string, // The legacy endpoint predates slots + openIDToken: IOpenIDToken, + delay: DelayParams, +): Promise<{ url: string; jwt: string }> { + const makeRequest = async (delayParts: DelayParams): Promise => + await fetch(livekitServiceURL + "/sfu/get", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + // The legacy JWT endpoint uses only the matrix room id to calculate + // the livekit room alias, which comes back in the JWT payload + room: matrixRoomId, + openid_token: openIDToken, + device_id: membership.deviceId, + ...delayParts, + }), + }); + + const res = await doNetworkOperationWithRetry(async () => { + let response = await makeRequest(delay); + // A service from before delegation rejects the delay fields as bad JSON; + // it still issues a token without them + if (response.status === 400 && Object.keys(delay).length > 0) { + try { + const errorBody = await response.json(); + if (errorBody.errcode === "M_BAD_JSON") + response = await makeRequest({}); + } catch { + // Not a JSON error body, so a real error + } + } + return response; + }); + + if (!res.ok) throw parseErrorResponse(res, await res.text()); + return await res.json(); +} + +class NotSupportedError extends Error { + public constructor(message: string) { + super(message); + this.name = "NotSupported"; + } +} + +/** The `/get_token` endpoint, which assigns a hashed identity. */ +async function getLiveKitJWT( + membership: CallMembershipIdentityParts, + livekitServiceURL: string, + matrixRoomId: string, + slotId: string, + openIDToken: IOpenIDToken, + delay: DelayParams, +): Promise<{ url: string; jwt: string }> { + const { userId, deviceId, memberId } = membership; + const res = await doNetworkOperationWithRetry( + async () => + await fetch(livekitServiceURL + "/get_token", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + room_id: matrixRoomId, + slot_id: slotId, + openid_token: openIDToken, + member: { + id: memberId, + claimed_user_id: userId, + claimed_device_id: deviceId, + }, + ...delay, + }), + }), + ); + + if (!res.ok) { + const msg = "SFU Config fetch failed with status code " + res.status; + if (res.status === 404) throw new NotSupportedError(msg); + throw parseErrorResponse(res, await res.text()); + } + return await res.json(); +} diff --git a/sdk/src/media-backend/livekit/auth/types.ts b/sdk/src/media-backend/livekit/auth/types.ts new file mode 100644 index 000000000..9c198f2d2 --- /dev/null +++ b/sdk/src/media-backend/livekit/auth/types.ts @@ -0,0 +1,58 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; + +/** + * A transport and the homeserver it belongs to. The server name is what our + * homeserver asks for a token for a transport with a `url`, over federation + * where the server is not ours; a JWT service is reached directly. + */ +export interface TransportLocator { + transport: UnstableLivekitTransport; + serverName: string; +} + +/** Whether we publish on a transport or only subscribe to it. */ +export type ConnectionRole = "publisher" | "subscriber"; + +/** What authenticating with a transport yields: where to connect and with what. */ +export interface SFUConfig { + /** + * The WebSocket URL of the SFU, as opposed to the JWT service URL in the + * transport. The legacy JWT service returns it; the client-server API flow + * has it from the transport. + */ + url: string; + jwt: string; + livekitAlias: string; + livekitIdentity: string; +} + +/** The parts of the JWT payload the backend reads. */ +interface SFUJWTPayload { + /** The subject: the participant identity the token carries. */ + sub: string; + video: { + room: string; + }; +} + +/** Completes the SFU websocket url and the JWT with what the JWT payload says. */ +export function extractFullConfigFromToken(sfuConfig: { + url: string; + jwt: string; +}): SFUConfig { + const [, payloadStr] = sfuConfig.jwt.split("."); + const payload = JSON.parse(global.atob(payloadStr)) as SFUJWTPayload; + return { + jwt: sfuConfig.jwt, + url: sfuConfig.url, + livekitAlias: payload.video.room, + livekitIdentity: payload.sub, + }; +} diff --git a/sdk/src/media-backend/livekit/connectionState.ts b/sdk/src/media-backend/livekit/connectionState.ts index eddbaddb2..444bd14d3 100644 --- a/sdk/src/media-backend/livekit/connectionState.ts +++ b/sdk/src/media-backend/livekit/connectionState.ts @@ -5,24 +5,34 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { MediaConnectionState } from "../api"; -import { ConnectionState } from "./Connection"; +import { ConnectionState as LivekitConnectionState } from "livekit-client"; -const states: Record = { - [ConnectionState.Initialized]: MediaConnectionState.Initialized, - [ConnectionState.FetchingConfig]: MediaConnectionState.Preparing, - [ConnectionState.Stopped]: MediaConnectionState.Stopped, - [ConnectionState.LivekitDisconnected]: MediaConnectionState.Disconnected, - [ConnectionState.LivekitConnecting]: MediaConnectionState.Connecting, - [ConnectionState.LivekitConnected]: MediaConnectionState.Connected, - [ConnectionState.LivekitReconnecting]: MediaConnectionState.Reconnecting, - [ConnectionState.LivekitSignalReconnecting]: +import { MediaConnectionState } from "../api"; +import { type ConnectionState } from "./Connection"; + +const livekitStates: Record = { + [LivekitConnectionState.Disconnected]: MediaConnectionState.Disconnected, + [LivekitConnectionState.Connecting]: MediaConnectionState.Connecting, + [LivekitConnectionState.Connected]: MediaConnectionState.Connected, + [LivekitConnectionState.Reconnecting]: MediaConnectionState.Reconnecting, + [LivekitConnectionState.SignalReconnecting]: MediaConnectionState.Reconnecting, }; /** A connection's state as the session sees it; an error passes through. */ export function toMediaConnectionState( - state: ConnectionState | Error, + state: ConnectionState, ): MediaConnectionState | Error { - return state instanceof Error ? state : states[state]; + switch (state.state) { + case "initialized": + return MediaConnectionState.Initialized; + case "authenticating": + return MediaConnectionState.Preparing; + case "authenticated": + return livekitStates[state.livekitState]; + case "error": + return state.error; + case "stopped": + return MediaConnectionState.Stopped; + } } diff --git a/sdk/src/media-backend/livekit/openIDSFU.test.ts b/sdk/src/media-backend/livekit/openIDSFU.test.ts deleted file mode 100644 index 73a52018a..000000000 --- a/sdk/src/media-backend/livekit/openIDSFU.test.ts +++ /dev/null @@ -1,332 +0,0 @@ -/* -Copyright 2025 Element Creations Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - beforeEach, - afterEach, - describe, - expect, - it, - type MockedObject, - vitest, -} from "vitest"; -import fetchMock from "fetch-mock"; -import { MatrixError } from "matrix-js-sdk"; - -import { getSFUConfigWithOpenID, type OpenIDClientParts } from "./openIDSFU"; -import { testJWTToken } from "../../utils/test-fixtures"; -import { ownMemberMock } from "../../utils/test"; -import { FailToGetOpenIdToken } from "../../errors"; - -const sfuUrl = "https://sfu.example.org"; - -describe("getSFUConfigWithOpenID", () => { - let matrixClient: MockedObject; - beforeEach(() => { - fetchMock.catch(404); - matrixClient = { - getOpenIdToken: vitest.fn(), - }; - }); - afterEach(() => { - vitest.clearAllMocks(); - fetchMock.reset(); - }); - - it("should handle fetching a token", async () => { - fetchMock.post("https://sfu.example.org/sfu/get", () => { - return { - status: 200, - body: { url: sfuUrl, jwt: testJWTToken }, - }; - }); - const config = await getSFUConfigWithOpenID( - matrixClient, - ownMemberMock, - "https://sfu.example.org", - "!example_room_id", - ); - expect(config).toEqual({ - jwt: testJWTToken, - url: sfuUrl, - livekitIdentity: "@me:example.org:ABCDEF", - livekitAlias: "!example_room_id", - }); - void (await fetchMock.flush()); - }); - - it("should fail if the SFU errors", async () => { - fetchMock.post("https://sfu.example.org/sfu/get", () => { - return { - status: 500, - body: { - errcode: "M_LOOKUP_FAILED", - error: "Failed to look up user info from homeserver", - }, - }; - }); - try { - await getSFUConfigWithOpenID( - matrixClient, - ownMemberMock, - "https://sfu.example.org", - "!example_room_id", - ); - } catch (ex: unknown) { - expect(ex).toBeInstanceOf(FailToGetOpenIdToken); - expect((ex as FailToGetOpenIdToken).cause).toBeInstanceOf(MatrixError); - const mxError = (ex as Error).cause as MatrixError; - expect(mxError.message).toEqual( - "MatrixError: [500] Failed to look up user info from homeserver", - ); - - void (await fetchMock.flush()); - return; - } - expect.fail("Expected test to throw;"); - }); - - it("should retry without delay params if the JWT service legacy endpoint returns M_BAD_JSON 400", async () => { - let callCount = 0; - - fetchMock.post( - "https://sfu.example.org/sfu/get", - (url, opts) => { - callCount++; - const body = JSON.parse(opts.body as string); - - // First call: check if it has delay parts and return 400 - if (callCount === 1) { - expect(body).toHaveProperty("delay_id", "mock_delay_id"); - return { - status: 400, - body: { errcode: "M_BAD_JSON", error: "Unsupported parameters" }, - }; - } - - // Second call: check if delay parts were stripped and return success - expect(body).not.toHaveProperty("delay_id"); - expect(body).not.toHaveProperty("delay_timeout"); - expect(body).not.toHaveProperty("delay_cs_api_url"); - - return { - status: 200, - body: { url: sfuUrl, jwt: testJWTToken }, - }; - }, - { overwriteRoutes: true }, - ); - - // Note: Assuming getSFUConfigWithOpenID eventually calls getLiveKitJWT - const config = await getSFUConfigWithOpenID( - matrixClient, - ownMemberMock, - "https://sfu.example.org", - "!example_room_id", - { - delayEndpointBaseUrl: "https://matrix.homeserverserver.org", - delayId: "mock_delay_id", - delayTimeoutMs: 3_600_000, - }, - ); - - expect(config.jwt).toBe(testJWTToken); - expect(callCount).toBe(2); - void (await fetchMock.flush()); - }); - - it("should successfully send delay parameters to the JWT service legacy endpoint", async () => { - fetchMock.post( - "https://sfu.example.org/sfu/get", - (url, opts) => { - const body = JSON.parse(opts.body as string); - - // Verify, that the request contains the expected delay parameters - if ( - body.delay_id === "mock_delay_id" && - body.delay_timeout === 3600000 && - body.delay_cs_api_url === "https://homeserverserver.org/cs_api" - ) { - return { - status: 200, - body: { url: sfuUrl, jwt: testJWTToken }, - }; - } - return { - status: 400, - body: { error: "Missing expected delay params" }, - }; - }, - { overwriteRoutes: true }, - ); - - const config = await getSFUConfigWithOpenID( - matrixClient, - ownMemberMock, - "https://sfu.example.org", - "!example_room_id", - { - delayEndpointBaseUrl: "https://homeserverserver.org/cs_api", - delayId: "mock_delay_id", - delayTimeoutMs: 3_600_000, - }, - ); - - // Prüfe das Ergebnis - expect(config).toMatchObject({ - jwt: testJWTToken, - url: sfuUrl, - }); - - void (await fetchMock.flush()); - }); - - it("should try legacy and then new endpoint with delay delegation", async () => { - fetchMock.post("https://sfu.example.org/get_token", () => { - return { - status: 500, - body: { - errcode: "M_LOOKUP_FAILED", - error: "Failed to look up user info from homeserver", - }, - }; - }); - fetchMock.post("https://sfu.example.org/sfu/get", () => { - return { - status: 500, - body: { - errcode: "M_LOOKUP_FAILED", - error: "Failed to look up user info from homeserver", - }, - }; - }); - try { - await getSFUConfigWithOpenID( - matrixClient, - ownMemberMock, - "https://sfu.example.org", - "!example_room_id", - { - delayEndpointBaseUrl: "https://matrix.homeserverserver.org", - delayId: "mock_delay_id", - delayTimeoutMs: 3_600_000, - }, - ); - } catch (ex) { - expect(ex).toBeInstanceOf(FailToGetOpenIdToken); - expect((ex as FailToGetOpenIdToken).cause).toBeInstanceOf(MatrixError); - const mxError = (ex as Error).cause as MatrixError; - expect(mxError.message).toEqual( - "MatrixError: [500] Failed to look up user info from homeserver", - ); - void (await fetchMock.flush()); - } - const calls = fetchMock.calls(); - expect(calls.length).toBe(2); - - expect(calls[0][0]).toStrictEqual("https://sfu.example.org/get_token"); - expect(calls[0][1]).toStrictEqual({ - // check if it uses correct delayID! - body: '{"room_id":"!example_room_id","slot_id":"m.call#ROOM","member":{"id":"@alice:example.org:DEVICE","claimed_user_id":"@alice:example.org","claimed_device_id":"DEVICE"},"delay_id":"mock_delay_id","delay_timeout":3600000,"delay_cs_api_url":"https://matrix.homeserverserver.org"}', - method: "POST", - headers: { - "Content-Type": "application/json", - }, - }); - - expect(calls[1][0]).toStrictEqual("https://sfu.example.org/sfu/get"); - - expect(calls[1][1]).toStrictEqual({ - body: '{"room":"!example_room_id","device_id":"DEVICE","delay_id":"mock_delay_id","delay_timeout":3600000,"delay_cs_api_url":"https://matrix.homeserverserver.org"}', - headers: { - "Content-Type": "application/json", - }, - method: "POST", - }); - }); - - it("dont try legacy if endpoint with delay delegation is sucessful", async () => { - fetchMock.post("https://sfu.example.org/get_token", () => { - return { - status: 200, - body: { url: sfuUrl, jwt: testJWTToken }, - }; - }); - fetchMock.post("https://sfu.example.org/sfu/get", () => { - return { - status: 500, - body: { error: "Test failure" }, - }; - }); - try { - await getSFUConfigWithOpenID( - matrixClient, - ownMemberMock, - "https://sfu.example.org", - "!example_room_id", - { - delayEndpointBaseUrl: "https://matrix.homeserverserver.org", - delayId: "mock_delay_id", - delayTimeoutMs: 3_600_000, - }, - ); - } catch (ex) { - expect(ex).toBeInstanceOf(FailToGetOpenIdToken); - expect((ex as FailToGetOpenIdToken).cause).toEqual( - new Error("SFU Config fetch failed with status code 500"), - ); - void (await fetchMock.flush()); - } - const calls = fetchMock.calls(); - expect(calls.length).toBe(1); - - expect(calls[0][0]).toStrictEqual("https://sfu.example.org/get_token"); - expect(calls[0][1]).toStrictEqual({ - // check if it uses correct delayID! - body: '{"room_id":"!example_room_id","slot_id":"m.call#ROOM","member":{"id":"@alice:example.org:DEVICE","claimed_user_id":"@alice:example.org","claimed_device_id":"DEVICE"},"delay_id":"mock_delay_id","delay_timeout":3600000,"delay_cs_api_url":"https://matrix.homeserverserver.org"}', - method: "POST", - headers: { - "Content-Type": "application/json", - }, - }); - }); - - it("should retry fetching the openid token", async () => { - let count = 0; - matrixClient.getOpenIdToken.mockImplementation(async () => { - count++; - if (count < 2) { - throw Error("Test failure"); - } - return Promise.resolve({ - token_type: "Bearer", - access_token: "foobar", - matrix_server_name: "example.org", - expires_in: 30, - }); - }); - fetchMock.post("https://sfu.example.org/sfu/get", () => { - return { - status: 200, - body: { url: sfuUrl, jwt: testJWTToken }, - }; - }); - const config = await getSFUConfigWithOpenID( - matrixClient, - ownMemberMock, - "https://sfu.example.org", - "!example_room_id", - ); - expect(config).toEqual({ - jwt: testJWTToken, - url: sfuUrl, - livekitIdentity: "@me:example.org:ABCDEF", - livekitAlias: "!example_room_id", - }); - void (await fetchMock.flush()); - }); -}); diff --git a/sdk/src/media-backend/livekit/openIDSFU.ts b/sdk/src/media-backend/livekit/openIDSFU.ts deleted file mode 100644 index d86989e23..000000000 --- a/sdk/src/media-backend/livekit/openIDSFU.ts +++ /dev/null @@ -1,321 +0,0 @@ -/* -Copyright 2023, 2024 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - type IOpenIDToken, - type MatrixClient, - parseErrorResponse, -} from "matrix-js-sdk"; -import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; -import { type Logger } from "matrix-js-sdk/lib/logger"; - -import { - FailToGetOpenIdToken, - NoMatrix2AuthorizationService, -} from "../../errors"; -import { doNetworkOperationWithRetry } from "../../utils/network"; - -/** - * Configuration and access tokens provided by the SFU on successful authentication. - */ -export interface SFUConfig { - url: string; - jwt: string; - livekitAlias: string; - // NOTE: Currently unused. - livekitIdentity: string; -} - -/** - * Decoded details from the JWT. - */ -interface SFUJWTPayload { - /** - * Expiration time for the JWT. - * Note: This value is in seconds since Unix epoch. - */ - exp: number; - /** - * Name of the instance which authored the JWT - */ - iss: string; - /** - * Time at which the JWT can start to be used. - * Note: This value is in seconds since Unix epoch. - */ - nbf: number; - /** - * Subject. The Livekit alias in this context. - */ - sub: string; - /** - * The set of permissions for the user. - */ - video: { - canPublish: boolean; - canSubscribe: boolean; - room: string; - roomJoin: boolean; - }; -} - -// The bits we need from MatrixClient -export type OpenIDClientParts = Pick; - -/** - * Which token service endpoint to use: the MSC4195 one, or the legacy one - * with the old identity format. Undefined tries the MSC4195 one and falls - * back to the legacy one, for transports we only subscribe on. - */ -export type TokenEndpoint = "msc4195" | "legacy"; - -/** - * Gets a bearer token from the homeserver and then use it to authenticate - * to the matrix RTC backend in order to get acces to the SFU. - * It has built-in retry for calls to the homeserver with a backoff policy. - * @param client The Matrix client - * @param membership Our own membership identity parts used to send to jwt service. - * @param serviceUrl The URL of the livekit SFU service - * @param roomId The room id used in the jwt request. This is NOT the livekit_alias. The jwt service will provide the alias. It maps matrix room ids <-> Livekit aliases. - * @param opts Additional options to modify which endpoint with which data will be used to acquire the jwt token. - * @param opts.tokenEndpoint Which version of the JWT endpoint to use, which affects whether the - * RTC backend identity is based on string concatenation (legacy) or a hash (MSC4195). - * This function by default uses whatever is possible with the current jwt service installed next to the SFU. - * For remote connections this does not matter, since we will not publish there we can rely on the newest option. - * @param opts.delayEndpointBaseUrl The URL of the matrix homeserver. - * @param opts.delayId The delay id used for the jwt service to manage. - * @param logger optional logger. - * @returns Object containing the token information - * @throws FailToGetOpenIdToken - */ -export async function getSFUConfigWithOpenID( - client: OpenIDClientParts, - membership: CallMembershipIdentityParts, - serviceUrl: string, - roomId: string, - opts?: { - tokenEndpoint?: TokenEndpoint; - delayEndpointBaseUrl?: string; - delayId?: string; - /** How long the SFU waits before sending the delegated leave. Needed with `delayId`. */ - delayTimeoutMs?: number; - }, - logger?: Logger, -): Promise { - let openIdToken: IOpenIDToken; - try { - openIdToken = await doNetworkOperationWithRetry(async () => - client.getOpenIdToken(), - ); - } catch (error) { - throw new FailToGetOpenIdToken( - error instanceof Error ? error : new Error("Unknown error"), - ); - } - logger?.debug("Got openID token", openIdToken); - let sfuConfig: { url: string; jwt: string } | undefined; - - const tryBothJwtEndpoints = opts?.tokenEndpoint === undefined; // This is for SFUs where we do not publish. - - const forceMatrix2Jwt = opts?.tokenEndpoint === "msc4195"; - - // We want to start using the new endpoint (with optional delay delegation) - // if we can use both or if we are forced to use the new one. - if (tryBothJwtEndpoints || forceMatrix2Jwt) { - try { - logger?.info( - `Trying to get JWT via default endpoint for focus ${serviceUrl}...`, - ); - const sfuConfig = await getLiveKitJWT( - membership, - serviceUrl, - roomId, - openIdToken, - opts?.delayEndpointBaseUrl, - opts?.delayId, - opts?.delayTimeoutMs, - ); - - return extractFullConfigFromToken(sfuConfig); - } catch (e) { - logger?.debug(`Failed fetching jwt with matrix 2.0 endpoint:`, e); - // Make this throw a hard error in case we force the matrix2.0 endpoint. - if (forceMatrix2Jwt) { - throw new NoMatrix2AuthorizationService(e as Error); - } - } - } - - // DEPRECATED - // here we either have a sfuConfig or we already exited because of `if (forceMatrix2) throw ...` - // The only case we can get into this condition is, if `forceMatrix2` is `false` - try { - logger?.info( - `Trying to get JWT with legacy endpoint for focus ${serviceUrl}...`, - ); - sfuConfig = await getLiveKitJWTLegacy( - membership.deviceId, - serviceUrl, - roomId, - openIdToken, - opts?.delayEndpointBaseUrl, - opts?.delayId, - opts?.delayTimeoutMs, - ); - logger?.info(`Got JWT from call's active focus URL.`); - return extractFullConfigFromToken(sfuConfig); - } catch (ex) { - throw new FailToGetOpenIdToken( - ex instanceof Error ? ex : new Error(`Unknown error ${ex}`), - ); - } -} - -function extractFullConfigFromToken(sfuConfig: { - url: string; - jwt: string; -}): SFUConfig { - const [, payloadStr] = sfuConfig.jwt.split("."); - const payload = JSON.parse(global.atob(payloadStr)) as SFUJWTPayload; - return { - jwt: sfuConfig.jwt, - url: sfuConfig.url, - livekitAlias: payload.video.room, - // NOTE: Currently unused. - // Probably also not helpful since we now compute the backendIdentity on joining the call so we can use it for the encryption manager. - // The only reason for us to know it locally is to connect the right users with the lk world. (and to set our own keys) - livekitIdentity: payload.sub, - }; -} - -async function getLiveKitJWTLegacy( - deviceId: string, - livekitServiceURL: string, - matrixRoomId: string, - openIDToken: IOpenIDToken, - delayEndpointBaseUrl?: string, - delayId?: string, - delayTimeoutMs?: number, -): Promise<{ url: string; jwt: string }> { - interface IDelayParams { - delay_id?: string; - delay_timeout?: number; - delay_cs_api_url?: string; - } - let bodyDalayParts: IDelayParams = {}; - // Also check for empty string - if (delayId && delayEndpointBaseUrl) { - bodyDalayParts = { - delay_id: delayId, - delay_timeout: delayTimeoutMs, - delay_cs_api_url: delayEndpointBaseUrl, - }; - } - - const makeRequest = async (delayParts: IDelayParams): Promise => { - return await fetch(livekitServiceURL + "/sfu/get", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ - // The legacy JWT endpoint uses only the matrix room id to calculate the livekit room alias. - // However, the livekit room alias is provided as part of the JWT payload. - room: matrixRoomId, - openid_token: openIDToken, - device_id: deviceId, - ...delayParts, - }), - }); - }; - - const res = await doNetworkOperationWithRetry(async () => { - let response = await makeRequest(bodyDalayParts); - - // Old service compatibility check - const oldServiceDoesNotSupportDelayParts = - response.status === 400 && Object.keys(bodyDalayParts).length > 0; - // If http status 400 with M_BAD_JSON and we sent delay parts, retry without them - if (oldServiceDoesNotSupportDelayParts) { - try { - const errorBody = await response.json(); - if (errorBody.errcode === "M_BAD_JSON") { - response = await makeRequest({}); - } - } catch { - // If we can't parse the error, treat as real error - } - } - - return response; - }); - - if (!res.ok) { - throw parseErrorResponse(res, await res.text()); - } - return await res.json(); -} - -class NotSupportedError extends Error { - public constructor(message: string) { - super(message); - this.name = "NotSupported"; - } -} - -export async function getLiveKitJWT( - membership: CallMembershipIdentityParts, - livekitServiceURL: string, - matrixRoomId: string, - openIDToken: IOpenIDToken, - delayEndpointBaseUrl?: string, - delayId?: string, - delayTimeoutMs?: number, -): Promise<{ url: string; jwt: string }> { - const { userId, deviceId, memberId } = membership; - - const body = { - room_id: matrixRoomId, - slot_id: "m.call#ROOM", - openid_token: openIDToken, - member: { - id: memberId, - claimed_user_id: userId, - claimed_device_id: deviceId, - }, - }; - - let bodyDalayParts = {}; - // Also check for empty string - if (delayId && delayEndpointBaseUrl) { - bodyDalayParts = { - delay_id: delayId, - delay_timeout: delayTimeoutMs, - delay_cs_api_url: delayEndpointBaseUrl, - }; - } - - const res = await doNetworkOperationWithRetry(async () => { - return await fetch(livekitServiceURL + "/get_token", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ ...body, ...bodyDalayParts }), - }); - }); - - if (!res.ok) { - const msg = "SFU Config fetch failed with status code " + res.status; - if (res.status === 404) { - throw new NotSupportedError(msg); - } else { - throw parseErrorResponse(res, await res.text()); - } - } - return await res.json(); -} diff --git a/sdk/src/media-backend/livekit/test.ts b/sdk/src/media-backend/livekit/test.ts index e19dd3952..84e439910 100644 --- a/sdk/src/media-backend/livekit/test.ts +++ b/sdk/src/media-backend/livekit/test.ts @@ -11,13 +11,15 @@ import { type RemoteParticipant, type RemoteTrackPublication, } from "livekit-client"; -import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { mockEmitter } from "../../utils/test"; -export const exampleTransport: LivekitTransport = { +export const exampleServiceUrl = "https://lk.example.org"; + +export const exampleTransport: UnstableLivekitTransport = { type: "livekit", - livekit_service_url: "https://lk.example.org", + livekit_service_url: exampleServiceUrl, }; export function mockRemoteParticipant( diff --git a/sdk/src/utils/network.ts b/sdk/src/utils/network.ts index ea8def05d..1d72ae6c5 100644 --- a/sdk/src/utils/network.ts +++ b/sdk/src/utils/network.ts @@ -12,9 +12,13 @@ import { sleep } from "matrix-js-sdk/lib/utils"; * Performs a network operation with retries on connection errors, honouring * the homeserver's rate limits. Rethrows once the error is not retryable or * the retries are used up. + * @param operation The operation to perform. + * @param shouldRetry Rules out errors that retrying cannot fix, on top of the + * standard backoff rule. Retries everything by default. */ export async function doNetworkOperationWithRetry( operation: () => Promise, + shouldRetry: (e: unknown) => boolean = () => true, ): Promise { let currentRetryCount = 0; // eslint-disable-next-line no-constant-condition @@ -23,7 +27,9 @@ export async function doNetworkOperationWithRetry( return await operation(); } catch (e) { currentRetryCount++; - const backoff = calculateRetryBackoff(e, currentRetryCount, true); + const backoff = shouldRetry(e) + ? calculateRetryBackoff(e, currentRetryCount, true) + : -1; if (backoff < 0) throw e; await sleep(backoff); } diff --git a/sdk/src/utils/serverName.ts b/sdk/src/utils/serverName.ts new file mode 100644 index 000000000..d6b6d685a --- /dev/null +++ b/sdk/src/utils/serverName.ts @@ -0,0 +1,11 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +/** The homeserver a Matrix user id belongs to: everything after the first colon. */ +export function serverNameOf(userId: string): string { + return userId.replace(/^.*?:/, ""); +} diff --git a/sdk/src/utils/test-fixtures.ts b/sdk/src/utils/test-fixtures.ts index 807357a5f..d3884b04e 100644 --- a/sdk/src/utils/test-fixtures.ts +++ b/sdk/src/utils/test-fixtures.ts @@ -48,16 +48,24 @@ export const daveRTL = mockMatrixRoomMember(daveRTLRtcMember, { rawDisplayName: "\u202eevaD", }); -export const testJWTToken = [ - {}, // header - { - // payload - sub: "@me:example.org:ABCDEF", - video: { - room: "!example_room_id", +/** An unsigned JWT with the given subject and LiveKit room, as a token service would issue it. */ +export function createJWTToken(sub: string, room: string): string { + return [ + {}, // header + { + // payload + sub, + video: { + room, + }, }, - }, - {}, // signature -] - .map((d) => global.btoa(JSON.stringify(d))) - .join("."); + {}, // signature + ] + .map((d) => global.btoa(JSON.stringify(d))) + .join("."); +} + +export const testJWTToken = createJWTToken( + "@me:example.org:ABCDEF", + "!example_room_id", +);