Matrix/element-call-Github
1
0
Fork 0
mirror of https://github.com/vector-im/element-call.git synced 2026-10-16 23:45:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

[SDK] Move SFU authentication into livekit/auth and choose the endpoint by role

Browse Source 
The OpenID flow moves to src/livekit/auth/openID.ts with the SFU config
types in a sibling file, takes an options object, and replaces the
MatrixRTC mode with the role we take on the transport plus the session's
slot ID instead of a hard-coded m.call#ROOM.

This changes behaviour: publishers on a livekit_service_url transport
always use the legacy /sfu/get endpoint, so that subscribers can predict
their user:device participant identity, and subscribers always use the
default /get_token endpoint. The MatrixRTC mode no longer influences which
endpoint is used.

On the SDK this lands as sdk/src/media-backend/livekit/auth/ with the connection map, the discriminated connection state and the client-server API flow of the following commits of element-hq/element-call#4331, which rewrite the same files:
- Model ConnectionState as a discriminated union
- Show the whole transport in debug UI and logs (SDK part)
- Let the publishing Connection authenticate itself
- Reduce ConnectionManager to a ConnectionMap
- Use the unstable LiveKit transport type
- Identify transports by server name as well as URLs
- Get SFU tokens and delegate delayed leaves through the MSC4195 CS API
- Fall back to get_token as subscriber on widget API errors
- Share a connection between homeservers using the same JWT service
- Make the auth choice mode independent again

Co-authored-by: Johannes Marbach <n0-0ne+github@mailbox.org>
(cherry picked from commit adb380cd9ed108856e42d992fc03ced8c44f1e56)
This commit is contained in:
Robin
2026-10-13 13:00:58 +02:00
committed by Timo K.
parent de8b5d98ec
commit b4bd1f81a0
29 changed files with 2495 additions and 1525 deletions
Download Patch File Download Diff File Expand all files Collapse all files
sdk/index.ts
+4 -4
View File
@@ -63,8 +63,8 @@ export {
} from "./src/media-backend/livekit/LivekitBackend";
// For a host that creates LiveKit tracks itself, such as a preview before joining
export { convertToLivekitProcessor } from "./src/media-backend/livekit/videoProcessor";
// For a developer panel to check a transport url before using it
// For a developer panel to check a transport before using it
export {
getSFUConfigWithOpenID as authenticateWithTransport,
type TokenEndpoint,
} from "./src/media-backend/livekit/openIDSFU";
getSFUConfig as authenticateWithTransport,
type GetSFUConfigParams as AuthenticateWithTransportParams,
} from "./src/media-backend/livekit/auth";
sdk/src/RTCParticipation.ts
+3 -8
View File
@@ -119,8 +119,9 @@ export function createRTCParticipation(
: `${userId}:${deviceId}`,
};
const backend = createBackend(scope, client, options, matrixRTCMode, {
const backend = createBackend(scope, client, options, {
roomId: room.roomId,
slotId: jsSdkSession.slotId,
ownMembershipIdentity,
publish: options.publish,
encryptionSystem,
@@ -285,16 +286,10 @@ function createBackend(
scope: ObservableScope,
client: MatrixClient,
{ backend, mediaQuality }: RTCParticipationOptions,
matrixRTCMode: MatrixRTCMode,
context: MediaBackendContext,
): MediaBackend {
if (backend) return backend(scope, context);
return createLivekitBackend(scope, context, {
client,
mediaQuality,
tokenEndpoint:
matrixRTCMode === MatrixRTCMode.Matrix_2_0 ? "msc4195" : "legacy",
});
return createLivekitBackend(scope, context, { client, mediaQuality });
}
/** The slot's per-participant keys: the ones it already holds on subscribe, then each new one. */
sdk/src/config.ts
+5 -4
View File
@@ -10,12 +10,13 @@ Please see LICENSE in the repository root for full details.
* backend and other participants. Resolved by the host.
*/
export enum MatrixRTCMode {
/** Multi-SFU transport, legacy JWT endpoint, state events. */
/** Multi-SFU transport, membership state events. */
Compatibility = "compatibility",
/**
* Multi-SFU transport with sticky events, a hashed RTC backend identity and
* the new JWT endpoint for the local membership. Remote memberships always
* try the new endpoint first, then the legacy one.
* Multi-SFU transport with sticky membership events and a uuid member id.
* The mode decides how the membership is written, not how the media
* transport is authenticated with: that follows the transport's shape and
* our role on it.
*/
Matrix_2_0 = "matrix_2_0",
}
sdk/src/matrixrtc/Transports.ts
+9 -5
View File
@@ -36,13 +36,17 @@ export function createTransportRegistry(
return transport;
};
// Deduplicated: a backend may hold more than one connection to a transport,
// one per homeserver it has to authenticate through
const connected$ = scope.behavior(
connections$.pipe(
map((connections) =>
connections
.filter(({ state }) => state === MediaConnectionState.Connected)
.map(({ transport }) => get(transport)),
),
map((connections) => [
...new Set(
connections
.filter(({ state }) => state === MediaConnectionState.Connected)
.map(({ transport }) => get(transport)),
),
]),
),
);
sdk/src/media-backend/api.ts
+2
View File
@@ -53,6 +53,8 @@ export interface MediaKey {
export interface MediaBackendContext {
roomId: string;
/** The slot the session is in, `application#id`; a token is issued for it. */
slotId: string;
ownMembershipIdentity: CallMembershipIdentityParts;
/** What the local member publishes at the join. */
publish: PublishRequest[];
sdk/src/media-backend/livekit/Connection.test.ts
+118 -139
View File
@@ -8,7 +8,6 @@ Please see LICENSE in the repository root for full details.
import {
afterEach,
beforeEach,
describe,
expect,
it,
@@ -28,25 +27,28 @@ import {
} from "livekit-client";
import fetchMock from "fetch-mock";
import EventEmitter from "events";
import { type IOpenIDToken } from "matrix-js-sdk";
import { MatrixError } from "matrix-js-sdk";
import { logger, type Logger } from "matrix-js-sdk/lib/logger";
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import {
type LivekitGetTokenResponse,
type UnstableLivekitTransport,
} from "matrix-js-sdk/lib/matrixrtc";
import {
Connection,
ConnectionState,
type ConnectionState,
type ConnectionOpts,
} from "./Connection.ts";
import { ObservableScope } from "../../reactive/ObservableScope";
import { type OpenIDClientParts } from "./openIDSFU";
import { MatrixRTCError, FailToGetOpenIdToken } from "../../errors";
import { type ClientGetTokenParts, type ClientOpenIDParts } from "./auth";
import { MatrixRTCError } from "../../errors";
import { testJWTToken } from "../../utils/test-fixtures";
import { ownMemberMock } from "../../utils/test";
import { mockRemoteParticipant } from "./test";
let testScope: ObservableScope;
let client: MockedObject<OpenIDClientParts>;
let client: MockedObject<ClientGetTokenParts & ClientOpenIDParts>;
let fakeLivekitRoom: MockedObject<LivekitRoom>;
@@ -54,23 +56,39 @@ let localParticipantEventEmiter: EventEmitter;
let fakeLocalParticipant: MockedObject<LocalParticipant>;
const ROOM_ID = "!roomID:example.org";
const SLOT_ID = "m.call#ROOM";
const livekitFocus: LivekitTransport = {
livekit_service_url: "https://matrix-rtc.example.org/livekit/jwt",
const transport: UnstableLivekitTransport = {
type: "livekit",
url: "wss://matrix-rtc.m.localhost/livekit/sfu",
};
/** The options of a subscriber connection to `transport`. */
function connectionOpts(): ConnectionOpts {
return {
client,
roomId: ROOM_ID,
slotId: SLOT_ID,
role: "subscriber",
transport,
serverName: "m.localhost",
scope: testScope,
ownMembershipIdentity: ownMemberMock,
livekitRoomFactory: () => fakeLivekitRoom,
};
}
function setupTest(): void {
testScope = new ObservableScope();
client = vi.mocked<OpenIDClientParts>({
client = vi.mocked<ClientGetTokenParts & ClientOpenIDParts>({
getOpenIdToken: vi.fn().mockResolvedValue({
access_token: "rYsmGUEwNjKgJYyeNUkZseJN",
token_type: "Bearer",
matrix_server_name: "example.org",
expires_in: 3600,
}),
getDeviceId: vi.fn().mockReturnValue("ABCDEF"),
} as unknown as OpenIDClientParts);
_unstable_getLivekitToken: vi.fn().mockResolvedValue({ jwt: testJWTToken }),
});
localParticipantEventEmiter = new EventEmitter();
@@ -113,24 +131,7 @@ function setupTest(): void {
}
function setupRemoteConnection(): Connection {
const opts: ConnectionOpts = {
client: client,
roomId: ROOM_ID,
transport: livekitFocus,
scope: testScope,
ownMembershipIdentity: ownMemberMock,
livekitRoomFactory: () => fakeLivekitRoom,
};
fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, () => {
return {
status: 200,
body: {
url: "wss://matrix-rtc.m.localhost/livekit/sfu",
jwt: testJWTToken,
},
};
});
const opts = connectionOpts();
fakeLivekitRoom.connect.mockImplementation(async (): Promise<void> => {
const changeEv = RoomEvent.ConnectionStateChanged;
@@ -153,60 +154,33 @@ afterEach(() => {
});
describe("Start connection states", () => {
beforeEach(() => {
fetchMock.post(
`https://matrix-rtc.example.org/livekit/jwt/get_token`,
() => {
return {
// Return a non-retryable error, if not, the retry logic will
// wait and fail the test with a timeout.
status: 404,
};
},
);
});
it("start in initialized state", () => {
setupTest();
const opts: ConnectionOpts = {
client: client,
roomId: ROOM_ID,
transport: livekitFocus,
scope: testScope,
ownMembershipIdentity: ownMemberMock,
livekitRoomFactory: () => fakeLivekitRoom,
};
const opts = connectionOpts();
const connection = new Connection(opts, logger);
expect(connection.state$.getValue()).toEqual("Initialized");
expect(connection.state$.getValue()).toEqual({ state: "initialized" });
});
it("fail to getOpenId token then error state", async () => {
it("fail to get SFU token then error state", async () => {
setupTest();
vi.useFakeTimers();
const opts: ConnectionOpts = {
client: client,
roomId: ROOM_ID,
transport: livekitFocus,
scope: testScope,
ownMembershipIdentity: ownMemberMock,
livekitRoomFactory: () => fakeLivekitRoom,
};
const opts = connectionOpts();
const connection = new Connection(opts, logger);
const capturedStates: (ConnectionState | Error)[] = [];
const capturedStates: ConnectionState[] = [];
const s = connection.state$.subscribe((value) => {
capturedStates.push(value);
});
onTestFinished(() => s.unsubscribe());
const deferred = Promise.withResolvers<IOpenIDToken>();
const deferred = Promise.withResolvers<LivekitGetTokenResponse>();
client.getOpenIdToken.mockImplementation(
async (): Promise<IOpenIDToken> => {
client._unstable_getLivekitToken.mockImplementation(
async (): Promise<LivekitGetTokenResponse> => {
return await deferred.promise;
},
);
@@ -217,18 +191,20 @@ describe("Start connection states", () => {
let capturedState = capturedStates.pop();
expect(capturedState).toBeDefined();
expect(capturedState!).toEqual("FetchingConfig");
expect(capturedState).toEqual({ state: "authenticating" });
deferred.reject(new FailToGetOpenIdToken(new Error("Failed to get token")));
deferred.reject(new MatrixRTCError("Failed to get token"));
await vi.runAllTimersAsync();
capturedState = capturedStates.pop();
if (capturedState instanceof Error) {
expect(capturedState).toBeInstanceOf(FailToGetOpenIdToken);
expect(connection.transport).toEqual(livekitFocus);
if (capturedState?.state === "error") {
expect(capturedState.error.message).toEqual("Failed to get token");
expect(connection.transport).toEqual(transport);
} else {
expect.fail("Expected FailedToStart state but got " + capturedState);
expect.fail(
"Expected error state but got " + JSON.stringify(capturedState),
);
}
});
@@ -236,34 +212,26 @@ describe("Start connection states", () => {
setupTest();
vi.useFakeTimers();
const opts: ConnectionOpts = {
client: client,
roomId: ROOM_ID,
transport: livekitFocus,
scope: testScope,
ownMembershipIdentity: ownMemberMock,
livekitRoomFactory: () => fakeLivekitRoom,
};
const opts = connectionOpts();
const connection = new Connection(opts, logger);
const capturedStates: (ConnectionState | Error)[] = [];
const capturedStates: ConnectionState[] = [];
const s = connection.state$.subscribe((value) => {
capturedStates.push(value);
});
onTestFinished(() => s.unsubscribe());
const deferredSFU = Promise.withResolvers<void>();
// mock the /sfu/get call
fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, async () => {
client._unstable_getLivekitToken.mockImplementation(async () => {
await deferredSFU.promise;
return {
status: 500,
body: {
throw new MatrixError(
{
errcode: "M_LOOKUP_FAILED",
error: "Failed to look up user info from homeserver",
},
};
500,
);
});
connection.start().catch(() => {
@@ -272,7 +240,7 @@ describe("Start connection states", () => {
let capturedState = capturedStates.pop();
expect(capturedState).toBeDefined();
expect(capturedState).toEqual(ConnectionState.FetchingConfig);
expect(capturedState).toEqual({ state: "authenticating" });
deferredSFU.resolve();
await vi.runAllTimersAsync();
@@ -280,15 +248,17 @@ describe("Start connection states", () => {
capturedState = capturedStates.pop();
if (
capturedState instanceof MatrixRTCError &&
capturedState.cause instanceof Error
capturedState?.state === "error" &&
capturedState.error.cause instanceof Error
) {
expect(capturedState.cause.message).toContain(
expect(capturedState.error.cause.message).toContain(
"Failed to look up user info from homeserver",
);
expect(connection.transport).toEqual(livekitFocus);
expect(connection.transport).toEqual(transport);
} else {
expect.fail("Expected FailedToStart state but got " + capturedState);
expect.fail(
"Expected error state but got " + JSON.stringify(capturedState),
);
}
});
@@ -296,35 +266,17 @@ describe("Start connection states", () => {
setupTest();
vi.useFakeTimers();
const opts: ConnectionOpts = {
client: client,
roomId: ROOM_ID,
transport: livekitFocus,
scope: testScope,
ownMembershipIdentity: ownMemberMock,
livekitRoomFactory: () => fakeLivekitRoom,
};
const opts = connectionOpts();
const connection = new Connection(opts, logger);
const capturedStates: (ConnectionState | Error)[] = [];
const capturedStates: ConnectionState[] = [];
const s = connection.state$.subscribe((value) => {
capturedStates.push(value);
});
onTestFinished(() => s.unsubscribe());
const deferredSFU = Promise.withResolvers<void>();
// mock the /sfu/get call
fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, () => {
return {
status: 200,
body: {
url: "wss://matrix-rtc.m.localhost/livekit/sfu",
jwt: testJWTToken,
},
};
});
fakeLivekitRoom.connect.mockImplementation(async () => {
await deferredSFU.promise;
throw new Error("Failed to connect to livekit");
@@ -337,7 +289,7 @@ describe("Start connection states", () => {
let capturedState = capturedStates.pop();
expect(capturedState).toBeDefined();
expect(capturedState).toEqual(ConnectionState.FetchingConfig);
expect(capturedState).toEqual({ state: "authenticating" });
deferredSFU.resolve();
await vi.runAllTimersAsync();
@@ -345,13 +297,13 @@ describe("Start connection states", () => {
capturedState = capturedStates.pop();
if (
capturedState instanceof MatrixRTCError &&
capturedState.cause instanceof Error
capturedState?.state === "error" &&
capturedState.error.cause instanceof Error
) {
expect(capturedState.cause.message).toContain(
expect(capturedState.error.cause.message).toContain(
"Failed to connect to livekit",
);
expect(connection.transport).toEqual(livekitFocus);
expect(connection.transport).toEqual(transport);
} else {
expect.fail(
"Expected FailedToStart state but got " + JSON.stringify(capturedState),
@@ -365,7 +317,7 @@ describe("Start connection states", () => {
const connection = setupRemoteConnection();
const capturedStates: (ConnectionState | Error)[] = [];
const capturedStates: ConnectionState[] = [];
const s = connection.state$.subscribe((value) => {
capturedStates.push(value);
});
@@ -375,22 +327,59 @@ describe("Start connection states", () => {
await vi.runAllTimersAsync();
const initialState = capturedStates.shift();
expect(initialState).toEqual(ConnectionState.Initialized);
expect(initialState).toEqual({ state: "initialized" });
const fetchingState = capturedStates.shift();
expect(fetchingState).toEqual(ConnectionState.FetchingConfig);
expect(fetchingState).toEqual({ state: "authenticating" });
const disconnectedState = capturedStates.shift();
expect(disconnectedState).toEqual(ConnectionState.LivekitDisconnected);
expect(disconnectedState).toEqual({
state: "authenticated",
livekitState: LivekitConnectionState.Disconnected,
});
const connectingState = capturedStates.shift();
expect(connectingState).toEqual(ConnectionState.LivekitConnecting);
expect(connectingState).toEqual({
state: "authenticated",
livekitState: LivekitConnectionState.Connecting,
});
const connectedState = capturedStates.shift();
expect(connectedState).toEqual(ConnectionState.LivekitConnected);
expect(connectedState).toEqual({
state: "authenticated",
livekitState: LivekitConnectionState.Connected,
});
});
it("authenticates a publisher on a livekit_service_url transport through the JWT service", async () => {
setupTest();
const serviceUrl = "https://matrix-rtc.example.org/livekit/jwt";
fetchMock.post(`${serviceUrl}/sfu/get`, {
status: 200,
body: { url: "wss://sfu", jwt: testJWTToken },
});
fakeLivekitRoom.connect.mockResolvedValue(undefined);
const connection = new Connection(
{
...connectionOpts(),
role: "publisher",
transport: { type: "livekit", livekit_service_url: serviceUrl },
},
logger,
);
await connection.start();
expect(client._unstable_getLivekitToken).not.toHaveBeenCalled();
expect(fetchMock.called(`${serviceUrl}/sfu/get`)).toBe(true);
expect(fakeLivekitRoom.connect).toHaveBeenCalledWith(
"wss://sfu",
testJWTToken,
);
expect(connection.livekitAlias).toBe("!example_room_id");
});
it("stopping while connecting does not report an error", async () => {
setupTest();
const connection = setupRemoteConnection();
const capturedStates: (ConnectionState | Error)[] = [];
const capturedStates: ConnectionState[] = [];
const s = connection.state$.subscribe((value) => {
capturedStates.push(value);
});
@@ -407,11 +396,11 @@ describe("Start connection states", () => {
pendingConnect.reject(new Error("Client initiated disconnect"));
await stopping;
// start() resolves rather than rejecting (it is not awaited by the
// ConnectionManager, so a rejection would be unhandled).
// start() resolves rather than rejecting (it is not awaited by the caller,
// so a rejection would be unhandled).
await expect(started).resolves.toBeUndefined();
expect(capturedStates.at(-1)).toEqual(ConnectionState.Stopped);
expect(capturedStates.some((st) => st instanceof Error)).toBe(false);
expect(capturedStates.at(-1)).toEqual({ state: "stopped" });
expect(capturedStates.some((st) => st.state === "error")).toBe(false);
});
it("shutting down the scope should stop the connection", async () => {
@@ -544,17 +533,7 @@ describe("remote track logging", () => {
warn,
error: vi.fn(),
} as unknown as Logger;
new Connection(
{
client,
roomId: ROOM_ID,
transport: livekitFocus,
scope: testScope,
ownMembershipIdentity: ownMemberMock,
livekitRoomFactory: () => fakeLivekitRoom,
},
testLogger,
);
new Connection(connectionOpts(), testLogger);
info.mockClear(); // drop the constructor log line
const bob = mockRemoteParticipant({
sdk/src/media-backend/livekit/Connection.ts
+81 -93
View File
@@ -13,6 +13,7 @@ import {
import {
ConnectionError,
ConnectionErrorReason,
type ConnectionState as LivekitConnectionState,
type Participant,
type RemoteParticipant,
type RemoteTrackPublication,
@@ -22,16 +23,18 @@ import {
type Track,
type TrackPublication,
} from "livekit-client";
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import { BehaviorSubject, map } from "rxjs";
import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import { BehaviorSubject } from "rxjs";
import { type Logger } from "matrix-js-sdk/lib/logger";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import {
getSFUConfigWithOpenID,
type OpenIDClientParts,
type ClientGetTokenParts,
type ClientOpenIDParts,
type ConnectionRole,
getSFUConfig,
type SFUConfig,
} from "./openIDSFU";
} from "./auth";
import { type Behavior } from "../../reactive/Behavior";
import { type ObservableScope } from "../../reactive/ObservableScope";
import {
@@ -44,19 +47,20 @@ import {
} from "../../errors";
export interface ConnectionOpts {
/**
* For the local transport we already do know the jwt token and url. We can reuse it.
* On top the local transport will send additional data to the jwt server to use delayed event delegation.
*/
existingSFUConfig?: SFUConfig;
/** The identity parts to use on this connection */
ownMembershipIdentity: CallMembershipIdentityParts;
/** Whether we publish on this connection or only subscribe. */
role: ConnectionRole;
/** The media transport to connect to. */
transport: LivekitTransport;
/** The Matrix client to use for OpenID and SFU config requests. */
client: OpenIDClientParts;
/** The room ID this connection is associated with. */
transport: UnstableLivekitTransport;
/** The homeserver the transport belongs to. */
serverName: string;
/** The Matrix client, for the token exchange. */
client: ClientGetTokenParts & ClientOpenIDParts;
/** The Matrix room the session is in. */
roomId: string;
/** The MatrixRTC slot the session is in. */
slotId: string;
/** The observable scope to use for this connection. */
scope: ObservableScope;
@@ -64,23 +68,15 @@ export interface ConnectionOpts {
livekitRoomFactory: () => LivekitRoom;
}
export enum ConnectionState {
/** The start state of a connection. It has been created but nothing has loaded yet. */
Initialized = "Initialized",
/** `start` has been called on the connection. It aquires the jwt info to conenct to the LK Room */
FetchingConfig = "FetchingConfig",
Stopped = "Stopped",
/** The same as ConnectionState.Disconnected from `livekit-client` */
LivekitDisconnected = "disconnected",
/** The same as ConnectionState.Connecting from `livekit-client` */
LivekitConnecting = "connecting",
/** The same as ConnectionState.Connected from `livekit-client` */
LivekitConnected = "connected",
/** The same as ConnectionState.Reconnecting from `livekit-client` */
LivekitReconnecting = "reconnecting",
/** The same as ConnectionState.SignalReconnecting from `livekit-client` */
LivekitSignalReconnecting = "signalReconnecting",
}
export type ConnectionState =
/** Created, nothing started yet. */
| { state: "initialized" }
/** `start` was called; the SFU token is being fetched. */
| { state: "authenticating" }
/** The token is in hand; the LiveKit room's own state from here. */
| { state: "authenticated"; livekitState: LivekitConnectionState }
| { state: "error"; error: MatrixRTCError }
| { state: "stopped" };
/**
* A connection to a Matrix RTC LiveKit backend.
@@ -89,19 +85,25 @@ export enum ConnectionState {
*/
export class Connection {
// Private Behavior
private readonly _state$ = new BehaviorSubject<
ConnectionState | MatrixRTCError
>(ConnectionState.Initialized);
private readonly _state$ = new BehaviorSubject<ConnectionState>({
state: "initialized",
});
/**
* The current state of the connection to the media transport.
*/
public readonly state$: Behavior<ConnectionState | Error> = this._state$;
public readonly state$: Behavior<ConnectionState> = this._state$;
/** Whether we publish on this connection or only subscribe. */
public readonly role: ConnectionRole;
/**
* The media transport to connect to.
*/
public readonly transport: LivekitTransport;
public readonly transport: UnstableLivekitTransport;
/** The homeserver the transport belongs to. */
public readonly serverName: string;
public readonly livekitRoom: LivekitRoom;
@@ -130,12 +132,11 @@ export class Connection {
* */
protected stopped = false;
// TODO: can we just keep the ConnectionOpts object instead of spreading?
private readonly client: OpenIDClientParts;
private readonly client: ClientGetTokenParts & ClientOpenIDParts;
private readonly roomId: string;
private readonly slotId: string;
private readonly logger: Logger;
private readonly ownMembershipIdentity: CallMembershipIdentityParts;
private readonly existingSFUConfig?: SFUConfig;
/**
* Creates a new connection to a matrix RTC LiveKit backend.
*
@@ -145,19 +146,19 @@ export class Connection {
*/
public constructor(opts: ConnectionOpts, logger: Logger) {
this.ownMembershipIdentity = opts.ownMembershipIdentity;
this.existingSFUConfig = opts.existingSFUConfig;
this.roomId = opts.roomId;
this.slotId = opts.slotId;
this.logger = logger.getChild(
"[Connection " + opts.transport.livekit_service_url + "]",
`[Connection ${JSON.stringify(opts.transport)}]`,
);
this.logger.info(
`constructor: ${opts.transport.livekit_service_url} roomId: ${this.roomId} withSfuConfig?: ${opts.existingSFUConfig ? JSON.stringify(opts.existingSFUConfig) : "undefined"}`,
);
const { transport, client, scope } = opts;
this.logger.debug(`constructor called`);
const { role, transport, serverName, client, scope } = opts;
this.scope = scope;
this.livekitRoom = opts.livekitRoomFactory();
this.role = role;
this.transport = transport;
this.serverName = serverName;
this.client = client;
this.remoteParticipants$ = scope.behavior(
@@ -284,9 +285,9 @@ export class Connection {
* Starts the connection.
*
* This will:
* 1. Request an OpenId token `request_token` (allows matrix users to verify their identity with a third-party service.)
* 2. Use this token to request the SFU config to the MatrixRtc authentication service.
* 3. Connect to the configured LiveKit room.
* 1. Authenticate with the transport for our role, through the homeserver
* or the JWT service.
* 2. Connect to the LiveKit room the token is for.
*
* The errors are also represented as a state in the `state$` observable.
* It is safe to ignore those errors and handle them accordingly via the `state$` observable.
@@ -298,16 +299,10 @@ export class Connection {
this.logger.debug("Starting Connection");
this.stopped = false;
try {
this._state$.next(ConnectionState.FetchingConfig);
// We should already have this information after creating the localTransport.
// only call getSFUConfigWithOpenID for connections where we do not have a token yet. (existingJwtTokenData === undefined)
const { url, jwt, livekitAlias } =
this.existingSFUConfig ??
(await this.getSFUConfigForRemoteConnection());
this._state$.next({ state: "authenticating" });
const { url, jwt, livekitAlias } = await this.getSFUConfig();
this.logger.debug(
"Starting Connection to: ",
this.transport.livekit_service_url,
"jwt: ",
"Starting Connection - jwt: ",
jwt,
"wss: ",
url,
@@ -318,16 +313,12 @@ export class Connection {
// If we were stopped while fetching the config, don't proceed to connect
if (this.stopped) return;
// Setup observer once we are done with getSFUConfigWithOpenID
// Setup observer once we are done with getSFUConfig
connectionStateObserver(this.livekitRoom)
.pipe(
this.scope.bind(),
map((s) => s as unknown as ConnectionState),
)
.subscribe((lkState) => {
// It is save to cast lkState to ConnectionState as they are fully overlapping.
this._state$.next(lkState);
});
.pipe(this.scope.bind())
.subscribe((livekitState) =>
this._state$.next({ state: "authenticated", livekitState }),
);
try {
this.logger.info(`livekitRoom.connect ${url}`);
@@ -371,36 +362,37 @@ export class Connection {
// stop() was called while we were connecting, which makes the pending
// connect reject. That is the abort we asked for, not a failure, so
// don't record an error state on a stopped connection or rethrow it
// (start() is not awaited by the ConnectionManager, so a throw here
// (start() is not necessarily awaited by the caller, so a throw here
// becomes an unhandled promise rejection).
this.logger.debug(`Connect aborted because the connection was stopped`);
return;
}
this.logger.debug(`Failed to connect to LiveKit room: ${error}`);
this._state$.next(
error instanceof MatrixRTCError
? error
: error instanceof Error
? new UnknownRTCError(error)
: new UnknownRTCError(new Error(`${error}`)),
);
this._state$.next({
state: "error",
error:
error instanceof MatrixRTCError
? error
: error instanceof Error
? new UnknownRTCError(error)
: new UnknownRTCError(new Error(`${error}`)),
});
// Its okay to ignore the throw. The error is part of the state.
throw error;
}
}
protected async getSFUConfigForRemoteConnection(): Promise<SFUConfig> {
// This will only be called for sfu's where we do not publish ourselves.
// For the local connection we will use the existingJwtTokenData
return await getSFUConfigWithOpenID(
this.client,
this.ownMembershipIdentity,
this.transport.livekit_service_url,
this.roomId,
// dont pass any custom opts for the subscribe only connections
{},
this.logger,
);
protected async getSFUConfig(): Promise<SFUConfig> {
return await getSFUConfig({
client: this.client,
membership: this.ownMembershipIdentity,
role: this.role,
transport: this.transport,
serverName: this.serverName,
roomId: this.roomId,
slotId: this.slotId,
logger: this.logger,
});
}
/**
@@ -410,17 +402,13 @@ export class Connection {
* If the connection is already stopped, this is a no-op.
*/
public async stop(): Promise<void> {
this.logger.debug(
`stop: disconnecing from lk room ${this.transport.livekit_service_url}`,
);
this.logger.debug("stop: disconnecting from lk room");
if (this.stopped) return;
// Mark as stopped before disconnecting so that a connect() aborted by the
// disconnect sees the flag and does not report the abort as an error.
this.stopped = true;
await this.livekitRoom.disconnect();
this._state$.next(ConnectionState.Stopped);
this.logger.debug(
`stop: DONE disconnecing from lk room ${this.transport.livekit_service_url}`,
);
this._state$.next({ state: "stopped" });
this.logger.debug("stop: DONE disconnecting from lk room");
}
}
sdk/src/media-backend/livekit/ConnectionFactory.ts
+31 -20
View File
@@ -14,18 +14,23 @@ import {
// origin than the page
import E2EEWorker from "livekit-client/e2ee-worker?worker&inline";
import { type Logger } from "matrix-js-sdk/lib/logger";
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import { BehaviorSubject, combineLatest, map } from "rxjs";
import { buildLiveKitOptions } from "./livekitOptions";
import { type OpenIDClientParts, type SFUConfig } from "./openIDSFU";
import { type Behavior } from "../../reactive/Behavior";
import { Connection, type ConnectionOpts, ConnectionState } from "./Connection";
import { type ObservableScope } from "../../reactive/ObservableScope";
import { type ResolvedTransport } from "../../api";
import { type PublishRequest } from "../../media-api";
import { type MediaQuality } from "../../config";
import { type PublishRequest } from "../../media-api";
import { type Behavior } from "../../reactive/Behavior";
import { type ObservableScope } from "../../reactive/ObservableScope";
import {
type ClientGetTokenParts,
type ClientOpenIDParts,
type ConnectionRole,
type SFUConfig,
} from "./auth";
import { Connection, type ConnectionOpts } from "./Connection";
import { buildLiveKitOptions } from "./livekitOptions";
import {
audioCaptureOptions,
type PublishRequestFor,
@@ -36,25 +41,27 @@ import {
export interface ConnectionFactory {
createConnection(
scope: ObservableScope,
transport: LivekitTransport,
role: ConnectionRole,
transport: UnstableLivekitTransport,
serverName: string,
ownMembershipIdentity: CallMembershipIdentityParts,
logger: Logger,
sfuConfig?: SFUConfig,
): Connection;
}
/**
* A connection that keeps what it fetched from the JWT service, so that the
* transport can report it as `resolved$`. Nothing while the config is still
* A connection that keeps what it fetched when authenticating, so that the
* transport can report it as `resolved$`. Nothing while the token is still
* being fetched, and nothing again once the connection has stopped or failed.
*/
export class ResolvedConnection extends Connection {
private readonly sfuConfig$: BehaviorSubject<SFUConfig | undefined>;
private readonly sfuConfig$ = new BehaviorSubject<SFUConfig | undefined>(
undefined,
);
public readonly resolved$: Behavior<ResolvedTransport | undefined>;
public constructor(opts: ConnectionOpts, logger: Logger) {
super(opts, logger);
this.sfuConfig$ = new BehaviorSubject(opts.existingSFUConfig);
// One object per fetch, so that a change of state does not read as a
// change of what was fetched
const resolved$ = this.sfuConfig$.pipe(
@@ -63,7 +70,7 @@ export class ResolvedConnection extends Connection {
this.resolved$ = opts.scope.behavior(
combineLatest([resolved$, this.state$]).pipe(
map(([resolved, state]) =>
state === ConnectionState.Stopped || state instanceof Error
state.state === "stopped" || state.state === "error"
? undefined
: resolved,
),
@@ -71,8 +78,8 @@ export class ResolvedConnection extends Connection {
);
}
protected override async getSFUConfigForRemoteConnection(): Promise<SFUConfig> {
const config = await super.getSFUConfigForRemoteConnection();
protected override async getSFUConfig(): Promise<SFUConfig> {
const config = await super.getSFUConfig();
this.sfuConfig$.next(config);
return config;
}
@@ -94,8 +101,9 @@ function resolvedTransport(config: SFUConfig): ResolvedTransport {
*/
export class LivekitConnectionFactory implements ConnectionFactory {
public constructor(
private readonly client: OpenIDClientParts,
private readonly client: ClientGetTokenParts & ClientOpenIDParts,
private readonly roomId: string,
private readonly slotId: string,
private readonly publish: PublishRequest[],
private readonly audioOutputDeviceId$: Behavior<string | undefined>,
private readonly keyProvider: BaseKeyProvider | undefined,
@@ -104,16 +112,19 @@ export class LivekitConnectionFactory implements ConnectionFactory {
public createConnection(
scope: ObservableScope,
transport: LivekitTransport,
role: ConnectionRole,
transport: UnstableLivekitTransport,
serverName: string,
ownMembershipIdentity: CallMembershipIdentityParts,
logger: Logger,
sfuConfig?: SFUConfig,
): Connection {
return new ResolvedConnection(
{
existingSFUConfig: sfuConfig,
roomId: this.roomId,
slotId: this.slotId,
role,
transport,
serverName,
client: this.client,
scope,
livekitRoomFactory: () =>
sdk/src/media-backend/livekit/ConnectionManager.ts
-297
View File
@@ -1,297 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
Copyright 2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import {
combineLatest,
map,
type Observable,
of,
switchMap,
startWith,
catchError,
NEVER,
} from "rxjs";
import { type Logger } from "matrix-js-sdk/lib/logger";
import { type RemoteParticipant } from "livekit-client";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import { type Behavior } from "../../reactive/Behavior";
import { type Connection } from "./Connection.ts";
import { Epoch, type ObservableScope } from "../../reactive/ObservableScope";
import { generateItemsWithEpoch } from "../../reactive/observable";
import { type ConnectionFactory } from "./ConnectionFactory.ts";
import { type SFUConfig } from "./openIDSFU";
export class ConnectionManagerData {
private readonly store: Map<
string,
{ connection: Connection; participants: RemoteParticipant[] }
> = new Map();
public constructor(private readonly logger?: Logger) {}
public add(connection: Connection, participants: RemoteParticipant[]): void {
const key = this.getKey(connection.transport);
const existing = this.store.get(key);
if (!existing) {
this.store.set(key, { connection, participants });
} else {
// Transports are deduplicated by URL upstream, so this should never
// happen; if it does, members may be matched against the wrong room.
this.logger?.warn(
`Merging participants from a second connection to ${key}: existing [${existing.participants.map((p) => p.identity).join(", ")}], adding [${participants.map((p) => p.identity).join(", ")}]`,
);
existing.participants.push(...participants);
}
}
private getKey(transport: LivekitTransport): string {
// This is enough as a key because the ConnectionManager is already scoped by room.
// We also do not need to consider the slotId at this point since each `MatrixRTCSession` is already scoped by `slotDescription: {id, application}`.
return transport.livekit_service_url;
}
public getConnections(): Connection[] {
return Array.from(this.store.values()).map(({ connection }) => connection);
}
public getConnectionForTransport(
transport: LivekitTransport,
): Connection | null {
return this.store.get(this.getKey(transport))?.connection ?? null;
}
public getParticipantsForTransport(
transport: LivekitTransport,
): RemoteParticipant[] {
const key = this.getKey(transport);
const existing = this.store.get(key);
if (existing) {
return existing.participants;
}
return [];
}
}
interface Props {
scope: ObservableScope;
connectionFactory: ConnectionFactory;
localTransport$: Observable<LocalTransport>;
remoteTransports$: Behavior<Epoch<LivekitTransport[]>>;
logger: Logger;
ownMembershipIdentity: CallMembershipIdentityParts;
}
// TODO - write test for scopes (do we really need to bind scope)
export interface IConnectionManager {
connectionManagerData$: Behavior<Epoch<ConnectionManagerData>>;
}
/**
* Crete a `ConnectionManager`
* @param props - Configuration object
* @param props.scope - The observable scope used by this object
* @param props.connectionFactory - Used to create new connections
* @param props.localTransport$ - The transport to publish local media on. (deduplicated with remoteTransports$)
* @param props.remoteTransports$ - All other transports. The connection manager will create connections for each transport. (deduplicated with localTransport$)
* @param props.ownMembershipIdentity - The own membership identity to use.
* @param props.logger - The logger to use.
*
* Each of these behaviors can be interpreted as subscribed list of transports.
*
* Using `registerTransports` independent external modules can control what connections
* are created by the ConnectionManager.
*
* The connection manager will remove all duplicate transports in each subscibed list.
*
* See `unregisterAllTransports` and `unregisterTransport` for details on how to unsubscribe.
*/
export function createConnectionManager$({
scope,
connectionFactory,
localTransport$,
remoteTransports$,
logger: parentLogger,
ownMembershipIdentity,
}: Props): IConnectionManager {
const logger = parentLogger.getChild("[ConnectionManager]");
// TODO logger: only construct one logger from the client and make it compatible via a EC specific sing
const localTransportAsArray$ = localTransport$.pipe(
// LocalMembership already surfaces local transport errors properly in the UI,
// here we can just swallow them
catchError(() => NEVER),
map((transport) => [transport]),
startWith([]),
);
/**
* All transports currently managed by the ConnectionManager.
*
* This list does not include duplicate transports.
*
* It is build based on the list of subscribed transports (`transportsSubscriptions$`).
* externally this is modified via `registerTransports()`.
*/
const localAndRemoteTransports$: Behavior<
Epoch<(LivekitTransport | LocalTransport)[]>
> = scope.behavior(
combineLatest([localTransportAsArray$, remoteTransports$]).pipe(
// Combine local and remote transports into one transport array
// and set the forceOldJwtEndpoint property on the local transport
map(([localTransportAsArray, remoteTransports]) => {
const dedupedRemote = removeDuplicateTransports(remoteTransports.value);
const remoteWithoutLocal = dedupedRemote.filter(
(transport) =>
!localTransportAsArray.find((l) =>
areLivekitTransportsEqual(l.transport, transport),
),
);
logger.debug(
"remoteWithoutLocal",
remoteWithoutLocal,
"localTransportAsArray",
localTransportAsArray,
);
return new Epoch(
[...localTransportAsArray, ...remoteWithoutLocal],
remoteTransports.epoch,
);
}),
),
);
/**
* Connections for each transport in use by one or more session members.
*/
const connections$ = scope.behavior(
localAndRemoteTransports$.pipe(
generateItemsWithEpoch(
"ConnectionManager connections$",
function* (transports) {
for (const transport of transports) {
if (isLocalTransport(transport)) {
// This is the local transport; only the `LocalTransport`
// interface has a `sfuConfig` field.
yield {
keys: [
transport.transport.livekit_service_url,
transport.sfuConfig,
],
data: undefined,
};
} else {
yield {
keys: [
transport.livekit_service_url,
undefined as SFUConfig | undefined,
],
data: undefined,
};
}
}
},
(scope, _data$, serviceUrl, sfuConfig) => {
const connection = connectionFactory.createConnection(
scope,
{
type: "livekit",
livekit_service_url: serviceUrl,
},
ownMembershipIdentity,
logger,
// TODO: This whole optional SFUConfig parameter is not particularly elegant.
// I would like it if connections always fetched the SFUConfig by themselves.
sfuConfig,
);
// Start the connection immediately
// Use connection state to track connection progress
void connection.start();
// TODO subscribe to connection state to retry or log issues?
return connection;
},
),
),
);
const connectionManagerData$ = scope.behavior(
connections$.pipe(
switchMap((connections) => {
const epoch = connections.epoch;
// Map the connections to list of {connection, participants}[]
const listOfConnectionsWithRemoteParticipants = connections.value.map(
(connection) => {
return connection.remoteParticipants$.pipe(
map((participants) => ({
connection,
participants,
})),
);
},
);
// probably not required
if (listOfConnectionsWithRemoteParticipants.length === 0) {
return of(new Epoch(new ConnectionManagerData(), epoch));
}
// combineLatest the several streams into a single stream with the ConnectionManagerData
return combineLatest(listOfConnectionsWithRemoteParticipants).pipe(
map(
(lists) =>
new Epoch(
lists.reduce((data, { connection, participants }) => {
data.add(connection, participants);
return data;
}, new ConnectionManagerData(logger)),
epoch,
),
),
);
}),
),
new Epoch(new ConnectionManagerData(), -1),
);
return { connectionManagerData$ };
}
function removeDuplicateTransports<T extends LivekitTransport>(
transports: T[],
): T[] {
return transports.reduce((acc, transport) => {
if (!acc.some((t) => areLivekitTransportsEqual(t, transport)))
acc.push(transport);
return acc;
}, [] as T[]);
}
/** The transport the local member publishes on, authenticated with. */
export interface LocalTransport {
transport: LivekitTransport;
sfuConfig: SFUConfig;
}
export function isLocalTransport(
obj: LivekitTransport | LocalTransport,
): obj is LocalTransport {
return "transport" in obj && "sfuConfig" in obj;
}
export function areLivekitTransportsEqual<T extends LivekitTransport>(
t1: T | null,
t2: T | null,
): boolean {
if (t1 && t2) return t1.livekit_service_url === t2.livekit_service_url;
return !t1 && !t2;
}
sdk/src/media-backend/livekit/ConnectionManager.test.ts → sdk/src/media-backend/livekit/ConnectionMap.test.ts
+194 -63
View File
@@ -6,8 +6,8 @@ Please see LICENSE in the repository root for full details.
*/
import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
import { BehaviorSubject, NEVER } from "rxjs";
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import { BehaviorSubject } from "rxjs";
import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import { type RemoteParticipant } from "livekit-client";
import { logger } from "matrix-js-sdk/lib/logger";
@@ -17,27 +17,40 @@ import {
ObservableScope,
} from "../../reactive/ObservableScope";
import {
createConnectionManager$,
ConnectionManagerData,
areLivekitTransportsEqual,
} from "./ConnectionManager.ts";
createConnectionMap$,
ConnectionMap,
keyFor,
} from "./ConnectionMap.ts";
import { type ConnectionFactory } from "./ConnectionFactory.ts";
import { type Connection } from "./Connection.ts";
import { ownMemberMock, withTestScheduler } from "../../utils/test";
import { type Behavior } from "../../reactive/Behavior";
import { type TransportLocator } from "./auth";
// Some test constants
const TRANSPORT_1: LivekitTransport = {
type: "livekit",
livekit_service_url: "https://lk.example.org",
const TRANSPORT_1: TransportLocator = {
transport: {
type: "livekit",
livekit_service_url: "https://lk.example.org",
},
serverName: "example.org",
};
const TRANSPORT_2: LivekitTransport = {
type: "livekit",
livekit_service_url: "https://lk.sample.com",
const TRANSPORT_2: TransportLocator = {
transport: {
type: "livekit",
livekit_service_url: "https://lk.sample.com",
},
serverName: "sample.com",
};
function findConnection(
transport: TransportLocator,
connections: Connection[],
): Connection | undefined {
const key = keyFor(transport);
return connections.find((c) => keyFor(c) === key);
}
let fakeConnectionFactory: ConnectionFactory;
let testScope: ObservableScope;
@@ -51,9 +64,15 @@ beforeEach(() => {
vi.mocked(fakeConnectionFactory).createConnection = vi
.fn()
.mockImplementation(
(scope: ObservableScope, transport: LivekitTransport) => {
(
scope: ObservableScope,
_role: unknown,
transport: UnstableLivekitTransport,
serverName: string,
) => {
const mockConnection = {
transport,
serverName,
remoteParticipants$: new BehaviorSubject([]),
} as unknown as Connection;
vi.mocked(mockConnection).start = vi.fn();
@@ -75,10 +94,10 @@ afterEach(() => {
describe("connections$ stream", () => {
test("Should create and start new connections for each transports", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const { connectionManagerData$ } = createConnectionManager$({
const connectionMap$ = createConnectionMap$({
scope: testScope,
connectionFactory: fakeConnectionFactory,
localTransport$: NEVER,
localTransport: TRANSPORT_1,
remoteTransports$: behavior("a", {
a: new Epoch([TRANSPORT_1, TRANSPORT_2], 0),
}),
@@ -87,7 +106,7 @@ describe("connections$ stream", () => {
});
expectObservable(
connectionManagerData$.pipe(mapEpoch((d) => d.getConnections())),
connectionMap$.pipe(mapEpoch((d) => d.getConnections())),
).toBe("a", {
a: expect.toSatisfy((e: Epoch<Connection[]>) => {
const connections = e.value;
@@ -97,15 +116,11 @@ describe("connections$ stream", () => {
vi.mocked(fakeConnectionFactory).createConnection,
).toHaveBeenCalledTimes(2);
const conn1 = connections.find((c) =>
areLivekitTransportsEqual(c.transport, TRANSPORT_1),
);
const conn1 = findConnection(TRANSPORT_1, connections);
expect(conn1).toBeDefined();
expect(conn1!.start).toHaveBeenCalled();
const conn2 = connections.find((c) =>
areLivekitTransportsEqual(c.transport, TRANSPORT_2),
);
const conn2 = findConnection(TRANSPORT_2, connections);
expect(conn2).toBeDefined();
expect(conn2!.start).toHaveBeenCalled();
return true;
@@ -114,12 +129,126 @@ describe("connections$ stream", () => {
});
});
test("Should start connection only once", () => {
test("Tags the local transport as the publisher and the rest as subscribers", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const { connectionManagerData$ } = createConnectionManager$({
const connectionMap$ = createConnectionMap$({
scope: testScope,
connectionFactory: fakeConnectionFactory,
localTransport$: NEVER,
localTransport: TRANSPORT_1,
remoteTransports$: behavior("a", {
a: new Epoch([TRANSPORT_2], 0),
}),
logger: logger,
ownMembershipIdentity: ownMemberMock,
});
expectObservable(connectionMap$).toBe("a", {
a: expect.toSatisfy(() => {
const { createConnection } = vi.mocked(fakeConnectionFactory);
expect(createConnection).toHaveBeenCalledWith(
expect.anything(),
"publisher",
TRANSPORT_1.transport,
TRANSPORT_1.serverName,
ownMemberMock,
expect.anything(),
);
expect(createConnection).toHaveBeenCalledWith(
expect.anything(),
"subscriber",
TRANSPORT_2.transport,
TRANSPORT_2.serverName,
ownMemberMock,
expect.anything(),
);
return true;
}),
});
});
});
test("Shares one connection between homeservers advertising the same JWT service", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const sameServiceElsewhere: TransportLocator = {
transport: TRANSPORT_1.transport,
serverName: "elsewhere.org",
};
const connectionMap$ = createConnectionMap$({
scope: testScope,
connectionFactory: fakeConnectionFactory,
localTransport: TRANSPORT_1,
remoteTransports$: behavior("a", {
a: new Epoch([sameServiceElsewhere], 0),
}),
logger: logger,
ownMembershipIdentity: ownMemberMock,
});
expectObservable(
connectionMap$.pipe(mapEpoch((d) => d.getConnections())),
).toBe("a", {
a: expect.toSatisfy((e: Epoch<Connection[]>) => {
expect(e.value.length).toBe(1);
// The JWT service is reached directly, so the server name plays no
// part and the local locator, which came first, is the one used
expect(
vi.mocked(fakeConnectionFactory).createConnection,
).toHaveBeenCalledExactlyOnceWith(
expect.anything(),
"publisher",
TRANSPORT_1.transport,
"example.org",
expect.anything(),
expect.anything(),
);
return true;
}),
});
});
});
test("Keeps one connection per homeserver for transports with a url", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const urlTransport: TransportLocator = {
transport: { type: "livekit", url: "wss://sfu.example.org" },
serverName: "example.org",
};
const sameUrlElsewhere: TransportLocator = {
transport: urlTransport.transport,
serverName: "elsewhere.org",
};
const connectionMap$ = createConnectionMap$({
scope: testScope,
connectionFactory: fakeConnectionFactory,
localTransport: urlTransport,
remoteTransports$: behavior("a", {
a: new Epoch([sameUrlElsewhere], 0),
}),
logger: logger,
ownMembershipIdentity: ownMemberMock,
});
expectObservable(
connectionMap$.pipe(mapEpoch((d) => d.getConnections())),
).toBe("a", {
a: expect.toSatisfy((e: Epoch<Connection[]>) => {
// Our homeserver asks the named server for the token, so each
// server name needs its own connection
expect(e.value.length).toBe(2);
expect(findConnection(urlTransport, e.value)).toBeDefined();
expect(findConnection(sameUrlElsewhere, e.value)).toBeDefined();
return true;
}),
});
});
});
test("Should start connection only once", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const connectionMap$ = createConnectionMap$({
scope: testScope,
connectionFactory: fakeConnectionFactory,
localTransport: TRANSPORT_1,
remoteTransports$: behavior("abcdef", {
a: new Epoch([TRANSPORT_1], 0),
b: new Epoch([TRANSPORT_1], 1),
@@ -133,7 +262,7 @@ describe("connections$ stream", () => {
});
expectObservable(
connectionManagerData$.pipe(mapEpoch((d) => d.getConnections())),
connectionMap$.pipe(mapEpoch((d) => d.getConnections())),
).toBe("xxxxxa", {
x: expect.anything(),
a: expect.toSatisfy((e: Epoch<Connection[]>) => {
@@ -144,14 +273,10 @@ describe("connections$ stream", () => {
vi.mocked(fakeConnectionFactory).createConnection,
).toHaveBeenCalledTimes(2);
const conn2 = connections.find((c) =>
areLivekitTransportsEqual(c.transport, TRANSPORT_2),
);
const conn2 = findConnection(TRANSPORT_2, connections);
expect(conn2).toBeDefined();
const conn1 = connections.find((c) =>
areLivekitTransportsEqual(c.transport, TRANSPORT_1),
);
const conn1 = findConnection(TRANSPORT_1, connections);
expect(conn1).toBeDefined();
expect(conn1!.start).toHaveBeenCalledOnce();
@@ -163,10 +288,10 @@ describe("connections$ stream", () => {
test("Should cleanup connections when not needed anymore", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const { connectionManagerData$ } = createConnectionManager$({
const connectionMap$ = createConnectionMap$({
scope: testScope,
connectionFactory: fakeConnectionFactory,
localTransport$: NEVER,
localTransport: TRANSPORT_1,
remoteTransports$: behavior("abc", {
a: new Epoch([TRANSPORT_1], 0),
b: new Epoch([TRANSPORT_1, TRANSPORT_2], 1),
@@ -177,7 +302,7 @@ describe("connections$ stream", () => {
});
expectObservable(
connectionManagerData$.pipe(mapEpoch((d) => d.getConnections())),
connectionMap$.pipe(mapEpoch((d) => d.getConnections())),
).toBe("xab", {
x: expect.anything(),
a: expect.toSatisfy((e: Epoch<Connection[]>) => {
@@ -190,8 +315,9 @@ describe("connections$ stream", () => {
expect(connections.length).toBe(1);
// The second connection should have been stopped has it is no longer needed.
const connection2 = allCreatedConnections.find((c) =>
areLivekitTransportsEqual(c.transport, TRANSPORT_2),
const connection2 = findConnection(
TRANSPORT_2,
allCreatedConnections,
);
expect(connection2).toBeDefined();
expect(connection2!.stop).toHaveBeenCalled();
@@ -207,40 +333,36 @@ describe("connections$ stream", () => {
});
});
describe("ConnectionManagerData", () => {
describe("ConnectionMap", () => {
test("warns when a second connection to the same URL is merged", () => {
const warn = vi.spyOn(logger, "warn").mockImplementation(() => {});
const data = new ConnectionManagerData(logger);
const connection = { transport: TRANSPORT_1 } as unknown as Connection;
const data = new ConnectionMap(logger);
const connection = TRANSPORT_1 as unknown as Connection;
const p = (identity: string): RemoteParticipant =>
({ identity }) as unknown as RemoteParticipant;
data.add(connection, [p("a")]);
data.add({ ...connection } as Connection, [p("b")]);
expect(warn).toHaveBeenCalledWith(
expect.stringContaining(
"second connection to https://lk.example.org: existing [a], adding [b]",
'second connection to [null,"https://lk.example.org",null]: existing [a], adding [b]',
),
);
expect(data.getParticipantsForTransport(TRANSPORT_1)).toHaveLength(2);
});
});
describe("connectionManagerData$ stream", () => {
describe("connectionMap$", () => {
// Used in test to control fake connections' remoteParticipants$ streams
let fakeRemoteParticipantsStreams: Map<string, Behavior<RemoteParticipant[]>>;
function keyForTransport(transport: LivekitTransport): string {
return `${transport.livekit_service_url}`;
}
beforeEach(() => {
fakeRemoteParticipantsStreams = new Map();
function getRemoteParticipantsFor(
transport: LivekitTransport,
transport: TransportLocator,
): Behavior<RemoteParticipant[]> {
return (
fakeRemoteParticipantsStreams.get(keyForTransport(transport)) ??
fakeRemoteParticipantsStreams.get(keyFor(transport)) ??
new BehaviorSubject([])
);
}
@@ -249,13 +371,22 @@ describe("connectionManagerData$ stream", () => {
vi.mocked(fakeConnectionFactory).createConnection = vi
.fn()
.mockImplementation(
(scope: ObservableScope, transport: LivekitTransport) => {
(
scope: ObservableScope,
_role: unknown,
transport: UnstableLivekitTransport,
serverName: string,
) => {
const fakeRemoteParticipants$ = new BehaviorSubject<
RemoteParticipant[]
>([]);
const mockConnection = {
transport,
remoteParticipants$: getRemoteParticipantsFor(transport),
serverName,
remoteParticipants$: getRemoteParticipantsFor({
transport,
serverName,
}),
} as unknown as Connection;
vi.mocked(mockConnection).start = vi.fn();
vi.mocked(mockConnection).stop = vi.fn();
@@ -265,7 +396,7 @@ describe("connectionManagerData$ stream", () => {
});
fakeRemoteParticipantsStreams.set(
keyForTransport(transport),
keyFor({ transport, serverName }),
fakeRemoteParticipants$,
);
return mockConnection;
@@ -274,11 +405,11 @@ describe("connectionManagerData$ stream", () => {
});
test("Should report connections with the remote participants", () => {
withTestScheduler(({ expectObservable, schedule, behavior }) => {
withTestScheduler(({ expectObservable, behavior }) => {
// Setup the fake participants streams behavior
// ==============================
fakeRemoteParticipantsStreams.set(
keyForTransport(TRANSPORT_1),
keyFor(TRANSPORT_1),
behavior("oa-b", {
o: [],
a: [{ identity: "user1A" } as RemoteParticipant],
@@ -290,7 +421,7 @@ describe("connectionManagerData$ stream", () => {
);
fakeRemoteParticipantsStreams.set(
keyForTransport(TRANSPORT_2),
keyFor(TRANSPORT_2),
behavior("o-a", {
o: [],
a: [{ identity: "user2A" } as RemoteParticipant],
@@ -298,10 +429,10 @@ describe("connectionManagerData$ stream", () => {
);
// ==============================
const { connectionManagerData$ } = createConnectionManager$({
const connectionMap$ = createConnectionMap$({
scope: testScope,
connectionFactory: fakeConnectionFactory,
localTransport$: NEVER,
localTransport: TRANSPORT_1,
remoteTransports$: behavior("a", {
a: new Epoch([TRANSPORT_1, TRANSPORT_2], 0),
}),
@@ -309,16 +440,16 @@ describe("connectionManagerData$ stream", () => {
ownMembershipIdentity: ownMemberMock,
});
expectObservable(connectionManagerData$).toBe("abcd", {
expectObservable(connectionMap$).toBe("abcd", {
a: expect.toSatisfy((e) => {
const data: ConnectionManagerData = e.value;
const data: ConnectionMap = e.value;
expect(data.getConnections().length).toBe(2);
expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(0);
expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(0);
return true;
}),
b: expect.toSatisfy((e) => {
const data: ConnectionManagerData = e.value;
const data: ConnectionMap = e.value;
expect(data.getConnections().length).toBe(2);
expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(1);
expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(0);
@@ -328,7 +459,7 @@ describe("connectionManagerData$ stream", () => {
return true;
}),
c: expect.toSatisfy((e) => {
const data: ConnectionManagerData = e.value;
const data: ConnectionMap = e.value;
expect(data.getConnections().length).toBe(2);
expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(1);
expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(1);
@@ -341,7 +472,7 @@ describe("connectionManagerData$ stream", () => {
return true;
}),
d: expect.toSatisfy((e) => {
const data: ConnectionManagerData = e.value;
const data: ConnectionMap = e.value;
expect(data.getConnections().length).toBe(2);
expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(2);
expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(1);
sdk/src/media-backend/livekit/ConnectionMap.ts
+194
View File
@@ -0,0 +1,194 @@
/*
Copyright 2025-2026 Element Creations Ltd.
Copyright 2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type RemoteParticipant } from "livekit-client";
import { type Logger } from "matrix-js-sdk/lib/logger";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import { combineLatest, map, of, switchMap, tap } from "rxjs";
import { type Behavior } from "../../reactive/Behavior";
import {
Epoch,
mapEpoch,
type ObservableScope,
} from "../../reactive/ObservableScope";
import { generateItemsWithEpoch } from "../../reactive/observable";
import { type ConnectionRole, type TransportLocator } from "./auth";
import { type Connection } from "./Connection";
import { type ConnectionFactory } from "./ConnectionFactory";
// Branded, so that only values returned by `keyFor` can be used as keys
export type TransportKey = string & { __transportKey: true };
/**
* The key a locator is identified by in a map or set. Locators with the same
* key share a connection: the server name only takes part in authentication
* for a transport with a `url`, where our homeserver asks that server for the
* token. A JWT service is reached directly, so members of several homeservers
* advertising the same one share a connection rather than each get their own.
*/
export function keyFor({
transport,
serverName,
}: TransportLocator): TransportKey {
const hasUrl = "url" in transport;
// Serialize the fields in a consistent order
return JSON.stringify([
hasUrl ? transport.url : null,
"livekit_service_url" in transport ? transport.livekit_service_url : null,
hasUrl ? serverName : null,
]) as TransportKey;
}
function removeDuplicateTransports(
transports: TransportLocator[],
): TransportLocator[] {
// The first locator wins, so the local one stays first in the list
const map = new Map<TransportKey, TransportLocator>();
for (const transport of transports) {
const key = keyFor(transport);
if (!map.has(key)) map.set(key, transport);
}
return [...map.values()];
}
/** The connections at one moment, with the remote participants seen on each. */
export class ConnectionMap {
private readonly map = new Map<
TransportKey,
{ connection: Connection; participants: RemoteParticipant[] }
>();
public constructor(private readonly logger?: Logger) {}
public add(connection: Connection, participants: RemoteParticipant[]): void {
const key = keyFor(connection);
const existing = this.map.get(key);
if (!existing) {
this.map.set(key, { connection, participants });
} else {
// Transports are deduplicated by key upstream, so this should never
// happen; if it does, members may be matched against the wrong room.
this.logger?.warn(
`Merging participants from a second connection to ${key}: existing [${existing.participants.map((p) => p.identity).join(", ")}], adding [${participants.map((p) => p.identity).join(", ")}]`,
);
existing.participants.push(...participants);
}
}
public getConnections(): Connection[] {
return Array.from(this.map.values()).map(({ connection }) => connection);
}
public getConnectionForTransport(
transport: TransportLocator,
): Connection | null {
return this.map.get(keyFor(transport))?.connection ?? null;
}
public getParticipantsForTransport(
transport: TransportLocator,
): RemoteParticipant[] {
return this.map.get(keyFor(transport))?.participants ?? [];
}
}
interface Props {
scope: ObservableScope;
connectionFactory: ConnectionFactory;
/** The transport we publish on; the one connection with the publisher role. */
localTransport: TransportLocator;
/** Every transport some member is on, deduplicated against the local one. */
remoteTransports$: Behavior<Epoch<TransportLocator[]>>;
logger: Logger;
ownMembershipIdentity: CallMembershipIdentityParts;
}
/**
* One connection per transport in use by one or more session members, with
* the remote participants each one sees. The connections start as they are
* created and stop as their transport drops out of the list.
*/
export function createConnectionMap$({
scope,
connectionFactory,
localTransport,
remoteTransports$,
logger: parentLogger,
ownMembershipIdentity,
}: Props): Behavior<Epoch<ConnectionMap>> {
const logger = parentLogger.getChild("[ConnectionMap]");
const localAndRemoteTransports$: Behavior<Epoch<TransportLocator[]>> =
scope.behavior(
remoteTransports$.pipe(
mapEpoch((remoteTransports) =>
removeDuplicateTransports([localTransport, ...remoteTransports]),
),
tap((transports) =>
logger.debug("localAndRemoteTransports$ = ", transports),
),
),
);
const localTransportKey = keyFor(localTransport);
const connections$ = scope.behavior(
localAndRemoteTransports$.pipe(
generateItemsWithEpoch(
"ConnectionMap connections$",
function* (transports) {
for (const transport of transports) {
const role: ConnectionRole =
keyFor(transport) === localTransportKey
? "publisher"
: "subscriber";
// The locator is an object, so a stable key stands in for it
yield { keys: [role, keyFor(transport)], data: transport };
}
},
(scope, locator$, role, _key) => {
const { transport, serverName } = locator$.value;
const connection = connectionFactory.createConnection(
scope,
role,
transport,
serverName,
ownMembershipIdentity,
logger,
);
// The state carries the progress, errors included
void connection.start();
return connection;
},
),
),
);
return scope.behavior(
connections$.pipe(
switchMap(({ value: connections, epoch }) => {
if (connections.length === 0)
return of(new Epoch(new ConnectionMap(), epoch));
return combineLatest(
connections.map((connection) =>
connection.remoteParticipants$.pipe(
map((participants) => ({ connection, participants })),
),
),
(...list) => {
const map = new ConnectionMap(logger);
for (const item of list)
map.add(item.connection, item.participants);
return new Epoch(map, epoch);
},
);
}),
),
);
}
sdk/src/media-backend/livekit/DataChannel.test.ts
+21 -12
View File
@@ -6,24 +6,33 @@ Please see LICENSE in the repository root for full details.
*/
import { EventEmitter } from "events";
import { type RemoteParticipant, RoomEvent } from "livekit-client";
import {
ConnectionState as LivekitConnectionState,
type RemoteParticipant,
RoomEvent,
} from "livekit-client";
import { BehaviorSubject } from "rxjs";
import { describe, expect, it, vi } from "vitest";
import { Epoch } from "../../reactive/ObservableScope";
import { constant } from "../../reactive/Behavior";
import { type DataPacket } from "../api";
import { type Connection, ConnectionState } from "./Connection";
import { ConnectionManagerData } from "./ConnectionManager";
import { type Connection, type ConnectionState } from "./Connection";
import { ConnectionMap } from "./ConnectionMap";
import { createDataChannel$ } from "./DataChannel";
import { testScope } from "../../utils/test";
import { exampleTransport } from "./test";
const encode = (text: string): Uint8Array => new TextEncoder().encode(text);
const connected: ConnectionState = {
state: "authenticated",
livekitState: LivekitConnectionState.Connected,
};
function fakeConnection(
url: string,
state = ConnectionState.LivekitConnected,
state: ConnectionState = connected,
): {
connection: Connection;
room: EventEmitter;
@@ -33,6 +42,7 @@ function fakeConnection(
const publishData = vi.fn(async () => Promise.resolve());
const connection = {
transport: { ...exampleTransport, livekit_service_url: url },
serverName: "example.org",
state$: constant(state),
livekitRoom: Object.assign(room, { localParticipant: { publishData } }),
} as unknown as Connection;
@@ -41,15 +51,15 @@ function fakeConnection(
function setup(connections: Connection[]): {
received: DataPacket[];
connections$: BehaviorSubject<Epoch<ConnectionManagerData>>;
connections$: BehaviorSubject<Epoch<ConnectionMap>>;
sendData: (topic: string, text: string) => Promise<void>;
} {
const data = new ConnectionManagerData();
const data = new ConnectionMap();
for (const connection of connections) data.add(connection, []);
const connections$ = new BehaviorSubject(new Epoch(data));
const { data$, sendData } = createDataChannel$({
scope: testScope(),
connectionManager: { connectionManagerData$: connections$ },
connectionMap$: connections$,
connection$: constant(connections[0] ?? null),
});
const received: DataPacket[] = [];
@@ -89,7 +99,7 @@ describe("data$", () => {
const second = fakeConnection("https://sfu-2");
const { received, connections$ } = setup([first.connection]);
const data = new ConnectionManagerData();
const data = new ConnectionMap();
data.add(first.connection, []);
data.add(second.connection, []);
connections$.next(new Epoch(data, 1));
@@ -122,10 +132,9 @@ describe("sendData", () => {
});
it("rejects while the local transport is not connected", async () => {
const { connection, publishData } = fakeConnection(
"https://sfu",
ConnectionState.Initialized,
);
const { connection, publishData } = fakeConnection("https://sfu", {
state: "initialized",
});
const { sendData } = setup([connection]);
await expect(sendData("chat", "hello")).rejects.toThrow("not connected");
sdk/src/media-backend/livekit/DataChannel.ts
+16 -7
View File
@@ -6,6 +6,7 @@ Please see LICENSE in the repository root for full details.
*/
import {
ConnectionState as LivekitConnectionState,
type RemoteParticipant,
type Room as LivekitRoom,
RoomEvent,
@@ -22,14 +23,17 @@ import {
} from "rxjs";
import { type Behavior } from "../../reactive/Behavior";
import { type ObservableScope } from "../../reactive/ObservableScope";
import {
type Epoch,
type ObservableScope,
} from "../../reactive/ObservableScope";
import { type DataPacket } from "../api";
import { type Connection, ConnectionState } from "./Connection";
import { type IConnectionManager } from "./ConnectionManager";
import { type Connection } from "./Connection";
import { type ConnectionMap } from "./ConnectionMap";
interface Props {
scope: ObservableScope;
connectionManager: IConnectionManager;
connectionMap$: Behavior<Epoch<ConnectionMap>>;
/** The local member's connection, which carries what we send. */
connection$: Behavior<Connection | null>;
}
@@ -47,13 +51,13 @@ export interface DataChannel {
*/
export function createDataChannel$({
scope,
connectionManager,
connectionMap$,
connection$,
}: Props): DataChannel {
const decoder = new TextDecoder();
const encoder = new TextEncoder();
const rooms$ = connectionManager.connectionManagerData$.pipe(
const rooms$ = connectionMap$.pipe(
map(({ value }) => value.getConnections().map((c) => c.livekitRoom)),
distinctUntilChanged(
(a, b) => a.length === b.length && a.every((room, i) => room === b[i]),
@@ -79,7 +83,12 @@ export function createDataChannel$({
const sendData = async (topic: string, text: string): Promise<void> => {
const connection = connection$.value;
if (connection?.state$.value !== ConnectionState.LivekitConnected)
const state = connection?.state$.value;
if (
connection === null ||
state?.state !== "authenticated" ||
state.livekitState !== LivekitConnectionState.Connected
)
throw new Error("Cannot send data: the local transport is not connected");
await connection.livekitRoom.localParticipant.publishData(
encoder.encode(text),
sdk/src/media-backend/livekit/LivekitBackend.test.ts
+356 -41
View File
@@ -5,21 +5,52 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
ConnectionState as LivekitConnectionState,
type Room as LivekitRoom,
type RemoteParticipant,
} from "livekit-client";
import fetchMock from "fetch-mock";
import { logger } from "matrix-js-sdk/lib/logger";
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import { BehaviorSubject } from "rxjs";
import { afterEach, describe, expect, it } from "vitest";
import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import { BehaviorSubject, NEVER } from "rxjs";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { E2eeType } from "../../encryption";
import { MatrixRTCError, NoMatrix2AuthorizationService } from "../../errors";
import { constant } from "../../reactive/Behavior";
import { ObservableScope } from "../../reactive/ObservableScope";
import { testScope } from "../../utils/test";
import { type RTCMember } from "../../api";
import { defaultSessionTimings } from "../../config";
import {
flushPromises,
mockEmitter,
ownMemberMock,
testScope,
} from "../../utils/test";
import { MediaConnectionState } from "../api";
import { ConnectionState } from "./Connection";
import * as auth from "./auth";
import { type TransportLocator } from "./auth";
import { type Connection, type ConnectionState } from "./Connection";
import * as connectionFactory from "./ConnectionFactory";
import { toMediaConnectionState } from "./connectionState";
import { createTransportRegistry, supportsDelegation } from "./LivekitBackend";
import { exampleTransport } from "./test";
import {
createLivekitBackend,
createTransportRegistry,
supportsDelegation,
} from "./LivekitBackend";
import {
exampleServiceUrl,
exampleTransport,
mockRemoteParticipant,
} from "./test";
const otherTransport: LivekitTransport = {
const locator = (
transport: UnstableLivekitTransport,
serverName = "example.org",
): TransportLocator => ({ transport, serverName });
const otherTransport: UnstableLivekitTransport = {
type: "livekit",
livekit_service_url: "https://other.example.org",
};
@@ -27,31 +58,31 @@ const otherTransport: LivekitTransport = {
describe("createTransportRegistry", () => {
it("connects to a transport once however many members are on it, and drops it with the last", () => {
const registry = createTransportRegistry(testScope());
const urls = (): string[] =>
registry.transports$.value.value.map((t) => t.livekit_service_url);
const transports = (): UnstableLivekitTransport[] =>
registry.transports$.value.value.map((t) => t.transport);
const alice = new ObservableScope();
const bob = new ObservableScope();
registry.follow(alice, new BehaviorSubject(exampleTransport));
registry.follow(bob, new BehaviorSubject(exampleTransport));
expect(urls()).toEqual([exampleTransport.livekit_service_url]);
registry.follow(alice, new BehaviorSubject(locator(exampleTransport)));
registry.follow(bob, new BehaviorSubject(locator(exampleTransport)));
expect(transports()).toEqual([exampleTransport]);
alice.end();
expect(urls()).toEqual([exampleTransport.livekit_service_url]);
expect(transports()).toEqual([exampleTransport]);
bob.end();
expect(urls()).toEqual([]);
expect(transports()).toEqual([]);
});
it("follows a member that changes transport", () => {
const registry = createTransportRegistry(testScope());
const transport$ = new BehaviorSubject<LivekitTransport | undefined>(
exampleTransport,
const transport$ = new BehaviorSubject<TransportLocator | undefined>(
locator(exampleTransport),
);
registry.follow(testScope(), transport$);
transport$.next(otherTransport);
expect(registry.transports$.value.value).toEqual([otherTransport]);
transport$.next(locator(otherTransport));
expect(registry.transports$.value.value).toEqual([locator(otherTransport)]);
transport$.next(undefined);
expect(registry.transports$.value.value).toEqual([]);
@@ -61,7 +92,7 @@ describe("createTransportRegistry", () => {
const scope = new ObservableScope();
const registry = createTransportRegistry(scope);
const member = new ObservableScope();
registry.follow(member, new BehaviorSubject(exampleTransport));
registry.follow(member, new BehaviorSubject(locator(exampleTransport)));
scope.end();
expect(() => member.end()).not.toThrow();
@@ -70,43 +101,327 @@ describe("createTransportRegistry", () => {
describe("supportsDelegation", () => {
afterEach(() => fetchMock.reset());
const homeserver = "https://matrix.example.org";
const homeserverEndpoint = `${homeserver}/_matrix/client/unstable/io.element.msc4195/rtc/livekit/delegate_delayed_leave`;
const transportEndpoint = `${exampleTransport.livekit_service_url}/delegate_delayed_leave`;
const endpoint = `${exampleServiceUrl}/delegate_delayed_leave`;
it.each([
["the homeserver supports it", 400, 404, true],
["only the transport supports it", 404, 400, true],
["neither supports it", 404, 404, false],
])("%s", async (_name, homeserverStatus, transportStatus, expected) => {
fetchMock.post(homeserverEndpoint, homeserverStatus);
fetchMock.post(transportEndpoint, transportStatus);
["anything but a 404", 401, true],
["a 404", 404, false],
])("reads %s as the answer", async (_name, status, expected) => {
fetchMock.post(endpoint, status);
await expect(
supportsDelegation(homeserver, exampleTransport, logger),
supportsDelegation(endpoint, "transport", logger),
).resolves.toBe(expected);
});
it("assumes no support where the probe itself fails", async () => {
fetchMock.post(homeserverEndpoint, { throws: new TypeError("CORS") });
fetchMock.post(transportEndpoint, { throws: new TypeError("CORS") });
fetchMock.post(endpoint, { throws: new TypeError("CORS") });
await expect(
supportsDelegation(homeserver, exampleTransport, logger),
supportsDelegation(endpoint, "transport", logger),
).resolves.toBe(false);
});
});
describe("toMediaConnectionState", () => {
it("maps every LiveKit state and passes an error through", () => {
expect(toMediaConnectionState(ConnectionState.FetchingConfig)).toBe(
it("maps every state and passes an error through", () => {
expect(toMediaConnectionState({ state: "authenticating" })).toBe(
MediaConnectionState.Preparing,
);
expect(toMediaConnectionState(ConnectionState.LivekitConnected)).toBe(
MediaConnectionState.Connected,
);
expect(
toMediaConnectionState(ConnectionState.LivekitSignalReconnecting),
toMediaConnectionState({
state: "authenticated",
livekitState: LivekitConnectionState.Connected,
}),
).toBe(MediaConnectionState.Connected);
expect(
toMediaConnectionState({
state: "authenticated",
livekitState: LivekitConnectionState.SignalReconnecting,
}),
).toBe(MediaConnectionState.Reconnecting);
const error = new Error("gone");
expect(toMediaConnectionState(error)).toBe(error);
expect(toMediaConnectionState({ state: "stopped" })).toBe(
MediaConnectionState.Stopped,
);
const error = new MatrixRTCError("gone");
expect(toMediaConnectionState({ state: "error", error })).toBe(error);
});
});
describe("createLivekitBackend", () => {
const homeserver = "https://matrix.example.org";
const homeserverEndpoint = `${homeserver}/_matrix/client/unstable/io.element.msc4195/rtc/livekit/delegate_delayed_leave`;
const urlTransport: UnstableLivekitTransport = {
type: "livekit",
url: "wss://sfu.example.org",
};
const bothTransport: UnstableLivekitTransport = {
...urlTransport,
...exampleTransport,
};
const client = {
baseUrl: homeserver,
getOpenIdToken: vi.fn(),
_unstable_getLivekitToken: vi.fn(),
_unstable_delegateDelayedLeave: vi.fn(),
};
/** The connections the stubbed factory handed out, with their participants to drive. */
let connections: {
connection: Connection;
participants$: BehaviorSubject<RemoteParticipant[]>;
state$: BehaviorSubject<ConnectionState>;
}[];
const connected: ConnectionState = {
state: "authenticated",
livekitState: LivekitConnectionState.Connected,
};
/** Stands in for the factory: connections whose state and participants the test drives. */
function stubFactory(initialState: ConnectionState): void {
vi.spyOn(connectionFactory, "LivekitConnectionFactory").mockImplementation(
function (this: unknown) {
return {
createConnection: (
scope: ObservableScope,
role: auth.ConnectionRole,
transport: UnstableLivekitTransport,
serverName: string,
): Connection => {
const participants$ = new BehaviorSubject<RemoteParticipant[]>([]);
const state$ = new BehaviorSubject<ConnectionState>(initialState);
const connection = {
role,
transport,
serverName,
state$,
remoteParticipants$: scope.behavior(participants$),
livekitRoom: fakeLivekitRoom(),
start: vi.fn(async () => Promise.resolve()),
stop: vi.fn(async () => Promise.resolve()),
} as unknown as Connection;
connections.push({ connection, participants$, state$ });
return connection;
},
} as unknown as connectionFactory.LivekitConnectionFactory;
},
);
}
beforeEach(() => {
connections = [];
stubFactory(connected);
});
afterEach(() => {
vi.restoreAllMocks();
fetchMock.reset();
});
function backend(): ReturnType<typeof createLivekitBackend> {
return createLivekitBackend(
testScope(),
{
roomId: "!room:example.org",
slotId: "m.call#ROOM",
ownMembershipIdentity: ownMemberMock,
publish: [],
encryptionSystem: { kind: E2eeType.NONE },
mediaKeys$: NEVER,
timings: defaultSessionTimings,
logger,
},
{ client },
);
}
// Delegation goes through whichever service authenticates us as a
// publisher: the JWT service wherever the transport names one, the
// homeserver for a `url` alone
it.each([
["the JWT service", exampleTransport, 404, 401, true],
["the JWT service, and only it", bothTransport, 401, 404, false],
["the homeserver", urlTransport, 401, 404, true],
["neither", exampleTransport, 404, 404, false],
])(
"prepares the local transport and asks %s about delegation",
async (_name, transport, homeserverStatus, serviceStatus, expected) => {
fetchMock.post(homeserverEndpoint, homeserverStatus);
fetchMock.post(
`${exampleServiceUrl}/delegate_delayed_leave`,
serviceStatus,
);
const capabilities = await backend().prepareLocalTransport(transport);
expect(capabilities).toEqual({ canDelegateDelayedLeave: expected });
expect(connections).toHaveLength(1);
expect(connections[0].connection.role).toBe("publisher");
expect(connections[0].connection.transport).toEqual(transport);
expect(connections[0].connection.serverName).toBe("example.org");
},
);
it("waits for the publisher connection to hold its token before the join", async () => {
fetchMock.post(homeserverEndpoint, 404);
fetchMock.post(`${exampleServiceUrl}/delegate_delayed_leave`, 404);
stubFactory({ state: "authenticating" });
let prepared = false;
const preparing = backend()
.prepareLocalTransport(exampleTransport)
.then(() => (prepared = true));
await flushPromises();
expect(prepared).toBe(false);
connections[0].state$.next(connected);
await preparing;
expect(prepared).toBe(true);
});
it("fails the preparation with the publisher connection's error", async () => {
fetchMock.post(homeserverEndpoint, 404);
fetchMock.post(`${exampleServiceUrl}/delegate_delayed_leave`, 404);
stubFactory({ state: "authenticating" });
const preparing = backend().prepareLocalTransport(exampleTransport);
await flushPromises();
const error = new NoMatrix2AuthorizationService(new Error("404"));
connections[0].state$.next({ state: "error", error });
await expect(preparing).rejects.toBe(error);
});
it("delegates the leave through the flow that authenticated the publisher", async () => {
fetchMock.post(homeserverEndpoint, 401);
const delegate = vi.spyOn(auth, "delegateDelayedLeave").mockResolvedValue();
const b = backend();
await expect(b.delegateDelayedLeave("delay-1")).rejects.toThrow(
"No local transport",
);
await b.prepareLocalTransport(urlTransport);
await b.delegateDelayedLeave("delay-1");
expect(delegate).toHaveBeenCalledWith(
expect.objectContaining({
client,
membership: ownMemberMock,
transport: urlTransport,
roomId: "!room:example.org",
slotId: "m.call#ROOM",
delayId: "delay-1",
delayTimeoutMs: defaultSessionTimings.delegatedDelayedLeave.delay_ms,
}),
);
});
describe("mediaFor", () => {
// A member advertising both a `url` and a `livekit_service_url` may have
// authenticated through either flow, so it has two candidate identities
const legacyId = "@bob:example.org:DEV000";
const hashedId = "2qkXfWz1S7jQmU0bJ6yN8aVtR3pLcDgHe9oZxKwMiA4";
function bob(
transport: UnstableLivekitTransport,
backendIdentities: string[],
): RTCMember {
return {
local: false,
rtcBackendIdentity: legacyId,
backendIdentities,
userId: "@bob:example.org",
deviceId: "DEV000",
memberId: legacyId,
transport$: constant({
type: "livekit",
id: JSON.stringify(transport),
raw: transport,
resolved$: constant(undefined),
}),
applicationData$: constant({}),
displayName$: constant("Bob"),
avatarUrl$: constant(undefined),
};
}
async function prepared(): Promise<
ReturnType<typeof createLivekitBackend>
> {
fetchMock.post(homeserverEndpoint, 404);
fetchMock.post(`${exampleServiceUrl}/delegate_delayed_leave`, 404);
const b = backend();
await b.prepareLocalTransport(bothTransport);
return b;
}
it.each([
["legacy", legacyId],
["hashed", hashedId],
])(
"matches a member with two candidate identities publishing under the %s one",
async (_name, identity) => {
const b = await prepared();
const media = b.mediaFor(
testScope(),
bob(bothTransport, [hashedId, legacyId]),
);
await flushPromises();
// Bob is on our transport, so his media arrives on the publisher
// connection
expect(connections).toHaveLength(1);
expect(media.tracks$.value).toBeNull();
connections[0].participants$.next([
mockRemoteParticipant({ identity, trackPublications: new Map() }),
]);
await flushPromises();
expect(media.tracks$.value).toEqual([]);
},
);
it("does not match a member with two candidate identities to anyone else", async () => {
const b = await prepared();
const media = b.mediaFor(
testScope(),
bob(bothTransport, [hashedId, legacyId]),
);
connections[0].participants$.next([
mockRemoteParticipant({
identity: "@stranger:example.org:DEV999",
trackPublications: new Map(),
}),
]);
await flushPromises();
expect(media.tracks$.value).toBeNull();
});
it("connects to a remote member's transport on the member's homeserver", async () => {
const b = await prepared();
const carlsTransport: UnstableLivekitTransport = {
type: "livekit",
url: "wss://sfu.sample.com",
};
b.mediaFor(testScope(), {
...bob(carlsTransport, ["carl-hashed"]),
userId: "@carl:sample.com",
});
await flushPromises();
const remote = connections.find(
(c) => c.connection.role === "subscriber",
);
expect(remote?.connection.transport).toEqual(carlsTransport);
expect(remote?.connection.serverName).toBe("sample.com");
});
});
});
/** The parts of a LiveKit room the local media and the member media read. */
function fakeLivekitRoom(): LivekitRoom {
return {
options: {},
setE2EEEnabled: vi.fn(),
localParticipant: {
...mockEmitter(),
trackPublications: new Map(),
getTrackPublication: () => undefined,
},
remoteParticipants: new Map(),
...mockEmitter(),
} as unknown as LivekitRoom;
}
sdk/src/media-backend/livekit/LivekitBackend.ts
+168 -130
View File
@@ -8,71 +8,86 @@ Please see LICENSE in the repository root for full details.
import { type MatrixClient } from "matrix-js-sdk";
import { type Logger } from "matrix-js-sdk/lib/logger";
import {
isLivekitTransport,
type LivekitTransport,
isUnstableLivekitTransport,
type Transport,
} from "matrix-js-sdk/lib/matrixrtc";
import {
BehaviorSubject,
combineLatest,
distinctUntilChanged,
filter,
firstValueFrom,
map,
type Observable,
of,
ReplaySubject,
switchMap,
} from "rxjs";
import { type ResolvedTransport, type RTCMember } from "../../api";
import { type MediaQuality } from "../../config";
import { MatrixRTCError } from "../../errors";
import { type Behavior } from "../../reactive/Behavior";
import { type MemberMedia } from "../../media-api";
import { type Behavior } from "../../reactive/Behavior";
import {
type Epoch,
Epoch,
type ObservableScope,
trackEpoch,
} from "../../reactive/ObservableScope";
import { mapScoped } from "../../utils/mapScoped";
import { serverNameOf } from "../../utils/serverName";
import {
type BackendConnection,
type MediaBackend,
type MediaBackendContext,
type TransportCapabilities,
} from "../api";
import {
chooseAuthFlow,
delegateDelayedLeave,
type TransportLocator,
} from "./auth";
import { type Connection } from "./Connection";
import {
ConnectionMap,
createConnectionMap$,
keyFor,
type TransportKey,
} from "./ConnectionMap";
import {
LivekitConnectionFactory,
ResolvedConnection,
} from "./ConnectionFactory";
import {
createConnectionManager$,
type LocalTransport,
} from "./ConnectionManager";
import { toMediaConnectionState } from "./connectionState";
import { createDataChannel$ } from "./DataChannel";
import { createKeyProvider } from "./KeyProvider";
import { createLivekitMemberMedia } from "./LivekitMemberMedia";
import { createLivekitLocalMedia } from "./LocalMedia";
import { getSFUConfigWithOpenID, type TokenEndpoint } from "./openIDSFU";
export interface LivekitBackendOptions {
/** `getOpenIdToken` for the token exchange, `baseUrl` for the delegation probe and handover. */
client: Pick<MatrixClient, "getOpenIdToken" | "baseUrl">;
/**
* `getOpenIdToken` for the JWT service flow, the MSC4195 methods for the
* client-server API flow, `baseUrl` for the delegation probe and handover.
*/
client: Pick<
MatrixClient,
| "getOpenIdToken"
| "_unstable_getLivekitToken"
| "_unstable_delegateDelayedLeave"
| "baseUrl"
>;
mediaQuality?: MediaQuality;
/** Which token service endpoint the local transport uses; remote ones try both. */
tokenEndpoint: TokenEndpoint;
}
/**
* MatrixRTC media over LiveKit SFUs: one room per transport, the local one
* published on, every remote one subscribed to, with the token service
* protocol in front of each.
* published on, every remote one subscribed to, each authenticated for its
* role through the homeserver or the transport's JWT service.
*/
export function createLivekitBackend(
scope: ObservableScope,
{
roomId,
slotId,
ownMembershipIdentity,
publish,
encryptionSystem,
@@ -80,102 +95,133 @@ export function createLivekitBackend(
timings,
logger: parentLogger,
}: MediaBackendContext,
{ client, mediaQuality, tokenEndpoint }: LivekitBackendOptions,
{ client, mediaQuality }: LivekitBackendOptions,
): MediaBackend {
const logger = parentLogger.getChild("[LivekitBackend]");
const keyProvider = createKeyProvider(encryptionSystem, mediaKeys$, logger);
const transports = createTransportRegistry(scope);
const localTransport$ = new ReplaySubject<LocalTransport>(1);
const localTransport$ = new BehaviorSubject<TransportLocator | null>(null);
const audioOutputDeviceId$ = new BehaviorSubject<string | undefined>(
undefined,
);
const connectionFactory = new LivekitConnectionFactory(
client,
roomId,
slotId,
publish,
audioOutputDeviceId$,
keyProvider,
mediaQuality,
);
const connectionManager = createConnectionManager$({
// A new map, in a scope of its own, whenever the local transport changes:
// the publisher role sits on one connection, and which one is fixed per map
const connectionMap$ = scope.behavior<Epoch<ConnectionMap>>(
mapScoped(scope, localTransport$, (mapScope, localTransport) =>
createConnectionMap$({
scope: mapScope,
connectionFactory,
localTransport,
remoteTransports$: transports.transports$,
logger,
ownMembershipIdentity,
}),
).pipe(switchMap((map$) => map$ ?? of(new Epoch(new ConnectionMap())))),
);
const localMedia_ = createLivekitLocalMedia({
scope,
connectionFactory: new LivekitConnectionFactory(
client,
roomId,
publish,
audioOutputDeviceId$,
keyProvider,
mediaQuality,
),
connectionMap$,
localTransport$,
remoteTransports$: transports.transports$,
publish,
audioOutputDeviceId$,
encryptionSystem,
logger,
ownMembershipIdentity,
});
let local: LivekitTransport | undefined;
// Probed once: the answer cannot change within a session
const homeserverSupportsDelegation = supportsDelegation(
`${client.baseUrl}/_matrix/client/unstable/io.element.msc4195/rtc/livekit/delegate_delayed_leave`,
"homeserver",
logger,
);
const prepareLocalTransport = async (
transport: Transport,
): Promise<TransportCapabilities> => {
if (!isLivekitTransport(transport))
if (!isUnstableLivekitTransport(transport))
throw new MatrixRTCError(`Not a LiveKit transport: ${transport.type}`);
const [sfuConfig, canDelegateDelayedLeave] = await Promise.all([
getSFUConfigWithOpenID(
client,
ownMembershipIdentity,
transport.livekit_service_url,
roomId,
{ tokenEndpoint },
logger,
),
supportsDelegation(client.baseUrl, transport, logger),
const locator: TransportLocator = {
transport,
serverName: serverNameOf(ownMembershipIdentity.userId),
};
// Delegation goes through whichever service authenticates us as a
// publisher, so only that service's support counts
const flow = chooseAuthFlow(transport, "publisher");
const canDelegate =
flow.kind === "openID"
? supportsDelegation(
`${flow.serviceUrl}/delegate_delayed_leave`,
`transport ${flow.serviceUrl}`,
logger,
)
: homeserverSupportsDelegation;
localTransport$.next(locator);
// The membership is only sent once the publisher connection holds its
// token, so that the LiveKit room exists before anyone subscribes to it:
// a subscriber lacks the permission to create it and would fail without
// retrying
const [, canDelegateDelayedLeave] = await Promise.all([
authenticated(localMedia_.connection$),
canDelegate,
]);
local = transport;
localTransport$.next({ transport, sfuConfig });
return { canDelegateDelayedLeave };
};
// The token this issues is discarded; the request is what hands the leave over
const delegateDelayedLeave = async (delayId: string): Promise<void> => {
if (local === undefined)
const delegateLeave = async (delayId: string): Promise<void> => {
const local = localTransport$.value;
if (local === null)
throw new MatrixRTCError("No local transport to delegate the leave to");
await getSFUConfigWithOpenID(
await delegateDelayedLeave({
client,
ownMembershipIdentity,
local.livekit_service_url,
membership: ownMembershipIdentity,
transport: local.transport,
roomId,
{
tokenEndpoint,
delayEndpointBaseUrl: client.baseUrl,
delayId,
delayTimeoutMs: timings.delegatedDelayedLeave.delay_ms,
},
slotId,
delayId,
delayTimeoutMs: timings.delegatedDelayedLeave.delay_ms,
logger,
);
});
};
const mediaFor = (
memberScope: ObservableScope,
member: RTCMember,
): MemberMedia => {
const identity = member.rtcBackendIdentity;
const transport$ = memberScope.behavior(
const { backendIdentities } = member;
const locator$ = memberScope.behavior(
member.transport$.pipe(
map((transport) => transport?.raw),
map((transport) =>
isLivekitTransport(transport) ? transport : undefined,
),
distinctUntilChanged(
(a, b) => a?.livekit_service_url === b?.livekit_service_url,
),
map((transport): TransportLocator | undefined => {
const raw = transport?.raw;
return isUnstableLivekitTransport(raw)
? { transport: raw, serverName: serverNameOf(member.userId) }
: undefined;
}),
distinctUntilChanged((a, b) => (a && keyFor(a)) === (b && keyFor(b))),
),
);
transports.follow(memberScope, transport$);
transports.follow(memberScope, locator$);
const source$ = memberScope.behavior(
combineLatest([
transport$,
connectionManager.connectionManagerData$,
]).pipe(
map(([transport, { value: connections }]) => {
if (transport === undefined) return null;
const connection = connections.getConnectionForTransport(transport);
combineLatest([locator$, connectionMap$]).pipe(
map(([locator, { value: connections }]) => {
if (locator === undefined) return null;
const connection = connections.getConnectionForTransport(locator);
// A member advertising both a `url` and a `livekit_service_url` may
// have authenticated through either flow, under either identity
const participant = connections
.getParticipantsForTransport(transport)
.find((p) => p.identity === identity);
.getParticipantsForTransport(locator)
.find((p) => backendIdentities.includes(p.identity));
return connection && participant
? { participant, room: connection.livekitRoom }
: null;
@@ -187,14 +233,14 @@ export function createLivekitBackend(
);
source$.pipe(memberScope.bind()).subscribe((source) => {
logger.info(
`${identity}: LiveKit participant ${source ? `matched (${source.participant.sid})` : "missing"}`,
`${member.rtcBackendIdentity}: LiveKit participant ${source ? `matched (${source.participant.sid})` : "missing"} on ${locator$.value ? JSON.stringify(locator$.value.transport) : "no transport"}`,
);
});
return createLivekitMemberMedia(memberScope, source$, encryptionSystem);
};
const connections$ = scope.behavior<BackendConnection[]>(
connectionManager.connectionManagerData$.pipe(
connectionMap$.pipe(
switchMap(({ value }) => {
const connections = value.getConnections();
if (connections.length === 0) return of([]);
@@ -203,26 +249,16 @@ export function createLivekitBackend(
),
);
const localMedia_ = createLivekitLocalMedia({
scope,
connectionManager,
localTransport$,
publish,
audioOutputDeviceId$,
encryptionSystem,
logger,
});
const { data$, sendData } = createDataChannel$({
scope,
connectionManager,
connectionMap$,
connection$: localMedia_.connection$,
});
return {
transportType: "livekit",
prepareLocalTransport,
delegateDelayedLeave,
delegateDelayedLeave: delegateLeave,
local: localMedia_,
mediaFor,
connections$,
@@ -232,6 +268,29 @@ export function createLivekitBackend(
};
}
/** Resolves once the connection holds its token; rejects with its error otherwise. */
async function authenticated(
connection$: Behavior<Connection | null>,
): Promise<void> {
const state = await firstValueFrom(
connection$.pipe(
switchMap((connection) => connection?.state$ ?? of(null)),
filter(
(state) =>
state !== null &&
(state.state === "authenticated" ||
state.state === "error" ||
state.state === "stopped"),
),
),
);
if (state?.state === "error") throw state.error;
if (state?.state === "stopped")
throw new MatrixRTCError(
"The local connection stopped before it was authenticated",
);
}
function backendConnection$(
connection: Connection,
): Observable<BackendConnection> {
@@ -252,10 +311,10 @@ export interface TransportRegistry {
/** Keeps a member's transport registered for as long as its scope lives. */
follow(
scope: ObservableScope,
transport$: Behavior<LivekitTransport | undefined>,
locator$: Behavior<TransportLocator | undefined>,
): void;
/** Every transport some member is on, for the connection manager. */
transports$: Behavior<Epoch<LivekitTransport[]>>;
/** Every transport some member is on, for the connection map. */
transports$: Behavior<Epoch<TransportLocator[]>>;
}
/**
@@ -267,34 +326,34 @@ export function createTransportRegistry(
scope: ObservableScope,
): TransportRegistry {
const counts = new Map<
string,
{ transport: LivekitTransport; count: number }
TransportKey,
{ locator: TransportLocator; count: number }
>();
const transports = new BehaviorSubject<LivekitTransport[]>([]);
const transports = new BehaviorSubject<TransportLocator[]>([]);
const publish = (): void =>
transports.next([...counts.values()].map(({ transport }) => transport));
transports.next([...counts.values()].map(({ locator }) => locator));
const register = (transport: LivekitTransport): (() => void) => {
const url = transport.livekit_service_url;
const entry = counts.get(url) ?? { transport, count: 0 };
const register = (locator: TransportLocator): (() => void) => {
const key = keyFor(locator);
const entry = counts.get(key) ?? { locator, count: 0 };
entry.count++;
if (entry.count === 1) {
counts.set(url, entry);
counts.set(key, entry);
publish();
}
return () => {
if (--entry.count > 0) return;
counts.delete(url);
counts.delete(key);
publish();
};
};
return {
follow: (memberScope, transport$) => {
follow: (memberScope, locator$) => {
let release: (() => void) | undefined;
transport$.pipe(memberScope.bind()).subscribe((transport) => {
locator$.pipe(memberScope.bind()).subscribe((locator) => {
release?.();
release = transport && register(transport);
release = locator && register(locator);
});
memberScope.onEnd(() => release?.());
},
@@ -303,33 +362,12 @@ export function createTransportRegistry(
}
/**
* Whether the SFU can take over restarting the delayed leave. Either the
* homeserver or the transport has to support it. Each endpoint is hit without
* credentials and read for a 404, not retried: many servers predate the
* endpoint altogether and answer with a CORS failure that a retry loop would
* only repeat.
* Whether a service can take over restarting the delayed leave. The endpoint
* is hit without credentials and read for a 404, not retried: many servers
* predate the endpoint altogether and answer with a CORS failure that a retry
* loop would only repeat.
*/
export async function supportsDelegation(
homeserverUrl: string,
transport: LivekitTransport,
logger: Logger,
): Promise<boolean> {
const [homeserver, sfu] = await Promise.all([
endpointExists(
`${homeserverUrl}/_matrix/client/unstable/io.element.msc4195/rtc/livekit/delegate_delayed_leave`,
"homeserver",
logger,
),
endpointExists(
`${transport.livekit_service_url}/delegate_delayed_leave`,
`transport ${transport.livekit_service_url}`,
logger,
),
]);
return homeserver || sfu;
}
async function endpointExists(
url: string,
serviceName: string,
logger: Logger,
sdk/src/media-backend/livekit/LocalMedia.ts
+16 -20
View File
@@ -35,13 +35,14 @@ import {
import { type EncryptionSystem } from "../../encryption";
import { FailToStartLivekitConnection, toMatrixRTCError } from "../../errors";
import { type Behavior } from "../../reactive/Behavior";
import { type ObservableScope } from "../../reactive/ObservableScope";
import { type LocalMediaBackend, MediaConnectionState } from "../api";
import { type Connection } from "./Connection";
import {
type IConnectionManager,
type LocalTransport,
} from "./ConnectionManager";
type Epoch,
type ObservableScope,
} from "../../reactive/ObservableScope";
import { type LocalMediaBackend, MediaConnectionState } from "../api";
import { type TransportLocator } from "./auth";
import { type Connection } from "./Connection";
import { type ConnectionMap } from "./ConnectionMap";
import { createLocalLivekitMemberMedia } from "./LivekitMemberMedia";
import { livekitSources } from "./LivekitMediaTrack";
import { type DesiredMedia, Publisher } from "./Publisher";
@@ -49,8 +50,9 @@ import { toMediaConnectionState } from "./connectionState";
interface Props {
scope: ObservableScope;
connectionManager: IConnectionManager;
localTransport$: Observable<LocalTransport>;
connectionMap$: Behavior<Epoch<ConnectionMap>>;
/** The transport we publish on; null until the session has found it. */
localTransport$: Behavior<TransportLocator | null>;
publish: PublishRequest[];
/** Read by the connection factory for the rooms it creates from now on. */
audioOutputDeviceId$: BehaviorSubject<string | undefined>;
@@ -71,7 +73,7 @@ export interface LivekitLocalMedia extends LocalMediaBackend {
*/
export function createLivekitLocalMedia({
scope,
connectionManager,
connectionMap$,
localTransport$,
publish: initialRequests,
audioOutputDeviceId$,
@@ -81,15 +83,11 @@ export function createLivekitLocalMedia({
const logger = parentLogger.getChild("[LocalMedia]");
const connection$ = scope.behavior(
combineLatest([
connectionManager.connectionManagerData$,
localTransport$,
]).pipe(
map(([{ value: connections }, { transport }]) =>
connections.getConnectionForTransport(transport),
),
combineLatest(
[connectionMap$, localTransport$],
({ value: connections }, transport) =>
transport && connections.getConnectionForTransport(transport),
),
null,
);
const connectionState$ = scope.behavior<MediaConnectionState | Error>(
@@ -117,9 +115,7 @@ export function createLivekitLocalMedia({
const publisher = new Publisher(
connection.livekitRoom,
desired,
logger.getChild(
`[Publisher ${connection.transport.livekit_service_url}]`,
),
logger.getChild(`[Publisher ${JSON.stringify(connection.transport)}]`),
);
publisher$.next(publisher);
return Promise.resolve(async (): Promise<void> => {
sdk/src/media-backend/livekit/auth/csApi.ts
+102
View File
@@ -0,0 +1,102 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { ConnectionError, HTTPError, type MatrixClient } from "matrix-js-sdk";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import { doNetworkOperationWithRetry } from "../../../utils/network";
import { extractFullConfigFromToken, type SFUConfig } from "./types";
/**
* In widget mode the host may refuse an action before any request is made,
* which surfaces as a plain error. Asking again cannot change its answer, so
* only errors from the network are worth retrying.
*/
function isNetworkError(e: unknown): boolean {
return e instanceof HTTPError || e instanceof ConnectionError;
}
export type ClientGetTokenParts = Pick<
MatrixClient,
"_unstable_getLivekitToken"
>;
export interface GetSFUConfigParams {
client: ClientGetTokenParts;
/** Our own membership identity; the member id is what the token is issued for. */
membership: CallMembershipIdentityParts;
/** The WebSocket URL of the SFU to get a token for. */
url: string;
/** The homeserver the SFU belongs to. */
serverName: string;
roomId: string;
slotId: string;
}
/** Gets an SFU token through the homeserver's MSC4195 `get_token` endpoint. */
export async function getSFUConfig({
client,
membership,
url,
serverName,
roomId,
slotId,
}: GetSFUConfigParams): Promise<SFUConfig> {
const res = await doNetworkOperationWithRetry(
async () =>
client._unstable_getLivekitToken({
url,
server_name: serverName,
room_id: roomId,
slot_id: slotId,
member_id: membership.memberId,
}),
isNetworkError,
);
return extractFullConfigFromToken({ url, jwt: res.jwt });
}
export type ClientDelegationParts = Pick<
MatrixClient,
"_unstable_delegateDelayedLeave" | "baseUrl"
>;
export interface DelegateDelayedLeaveParams {
client: ClientDelegationParts;
membership: CallMembershipIdentityParts;
/** The WebSocket URL of the SFU we publish on. */
url: string;
roomId: string;
slotId: string;
/** The delay ID of the leave event to hand over. */
delayId: string;
}
/**
* Hands the delayed leave to the homeserver through the MSC4195
* `delegate_delayed_leave` endpoint, which sends it once we drop off the SFU.
*/
export async function delegateDelayedLeave({
client,
membership,
url,
roomId,
slotId,
delayId,
}: DelegateDelayedLeaveParams): Promise<void> {
await doNetworkOperationWithRetry(
async () =>
client._unstable_delegateDelayedLeave({
url,
room_id: roomId,
slot_id: slotId,
member_id: membership.memberId,
delay_id: delayId,
}),
isNetworkError,
);
}
sdk/src/media-backend/livekit/auth/index.test.ts
+407
View File
@@ -0,0 +1,407 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { afterEach, describe, expect, it, type Mocked, vi } from "vitest";
import fetchMock from "fetch-mock";
import { ConnectionError, MatrixError } from "matrix-js-sdk";
import { logger } from "matrix-js-sdk/lib/logger";
import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import { NoMatrix2AuthorizationService } from "../../../errors";
import {
aliceDeviceId,
aliceUserId,
createJWTToken,
} from "../../../utils/test-fixtures";
import type * as csApi from "./csApi";
import type * as openID from "./openID";
import { chooseAuthFlow, delegateDelayedLeave, getSFUConfig } from ".";
const aliceId = `${aliceUserId}:${aliceDeviceId}`;
const csApiJWTToken = createJWTToken(aliceId, "via CS API");
const legacyJWTToken = createJWTToken(aliceId, "via legacy endpoint");
const defaultJWTToken = createJWTToken(aliceId, "via default endpoint");
const client: Mocked<
csApi.ClientGetTokenParts &
csApi.ClientDelegationParts &
openID.ClientOpenIDParts
> = {
baseUrl: "https://matrix.example.org",
_unstable_getLivekitToken: vi.fn().mockResolvedValue({ jwt: csApiJWTToken }),
_unstable_delegateDelayedLeave: vi.fn().mockResolvedValue({}),
getOpenIdToken: vi
.fn()
.mockResolvedValue({ access_token: "mock-openID-token" }),
};
const membership: CallMembershipIdentityParts = {
userId: aliceUserId,
deviceId: aliceDeviceId,
memberId: "123",
};
const roomId = "!room:example.org";
const slotId = "m.call#ROOM";
const unsupportedError = new MatrixError({ errcode: "M_UNSUPPORTED" }, 404);
const urlTransport: UnstableLivekitTransport = {
type: "livekit",
url: "wss://livekit.example.org",
};
const serviceUrl = "https://livekit-jwt.example.org";
const serviceUrlTransport: UnstableLivekitTransport = {
type: "livekit",
livekit_service_url: serviceUrl,
};
const urlAndServiceUrlTransport: UnstableLivekitTransport = {
...urlTransport,
...serviceUrlTransport,
};
/** The JWT service, answering each endpoint with a token that names it. */
function mockJWTService(): void {
fetchMock.post(`${serviceUrl}/sfu/get`, {
status: 200,
body: { url: "wss://sfu", jwt: legacyJWTToken },
});
fetchMock.post(`${serviceUrl}/get_token`, {
status: 200,
body: { url: "wss://sfu", jwt: defaultJWTToken },
});
}
/** Whether the JWT service was asked at all, and which endpoint. */
function jwtServiceCalls(): string[] {
return fetchMock.calls().map(([url]) => url.replace(serviceUrl, ""));
}
afterEach(() => {
vi.clearAllMocks();
fetchMock.reset();
});
describe("getSFUConfig", () => {
const defaultParams = {
client,
membership,
serverName: "example.org",
role: "subscriber" as const,
roomId,
slotId,
logger,
};
it("uses CS API with url transport", async () => {
mockJWTService();
expect(
await getSFUConfig({ ...defaultParams, transport: urlTransport }),
).toEqual(
expect.objectContaining({
jwt: csApiJWTToken,
livekitAlias: "via CS API",
}),
);
expect(client._unstable_getLivekitToken).toHaveBeenCalledWith({
url: urlTransport.url,
server_name: "example.org",
room_id: roomId,
slot_id: slotId,
member_id: "123",
});
expect(jwtServiceCalls()).toEqual([]);
});
it("does not fall back to OpenID with url transport", async () => {
mockJWTService();
client._unstable_getLivekitToken.mockRejectedValueOnce(unsupportedError);
await expect(
getSFUConfig({ ...defaultParams, transport: urlTransport }),
).rejects.toThrow(expect.any(NoMatrix2AuthorizationService));
expect(client._unstable_getLivekitToken).toHaveBeenCalled();
expect(jwtServiceCalls()).toEqual([]);
});
it("uses OpenID with livekit_service_url transport", async () => {
mockJWTService();
expect(
await getSFUConfig({ ...defaultParams, transport: serviceUrlTransport }),
).toEqual(
expect.objectContaining({
jwt: defaultJWTToken,
livekitAlias: "via default endpoint",
}),
);
expect(client._unstable_getLivekitToken).not.toHaveBeenCalled();
expect(jwtServiceCalls()).toEqual(["/get_token"]);
});
it("uses CS API when subscribing with url + livekit_service_url transport", async () => {
mockJWTService();
expect(
await getSFUConfig({
...defaultParams,
transport: urlAndServiceUrlTransport,
}),
).toEqual(
expect.objectContaining({
jwt: csApiJWTToken,
livekitAlias: "via CS API",
}),
);
expect(client._unstable_getLivekitToken).toHaveBeenCalled();
expect(jwtServiceCalls()).toEqual([]);
});
it("falls back to OpenID with url + livekit_service_url transport", async () => {
mockJWTService();
client._unstable_getLivekitToken.mockRejectedValueOnce(unsupportedError);
expect(
await getSFUConfig({
...defaultParams,
transport: urlAndServiceUrlTransport,
}),
).toEqual(
expect.objectContaining({
jwt: defaultJWTToken,
livekitAlias: "via default endpoint",
}),
);
expect(client._unstable_getLivekitToken).toHaveBeenCalled();
expect(jwtServiceCalls()).toEqual(["/get_token"]);
});
it("falls back to OpenID when the widget host refuses the CS API", async () => {
mockJWTService();
// In widget mode a host without MSC4533 rejects the request before it
// reaches the homeserver, with a plain error rather than a Matrix error
client._unstable_getLivekitToken.mockRejectedValueOnce(
new Error(
"The rtc_livekit_get_token action is not supported by the client.",
),
);
expect(
await getSFUConfig({
...defaultParams,
transport: urlAndServiceUrlTransport,
}),
).toEqual(expect.objectContaining({ jwt: defaultJWTToken }));
expect(jwtServiceCalls()).toEqual(["/get_token"]);
});
it("surfaces the widget host's refusal with url transport", async () => {
mockJWTService();
const error = new Error(
"The rtc_livekit_get_token action is not supported by the client.",
);
client._unstable_getLivekitToken.mockRejectedValueOnce(error);
await expect(
getSFUConfig({ ...defaultParams, transport: urlTransport }),
).rejects.toThrow(error);
expect(jwtServiceCalls()).toEqual([]);
});
it("uses CS API when publishing with url transport", async () => {
mockJWTService();
expect(
await getSFUConfig({
...defaultParams,
transport: urlTransport,
role: "publisher",
}),
).toEqual(expect.objectContaining({ jwt: csApiJWTToken }));
expect(client._unstable_getLivekitToken).toHaveBeenCalled();
expect(jwtServiceCalls()).toEqual([]);
});
it("does not fall back to OpenID when publishing", async () => {
mockJWTService();
client._unstable_getLivekitToken.mockRejectedValueOnce(unsupportedError);
await expect(
getSFUConfig({
...defaultParams,
transport: urlTransport,
role: "publisher",
}),
).rejects.toThrow(expect.any(NoMatrix2AuthorizationService));
expect(client._unstable_getLivekitToken).toHaveBeenCalled();
expect(jwtServiceCalls()).toEqual([]);
});
it("uses the legacy OpenID endpoint when publishing with url + livekit_service_url transport", async () => {
mockJWTService();
// Subscribers predating MSC4195 expect a legacy participant identity from
// anyone advertising a membership, which only the JWT service assigns
expect(
await getSFUConfig({
...defaultParams,
transport: urlAndServiceUrlTransport,
role: "publisher",
}),
).toEqual(
expect.objectContaining({
jwt: legacyJWTToken,
livekitAlias: "via legacy endpoint",
}),
);
expect(client._unstable_getLivekitToken).not.toHaveBeenCalled();
expect(jwtServiceCalls()).toEqual(["/sfu/get"]);
});
it("does not fall back to OpenID on connection error", async () => {
vi.useFakeTimers();
try {
mockJWTService();
const client_ = {
...client,
_unstable_getLivekitToken: vi
.fn()
.mockRejectedValue(new ConnectionError("No connection")),
};
await Promise.all([
expect(
getSFUConfig({
...defaultParams,
client: client_,
transport: urlAndServiceUrlTransport,
}),
).rejects.toThrow(expect.any(ConnectionError)),
vi.runAllTimersAsync(), // Allow request to be retried
]);
expect(client_._unstable_getLivekitToken).toHaveBeenCalled();
expect(jwtServiceCalls()).toEqual([]);
} finally {
vi.useRealTimers();
}
});
});
describe("delegateDelayedLeave", () => {
const defaultParams = {
client,
membership,
roomId,
slotId,
delayId: "delay123",
delayTimeoutMs: 3_600_000,
logger,
};
/** The delay fields the JWT service was handed, if any. */
function delegationViaJWTService(): object | undefined {
const call = fetchMock.calls().find(([url]) => url.endsWith("/sfu/get"));
if (call === undefined) return undefined;
const { delay_id, delay_timeout, delay_cs_api_url } = JSON.parse(
call[1]?.body as string,
);
return { delay_id, delay_timeout, delay_cs_api_url };
}
it("uses CS API with url transport", async () => {
mockJWTService();
await delegateDelayedLeave({ ...defaultParams, transport: urlTransport });
expect(client._unstable_delegateDelayedLeave).toHaveBeenCalledWith({
url: urlTransport.url,
room_id: roomId,
slot_id: slotId,
member_id: "123",
delay_id: "delay123",
});
expect(jwtServiceCalls()).toEqual([]);
});
it("does not fall back to OpenID with url transport", async () => {
mockJWTService();
client._unstable_delegateDelayedLeave.mockRejectedValueOnce(
unsupportedError,
);
await expect(
delegateDelayedLeave({ ...defaultParams, transport: urlTransport }),
).rejects.toThrow();
expect(client._unstable_delegateDelayedLeave).toHaveBeenCalled();
expect(jwtServiceCalls()).toEqual([]);
});
it("uses OpenID with livekit_service_url transport", async () => {
mockJWTService();
await delegateDelayedLeave({
...defaultParams,
transport: serviceUrlTransport,
});
expect(client._unstable_delegateDelayedLeave).not.toHaveBeenCalled();
expect(delegationViaJWTService()).toEqual({
delay_id: "delay123",
delay_timeout: 3_600_000,
delay_cs_api_url: client.baseUrl,
});
});
it("uses OpenID with url + livekit_service_url transport", async () => {
mockJWTService();
// Must match the flow that got the token, so the identities line up
await delegateDelayedLeave({
...defaultParams,
transport: urlAndServiceUrlTransport,
});
expect(client._unstable_delegateDelayedLeave).not.toHaveBeenCalled();
expect(delegationViaJWTService()).toEqual({
delay_id: "delay123",
delay_timeout: 3_600_000,
delay_cs_api_url: client.baseUrl,
});
});
});
describe("chooseAuthFlow", () => {
it.each([
[
"url",
"publisher",
urlTransport,
{ kind: "csApi", url: urlTransport.url },
],
[
"url",
"subscriber",
urlTransport,
{ kind: "csApi", url: urlTransport.url },
],
[
"livekit_service_url",
"publisher",
serviceUrlTransport,
{ kind: "openID", serviceUrl },
],
[
"livekit_service_url",
"subscriber",
serviceUrlTransport,
{ kind: "openID", serviceUrl },
],
[
"url + livekit_service_url",
"publisher",
urlAndServiceUrlTransport,
{ kind: "openID", serviceUrl },
],
[
"url + livekit_service_url",
"subscriber",
urlAndServiceUrlTransport,
{ kind: "csApi", url: urlTransport.url },
],
] as const)(
"chooses the flow for a %s transport as %s",
(_shape, role, transport, flow) => {
expect(chooseAuthFlow(transport, role)).toEqual(flow);
},
);
});
sdk/src/media-backend/livekit/auth/index.ts
+212
View File
@@ -0,0 +1,212 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
/**
* Authentication with a LiveKit transport: the MSC4195 client-server API
* through our homeserver for a transport with an SFU `url`, the OpenID flow
* against the JWT service for one with a `livekit_service_url`, and the
* choice between them where a transport carries both.
*/
import { ConnectionError, MatrixError } from "matrix-js-sdk";
import { type Logger } from "matrix-js-sdk/lib/logger";
import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import { NoMatrix2AuthorizationService } from "../../../errors";
import * as csApi from "./csApi";
import * as openID from "./openID";
import { type ConnectionRole, type SFUConfig } from "./types";
export type { ClientOpenIDParts } from "./openID";
export type { ClientGetTokenParts, ClientDelegationParts } from "./csApi";
export type * from "./types";
/**
* Whether an error means the homeserver does not implement the MSC4195
* endpoints. The MSC says `M_NOT_FOUND`; a homeserver without support answers
* with whatever it uses for unknown routes (Synapse: 404 with
* `M_UNRECOGNIZED`), so any 404 counts.
*/
function isEndpointUnsupported(e: unknown): boolean {
return (
e instanceof MatrixError &&
(e.httpStatus === 404 ||
e.errcode === "M_NOT_FOUND" ||
e.errcode === "M_UNRECOGNIZED")
);
}
/**
* Whether an error means the MSC4195 client-server API cannot be used from
* here: the homeserver does not implement it, or the request never reached
* the homeserver because a widget host does not expose the endpoint
* (MSC4533), which the widget API reports as a plain error. A connection
* error is neither: the endpoint may well exist.
*/
function isCsApiUnavailable(e: unknown): boolean {
if (isEndpointUnsupported(e)) return true;
return !(e instanceof MatrixError) && !(e instanceof ConnectionError);
}
export interface GetSFUConfigParams {
client: csApi.ClientGetTokenParts & openID.ClientOpenIDParts;
membership: CallMembershipIdentityParts;
transport: UnstableLivekitTransport;
/** The homeserver the transport belongs to. */
serverName: string;
roomId: string;
slotId: string;
role: ConnectionRole;
logger: Logger;
}
/** Authenticates with a transport, through whichever flow suits it and our role. */
export async function getSFUConfig({
client,
membership,
transport,
serverName,
roomId,
slotId,
role,
logger,
}: GetSFUConfigParams): Promise<SFUConfig> {
const viaOpenID = async (serviceUrl: string): Promise<SFUConfig> =>
await openID.getSFUConfig({
client,
membership,
serviceUrl,
roomId,
slotId,
role,
logger,
});
const flow = chooseAuthFlow(transport, role);
if (flow.kind === "openID") return await viaOpenID(flow.serviceUrl);
try {
return await csApi.getSFUConfig({
client,
membership,
url: flow.url,
serverName,
roomId,
slotId,
});
} catch (e) {
// A publisher never falls back: a homeserver that advertises a `url` but
// does not implement MSC4195 is misconfigured, and the admin should hear
// about it before `livekit_service_url` is deprecated
const mayFallBack =
role === "subscriber" && "livekit_service_url" in transport;
if (mayFallBack && isCsApiUnavailable(e)) {
logger.warn(
`Cannot use the MSC4195 get_token endpoint (${e}), falling back to the OpenID flow with service ${transport.livekit_service_url}`,
);
return await viaOpenID(transport.livekit_service_url);
}
if (isEndpointUnsupported(e)) {
logger.error(
`Homeserver does not support the MSC4195 get_token endpoint, cannot get token for transport ${flow.url}`,
);
throw new NoMatrix2AuthorizationService(e as Error);
}
throw e;
}
}
export interface DelegateDelayedLeaveParams {
client: csApi.ClientDelegationParts & openID.ClientOpenIDParts;
membership: CallMembershipIdentityParts;
/** The transport we publish on. */
transport: UnstableLivekitTransport;
roomId: string;
slotId: string;
delayId: string;
/** How long the service waits before sending the leave, for the JWT service flow. */
delayTimeoutMs: number;
logger: Logger;
}
/**
* Hands the delayed leave to the service that authenticated us as a
* publisher, so that the identity it watches for on the SFU is the one we
* publish under.
*/
export async function delegateDelayedLeave({
client,
membership,
transport,
roomId,
slotId,
delayId,
delayTimeoutMs,
logger,
}: DelegateDelayedLeaveParams): Promise<void> {
const flow = chooseAuthFlow(transport, "publisher");
if (flow.kind === "csApi") {
try {
await csApi.delegateDelayedLeave({
client,
membership,
url: flow.url,
roomId,
slotId,
delayId,
});
} catch (e) {
if (isEndpointUnsupported(e))
logger.error(
"Server misconfigured: transport contains a 'url' field but server does not support the MSC4195 delegate_delayed_leave endpoint",
);
throw e;
}
} else {
// The token this issues is discarded; the request is what hands the leave over
await openID.getSFUConfig({
client,
membership,
serviceUrl: flow.serviceUrl,
roomId,
slotId,
role: "publisher",
delayEndpointBaseUrl: client.baseUrl,
delayId,
delayTimeoutMs,
logger,
});
}
}
/** The flow with which the local member authenticates against a transport. */
export type AuthFlow =
| { kind: "csApi"; url: string }
| { kind: "openID"; serviceUrl: string };
/**
* Chooses between the MSC4195 client-server API and the OpenID flow against
* the transport's JWT service.
*
* Only the JWT service's `/sfu/get` endpoint assigns the legacy participant
* identity that clients predating MSC4195 expect from anyone advertising a
* membership, so a publisher keeps using the JWT service wherever the
* transport still offers one, even next to a `url`. A subscriber may use the
* client-server API regardless, as nobody matches its identity.
*/
export function chooseAuthFlow(
transport: UnstableLivekitTransport,
role: ConnectionRole,
): AuthFlow {
if ("url" in transport) {
const useCsApi =
!("livekit_service_url" in transport) || role === "subscriber";
if (useCsApi) return { kind: "csApi", url: transport.url };
}
return { kind: "openID", serviceUrl: transport.livekit_service_url };
}
sdk/src/media-backend/livekit/auth/openID.test.ts
+229
View File
@@ -0,0 +1,229 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
beforeEach,
afterEach,
describe,
expect,
it,
type MockedObject,
vitest,
} from "vitest";
import fetchMock from "fetch-mock";
import { MatrixError } from "matrix-js-sdk";
import { logger } from "matrix-js-sdk/lib/logger";
import {
getSFUConfig,
type GetSFUConfigParams,
type ClientOpenIDParts,
} from "./openID";
import { testJWTToken } from "../../../utils/test-fixtures";
import { ownMemberMock } from "../../../utils/test";
import { FailToGetOpenIdToken } from "../../../errors";
const serviceUrl = "https://sfu.example.org";
const delayParams = {
delayEndpointBaseUrl: "https://matrix.homeserverserver.org",
delayId: "mock_delay_id",
delayTimeoutMs: 3_600_000,
};
describe("getSFUConfig", () => {
let matrixClient: MockedObject<ClientOpenIDParts>;
let params: GetSFUConfigParams;
beforeEach(() => {
fetchMock.catch(404);
matrixClient = { getOpenIdToken: vitest.fn() };
params = {
client: matrixClient,
membership: ownMemberMock,
serviceUrl: "https://sfu.example.org",
roomId: "!example_room_id",
slotId: "m.call#ROOM",
role: "publisher",
logger,
};
});
afterEach(async () => {
void (await fetchMock.flush());
vitest.clearAllMocks();
fetchMock.reset();
});
it("should handle fetching a token", async () => {
fetchMock.post("https://sfu.example.org/sfu/get", () => {
return {
status: 200,
body: { url: serviceUrl, jwt: testJWTToken },
};
});
const config = await getSFUConfig(params);
expect(config).toEqual({
jwt: testJWTToken,
url: serviceUrl,
livekitIdentity: "@me:example.org:ABCDEF",
livekitAlias: "!example_room_id",
});
});
it("should fail if the SFU errors", async () => {
fetchMock.post("https://sfu.example.org/sfu/get", () => {
return {
status: 500,
body: {
errcode: "M_LOOKUP_FAILED",
error: "Failed to look up user info from homeserver",
},
};
});
try {
await getSFUConfig(params);
} catch (ex: unknown) {
expect(ex).toBeInstanceOf(FailToGetOpenIdToken);
expect((ex as FailToGetOpenIdToken).cause).toBeInstanceOf(MatrixError);
const mxError = (ex as Error).cause as MatrixError;
expect(mxError.message).toEqual(
"MatrixError: [500] Failed to look up user info from homeserver",
);
return;
}
expect.fail("Expected test to throw;");
});
it("should retry without delay params if the JWT service legacy endpoint returns M_BAD_JSON 400", async () => {
let callCount = 0;
fetchMock.post(
"https://sfu.example.org/sfu/get",
(url, opts) => {
callCount++;
const body = JSON.parse(opts.body as string);
// First call: check if it has delay parts and return 400
if (callCount === 1) {
expect(body).toHaveProperty("delay_id", "mock_delay_id");
return {
status: 400,
body: { errcode: "M_BAD_JSON", error: "Unsupported parameters" },
};
}
// Second call: check if delay parts were stripped and return success
expect(body).not.toHaveProperty("delay_id");
expect(body).not.toHaveProperty("delay_timeout");
expect(body).not.toHaveProperty("delay_cs_api_url");
return {
status: 200,
body: { url: serviceUrl, jwt: testJWTToken },
};
},
{ overwriteRoutes: true },
);
const config = await getSFUConfig({ ...params, ...delayParams });
expect(config.jwt).toBe(testJWTToken);
expect(callCount).toBe(2);
});
it("should successfully send delay parameters to the JWT service legacy endpoint", async () => {
fetchMock.post(
"https://sfu.example.org/sfu/get",
(url, opts) => {
const body = JSON.parse(opts.body as string);
// Verify, that the request contains the expected delay parameters
if (
body.delay_id === "mock_delay_id" &&
body.delay_timeout === 3600000 &&
body.delay_cs_api_url === "https://matrix.homeserverserver.org"
) {
return {
status: 200,
body: { url: serviceUrl, jwt: testJWTToken },
};
}
return {
status: 400,
body: { error: "Missing expected delay params" },
};
},
{ overwriteRoutes: true },
);
const config = await getSFUConfig({ ...params, ...delayParams });
expect(config).toMatchObject({
jwt: testJWTToken,
url: serviceUrl,
});
});
it("should use the default endpoint when subscribing", async () => {
fetchMock.post("https://sfu.example.org/get_token", (url, opts) => {
const body = JSON.parse(opts.body as string);
expect(body).toMatchObject({
room_id: "!example_room_id",
slot_id: "m.call#ROOM",
member: {
id: ownMemberMock.memberId,
claimed_user_id: ownMemberMock.userId,
claimed_device_id: ownMemberMock.deviceId,
},
});
return { status: 200, body: { url: serviceUrl, jwt: testJWTToken } };
});
const config = await getSFUConfig({ ...params, role: "subscriber" });
expect(config.jwt).toBe(testJWTToken);
expect(fetchMock.called("https://sfu.example.org/sfu/get")).toBe(false);
});
it("should fail when subscribing if the default endpoint is missing", async () => {
// Nothing but the catch-all 404 answers the default endpoint. Subscribers
// must not fall back to the legacy endpoint, which would hand them a
// legacy identity.
await expect(
getSFUConfig({ ...params, role: "subscriber" }),
).rejects.toThrow(FailToGetOpenIdToken);
expect(fetchMock.called("https://sfu.example.org/get_token")).toBe(true);
expect(fetchMock.called("https://sfu.example.org/sfu/get")).toBe(false);
});
it("should retry fetching the openid token", async () => {
let count = 0;
matrixClient.getOpenIdToken.mockImplementation(async () => {
count++;
if (count < 2) {
throw Error("Test failure");
}
return Promise.resolve({
token_type: "Bearer",
access_token: "foobar",
matrix_server_name: "example.org",
expires_in: 30,
});
});
fetchMock.post("https://sfu.example.org/sfu/get", () => {
return {
status: 200,
body: { url: serviceUrl, jwt: testJWTToken },
};
});
const config = await getSFUConfig({ ...params, ...delayParams });
expect(config).toEqual({
jwt: testJWTToken,
url: serviceUrl,
livekitIdentity: "@me:example.org:ABCDEF",
livekitAlias: "!example_room_id",
});
});
});
sdk/src/media-backend/livekit/auth/openID.ts
+203
View File
@@ -0,0 +1,203 @@
/*
Copyright 2023, 2024 New Vector Ltd.
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type IOpenIDToken,
type MatrixClient,
parseErrorResponse,
} from "matrix-js-sdk";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import { type Logger } from "matrix-js-sdk/lib/logger";
import { FailToGetOpenIdToken } from "../../../errors";
import { doNetworkOperationWithRetry } from "../../../utils/network";
import {
type ConnectionRole,
extractFullConfigFromToken,
type SFUConfig,
} from "./types";
export type ClientOpenIDParts = Pick<MatrixClient, "getOpenIdToken">;
export interface GetSFUConfigParams {
client: ClientOpenIDParts;
/** Our own membership identity, as the JWT service wants it. */
membership: CallMembershipIdentityParts;
/** The base URL of the LiveKit JWT service. */
serviceUrl: string;
/** The Matrix room the session is in; the JWT service maps it to a LiveKit room. */
roomId: string;
/** The MatrixRTC slot the session is in, e.g. `m.call#ROOM`. */
slotId: string;
role: ConnectionRole;
/** The base URL of the Matrix homeserver, for delayed leave delegation. */
delayEndpointBaseUrl?: string;
/** The delay ID of the leave event to delegate to the JWT service. */
delayId?: string;
/** How long the SFU waits before sending the delegated leave. Needed with `delayId`. */
delayTimeoutMs?: number;
logger: Logger;
}
/**
* Authenticates with a LiveKit JWT service through the OpenID flow: an OpenID
* token from the homeserver, exchanged at the service for an SFU token.
*
* A publisher uses the legacy `/sfu/get` endpoint, the only one that assigns
* the `<user_id>:<device_id>` identity that clients predating MSC4195 match a
* membership against. A subscriber uses the `/get_token` endpoint, which
* hashes the identity: nobody matches a subscriber.
* @throws {@link FailToGetOpenIdToken}
*/
export async function getSFUConfig({
client,
membership,
serviceUrl,
roomId,
slotId,
role,
delayEndpointBaseUrl,
delayId,
delayTimeoutMs,
logger,
}: GetSFUConfigParams): Promise<SFUConfig> {
let openIdToken: IOpenIDToken;
try {
openIdToken = await doNetworkOperationWithRetry(async () =>
client.getOpenIdToken(),
);
} catch (error) {
throw new FailToGetOpenIdToken(
error instanceof Error ? error : new Error("Unknown error"),
);
}
logger.debug("Got openID token", openIdToken);
const delay: DelayParams =
// Also check for empty string
delayId && delayEndpointBaseUrl
? {
delay_id: delayId,
delay_timeout: delayTimeoutMs,
delay_cs_api_url: delayEndpointBaseUrl,
}
: {};
const endpoint = role === "publisher" ? "legacy" : "default";
try {
logger.info(
`Trying to get JWT with ${endpoint} endpoint for focus ${serviceUrl}...`,
);
const sfuConfig = await (
endpoint === "legacy" ? getLiveKitJWTLegacy : getLiveKitJWT
)(membership, serviceUrl, roomId, slotId, openIdToken, delay);
logger.info(`Got JWT from call's active focus URL.`);
return extractFullConfigFromToken(sfuConfig);
} catch (e) {
logger.error(`Failed fetching jwt with ${endpoint} endpoint:`, e);
throw new FailToGetOpenIdToken(
e instanceof Error ? e : new Error(`Unknown error ${e}`),
);
}
}
interface DelayParams {
delay_id?: string;
delay_timeout?: number;
delay_cs_api_url?: string;
}
/** The legacy `/sfu/get` endpoint, which assigns the `<user_id>:<device_id>` identity. */
async function getLiveKitJWTLegacy(
membership: CallMembershipIdentityParts,
livekitServiceURL: string,
matrixRoomId: string,
_slotId: string, // The legacy endpoint predates slots
openIDToken: IOpenIDToken,
delay: DelayParams,
): Promise<{ url: string; jwt: string }> {
const makeRequest = async (delayParts: DelayParams): Promise<Response> =>
await fetch(livekitServiceURL + "/sfu/get", {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify({
// The legacy JWT endpoint uses only the matrix room id to calculate
// the livekit room alias, which comes back in the JWT payload
room: matrixRoomId,
openid_token: openIDToken,
device_id: membership.deviceId,
...delayParts,
}),
});
const res = await doNetworkOperationWithRetry(async () => {
let response = await makeRequest(delay);
// A service from before delegation rejects the delay fields as bad JSON;
// it still issues a token without them
if (response.status === 400 && Object.keys(delay).length > 0) {
try {
const errorBody = await response.json();
if (errorBody.errcode === "M_BAD_JSON")
response = await makeRequest({});
} catch {
// Not a JSON error body, so a real error
}
}
return response;
});
if (!res.ok) throw parseErrorResponse(res, await res.text());
return await res.json();
}
class NotSupportedError extends Error {
public constructor(message: string) {
super(message);
this.name = "NotSupported";
}
}
/** The `/get_token` endpoint, which assigns a hashed identity. */
async function getLiveKitJWT(
membership: CallMembershipIdentityParts,
livekitServiceURL: string,
matrixRoomId: string,
slotId: string,
openIDToken: IOpenIDToken,
delay: DelayParams,
): Promise<{ url: string; jwt: string }> {
const { userId, deviceId, memberId } = membership;
const res = await doNetworkOperationWithRetry(
async () =>
await fetch(livekitServiceURL + "/get_token", {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify({
room_id: matrixRoomId,
slot_id: slotId,
openid_token: openIDToken,
member: {
id: memberId,
claimed_user_id: userId,
claimed_device_id: deviceId,
},
...delay,
}),
}),
);
if (!res.ok) {
const msg = "SFU Config fetch failed with status code " + res.status;
if (res.status === 404) throw new NotSupportedError(msg);
throw parseErrorResponse(res, await res.text());
}
return await res.json();
}
sdk/src/media-backend/livekit/auth/types.ts
+58
View File
@@ -0,0 +1,58 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
/**
* A transport and the homeserver it belongs to. The server name is what our
* homeserver asks for a token for a transport with a `url`, over federation
* where the server is not ours; a JWT service is reached directly.
*/
export interface TransportLocator {
transport: UnstableLivekitTransport;
serverName: string;
}
/** Whether we publish on a transport or only subscribe to it. */
export type ConnectionRole = "publisher" | "subscriber";
/** What authenticating with a transport yields: where to connect and with what. */
export interface SFUConfig {
/**
* The WebSocket URL of the SFU, as opposed to the JWT service URL in the
* transport. The legacy JWT service returns it; the client-server API flow
* has it from the transport.
*/
url: string;
jwt: string;
livekitAlias: string;
livekitIdentity: string;
}
/** The parts of the JWT payload the backend reads. */
interface SFUJWTPayload {
/** The subject: the participant identity the token carries. */
sub: string;
video: {
room: string;
};
}
/** Completes the SFU websocket url and the JWT with what the JWT payload says. */
export function extractFullConfigFromToken(sfuConfig: {
url: string;
jwt: string;
}): SFUConfig {
const [, payloadStr] = sfuConfig.jwt.split(".");
const payload = JSON.parse(global.atob(payloadStr)) as SFUJWTPayload;
return {
jwt: sfuConfig.jwt,
url: sfuConfig.url,
livekitAlias: payload.video.room,
livekitIdentity: payload.sub,
};
}
sdk/src/media-backend/livekit/connectionState.ts
+23 -13
View File
@@ -5,24 +5,34 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { MediaConnectionState } from "../api";
import { ConnectionState } from "./Connection";
import { ConnectionState as LivekitConnectionState } from "livekit-client";
const states: Record<ConnectionState, MediaConnectionState> = {
[ConnectionState.Initialized]: MediaConnectionState.Initialized,
[ConnectionState.FetchingConfig]: MediaConnectionState.Preparing,
[ConnectionState.Stopped]: MediaConnectionState.Stopped,
[ConnectionState.LivekitDisconnected]: MediaConnectionState.Disconnected,
[ConnectionState.LivekitConnecting]: MediaConnectionState.Connecting,
[ConnectionState.LivekitConnected]: MediaConnectionState.Connected,
[ConnectionState.LivekitReconnecting]: MediaConnectionState.Reconnecting,
[ConnectionState.LivekitSignalReconnecting]:
import { MediaConnectionState } from "../api";
import { type ConnectionState } from "./Connection";
const livekitStates: Record<LivekitConnectionState, MediaConnectionState> = {
[LivekitConnectionState.Disconnected]: MediaConnectionState.Disconnected,
[LivekitConnectionState.Connecting]: MediaConnectionState.Connecting,
[LivekitConnectionState.Connected]: MediaConnectionState.Connected,
[LivekitConnectionState.Reconnecting]: MediaConnectionState.Reconnecting,
[LivekitConnectionState.SignalReconnecting]:
MediaConnectionState.Reconnecting,
};
/** A connection's state as the session sees it; an error passes through. */
export function toMediaConnectionState(
state: ConnectionState | Error,
state: ConnectionState,
): MediaConnectionState | Error {
return state instanceof Error ? state : states[state];
switch (state.state) {
case "initialized":
return MediaConnectionState.Initialized;
case "authenticating":
return MediaConnectionState.Preparing;
case "authenticated":
return livekitStates[state.livekitState];
case "error":
return state.error;
case "stopped":
return MediaConnectionState.Stopped;
}
}
sdk/src/media-backend/livekit/openIDSFU.test.ts
-332
View File
@@ -1,332 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
beforeEach,
afterEach,
describe,
expect,
it,
type MockedObject,
vitest,
} from "vitest";
import fetchMock from "fetch-mock";
import { MatrixError } from "matrix-js-sdk";
import { getSFUConfigWithOpenID, type OpenIDClientParts } from "./openIDSFU";
import { testJWTToken } from "../../utils/test-fixtures";
import { ownMemberMock } from "../../utils/test";
import { FailToGetOpenIdToken } from "../../errors";
const sfuUrl = "https://sfu.example.org";
describe("getSFUConfigWithOpenID", () => {
let matrixClient: MockedObject<OpenIDClientParts>;
beforeEach(() => {
fetchMock.catch(404);
matrixClient = {
getOpenIdToken: vitest.fn(),
};
});
afterEach(() => {
vitest.clearAllMocks();
fetchMock.reset();
});
it("should handle fetching a token", async () => {
fetchMock.post("https://sfu.example.org/sfu/get", () => {
return {
status: 200,
body: { url: sfuUrl, jwt: testJWTToken },
};
});
const config = await getSFUConfigWithOpenID(
matrixClient,
ownMemberMock,
"https://sfu.example.org",
"!example_room_id",
);
expect(config).toEqual({
jwt: testJWTToken,
url: sfuUrl,
livekitIdentity: "@me:example.org:ABCDEF",
livekitAlias: "!example_room_id",
});
void (await fetchMock.flush());
});
it("should fail if the SFU errors", async () => {
fetchMock.post("https://sfu.example.org/sfu/get", () => {
return {
status: 500,
body: {
errcode: "M_LOOKUP_FAILED",
error: "Failed to look up user info from homeserver",
},
};
});
try {
await getSFUConfigWithOpenID(
matrixClient,
ownMemberMock,
"https://sfu.example.org",
"!example_room_id",
);
} catch (ex: unknown) {
expect(ex).toBeInstanceOf(FailToGetOpenIdToken);
expect((ex as FailToGetOpenIdToken).cause).toBeInstanceOf(MatrixError);
const mxError = (ex as Error).cause as MatrixError;
expect(mxError.message).toEqual(
"MatrixError: [500] Failed to look up user info from homeserver",
);
void (await fetchMock.flush());
return;
}
expect.fail("Expected test to throw;");
});
it("should retry without delay params if the JWT service legacy endpoint returns M_BAD_JSON 400", async () => {
let callCount = 0;
fetchMock.post(
"https://sfu.example.org/sfu/get",
(url, opts) => {
callCount++;
const body = JSON.parse(opts.body as string);
// First call: check if it has delay parts and return 400
if (callCount === 1) {
expect(body).toHaveProperty("delay_id", "mock_delay_id");
return {
status: 400,
body: { errcode: "M_BAD_JSON", error: "Unsupported parameters" },
};
}
// Second call: check if delay parts were stripped and return success
expect(body).not.toHaveProperty("delay_id");
expect(body).not.toHaveProperty("delay_timeout");
expect(body).not.toHaveProperty("delay_cs_api_url");
return {
status: 200,
body: { url: sfuUrl, jwt: testJWTToken },
};
},
{ overwriteRoutes: true },
);
// Note: Assuming getSFUConfigWithOpenID eventually calls getLiveKitJWT
const config = await getSFUConfigWithOpenID(
matrixClient,
ownMemberMock,
"https://sfu.example.org",
"!example_room_id",
{
delayEndpointBaseUrl: "https://matrix.homeserverserver.org",
delayId: "mock_delay_id",
delayTimeoutMs: 3_600_000,
},
);
expect(config.jwt).toBe(testJWTToken);
expect(callCount).toBe(2);
void (await fetchMock.flush());
});
it("should successfully send delay parameters to the JWT service legacy endpoint", async () => {
fetchMock.post(
"https://sfu.example.org/sfu/get",
(url, opts) => {
const body = JSON.parse(opts.body as string);
// Verify, that the request contains the expected delay parameters
if (
body.delay_id === "mock_delay_id" &&
body.delay_timeout === 3600000 &&
body.delay_cs_api_url === "https://homeserverserver.org/cs_api"
) {
return {
status: 200,
body: { url: sfuUrl, jwt: testJWTToken },
};
}
return {
status: 400,
body: { error: "Missing expected delay params" },
};
},
{ overwriteRoutes: true },
);
const config = await getSFUConfigWithOpenID(
matrixClient,
ownMemberMock,
"https://sfu.example.org",
"!example_room_id",
{
delayEndpointBaseUrl: "https://homeserverserver.org/cs_api",
delayId: "mock_delay_id",
delayTimeoutMs: 3_600_000,
},
);
// Prüfe das Ergebnis
expect(config).toMatchObject({
jwt: testJWTToken,
url: sfuUrl,
});
void (await fetchMock.flush());
});
it("should try legacy and then new endpoint with delay delegation", async () => {
fetchMock.post("https://sfu.example.org/get_token", () => {
return {
status: 500,
body: {
errcode: "M_LOOKUP_FAILED",
error: "Failed to look up user info from homeserver",
},
};
});
fetchMock.post("https://sfu.example.org/sfu/get", () => {
return {
status: 500,
body: {
errcode: "M_LOOKUP_FAILED",
error: "Failed to look up user info from homeserver",
},
};
});
try {
await getSFUConfigWithOpenID(
matrixClient,
ownMemberMock,
"https://sfu.example.org",
"!example_room_id",
{
delayEndpointBaseUrl: "https://matrix.homeserverserver.org",
delayId: "mock_delay_id",
delayTimeoutMs: 3_600_000,
},
);
} catch (ex) {
expect(ex).toBeInstanceOf(FailToGetOpenIdToken);
expect((ex as FailToGetOpenIdToken).cause).toBeInstanceOf(MatrixError);
const mxError = (ex as Error).cause as MatrixError;
expect(mxError.message).toEqual(
"MatrixError: [500] Failed to look up user info from homeserver",
);
void (await fetchMock.flush());
}
const calls = fetchMock.calls();
expect(calls.length).toBe(2);
expect(calls[0][0]).toStrictEqual("https://sfu.example.org/get_token");
expect(calls[0][1]).toStrictEqual({
// check if it uses correct delayID!
body: '{"room_id":"!example_room_id","slot_id":"m.call#ROOM","member":{"id":"@alice:example.org:DEVICE","claimed_user_id":"@alice:example.org","claimed_device_id":"DEVICE"},"delay_id":"mock_delay_id","delay_timeout":3600000,"delay_cs_api_url":"https://matrix.homeserverserver.org"}',
method: "POST",
headers: {
"Content-Type": "application/json",
},
});
expect(calls[1][0]).toStrictEqual("https://sfu.example.org/sfu/get");
expect(calls[1][1]).toStrictEqual({
body: '{"room":"!example_room_id","device_id":"DEVICE","delay_id":"mock_delay_id","delay_timeout":3600000,"delay_cs_api_url":"https://matrix.homeserverserver.org"}',
headers: {
"Content-Type": "application/json",
},
method: "POST",
});
});
it("dont try legacy if endpoint with delay delegation is sucessful", async () => {
fetchMock.post("https://sfu.example.org/get_token", () => {
return {
status: 200,
body: { url: sfuUrl, jwt: testJWTToken },
};
});
fetchMock.post("https://sfu.example.org/sfu/get", () => {
return {
status: 500,
body: { error: "Test failure" },
};
});
try {
await getSFUConfigWithOpenID(
matrixClient,
ownMemberMock,
"https://sfu.example.org",
"!example_room_id",
{
delayEndpointBaseUrl: "https://matrix.homeserverserver.org",
delayId: "mock_delay_id",
delayTimeoutMs: 3_600_000,
},
);
} catch (ex) {
expect(ex).toBeInstanceOf(FailToGetOpenIdToken);
expect((ex as FailToGetOpenIdToken).cause).toEqual(
new Error("SFU Config fetch failed with status code 500"),
);
void (await fetchMock.flush());
}
const calls = fetchMock.calls();
expect(calls.length).toBe(1);
expect(calls[0][0]).toStrictEqual("https://sfu.example.org/get_token");
expect(calls[0][1]).toStrictEqual({
// check if it uses correct delayID!
body: '{"room_id":"!example_room_id","slot_id":"m.call#ROOM","member":{"id":"@alice:example.org:DEVICE","claimed_user_id":"@alice:example.org","claimed_device_id":"DEVICE"},"delay_id":"mock_delay_id","delay_timeout":3600000,"delay_cs_api_url":"https://matrix.homeserverserver.org"}',
method: "POST",
headers: {
"Content-Type": "application/json",
},
});
});
it("should retry fetching the openid token", async () => {
let count = 0;
matrixClient.getOpenIdToken.mockImplementation(async () => {
count++;
if (count < 2) {
throw Error("Test failure");
}
return Promise.resolve({
token_type: "Bearer",
access_token: "foobar",
matrix_server_name: "example.org",
expires_in: 30,
});
});
fetchMock.post("https://sfu.example.org/sfu/get", () => {
return {
status: 200,
body: { url: sfuUrl, jwt: testJWTToken },
};
});
const config = await getSFUConfigWithOpenID(
matrixClient,
ownMemberMock,
"https://sfu.example.org",
"!example_room_id",
);
expect(config).toEqual({
jwt: testJWTToken,
url: sfuUrl,
livekitIdentity: "@me:example.org:ABCDEF",
livekitAlias: "!example_room_id",
});
void (await fetchMock.flush());
});
});
sdk/src/media-backend/livekit/openIDSFU.ts
-321
View File
@@ -1,321 +0,0 @@
/*
Copyright 2023, 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type IOpenIDToken,
type MatrixClient,
parseErrorResponse,
} from "matrix-js-sdk";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import { type Logger } from "matrix-js-sdk/lib/logger";
import {
FailToGetOpenIdToken,
NoMatrix2AuthorizationService,
} from "../../errors";
import { doNetworkOperationWithRetry } from "../../utils/network";
/**
* Configuration and access tokens provided by the SFU on successful authentication.
*/
export interface SFUConfig {
url: string;
jwt: string;
livekitAlias: string;
// NOTE: Currently unused.
livekitIdentity: string;
}
/**
* Decoded details from the JWT.
*/
interface SFUJWTPayload {
/**
* Expiration time for the JWT.
* Note: This value is in seconds since Unix epoch.
*/
exp: number;
/**
* Name of the instance which authored the JWT
*/
iss: string;
/**
* Time at which the JWT can start to be used.
* Note: This value is in seconds since Unix epoch.
*/
nbf: number;
/**
* Subject. The Livekit alias in this context.
*/
sub: string;
/**
* The set of permissions for the user.
*/
video: {
canPublish: boolean;
canSubscribe: boolean;
room: string;
roomJoin: boolean;
};
}
// The bits we need from MatrixClient
export type OpenIDClientParts = Pick<MatrixClient, "getOpenIdToken">;
/**
* Which token service endpoint to use: the MSC4195 one, or the legacy one
* with the old identity format. Undefined tries the MSC4195 one and falls
* back to the legacy one, for transports we only subscribe on.
*/
export type TokenEndpoint = "msc4195" | "legacy";
/**
* Gets a bearer token from the homeserver and then use it to authenticate
* to the matrix RTC backend in order to get acces to the SFU.
* It has built-in retry for calls to the homeserver with a backoff policy.
* @param client The Matrix client
* @param membership Our own membership identity parts used to send to jwt service.
* @param serviceUrl The URL of the livekit SFU service
* @param roomId The room id used in the jwt request. This is NOT the livekit_alias. The jwt service will provide the alias. It maps matrix room ids <-> Livekit aliases.
* @param opts Additional options to modify which endpoint with which data will be used to acquire the jwt token.
* @param opts.tokenEndpoint Which version of the JWT endpoint to use, which affects whether the
* RTC backend identity is based on string concatenation (legacy) or a hash (MSC4195).
* This function by default uses whatever is possible with the current jwt service installed next to the SFU.
* For remote connections this does not matter, since we will not publish there we can rely on the newest option.
* @param opts.delayEndpointBaseUrl The URL of the matrix homeserver.
* @param opts.delayId The delay id used for the jwt service to manage.
* @param logger optional logger.
* @returns Object containing the token information
* @throws FailToGetOpenIdToken
*/
export async function getSFUConfigWithOpenID(
client: OpenIDClientParts,
membership: CallMembershipIdentityParts,
serviceUrl: string,
roomId: string,
opts?: {
tokenEndpoint?: TokenEndpoint;
delayEndpointBaseUrl?: string;
delayId?: string;
/** How long the SFU waits before sending the delegated leave. Needed with `delayId`. */
delayTimeoutMs?: number;
},
logger?: Logger,
): Promise<SFUConfig> {
let openIdToken: IOpenIDToken;
try {
openIdToken = await doNetworkOperationWithRetry(async () =>
client.getOpenIdToken(),
);
} catch (error) {
throw new FailToGetOpenIdToken(
error instanceof Error ? error : new Error("Unknown error"),
);
}
logger?.debug("Got openID token", openIdToken);
let sfuConfig: { url: string; jwt: string } | undefined;
const tryBothJwtEndpoints = opts?.tokenEndpoint === undefined; // This is for SFUs where we do not publish.
const forceMatrix2Jwt = opts?.tokenEndpoint === "msc4195";
// We want to start using the new endpoint (with optional delay delegation)
// if we can use both or if we are forced to use the new one.
if (tryBothJwtEndpoints || forceMatrix2Jwt) {
try {
logger?.info(
`Trying to get JWT via default endpoint for focus ${serviceUrl}...`,
);
const sfuConfig = await getLiveKitJWT(
membership,
serviceUrl,
roomId,
openIdToken,
opts?.delayEndpointBaseUrl,
opts?.delayId,
opts?.delayTimeoutMs,
);
return extractFullConfigFromToken(sfuConfig);
} catch (e) {
logger?.debug(`Failed fetching jwt with matrix 2.0 endpoint:`, e);
// Make this throw a hard error in case we force the matrix2.0 endpoint.
if (forceMatrix2Jwt) {
throw new NoMatrix2AuthorizationService(e as Error);
}
}
}
// DEPRECATED
// here we either have a sfuConfig or we already exited because of `if (forceMatrix2) throw ...`
// The only case we can get into this condition is, if `forceMatrix2` is `false`
try {
logger?.info(
`Trying to get JWT with legacy endpoint for focus ${serviceUrl}...`,
);
sfuConfig = await getLiveKitJWTLegacy(
membership.deviceId,
serviceUrl,
roomId,
openIdToken,
opts?.delayEndpointBaseUrl,
opts?.delayId,
opts?.delayTimeoutMs,
);
logger?.info(`Got JWT from call's active focus URL.`);
return extractFullConfigFromToken(sfuConfig);
} catch (ex) {
throw new FailToGetOpenIdToken(
ex instanceof Error ? ex : new Error(`Unknown error ${ex}`),
);
}
}
function extractFullConfigFromToken(sfuConfig: {
url: string;
jwt: string;
}): SFUConfig {
const [, payloadStr] = sfuConfig.jwt.split(".");
const payload = JSON.parse(global.atob(payloadStr)) as SFUJWTPayload;
return {
jwt: sfuConfig.jwt,
url: sfuConfig.url,
livekitAlias: payload.video.room,
// NOTE: Currently unused.
// Probably also not helpful since we now compute the backendIdentity on joining the call so we can use it for the encryption manager.
// The only reason for us to know it locally is to connect the right users with the lk world. (and to set our own keys)
livekitIdentity: payload.sub,
};
}
async function getLiveKitJWTLegacy(
deviceId: string,
livekitServiceURL: string,
matrixRoomId: string,
openIDToken: IOpenIDToken,
delayEndpointBaseUrl?: string,
delayId?: string,
delayTimeoutMs?: number,
): Promise<{ url: string; jwt: string }> {
interface IDelayParams {
delay_id?: string;
delay_timeout?: number;
delay_cs_api_url?: string;
}
let bodyDalayParts: IDelayParams = {};
// Also check for empty string
if (delayId && delayEndpointBaseUrl) {
bodyDalayParts = {
delay_id: delayId,
delay_timeout: delayTimeoutMs,
delay_cs_api_url: delayEndpointBaseUrl,
};
}
const makeRequest = async (delayParts: IDelayParams): Promise<Response> => {
return await fetch(livekitServiceURL + "/sfu/get", {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify({
// The legacy JWT endpoint uses only the matrix room id to calculate the livekit room alias.
// However, the livekit room alias is provided as part of the JWT payload.
room: matrixRoomId,
openid_token: openIDToken,
device_id: deviceId,
...delayParts,
}),
});
};
const res = await doNetworkOperationWithRetry(async () => {
let response = await makeRequest(bodyDalayParts);
// Old service compatibility check
const oldServiceDoesNotSupportDelayParts =
response.status === 400 && Object.keys(bodyDalayParts).length > 0;
// If http status 400 with M_BAD_JSON and we sent delay parts, retry without them
if (oldServiceDoesNotSupportDelayParts) {
try {
const errorBody = await response.json();
if (errorBody.errcode === "M_BAD_JSON") {
response = await makeRequest({});
}
} catch {
// If we can't parse the error, treat as real error
}
}
return response;
});
if (!res.ok) {
throw parseErrorResponse(res, await res.text());
}
return await res.json();
}
class NotSupportedError extends Error {
public constructor(message: string) {
super(message);
this.name = "NotSupported";
}
}
export async function getLiveKitJWT(
membership: CallMembershipIdentityParts,
livekitServiceURL: string,
matrixRoomId: string,
openIDToken: IOpenIDToken,
delayEndpointBaseUrl?: string,
delayId?: string,
delayTimeoutMs?: number,
): Promise<{ url: string; jwt: string }> {
const { userId, deviceId, memberId } = membership;
const body = {
room_id: matrixRoomId,
slot_id: "m.call#ROOM",
openid_token: openIDToken,
member: {
id: memberId,
claimed_user_id: userId,
claimed_device_id: deviceId,
},
};
let bodyDalayParts = {};
// Also check for empty string
if (delayId && delayEndpointBaseUrl) {
bodyDalayParts = {
delay_id: delayId,
delay_timeout: delayTimeoutMs,
delay_cs_api_url: delayEndpointBaseUrl,
};
}
const res = await doNetworkOperationWithRetry(async () => {
return await fetch(livekitServiceURL + "/get_token", {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify({ ...body, ...bodyDalayParts }),
});
});
if (!res.ok) {
const msg = "SFU Config fetch failed with status code " + res.status;
if (res.status === 404) {
throw new NotSupportedError(msg);
} else {
throw parseErrorResponse(res, await res.text());
}
}
return await res.json();
}
sdk/src/media-backend/livekit/test.ts
+5 -3
View File
@@ -11,13 +11,15 @@ import {
type RemoteParticipant,
type RemoteTrackPublication,
} from "livekit-client";
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import { mockEmitter } from "../../utils/test";
export const exampleTransport: LivekitTransport = {
export const exampleServiceUrl = "https://lk.example.org";
export const exampleTransport: UnstableLivekitTransport = {
type: "livekit",
livekit_service_url: "https://lk.example.org",
livekit_service_url: exampleServiceUrl,
};
export function mockRemoteParticipant(
sdk/src/utils/network.ts
+7 -1
View File
@@ -12,9 +12,13 @@ import { sleep } from "matrix-js-sdk/lib/utils";
* Performs a network operation with retries on connection errors, honouring
* the homeserver's rate limits. Rethrows once the error is not retryable or
* the retries are used up.
* @param operation The operation to perform.
* @param shouldRetry Rules out errors that retrying cannot fix, on top of the
* standard backoff rule. Retries everything by default.
*/
export async function doNetworkOperationWithRetry<T>(
operation: () => Promise<T>,
shouldRetry: (e: unknown) => boolean = () => true,
): Promise<T> {
let currentRetryCount = 0;
// eslint-disable-next-line no-constant-condition
@@ -23,7 +27,9 @@ export async function doNetworkOperationWithRetry<T>(
return await operation();
} catch (e) {
currentRetryCount++;
const backoff = calculateRetryBackoff(e, currentRetryCount, true);
const backoff = shouldRetry(e)
? calculateRetryBackoff(e, currentRetryCount, true)
: -1;
if (backoff < 0) throw e;
await sleep(backoff);
}
sdk/src/utils/serverName.ts
+11
View File
@@ -0,0 +1,11 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
/** The homeserver a Matrix user id belongs to: everything after the first colon. */
export function serverNameOf(userId: string): string {
return userId.replace(/^.*?:/, "");
}
sdk/src/utils/test-fixtures.ts
+20 -12
View File
@@ -48,16 +48,24 @@ export const daveRTL = mockMatrixRoomMember(daveRTLRtcMember, {
rawDisplayName: "\u202eevaD",
});
export const testJWTToken = [
{}, // header
{
// payload
sub: "@me:example.org:ABCDEF",
video: {
room: "!example_room_id",
/** An unsigned JWT with the given subject and LiveKit room, as a token service would issue it. */
export function createJWTToken(sub: string, room: string): string {
return [
{}, // header
{
// payload
sub,
video: {
room,
},
},
},
{}, // signature
]
.map((d) => global.btoa(JSON.stringify(d)))
.join(".");
{}, // signature
]
.map((d) => global.btoa(JSON.stringify(d)))
.join(".");
}
export const testJWTToken = createJWTToken(
"@me:example.org:ABCDEF",
"!example_room_id",
);