From 0030371ea72ab795c3fc89ef9d2fb920d8dec1bd Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 2 Oct 2026 23:26:21 +0200 Subject: [PATCH] WIP --- package.json | 2 +- pnpm-lock.yaml | 20 ++- src/livekit/auth/csApi.ts | 71 +++++++++ src/livekit/auth/index.ts | 148 ++++++++++++++++++ .../openID.test.ts} | 0 src/livekit/{openIDSFU.ts => auth/openID.ts} | 74 +-------- src/livekit/auth/types.ts | 79 ++++++++++ .../CallViewModel/localMember/LocalMember.ts | 52 +++--- src/widget.ts | 2 + 9 files changed, 349 insertions(+), 99 deletions(-) create mode 100644 src/livekit/auth/csApi.ts create mode 100644 src/livekit/auth/index.ts rename src/livekit/{openIDSFU.test.ts => auth/openID.test.ts} (100%) rename src/livekit/{openIDSFU.ts => auth/openID.ts} (81%) create mode 100644 src/livekit/auth/types.ts diff --git a/package.json b/package.json index b30f57ce3..0a76e0c01 100644 --- a/package.json +++ b/package.json @@ -103,7 +103,7 @@ "livekit-client": "^2.18.1", "lodash-es": "^4.17.21", "loglevel": "^1.9.1", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#develop", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#robin/livekit-updates", "matrix-widget-api": "^1.18.0", "node-stdlib-browser": "^1.3.1", "normalize.css": "^8.0.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 73a7796ba..9968c64a8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -191,8 +191,8 @@ importers: specifier: ^1.9.1 version: 1.9.2 matrix-js-sdk: - specifier: github:matrix-org/matrix-js-sdk#develop - version: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/fa4772a4ddd335a5a8bfbf8bb21df49b70f7a999 + specifier: github:matrix-org/matrix-js-sdk#robin/livekit-updates + version: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/621b225e227dc6dc2b1c9d53413235d8999e6d54 matrix-widget-api: specifier: ^1.18.0 version: 1.19.0 @@ -4673,14 +4673,17 @@ packages: matrix-events-sdk@0.0.1: resolution: {integrity: sha512-1QEOsXO+bhyCroIe2/A5OwaxHvBm7EsSQ46DEDn8RBIfQwN5HWBpFvyWWR4QY0KHPPnnJdI99wgRiAl7Ad5qaA==} - matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/fa4772a4ddd335a5a8bfbf8bb21df49b70f7a999: - resolution: {gitHosted: true, integrity: sha512-qF8++74MX36KS1KEw6Lc8HHXgZ6nHD1oUU4yBZUj5vVPvZTIspk1lOGztkFmizE5vmLAHd5tdadL9Xu1Asy4bA==, tarball: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/fa4772a4ddd335a5a8bfbf8bb21df49b70f7a999} + matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/621b225e227dc6dc2b1c9d53413235d8999e6d54: + resolution: {gitHosted: true, integrity: sha512-HeTjG3NPGeCMkfPfGPaYl1QBnn7XYRiC1NpBgOxflujzQmTBNTPshGSTzMsMQKH/hpfi8QtjPE1SddNzzax3vA==, tarball: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/621b225e227dc6dc2b1c9d53413235d8999e6d54} version: 42.3.0 engines: {node: '>=22.0.0'} matrix-widget-api@1.19.0: resolution: {integrity: sha512-Qe7L6G1mQJphBC47/Bzp5ljPIK1UARgLiuGgS+/KmEs/eaNHkQ3+ArD0/0kvMV+d6kCGAZ9OKRqsnuPjizI+tw==} + matrix-widget-api@1.20.0: + resolution: {integrity: sha512-qwFP37rgns7mPesM5/vvZgknzb+kvyRB5BISv5sCuBxsGMceMr+Kogf7OkjRxBXdMCPBFuf6gIuYRe5S+lpzQw==} + md5.js@1.3.5: resolution: {integrity: sha512-xitP+WxNPcTTOgnTJcrhM0xvdPepipPSf3I8EIpGKeFLjt3PlJLIDG3u8EX53ZIubkb+5U2+3rELYpEhHhzdkg==} @@ -10227,7 +10230,7 @@ snapshots: matrix-events-sdk@0.0.1: {} - matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/fa4772a4ddd335a5a8bfbf8bb21df49b70f7a999: + matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/621b225e227dc6dc2b1c9d53413235d8999e6d54: dependencies: '@babel/runtime': 8.0.0 '@matrix-org/matrix-sdk-crypto-wasm': 18.9.0 @@ -10236,7 +10239,7 @@ snapshots: content-type: 3.1.1 loglevel: 1.9.2 matrix-events-sdk: 0.0.1 - matrix-widget-api: 1.19.0 + matrix-widget-api: 1.20.0 p-retry: 8.0.1 sdp-transform: 3.0.0 unhomoglyph: 1.0.6 @@ -10246,6 +10249,11 @@ snapshots: '@types/events': 3.0.3 events: 3.3.0 + matrix-widget-api@1.20.0: + dependencies: + '@types/events': 3.0.3 + events: 3.3.0 + md5.js@1.3.5: dependencies: hash-base: 3.0.5 diff --git a/src/livekit/auth/csApi.ts b/src/livekit/auth/csApi.ts new file mode 100644 index 000000000..728201e86 --- /dev/null +++ b/src/livekit/auth/csApi.ts @@ -0,0 +1,71 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type MatrixClient } from "matrix-js-sdk"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; + +import { type OpenIDClientParts } from "./openID.ts"; +import { extractFullConfigFromToken, type SFUConfig } from "./types.ts"; +import { doNetworkOperationWithRetry } from "../../utils/matrix.ts"; + +// TODO: This should come from the `MatrixRTCSession`s slot description instead +// of being hardcoded here. (the legacy flow hardcodes it as well) +const SLOT_ID = "m.call#ROOM"; + +export type ClientGetTokenParts = OpenIDClientParts & + Pick; + +export async function getSFUConfig({ + client, + membership, + url, + roomId, +}: { + client: ClientGetTokenParts; + membership: CallMembershipIdentityParts; + url: string; + roomId: string; +}): Promise { + const res = await doNetworkOperationWithRetry(async () => + client._unstable_getLivekitToken({ + url, + room_id: roomId, + slot_id: "m.call#ROOM", + member: { id: membership.memberId }, + }), + ); + return extractFullConfigFromToken({ url, jwt: res.jwt }); +} + +export type ClientDelegationParts = Pick< + MatrixClient, + "_unstable_delegateDelayedLeave" | "baseUrl" +>; + +export async function delegateDelayedLeave({ + client, + membership, + url, + roomId, + delayId, +}: { + client: ClientDelegationParts; + membership: CallMembershipIdentityParts; + url: string; + roomId: string; + delayId: string; +}): Promise { + await doNetworkOperationWithRetry(async () => + client._unstable_delegateDelayedLeave({ + url, + room_id: roomId, + slot_id: SLOT_ID, + member: { id: membership.memberId }, + delay_id: delayId, + }), + ); +} diff --git a/src/livekit/auth/index.ts b/src/livekit/auth/index.ts new file mode 100644 index 000000000..9af2bb9fb --- /dev/null +++ b/src/livekit/auth/index.ts @@ -0,0 +1,148 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; +import { type Logger } from "matrix-js-sdk/lib/logger"; + +import { + getSFUConfig as getSFUConfigWithOpenID, + type OpenIDClientParts, +} from "./openID"; +import { + type ClientDelegationParts, + delegateDelayedLeave as delegateDelayedLeaveWithCSApi, + type ClientGetTokenParts, + getSFUConfig as getSFUConfigWithCSApi, +} from "./csApi"; +import { type MatrixRTCMode } from "../../config/ConfigOptions"; +import { MatrixError } from "matrix-js-sdk"; +import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { type SFUConfig } from "./types"; + +/** + * Checks whether an error means "this homeserver does not implement the + * (unstable) MSC4195 endpoints". + * + * MSC4195 documents `M_NOT_FOUND` for this, but a homeserver without support + * answers with whatever it uses for unknown endpoints (Synapse: 404 with + * `M_UNRECOGNIZED`), so we accept any 404 as "unsupported". + * @param e The error to check. + */ +function isEndpointUnsupported(e: unknown): boolean { + return ( + e instanceof MatrixError && + (e.httpStatus === 404 || + e.errcode === "M_NOT_FOUND" || + e.errcode === "M_UNRECOGNIZED") + ); +} + +export async function getSFUConfig({ + client, + membership, + transport, + roomId, + matrixRTCMode, + logger, +}: { + client: ClientGetTokenParts & OpenIDClientParts; + membership: CallMembershipIdentityParts; + transport: LivekitTransport; + roomId: string; + matrixRTCMode: MatrixRTCMode; + logger: Logger; +}): Promise { + if ("url" in transport) { + try { + return await getSFUConfigWithCSApi({ + client, + membership, + url: transport.url, + roomId, + }); + } catch (e) { + if (isEndpointUnsupported(e)) { + // Alias transport before checking whether we can fall back, otherwise + // TypeScript gets confused about its type + const transport_ = transport; + if ("livekit_service_url" in transport_) { + logger.warn( + `Homeserver does not support the MSC4195 get_token endpoint, falling back to the OpenID flow with service ${transport_.livekit_service_url}`, + ); + // Fall through + } else { + logger.error( + `Homeserver does not support the MSC4195 get_token endpoint, cannot get token for transport ${transport.url}`, + ); + throw e; + } + } else { + throw e; + } + } + } + + if ("livekit_service_url" in transport) { + return await getSFUConfigWithOpenID( + client, + membership, + transport.livekit_service_url, + roomId, + { matrixRTCMode }, + logger, + ); + } + + throw new Error(`Unrecognized transport: ${JSON.stringify(transport)}`); +} + +export async function delegateDelayedLeave({ + client, + membership, + transport, + roomId, + delayId, + matrixRTCMode, + logger, +}: { + client: ClientDelegationParts & OpenIDClientParts; + membership: CallMembershipIdentityParts; + transport: LivekitTransport; + roomId: string; + delayId: string; + matrixRTCMode: MatrixRTCMode; + logger: Logger; +}): Promise { + if ("url" in transport) { + try { + await delegateDelayedLeaveWithCSApi({ + client, + membership, + url: transport.url, + roomId, + delayId, + }); + } catch (e) { + if (isEndpointUnsupported(e)) + logger.error( + "Server misconfigured: transport contains a 'url' field but server does not support the MSC4195 delegate_delayed_leave endpoint", + ); + throw e; + } + } else { + // This will technically cause the service to issue a new JWT token, but + // it's safe to discard. We're only interested in triggering delegation. + await getSFUConfigWithOpenID( + client, + membership, + transport.livekit_service_url, + roomId, + { matrixRTCMode, delayEndpointBaseUrl: client.baseUrl, delayId }, + logger, + ); + } +} diff --git a/src/livekit/openIDSFU.test.ts b/src/livekit/auth/openID.test.ts similarity index 100% rename from src/livekit/openIDSFU.test.ts rename to src/livekit/auth/openID.test.ts diff --git a/src/livekit/openIDSFU.ts b/src/livekit/auth/openID.ts similarity index 81% rename from src/livekit/openIDSFU.ts rename to src/livekit/auth/openID.ts index cf1f3c951..a442613c8 100644 --- a/src/livekit/openIDSFU.ts +++ b/src/livekit/auth/openID.ts @@ -16,54 +16,11 @@ import { type Logger } from "matrix-js-sdk/lib/logger"; import { FailToGetOpenIdToken, NoMatrix2AuthorizationService, -} from "../utils/errors"; -import { doNetworkOperationWithRetry } from "../utils/matrix"; -import { Config } from "../config/Config"; -import { MatrixRTCMode } from "../config/ConfigOptions"; - -/** - * Configuration and access tokens provided by the SFU on successful authentication. - */ -export interface SFUConfig { - url: string; - jwt: string; - livekitAlias: string; - // NOTE: Currently unused. - livekitIdentity: string; -} - -/** - * Decoded details from the JWT. - */ -interface SFUJWTPayload { - /** - * Expiration time for the JWT. - * Note: This value is in seconds since Unix epoch. - */ - exp: number; - /** - * Name of the instance which authored the JWT - */ - iss: string; - /** - * Time at which the JWT can start to be used. - * Note: This value is in seconds since Unix epoch. - */ - nbf: number; - /** - * Subject. The Livekit alias in this context. - */ - sub: string; - /** - * The set of permissions for the user. - */ - video: { - canPublish: boolean; - canSubscribe: boolean; - room: string; - roomJoin: boolean; - }; -} +} from "../../utils/errors"; +import { doNetworkOperationWithRetry } from "../../utils/matrix"; +import { Config } from "../../config/Config"; +import { MatrixRTCMode } from "../../config/ConfigOptions"; +import { extractFullConfigFromToken, type SFUConfig } from "./types"; // The bits we need from MatrixClient export type OpenIDClientParts = Pick< @@ -90,7 +47,7 @@ export type OpenIDClientParts = Pick< * @returns Object containing the token information * @throws FailToGetOpenIdToken */ -export async function getSFUConfigWithOpenID( +export async function getSFUConfig( client: OpenIDClientParts, membership: CallMembershipIdentityParts, serviceUrl: string, @@ -169,23 +126,6 @@ export async function getSFUConfigWithOpenID( } } -function extractFullConfigFromToken(sfuConfig: { - url: string; - jwt: string; -}): SFUConfig { - const [, payloadStr] = sfuConfig.jwt.split("."); - const payload = JSON.parse(global.atob(payloadStr)) as SFUJWTPayload; - return { - jwt: sfuConfig.jwt, - url: sfuConfig.url, - livekitAlias: payload.video.room, - // NOTE: Currently unused. - // Probably also not helpful since we now compute the backendIdentity on joining the call so we can use it for the encryption manager. - // The only reason for us to know it locally is to connect the right users with the lk world. (and to set our own keys) - livekitIdentity: payload.sub, - }; -} - async function getLiveKitJWTLegacy( deviceId: string, livekitServiceURL: string, @@ -261,7 +201,7 @@ class NotSupportedError extends Error { } } -export async function getLiveKitJWT( +async function getLiveKitJWT( membership: CallMembershipIdentityParts, livekitServiceURL: string, matrixRoomId: string, diff --git a/src/livekit/auth/types.ts b/src/livekit/auth/types.ts new file mode 100644 index 000000000..81982d689 --- /dev/null +++ b/src/livekit/auth/types.ts @@ -0,0 +1,79 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +/** + * Configuration and access tokens provided by the SFU on successful authentication. + */ +export interface SFUConfig { + /** + * The WebSocket URL of the LiveKit SFU. This is what we connect the LiveKit + * room to. Note that this is NOT the JWT service URL + * (`livekit_service_url`): the legacy JWT service returns this URL as part of + * its response, while the CS-Api based flow requires us to know it upfront. + */ + url: string; + jwt: string; + livekitAlias: string; + // NOTE: Currently unused. + livekitIdentity: string; +} + +/** + * Decoded details from the JWT. + */ +interface SFUJWTPayload { + /** + * Expiration time for the JWT. + * Note: This value is in seconds since Unix epoch. + */ + exp: number; + /** + * Name of the instance which authored the JWT + */ + iss: string; + /** + * Time at which the JWT can start to be used. + * Note: This value is in seconds since Unix epoch. + */ + nbf: number; + /** + * Subject. The Livekit alias in this context. + */ + sub: string; + /** + * The set of permissions for the user. + */ + video: { + canPublish: boolean; + canSubscribe: boolean; + room: string; + roomJoin: boolean; + }; +} + +/** + * Complements the SFU websocket url and the JWT with the information encoded in + * the JWT payload itself. + * @param sfuConfig The SFU websocket url and the JWT to connect with. + * @returns The full SFU config, including the LiveKit alias and identity. + */ +export function extractFullConfigFromToken(sfuConfig: { + url: string; + jwt: string; +}): SFUConfig { + const [, payloadStr] = sfuConfig.jwt.split("."); + const payload = JSON.parse(global.atob(payloadStr)) as SFUJWTPayload; + return { + jwt: sfuConfig.jwt, + url: sfuConfig.url, + livekitAlias: payload.video.room, + // NOTE: Currently unused. + // Probably also not helpful since we now compute the backendIdentity on joining the call so we can use it for the encryption manager. + // The only reason for us to know it locally is to connect the right users with the lk world. (and to set our own keys) + livekitIdentity: payload.sub, + }; +} diff --git a/src/state/CallViewModel/localMember/LocalMember.ts b/src/state/CallViewModel/localMember/LocalMember.ts index feb49554c..08dcc7549 100644 --- a/src/state/CallViewModel/localMember/LocalMember.ts +++ b/src/state/CallViewModel/localMember/LocalMember.ts @@ -15,7 +15,6 @@ import { MediaDeviceFailure, } from "livekit-client"; import { observeParticipantEvents } from "@livekit/components-core"; -import { type MatrixClient } from "matrix-js-sdk"; import { Status as RTCSessionStatus, type LivekitTransport, @@ -80,7 +79,10 @@ import { } from "../remoteMembers/Connection.ts"; import { type HomeserverConnected } from "./HomeserverConnected.ts"; import { type LocalTransport } from "./LocalTransport.ts"; -import { getSFUConfigWithOpenID } from "../../../livekit/openIDSFU.ts"; +import { + type ClientDelegationParts, + delegateDelayedLeave, +} from "../../../livekit/auth/index.ts"; export enum TransportState { /** Not even a transport is available to the LocalMembership */ @@ -150,7 +152,7 @@ interface Props { roomId: string; ownMembershipIdentity: CallMembershipIdentityParts; localTransport$: Observable; - client: Pick; + client: ClientDelegationParts; matrixRTCSession: Pick; /** Whether to hide the screen-sharing button. */ hideScreensharing: boolean; @@ -308,15 +310,16 @@ export const createLocalMembership$ = ({ const joinParams$ = scope.behavior( localTransport$.pipe( switchMap(async ({ transport }) => { - const transportSupportsDelegation = checkDelegationSupport( - transport.livekit_service_url + "/delegate_delayed_leave", - `transport ${transport.livekit_service_url}`, - ); + const transportSupportsDelegation = + "livekit_service_url" in transport && + (await checkDelegationSupport( + transport.livekit_service_url + "/delegate_delayed_leave", + `transport ${transport.livekit_service_url}`, + )); return { transport, delegationSupported: - (await homeserverSupportsDelegation) || - (await transportSupportsDelegation), + transportSupportsDelegation || (await homeserverSupportsDelegation), }; }), ), @@ -334,7 +337,7 @@ export const createLocalMembership$ = ({ ), tap((connection) => { logger.info( - `Local connection updated: ${connection?.transport?.livekit_service_url}`, + `Local connection updated: ${JSON.stringify(connection?.transport)}`, ); }), ), @@ -406,7 +409,7 @@ export const createLocalMembership$ = ({ scope.reconcile(localConnection$, async (connection) => { logger.info( "reconcile based on new localConnection:", - connection?.transport.livekit_service_url, + JSON.stringify(connection?.transport), ); if (connection !== null) { const publisher = createPublisherFactory(connection); @@ -702,23 +705,22 @@ export const createLocalMembership$ = ({ async ([joinParams, delayId]) => { if (joinParams?.delegationSupported && delayId !== null) { try { - // This will technically cause the service to issue a new JWT token, - // but it's safe to discard. We're only interested in triggering - // delegation. - await getSFUConfigWithOpenID( - client, - ownMembershipIdentity, - joinParams.transport.livekit_service_url, - roomId, - { matrixRTCMode, delayEndpointBaseUrl: baseUrl, delayId }, - logger, + logger.info( + `Delegating delayed leave to ${JSON.stringify(joinParams.transport)}…`, ); + await delegateDelayedLeave({ + client, + membership: ownMembershipIdentity, + transport: joinParams.transport, + roomId, + delayId, + matrixRTCMode, + logger, + }); + logger.info("Delayed leave successfully delegated"); } catch (e) { // TODO: Surface this to the user as a service interruption? - logger.error( - `Failed to delegate leave to ${joinParams.transport.livekit_service_url}`, - e, - ); + logger.error(`Failed to delegate leave`, e); } } }, diff --git a/src/widget.ts b/src/widget.ts index b35e335bc..f1847004b 100644 --- a/src/widget.ts +++ b/src/widget.ts @@ -175,6 +175,8 @@ export const initializeWidget = ( receiveToDevice: sendRecvToDevice, turnServers: false, rtcTransports: true, + rtcLivekitGetToken: true, + rtcLivekitDelegateDelayedLeave: true, sendDelayedEvents: true, updateDelayedEvents: true, sendSticky: true,