WIP

package.json

@@ -103,7 +103,7 @@
     "livekit-client": "^2.18.1",
     "lodash-es": "^4.17.21",
     "loglevel": "^1.9.1",
-    "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#develop",
+    "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#robin/livekit-updates",
     "matrix-widget-api": "^1.18.0",
     "node-stdlib-browser": "^1.3.1",
     "normalize.css": "^8.0.1",

pnpm-lock.yaml

@@ -191,8 +191,8 @@ importers:
         specifier: ^1.9.1
         version: 1.9.2
       matrix-js-sdk:
-        specifier: github:matrix-org/matrix-js-sdk#develop
+        specifier: github:matrix-org/matrix-js-sdk#robin/livekit-updates
-        version: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/fa4772a4ddd335a5a8bfbf8bb21df49b70f7a999
+        version: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/621b225e227dc6dc2b1c9d53413235d8999e6d54
       matrix-widget-api:
         specifier: ^1.18.0
         version: 1.19.0
@@ -4673,14 +4673,17 @@ packages:
   matrix-events-sdk@0.0.1:
     resolution: {integrity: sha512-1QEOsXO+bhyCroIe2/A5OwaxHvBm7EsSQ46DEDn8RBIfQwN5HWBpFvyWWR4QY0KHPPnnJdI99wgRiAl7Ad5qaA==}
 
-  matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/fa4772a4ddd335a5a8bfbf8bb21df49b70f7a999:
+  matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/621b225e227dc6dc2b1c9d53413235d8999e6d54:
-    resolution: {gitHosted: true, integrity: sha512-qF8++74MX36KS1KEw6Lc8HHXgZ6nHD1oUU4yBZUj5vVPvZTIspk1lOGztkFmizE5vmLAHd5tdadL9Xu1Asy4bA==, tarball: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/fa4772a4ddd335a5a8bfbf8bb21df49b70f7a999}
+    resolution: {gitHosted: true, integrity: sha512-HeTjG3NPGeCMkfPfGPaYl1QBnn7XYRiC1NpBgOxflujzQmTBNTPshGSTzMsMQKH/hpfi8QtjPE1SddNzzax3vA==, tarball: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/621b225e227dc6dc2b1c9d53413235d8999e6d54}
     version: 42.3.0
     engines: {node: '>=22.0.0'}
 
   matrix-widget-api@1.19.0:
     resolution: {integrity: sha512-Qe7L6G1mQJphBC47/Bzp5ljPIK1UARgLiuGgS+/KmEs/eaNHkQ3+ArD0/0kvMV+d6kCGAZ9OKRqsnuPjizI+tw==}
 
+  matrix-widget-api@1.20.0:
+    resolution: {integrity: sha512-qwFP37rgns7mPesM5/vvZgknzb+kvyRB5BISv5sCuBxsGMceMr+Kogf7OkjRxBXdMCPBFuf6gIuYRe5S+lpzQw==}
+
   md5.js@1.3.5:
     resolution: {integrity: sha512-xitP+WxNPcTTOgnTJcrhM0xvdPepipPSf3I8EIpGKeFLjt3PlJLIDG3u8EX53ZIubkb+5U2+3rELYpEhHhzdkg==}
 
@@ -10227,7 +10230,7 @@ snapshots:
 
   matrix-events-sdk@0.0.1: {}
 
-  matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/fa4772a4ddd335a5a8bfbf8bb21df49b70f7a999:
+  matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/621b225e227dc6dc2b1c9d53413235d8999e6d54:
     dependencies:
       '@babel/runtime': 8.0.0
       '@matrix-org/matrix-sdk-crypto-wasm': 18.9.0
@@ -10236,7 +10239,7 @@ snapshots:
       content-type: 3.1.1
       loglevel: 1.9.2
       matrix-events-sdk: 0.0.1
-      matrix-widget-api: 1.19.0
+      matrix-widget-api: 1.20.0
       p-retry: 8.0.1
       sdp-transform: 3.0.0
       unhomoglyph: 1.0.6
@@ -10246,6 +10249,11 @@ snapshots:
       '@types/events': 3.0.3
       events: 3.3.0
 
+  matrix-widget-api@1.20.0:
+    dependencies:
+      '@types/events': 3.0.3
+      events: 3.3.0
+
   md5.js@1.3.5:
     dependencies:
       hash-base: 3.0.5

src/livekit/auth/csApi.ts

@@ -0,0 +1,71 @@
+/*
+Copyright 2026 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { type MatrixClient } from "matrix-js-sdk";
+import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
+
+import { type OpenIDClientParts } from "./openID.ts";
+import { extractFullConfigFromToken, type SFUConfig } from "./types.ts";
+import { doNetworkOperationWithRetry } from "../../utils/matrix.ts";
+
+// TODO: This should come from the `MatrixRTCSession`s slot description instead
+// of being hardcoded here. (the legacy flow hardcodes it as well)
+const SLOT_ID = "m.call#ROOM";
+
+export type ClientGetTokenParts = OpenIDClientParts &
+  Pick<MatrixClient, "_unstable_getLivekitToken">;
+
+export async function getSFUConfig({
+  client,
+  membership,
+  url,
+  roomId,
+}: {
+  client: ClientGetTokenParts;
+  membership: CallMembershipIdentityParts;
+  url: string;
+  roomId: string;
+}): Promise<SFUConfig> {
+  const res = await doNetworkOperationWithRetry(async () =>
+    client._unstable_getLivekitToken({
+      url,
+      room_id: roomId,
+      slot_id: "m.call#ROOM",
+      member: { id: membership.memberId },
+    }),
+  );
+  return extractFullConfigFromToken({ url, jwt: res.jwt });
+}
+
+export type ClientDelegationParts = Pick<
+  MatrixClient,
+  "_unstable_delegateDelayedLeave" | "baseUrl"
+>;
+
+export async function delegateDelayedLeave({
+  client,
+  membership,
+  url,
+  roomId,
+  delayId,
+}: {
+  client: ClientDelegationParts;
+  membership: CallMembershipIdentityParts;
+  url: string;
+  roomId: string;
+  delayId: string;
+}): Promise<void> {
+  await doNetworkOperationWithRetry(async () =>
+    client._unstable_delegateDelayedLeave({
+      url,
+      room_id: roomId,
+      slot_id: SLOT_ID,
+      member: { id: membership.memberId },
+      delay_id: delayId,
+    }),
+  );
+}

src/livekit/auth/index.ts

@@ -0,0 +1,148 @@
+/*
+Copyright 2026 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
+import { type Logger } from "matrix-js-sdk/lib/logger";
+
+import {
+  getSFUConfig as getSFUConfigWithOpenID,
+  type OpenIDClientParts,
+} from "./openID";
+import {
+  type ClientDelegationParts,
+  delegateDelayedLeave as delegateDelayedLeaveWithCSApi,
+  type ClientGetTokenParts,
+  getSFUConfig as getSFUConfigWithCSApi,
+} from "./csApi";
+import { type MatrixRTCMode } from "../../config/ConfigOptions";
+import { MatrixError } from "matrix-js-sdk";
+import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
+import { type SFUConfig } from "./types";
+
+/**
+ * Checks whether an error means "this homeserver does not implement the
+ * (unstable) MSC4195 endpoints".
+ *
+ * MSC4195 documents `M_NOT_FOUND` for this, but a homeserver without support
+ * answers with whatever it uses for unknown endpoints (Synapse: 404 with
+ * `M_UNRECOGNIZED`), so we accept any 404 as "unsupported".
+ * @param e The error to check.
+ */
+function isEndpointUnsupported(e: unknown): boolean {
+  return (
+    e instanceof MatrixError &&
+    (e.httpStatus === 404 ||
+      e.errcode === "M_NOT_FOUND" ||
+      e.errcode === "M_UNRECOGNIZED")
+  );
+}
+
+export async function getSFUConfig({
+  client,
+  membership,
+  transport,
+  roomId,
+  matrixRTCMode,
+  logger,
+}: {
+  client: ClientGetTokenParts & OpenIDClientParts;
+  membership: CallMembershipIdentityParts;
+  transport: LivekitTransport;
+  roomId: string;
+  matrixRTCMode: MatrixRTCMode;
+  logger: Logger;
+}): Promise<SFUConfig> {
+  if ("url" in transport) {
+    try {
+      return await getSFUConfigWithCSApi({
+        client,
+        membership,
+        url: transport.url,
+        roomId,
+      });
+    } catch (e) {
+      if (isEndpointUnsupported(e)) {
+        // Alias transport before checking whether we can fall back, otherwise
+        // TypeScript gets confused about its type
+        const transport_ = transport;
+        if ("livekit_service_url" in transport_) {
+          logger.warn(
+            `Homeserver does not support the MSC4195 get_token endpoint, falling back to the OpenID flow with service ${transport_.livekit_service_url}`,
+          );
+          // Fall through
+        } else {
+          logger.error(
+            `Homeserver does not support the MSC4195 get_token endpoint, cannot get token for transport ${transport.url}`,
+          );
+          throw e;
+        }
+      } else {
+        throw e;
+      }
+    }
+  }
+
+  if ("livekit_service_url" in transport) {
+    return await getSFUConfigWithOpenID(
+      client,
+      membership,
+      transport.livekit_service_url,
+      roomId,
+      { matrixRTCMode },
+      logger,
+    );
+  }
+
+  throw new Error(`Unrecognized transport: ${JSON.stringify(transport)}`);
+}
+
+export async function delegateDelayedLeave({
+  client,
+  membership,
+  transport,
+  roomId,
+  delayId,
+  matrixRTCMode,
+  logger,
+}: {
+  client: ClientDelegationParts & OpenIDClientParts;
+  membership: CallMembershipIdentityParts;
+  transport: LivekitTransport;
+  roomId: string;
+  delayId: string;
+  matrixRTCMode: MatrixRTCMode;
+  logger: Logger;
+}): Promise<void> {
+  if ("url" in transport) {
+    try {
+      await delegateDelayedLeaveWithCSApi({
+        client,
+        membership,
+        url: transport.url,
+        roomId,
+        delayId,
+      });
+    } catch (e) {
+      if (isEndpointUnsupported(e))
+        logger.error(
+          "Server misconfigured: transport contains a 'url' field but server does not support the MSC4195 delegate_delayed_leave endpoint",
+        );
+      throw e;
+    }
+  } else {
+    // This will technically cause the service to issue a new JWT token, but
+    // it's safe to discard. We're only interested in triggering delegation.
+    await getSFUConfigWithOpenID(
+      client,
+      membership,
+      transport.livekit_service_url,
+      roomId,
+      { matrixRTCMode, delayEndpointBaseUrl: client.baseUrl, delayId },
+      logger,
+    );
+  }
+}

src/livekit/auth/openID.ts

@@ -16,54 +16,11 @@ import { type Logger } from "matrix-js-sdk/lib/logger";
 import {
   FailToGetOpenIdToken,
   NoMatrix2AuthorizationService,
-} from "../utils/errors";
+} from "../../utils/errors";
-import { doNetworkOperationWithRetry } from "../utils/matrix";
+import { doNetworkOperationWithRetry } from "../../utils/matrix";
-import { Config } from "../config/Config";
+import { Config } from "../../config/Config";
-import { MatrixRTCMode } from "../config/ConfigOptions";
+import { MatrixRTCMode } from "../../config/ConfigOptions";
-
+import { extractFullConfigFromToken, type SFUConfig } from "./types";
-/**
- * Configuration and access tokens provided by the SFU on successful authentication.
- */
-export interface SFUConfig {
-  url: string;
-  jwt: string;
-  livekitAlias: string;
-  // NOTE: Currently unused.
-  livekitIdentity: string;
-}
-
-/**
- * Decoded details from the JWT.
- */
-interface SFUJWTPayload {
-  /**
-   * Expiration time for the JWT.
-   * Note: This value is in seconds since Unix epoch.
-   */
-  exp: number;
-  /**
-   * Name of the instance which authored the JWT
-   */
-  iss: string;
-  /**
-   * Time at which the JWT can start to be used.
-   * Note: This value is in seconds since Unix epoch.
-   */
-  nbf: number;
-  /**
-   * Subject. The Livekit alias in this context.
-   */
-  sub: string;
-  /**
-   * The set of permissions for the user.
-   */
-  video: {
-    canPublish: boolean;
-    canSubscribe: boolean;
-    room: string;
-    roomJoin: boolean;
-  };
-}
 
 // The bits we need from MatrixClient
 export type OpenIDClientParts = Pick<
@@ -90,7 +47,7 @@ export type OpenIDClientParts = Pick<
  * @returns Object containing the token information
  * @throws FailToGetOpenIdToken
  */
-export async function getSFUConfigWithOpenID(
+export async function getSFUConfig(
   client: OpenIDClientParts,
   membership: CallMembershipIdentityParts,
   serviceUrl: string,
@@ -169,23 +126,6 @@ export async function getSFUConfigWithOpenID(
   }
 }
 
-function extractFullConfigFromToken(sfuConfig: {
-  url: string;
-  jwt: string;
-}): SFUConfig {
-  const [, payloadStr] = sfuConfig.jwt.split(".");
-  const payload = JSON.parse(global.atob(payloadStr)) as SFUJWTPayload;
-  return {
-    jwt: sfuConfig.jwt,
-    url: sfuConfig.url,
-    livekitAlias: payload.video.room,
-    // NOTE: Currently unused.
-    // Probably also not helpful since we now compute the backendIdentity on joining the call so we can use it for the encryption manager.
-    // The only reason for us to know it locally is to connect the right users with the lk world. (and to set our own keys)
-    livekitIdentity: payload.sub,
-  };
-}
-
 async function getLiveKitJWTLegacy(
   deviceId: string,
   livekitServiceURL: string,
@@ -261,7 +201,7 @@ class NotSupportedError extends Error {
   }
 }
 
-export async function getLiveKitJWT(
+async function getLiveKitJWT(
   membership: CallMembershipIdentityParts,
   livekitServiceURL: string,
   matrixRoomId: string,

src/livekit/auth/types.ts

@@ -0,0 +1,79 @@
+/*
+Copyright 2026 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+/**
+ * Configuration and access tokens provided by the SFU on successful authentication.
+ */
+export interface SFUConfig {
+  /**
+   * The WebSocket URL of the LiveKit SFU. This is what we connect the LiveKit
+   * room to. Note that this is NOT the JWT service URL
+   * (`livekit_service_url`): the legacy JWT service returns this URL as part of
+   * its response, while the CS-Api based flow requires us to know it upfront.
+   */
+  url: string;
+  jwt: string;
+  livekitAlias: string;
+  // NOTE: Currently unused.
+  livekitIdentity: string;
+}
+
+/**
+ * Decoded details from the JWT.
+ */
+interface SFUJWTPayload {
+  /**
+   * Expiration time for the JWT.
+   * Note: This value is in seconds since Unix epoch.
+   */
+  exp: number;
+  /**
+   * Name of the instance which authored the JWT
+   */
+  iss: string;
+  /**
+   * Time at which the JWT can start to be used.
+   * Note: This value is in seconds since Unix epoch.
+   */
+  nbf: number;
+  /**
+   * Subject. The Livekit alias in this context.
+   */
+  sub: string;
+  /**
+   * The set of permissions for the user.
+   */
+  video: {
+    canPublish: boolean;
+    canSubscribe: boolean;
+    room: string;
+    roomJoin: boolean;
+  };
+}
+
+/**
+ * Complements the SFU websocket url and the JWT with the information encoded in
+ * the JWT payload itself.
+ * @param sfuConfig The SFU websocket url and the JWT to connect with.
+ * @returns The full SFU config, including the LiveKit alias and identity.
+ */
+export function extractFullConfigFromToken(sfuConfig: {
+  url: string;
+  jwt: string;
+}): SFUConfig {
+  const [, payloadStr] = sfuConfig.jwt.split(".");
+  const payload = JSON.parse(global.atob(payloadStr)) as SFUJWTPayload;
+  return {
+    jwt: sfuConfig.jwt,
+    url: sfuConfig.url,
+    livekitAlias: payload.video.room,
+    // NOTE: Currently unused.
+    // Probably also not helpful since we now compute the backendIdentity on joining the call so we can use it for the encryption manager.
+    // The only reason for us to know it locally is to connect the right users with the lk world. (and to set our own keys)
+    livekitIdentity: payload.sub,
+  };
+}

src/state/CallViewModel/localMember/LocalMember.ts

@@ -15,7 +15,6 @@ import {
   MediaDeviceFailure,
 } from "livekit-client";
 import { observeParticipantEvents } from "@livekit/components-core";
-import { type MatrixClient } from "matrix-js-sdk";
 import {
   Status as RTCSessionStatus,
   type LivekitTransport,
@@ -80,7 +79,10 @@ import {
 } from "../remoteMembers/Connection.ts";
 import { type HomeserverConnected } from "./HomeserverConnected.ts";
 import { type LocalTransport } from "./LocalTransport.ts";
-import { getSFUConfigWithOpenID } from "../../../livekit/openIDSFU.ts";
+import {
+  type ClientDelegationParts,
+  delegateDelayedLeave,
+} from "../../../livekit/auth/index.ts";
 
 export enum TransportState {
   /** Not even a transport is available to the LocalMembership */
@@ -150,7 +152,7 @@ interface Props {
   roomId: string;
   ownMembershipIdentity: CallMembershipIdentityParts;
   localTransport$: Observable<LocalTransport>;
-  client: Pick<MatrixClient, "getDeviceId" | "getOpenIdToken">;
+  client: ClientDelegationParts;
   matrixRTCSession: Pick<MatrixRTCSession, "updateCallIntent" | "leave">;
   /** Whether to hide the screen-sharing button. */
   hideScreensharing: boolean;
@@ -308,15 +310,16 @@ export const createLocalMembership$ = ({
   const joinParams$ = scope.behavior(
     localTransport$.pipe(
       switchMap(async ({ transport }) => {
-        const transportSupportsDelegation = checkDelegationSupport(
+        const transportSupportsDelegation =
-          transport.livekit_service_url + "/delegate_delayed_leave",
+          "livekit_service_url" in transport &&
-          `transport ${transport.livekit_service_url}`,
+          (await checkDelegationSupport(
-        );
+            transport.livekit_service_url + "/delegate_delayed_leave",
+            `transport ${transport.livekit_service_url}`,
+          ));
         return {
           transport,
           delegationSupported:
-            (await homeserverSupportsDelegation) ||
+            transportSupportsDelegation || (await homeserverSupportsDelegation),
-            (await transportSupportsDelegation),
         };
       }),
     ),
@@ -334,7 +337,7 @@ export const createLocalMembership$ = ({
       ),
       tap((connection) => {
         logger.info(
-          `Local connection updated: ${connection?.transport?.livekit_service_url}`,
+          `Local connection updated: ${JSON.stringify(connection?.transport)}`,
         );
       }),
     ),
@@ -406,7 +409,7 @@ export const createLocalMembership$ = ({
   scope.reconcile(localConnection$, async (connection) => {
     logger.info(
       "reconcile based on new localConnection:",
-      connection?.transport.livekit_service_url,
+      JSON.stringify(connection?.transport),
     );
     if (connection !== null) {
       const publisher = createPublisherFactory(connection);
@@ -702,23 +705,22 @@ export const createLocalMembership$ = ({
     async ([joinParams, delayId]) => {
       if (joinParams?.delegationSupported && delayId !== null) {
         try {
-          // This will technically cause the service to issue a new JWT token,
+          logger.info(
-          // but it's safe to discard. We're only interested in triggering
+            `Delegating delayed leave to ${JSON.stringify(joinParams.transport)}…`,
-          // delegation.
-          await getSFUConfigWithOpenID(
-            client,
-            ownMembershipIdentity,
-            joinParams.transport.livekit_service_url,
-            roomId,
-            { matrixRTCMode, delayEndpointBaseUrl: baseUrl, delayId },
-            logger,
           );
+          await delegateDelayedLeave({
+            client,
+            membership: ownMembershipIdentity,
+            transport: joinParams.transport,
+            roomId,
+            delayId,
+            matrixRTCMode,
+            logger,
+          });
+          logger.info("Delayed leave successfully delegated");
         } catch (e) {
           // TODO: Surface this to the user as a service interruption?
-          logger.error(
+          logger.error(`Failed to delegate leave`, e);
-            `Failed to delegate leave to ${joinParams.transport.livekit_service_url}`,
-            e,
-          );
         }
       }
     },

src/widget.ts

@@ -175,6 +175,8 @@ export const initializeWidget = (
           receiveToDevice: sendRecvToDevice,
           turnServers: false,
           rtcTransports: true,
+          rtcLivekitGetToken: true,
+          rtcLivekitDelegateDelayedLeave: true,
           sendDelayedEvents: true,
           updateDelayedEvents: true,
           sendSticky: true,