Update SDK to be consumed by Element Call's CallViewModel

Update the public API to export the primitives and operators the call
model needs, and change `LocalMediaInputs` from observable flags to
immediate booleans with imperative setters. Add session timings, media
quality and capture settings as configuration options, and allow the
host to override the transport discovery with explicit URLs.

sdk/src/api.ts

@@ -15,12 +15,20 @@ import {
   type RTCNotificationType,
   type Transport,
 } from "matrix-js-sdk/lib/matrixrtc";
-import { type Track, type TrackProcessor } from "livekit-client";
+import {
+  type Track,
+  type TrackProcessor,
+  type VideoCodec,
+} from "livekit-client";
 import { type Observable } from "rxjs";
 
 import { type Behavior } from "./reactive/Behavior";
 import { type EncryptionSystem } from "./encryption";
-import { type MatrixRTCMode } from "./config";
+import {
+  type MatrixRTCMode,
+  type MediaQuality,
+  type SessionTimings,
+} from "./config";
 
 // ---------------------------------------------------------------------------
 // Session
@@ -45,14 +53,48 @@ export interface MatrixRTCClientOptions {
    * (`"audio"` or `"video"`); anything else is dropped until it can.
    */
   applicationData?: Record<string, unknown>;
+  /** Session timings the host has configured; the defaults otherwise. */
+  timings?: Partial<SessionTimings>;
+  /** Limits on what is published; LiveKit's defaults otherwise. */
+  mediaQuality?: MediaQuality;
+  /** How the local tracks are captured and encoded. */
+  capture?: CaptureSettings;
+  /** Use this transport instead of asking the homeserver. */
+  transportUrl?: string;
+  /** Use this transport when the homeserver advertises none. */
+  fallbackTransportUrl?: string;
 }
 
-/** What the local member publishes. */
+/** Capture and encoding settings for the local tracks, applied at creation. */
+export interface CaptureSettings {
+  audio?: {
+    echoCancellation?: boolean;
+    noiseSuppression?: boolean;
+    autoGainControl?: boolean;
+  };
+  camera?: VideoCaptureSettings;
+  screenShare?: VideoCaptureSettings;
+}
+
+export interface VideoCaptureSettings {
+  resolution?: { width: number; height: number; frameRate?: number };
+  maxBitrate?: number;
+  maxFramerate?: number;
+  codec?: VideoCodec;
+}
+
+/**
+ * What the local member publishes. The enabled flags are the initial state;
+ * from then on `setMicrophoneEnabled` and `setCameraEnabled` on the client
+ * change it, and report what the device allowed.
+ */
 export interface LocalMediaInputs {
-  microphoneEnabled$: Behavior<boolean>;
-  cameraEnabled$: Behavior<boolean>;
+  microphoneEnabled: boolean;
+  cameraEnabled: boolean;
   audioInputDeviceId$: Behavior<string | undefined>;
   videoInputDeviceId$: Behavior<string | undefined>;
+  /** Undefined where the host routes audio itself, or to leave the browser's choice. */
+  audioOutputDeviceId$: Behavior<string | undefined>;
   /** Background blur and the like. */
   videoProcessor$: Behavior<TrackProcessor<Track.Kind.Video> | undefined>;
 }
@@ -64,6 +106,9 @@ export type ConnectionStatus =
   | "reconnecting"
   | "disconnected";
 
+/** Why the client is not connected: the first failing of its three links. */
+export type DisconnectReason = "sync" | "membership" | "probablyLeft" | "media";
+
 import { type MatrixRTCError } from "./errors";
 
 export { MatrixRTCError } from "./errors";
@@ -81,6 +126,17 @@ export interface MatrixRTCClient {
   status$: Behavior<ConnectionStatus>;
   connected$: Behavior<boolean>;
   reconnecting$: Behavior<boolean>;
+  /** Null while connected, the first failing link otherwise. */
+  disconnectReason$: Behavior<DisconnectReason | null>;
+
+  /**
+   * Publishes or mutes the microphone. Resolves with the state that
+   * resulted, which differs from the request where the device could not be
+   * used. Before the transport is connected the request is remembered and
+   * applied once it is.
+   */
+  setMicrophoneEnabled(enabled: boolean): Promise<boolean>;
+  setCameraEnabled(enabled: boolean): Promise<boolean>;
   /** A transport, Matrix or connection error that stops the session. */
   fatalError$: Behavior<MatrixRTCError | null>;
 
@@ -239,4 +295,11 @@ export interface MemberMedia {
 
 export interface LocalMemberMedia extends MemberMedia {
   local: true;
+  /**
+   * Restarts the camera facing the other way, on devices with a front and a
+   * back camera, and resolves with the id of the device now in use. Does
+   * nothing without a camera track or where the facing mode is unknown (see
+   * `VideoMediaTrack.facingMode$`).
+   */
+  switchCamera(): Promise<string | undefined>;
 }

sdk/src/config.ts

@@ -24,7 +24,7 @@ export interface DelayedLeaveTimings {
   /** How long (in milliseconds) the homeserver waits before sending the delayed leave. */
   delay_ms: number;
   /** How often (in milliseconds) the client restarts the delayed leave. */
-  restart_ms: number;
+  restart_ms?: number;
   /** How long (in milliseconds) a restart may take before it counts as failed. */
   restart_timeout_ms?: number;
 }
@@ -45,12 +45,18 @@ export interface SessionTimings {
   /** The session size at which the media key stops being rotated. */
   keyRotationParticipantLimit?: number;
   delayedLeave: DelayedLeaveTimings;
+  /**
+   * The timings used where the SFU restarts the delayed leave on the
+   * client's behalf, which lets the client stop renewing it itself.
+   */
+  delegatedDelayedLeave: DelayedLeaveTimings;
 }
 
 export const defaultSessionTimings: SessionTimings = {
   syncDisconnectGracePeriodMs: 10_000,
   networkErrorRetryMs: 1_000,
   delayedLeave: { delay_ms: 18_000, restart_ms: 4_000 },
+  delegatedDelayedLeave: { delay_ms: 3_600_000, restart_ms: 300_000 },
 };
 
 /** Media quality settings a host may pass; everything is optional. */

sdk/src/media/LivekitMemberMedia.ts

@@ -11,11 +11,14 @@ import {
   roomEventSelector,
 } from "@livekit/components-core";
 import {
+  facingModeFromLocalTrack,
   type LocalParticipant,
+  LocalVideoTrack,
   type Participant,
   ParticipantEvent,
   type Room as LivekitRoom,
   RoomEvent,
+  Track,
   type TrackPublication,
 } from "livekit-client";
 import { distinctUntilChanged, filter, map, type Observable } from "rxjs";
@@ -89,6 +92,16 @@ export function createLocalLivekitMemberMedia(
   return {
     ...createLivekitMemberMedia(scope, participant, room, encryptionSystem),
     local: true,
+    switchCamera: async () => {
+      const track = participant.getTrackPublication(Track.Source.Camera)?.track;
+      if (!(track instanceof LocalVideoTrack)) return;
+      const { facingMode } = facingModeFromLocalTrack(track);
+      if (facingMode !== "user" && facingMode !== "environment") return;
+      await track.restartTrack({
+        facingMode: facingMode === "user" ? "environment" : "user",
+      });
+      return track.mediaStreamTrack.getSettings().deviceId;
+    },
   };
 }
 

sdk/src/session/ConnectionFactory.ts

@@ -23,7 +23,12 @@ import { type OpenIDClientParts, type SFUConfig } from "./openIDSFU";
 import { type Behavior } from "../reactive/Behavior";
 import { Connection, type ConnectionOpts, ConnectionState } from "./Connection";
 import { type ObservableScope } from "../reactive/ObservableScope";
-import { type LocalMediaInputs, type ResolvedTransport } from "../api";
+import {
+  type CaptureSettings,
+  type LocalMediaInputs,
+  type ResolvedTransport,
+} from "../api";
+import { type MediaQuality } from "../config";
 
 export interface ConnectionFactory {
   createConnection(
@@ -87,6 +92,8 @@ export class LivekitConnectionFactory implements ConnectionFactory {
     private readonly roomId: string,
     private readonly localMedia: LocalMediaInputs,
     private readonly keyProvider: BaseKeyProvider | undefined,
+    private readonly mediaQuality: MediaQuality | undefined,
+    private readonly capture: CaptureSettings | undefined,
   ) {}
 
   public createConnection(
@@ -104,7 +111,14 @@ export class LivekitConnectionFactory implements ConnectionFactory {
         client: this.client,
         scope,
         livekitRoomFactory: () =>
-          new LivekitRoom(roomOptions(this.localMedia, this.keyProvider)),
+          new LivekitRoom(
+            roomOptions(
+              this.localMedia,
+              this.keyProvider,
+              this.mediaQuality,
+              this.capture,
+            ),
+          ),
         ownMembershipIdentity,
       },
       logger,
@@ -115,19 +129,35 @@ export class LivekitConnectionFactory implements ConnectionFactory {
 function roomOptions(
   localMedia: LocalMediaInputs,
   keyProvider: BaseKeyProvider | undefined,
+  mediaQuality: MediaQuality | undefined,
+  capture: CaptureSettings | undefined,
 ): RoomOptions {
-  const base = buildLiveKitOptions();
+  const base = buildLiveKitOptions(mediaQuality);
+  const camera = capture?.camera;
   return {
     ...base,
     videoCaptureDefaults: {
       ...base.videoCaptureDefaults,
       deviceId: localMedia.videoInputDeviceId$.value,
       processor: localMedia.videoProcessor$.value,
+      ...(camera?.resolution && { resolution: camera.resolution }),
+    },
+    publishDefaults: {
+      ...base.publishDefaults,
+      ...(camera?.maxBitrate !== undefined && {
+        videoEncoding: {
+          maxBitrate: camera.maxBitrate,
+          maxFramerate: camera.maxFramerate,
+        },
+      }),
+      ...(camera?.codec && { videoCodec: camera.codec }),
     },
     audioCaptureDefaults: {
       ...base.audioCaptureDefaults,
       deviceId: localMedia.audioInputDeviceId$.value,
+      ...capture?.audio,
     },
+    audioOutput: { deviceId: localMedia.audioOutputDeviceId$.value },
     // Every room needs a worker of its own: one gets confused by streams from
     // several rooms
     e2ee: keyProvider && { keyProvider, worker: new E2EEWorker() },

sdk/src/session/LocalMember.ts

@@ -11,13 +11,17 @@ import {
   type Participant,
   ParticipantEvent,
   type ScreenShareCaptureOptions,
+  Track,
+  type TrackPublishOptions,
 } from "livekit-client";
+import { type MatrixClient } from "matrix-js-sdk";
 import { type Logger } from "matrix-js-sdk/lib/logger";
 import {
   type LivekitTransport,
   type MatrixRTCSession as JsSdkRTCSession,
   type Status as RTCSessionStatus,
 } from "matrix-js-sdk/lib/matrixrtc";
+import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
 import { deepCompare } from "matrix-js-sdk/lib/utils";
 import {
   BehaviorSubject,
@@ -37,17 +41,24 @@ import {
 
 import { type Behavior } from "../reactive/Behavior";
 import { type ObservableScope } from "../reactive/ObservableScope";
-import { type HomeserverConnected } from "./HomeserverConnected";
-import { type Connection, ConnectionState } from "./Connection";
-import { type IConnectionManager } from "./ConnectionManager";
+import {
+  type DelayedLeaveTimings,
+  type MatrixRTCMode,
+  type SessionTimings,
+} from "../config";
+import { type DisconnectReason, type VideoCaptureSettings } from "../api";
 import {
   FailToStartLivekitConnection,
   type MatrixRTCError,
   MembershipManagerError,
   toMatrixRTCError,
 } from "../errors";
+import { type HomeserverConnected } from "./HomeserverConnected";
+import { type Connection, ConnectionState } from "./Connection";
+import { type IConnectionManager } from "./ConnectionManager";
 import { type LocalTransport } from "./LocalTransport";
-import { type Publisher } from "./Publisher";
+import { getSFUConfigWithOpenID } from "./openIDSFU";
+import { type DesiredMedia, type Publisher } from "./Publisher";
 
 export enum TransportState {
   Waiting = "transport_waiting",
@@ -66,10 +77,7 @@ export type LocalMemberMediaState =
 export type LocalMemberState =
   | MatrixRTCError
   | TransportState.Waiting
-  | {
-      media: LocalMemberMediaState;
-      matrix: MatrixRTCError | RTCSessionStatus;
-    };
+  | { media: LocalMemberMediaState; matrix: MatrixRTCError | RTCSessionStatus };
 
 interface Props {
   scope: ObservableScope;
@@ -77,14 +85,22 @@ interface Props {
   localTransport$: Observable<LocalTransport>;
   homeserverConnected: HomeserverConnected;
   createPublisher: (connection: Connection) => Publisher;
-  joinMatrixRTC: (transport: LivekitTransport) => void;
+  joinMatrixRTC: (
+    transport: LivekitTransport,
+    delayedLeave: DelayedLeaveTimings,
+  ) => void;
   /** The membership manager giving up on keeping the membership alive. */
   membershipManagerError$: Observable<unknown>;
-  matrixRTCSession: Pick<
-    JsSdkRTCSession,
-    "updateCallIntent" | "leaveRoomSession"
-  >;
-  cameraEnabled$: Behavior<boolean>;
+  matrixRTCSession: Pick<JsSdkRTCSession, "leaveRoomSession">;
+  /** The id of the delayed leave event, for the SFU to take over. */
+  delayId$: Behavior<string | null>;
+  client: Pick<MatrixClient, "getOpenIdToken" | "getDeviceId" | "baseUrl">;
+  roomId: string;
+  ownMembershipIdentity: CallMembershipIdentityParts;
+  matrixRTCMode: MatrixRTCMode;
+  timings: SessionTimings;
+  desired: DesiredMedia;
+  screenShare?: VideoCaptureSettings;
   logger: Logger;
 }
 
@@ -99,6 +115,9 @@ export interface LocalMembership {
   connected$: Behavior<boolean>;
   /** Connected once, and currently not. */
   reconnecting$: Behavior<boolean>;
+  disconnectReason$: Behavior<DisconnectReason | null>;
+  setMicrophoneEnabled: (enabled: boolean) => Promise<boolean>;
+  setCameraEnabled: (enabled: boolean) => Promise<boolean>;
   sharingScreen$: Behavior<boolean>;
   toggleScreenSharing: (() => void) | null;
   screenShareError$: Behavior<Error | null>;
@@ -119,7 +138,14 @@ export function createLocalMembership$({
   joinMatrixRTC,
   membershipManagerError$,
   matrixRTCSession,
-  cameraEnabled$,
+  delayId$,
+  client,
+  roomId,
+  ownMembershipIdentity,
+  matrixRTCMode,
+  timings,
+  desired,
+  screenShare,
   logger: parentLogger,
 }: Props): LocalMembership {
   const logger = parentLogger.getChild("[LocalMember]");
@@ -131,8 +157,28 @@ export function createLocalMembership$({
       return NEVER;
     }),
   );
-  const transport$ = scope.behavior<LocalTransport | null>(
-    localTransport$,
+
+  // Whether the SFU can take over restarting the delayed leave, so that a
+  // client that vanishes is removed by the SFU rather than by the timeout.
+  // Either the homeserver or the transport has to support it.
+  const homeserverSupportsDelegation = checkDelegationSupport(
+    `${client.baseUrl}/_matrix/client/unstable/io.element.msc4195/rtc/livekit/delegate_delayed_leave`,
+    "homeserver",
+    logger,
+  );
+  const joinParams$ = scope.behavior(
+    localTransport$.pipe(
+      switchMap(async ({ transport }) => ({
+        transport,
+        delegationSupported:
+          (await homeserverSupportsDelegation) ||
+          (await checkDelegationSupport(
+            `${transport.livekit_service_url}/delegate_delayed_leave`,
+            `transport ${transport.livekit_service_url}`,
+            logger,
+          )),
+      })),
+    ),
     null,
   );
 
@@ -185,11 +231,16 @@ export function createLocalMembership$({
   );
 
   scope.reconcile(
-    scope.behavior(combineLatest([transport$, joinRequested$])),
-    async ([transport, shouldJoin]) => {
-      if (transport === null || !shouldJoin) return;
+    scope.behavior(combineLatest([joinParams$, joinRequested$])),
+    async ([joinParams, shouldJoin]) => {
+      if (joinParams === null || !shouldJoin) return;
       try {
-        joinMatrixRTC(transport.transport);
+        joinMatrixRTC(
+          joinParams.transport,
+          joinParams.delegationSupported
+            ? timings.delegatedDelayedLeave
+            : timings.delayedLeave,
+        );
       } catch (e) {
         logger.error("Failed to enter the session", e);
         if (matrixError$.value === null) matrixError$.next(toMatrixRTCError(e));
@@ -204,6 +255,35 @@ export function createLocalMembership$({
     },
   );
 
+  // Hand the delayed leave to the SFU. The token this issues is discarded;
+  // the request is what triggers the delegation.
+  scope.reconcile(
+    scope.behavior(combineLatest([joinParams$, delayId$])),
+    async ([joinParams, delayId]) => {
+      if (!joinParams?.delegationSupported || delayId === null) return;
+      try {
+        await getSFUConfigWithOpenID(
+          client,
+          ownMembershipIdentity,
+          joinParams.transport.livekit_service_url,
+          roomId,
+          {
+            matrixRTCMode,
+            delayEndpointBaseUrl: client.baseUrl,
+            delayId,
+            delayTimeoutMs: timings.delegatedDelayedLeave.delay_ms,
+          },
+          logger,
+        );
+      } catch (e) {
+        logger.error(
+          `Failed to delegate the leave to ${joinParams.transport.livekit_service_url}`,
+          e,
+        );
+      }
+    },
+  );
+
   membershipManagerError$.pipe(scope.bind()).subscribe((e) => {
     logger.error("The membership manager stopped", e);
     if (matrixError$.value === null)
@@ -214,16 +294,6 @@ export function createLocalMembership$({
       );
   });
 
-  cameraEnabled$.pipe(scope.bind()).subscribe((videoEnabled) => {
-    matrixRTCSession
-      .updateCallIntent(videoEnabled ? "video" : "audio")
-      .catch((e) => {
-        // Expected before the join: the intent is sent with it instead
-        if (e instanceof Error && e.message === "Not connected yet") return;
-        logger.error("Failed to update the call intent", e);
-      });
-  });
-
   const connectionState$ = connection$.pipe(
     switchMap((connection) => connection?.state$ ?? of(null)),
   );
@@ -271,15 +341,25 @@ export function createLocalMembership$({
     ),
   );
 
-  const connected$ = scope.behavior(
+  const disconnectReason$ = scope.behavior(
     combineLatest(
       [homeserverConnected.combined$, connectionState$],
-      ([homeserverConnected], connectionState) =>
-        homeserverConnected &&
-        connectionState === ConnectionState.LivekitConnected,
+      (
+        [homeserverConnected, reason],
+        connectionState,
+      ): DisconnectReason | null => {
+        if (!homeserverConnected) return reason ?? "sync";
+        if (connectionState !== ConnectionState.LivekitConnected)
+          return "media";
+        return null;
+      },
     ),
   );
 
+  const connected$ = scope.behavior(
+    disconnectReason$.pipe(map((reason) => reason === null)),
+  );
+
   const reconnecting$ = scope.behavior(
     connected$.pipe(
       pairwise(),
@@ -311,6 +391,23 @@ export function createLocalMembership$({
       }
     });
 
+  const setEnabled = async (
+    source: Track.Source.Microphone | Track.Source.Camera,
+    enabled: boolean,
+  ): Promise<boolean> => {
+    const desired$ =
+      source === Track.Source.Microphone
+        ? desired.microphone$
+        : desired.camera$;
+    desired$.next(enabled);
+    // Without a publisher the request waits for the tracks to be created
+    const publisher = publisher$.value;
+    if (publisher === null) return enabled;
+    const result = await publisher.setEnabled(source, enabled);
+    if (result !== enabled) desired$.next(result);
+    return result;
+  };
+
   const sharingScreen$ = scope.behavior(
     participant$.pipe(
       switchMap((p) => (p === null ? of(false) : observeSharingScreen$(p))),
@@ -324,7 +421,11 @@ export function createLocalMembership$({
           if (participant === null) return;
           const enable = !sharingScreen$.value;
           participant
-            .setScreenShareEnabled(enable, screenShareCaptureOptions)
+            .setScreenShareEnabled(
+              enable,
+              screenShareCaptureOptions(screenShare),
+              screenSharePublishOptions(screenShare),
+            )
             .catch((e: unknown) => {
               logger.error(
                 `Screen share ${enable ? "start" : "stop"} failed`,
@@ -349,6 +450,11 @@ export function createLocalMembership$({
     connection$,
     connected$,
     reconnecting$,
+    disconnectReason$,
+    setMicrophoneEnabled: async (enabled) =>
+      setEnabled(Track.Source.Microphone, enabled),
+    setCameraEnabled: async (enabled) =>
+      setEnabled(Track.Source.Camera, enabled),
     sharingScreen$,
     toggleScreenSharing,
     screenShareError$,
@@ -356,18 +462,64 @@ export function createLocalMembership$({
   };
 }
 
-const screenShareCaptureOptions: ScreenShareCaptureOptions = {
-  // No echo cancellation: it would cancel the other members' voices out of
-  // the shared audio
-  audio: {
-    autoGainControl: false,
-    noiseSuppression: false,
-    voiceIsolation: false,
-  },
-  selfBrowserSurface: "include",
-  surfaceSwitching: "include",
-  systemAudio: "include",
-};
+/**
+ * Whether an endpoint exists, by hitting it without credentials and reading
+ * the status. Not retried: many servers predate the endpoint altogether and
+ * answer with a CORS failure that a retry loop would only repeat.
+ */
+async function checkDelegationSupport(
+  endpointUrl: string,
+  serviceName: string,
+  logger: Logger,
+): Promise<boolean> {
+  try {
+    const res = await fetch(endpointUrl, { method: "POST" });
+    const supported = res.status !== 404;
+    logger.info(
+      `${serviceName} ${supported ? "supports" : "does not support"} delegation`,
+    );
+    return supported;
+  } catch (e) {
+    logger.warn(
+      `Failed to determine whether ${serviceName} supports delegation, assuming no support`,
+      e,
+    );
+    return false;
+  }
+}
+
+function screenShareCaptureOptions(
+  settings: VideoCaptureSettings | undefined,
+): ScreenShareCaptureOptions {
+  return {
+    // No echo cancellation: it would cancel the other members' voices out of
+    // the shared audio
+    audio: {
+      autoGainControl: false,
+      noiseSuppression: false,
+      voiceIsolation: false,
+    },
+    selfBrowserSurface: "include",
+    surfaceSwitching: "include",
+    systemAudio: "include",
+    ...(settings?.resolution && { resolution: settings.resolution }),
+  };
+}
+
+function screenSharePublishOptions(
+  settings: VideoCaptureSettings | undefined,
+): TrackPublishOptions | undefined {
+  if (settings === undefined) return undefined;
+  return {
+    ...(settings.maxBitrate !== undefined && {
+      screenShareEncoding: {
+        maxBitrate: settings.maxBitrate,
+        maxFramerate: settings.maxFramerate,
+      },
+    }),
+    ...(settings.codec && { videoCodec: settings.codec }),
+  };
+}
 
 function observeSharingScreen$(participant: Participant): Observable<boolean> {
   return observeParticipantEvents(

sdk/src/session/LocalTransport.ts

@@ -38,26 +38,34 @@ interface Props {
   ownMembershipIdentity: CallMembershipIdentityParts;
   roomId: string;
   matrixRTCMode: MatrixRTCMode;
+  /** Skip discovery and use this transport. */
+  transportUrl?: string;
+  /** Use this transport when the homeserver advertises none. */
+  fallbackTransportUrl?: string;
   logger: Logger;
 }
 
 /**
- * The transport the local member publishes on: the homeserver's preferred
- * one, authenticated with so that the session can be joined with a token in
- * hand. Only the homeserver is asked; the SDK has no configuration of its own
- * to fall back on.
+ * The transport the local member publishes on: the one the host names, else
+ * the homeserver's preferred one, else the host's fallback; authenticated
+ * with, so that the session can be joined with a token in hand.
  */
 export async function getLocalTransport({
   client,
   ownMembershipIdentity,
   roomId,
   matrixRTCMode,
+  transportUrl,
+  fallbackTransportUrl,
   logger,
 }: Props): Promise<LocalTransport> {
-  const transport = await new RtcTransportAutoDiscovery({
-    client,
-    logger,
-  }).discoverPreferredTransport();
+  const transport: LivekitTransport | null = transportUrl
+    ? { type: "livekit", livekit_service_url: transportUrl }
+    : await new RtcTransportAutoDiscovery({
+        client,
+        fallbackTransportUrl,
+        logger,
+      }).discoverPreferredTransport();
   if (transport === null)
     throw new MatrixRTCTransportMissingError(client.getDomain() ?? "");
   const sfuConfig = await getSFUConfigWithOpenID(

sdk/src/session/MatrixRTCClient.ts

@@ -10,46 +10,55 @@ import { type Logger, logger as rootLogger } from "matrix-js-sdk/lib/logger";
 import {
   MatrixRTCSessionEvent,
   MatrixRTCSessionManager,
+  MembershipManagerEvent,
 } from "matrix-js-sdk/lib/matrixrtc";
+import { type IMembershipManager } from "matrix-js-sdk/lib/matrixrtc/IMembershipManager";
 import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
 import { v4 as uuidv4 } from "uuid";
-import { combineLatest, from, fromEvent, map } from "rxjs";
-
-import { MatrixRTCMode } from "../config";
-import { type ObservableScope } from "../reactive/ObservableScope";
 import {
-  createKeyRotationSuppressed$,
-  createMemberships$,
-  membershipsAndTransports$,
-} from "./SessionBehaviors";
+  BehaviorSubject,
+  combineLatest,
+  from,
+  fromEvent,
+  map,
+  type Observable,
+} from "rxjs";
+
+import { defaultSessionTimings, MatrixRTCMode } from "../config";
+import { type ObservableScope } from "../reactive/ObservableScope";
+import { filterBehavior, generateItems } from "../reactive/observable";
+import { mapScoped } from "../utils/mapScoped";
+import {
+  type LocalMediaInputs,
+  type LocalRTCMember,
+  type MatrixRTCClient,
+  type MatrixRTCClientOptions,
+  type RemoteRTCMember,
+} from "../api";
+import { MatrixRTCError } from "../errors";
+import { LivekitConnectionFactory } from "./ConnectionFactory";
 import { createHomeserverConnected$ } from "./HomeserverConnected";
-import { createConnectionManager$ } from "./ConnectionManager";
+import { joinJsSdkSession } from "./joinJsSdkSession";
+import { createKeyProvider } from "./KeyProvider";
+import { createLocalMembership$ } from "./LocalMember";
+import { getLocalTransport } from "./LocalTransport";
 import { createRemoteMatrixLivekitMembers$ } from "./MatrixLivekitMembers";
 import {
   createMatrixMemberMetadata$,
   createRoomMembers$,
 } from "./MatrixMemberMetadata";
-import { filterBehavior, generateItems } from "../reactive/observable";
-import {
-  type LocalMediaInputs,
-  type LocalRTCMember,
-  type RemoteRTCMember,
-  type MatrixRTCClient,
-  MatrixRTCError,
-  type MatrixRTCClientOptions,
-} from "../api";
-import { LivekitConnectionFactory } from "./ConnectionFactory";
-import { joinJsSdkSession, sessionTimings } from "./joinJsSdkSession";
-import { createKeyProvider } from "./KeyProvider";
-import { createLocalMembership$ } from "./LocalMember";
-import { getLocalTransport } from "./LocalTransport";
 import {
   createLocalRTCMember,
   createRemoteRTCMember,
   membershipKeys,
 } from "./Members";
-import { Publisher } from "./Publisher";
-import { mapScoped } from "../utils/mapScoped";
+import { createConnectionManager$ } from "./ConnectionManager";
+import { type DesiredMedia, Publisher } from "./Publisher";
+import {
+  createKeyRotationSuppressed$,
+  createMemberships$,
+  membershipsAndTransports$,
+} from "./SessionBehaviors";
 import { fatalError, sessionStatus } from "./status";
 import { createTransportRegistry } from "./Transports";
 
@@ -73,6 +82,7 @@ export function createMatrixRTCClient(
   if (!(userId && deviceId))
     throw new MatrixRTCError("The client has to be logged in");
   const { encryptionSystem, matrixRTCMode } = options;
+  const timings = { ...defaultSessionTimings, ...options.timings };
 
   const jsSdkSession = sessionManager(
     scope,
@@ -106,10 +116,17 @@ export function createMatrixRTCClient(
       ownMembershipIdentity,
       roomId: room.roomId,
       matrixRTCMode,
+      transportUrl: options.transportUrl,
+      fallbackTransportUrl: options.fallbackTransportUrl,
       logger,
     }),
   );
 
+  const desired: DesiredMedia = {
+    microphone$: new BehaviorSubject(localMedia.microphoneEnabled),
+    camera$: new BehaviorSubject(localMedia.cameraEnabled),
+  };
+
   const connectionManager = createConnectionManager$({
     scope,
     connectionFactory: new LivekitConnectionFactory(
@@ -117,6 +134,8 @@ export function createMatrixRTCClient(
       room.roomId,
       localMedia,
       keyProvider,
+      options.mediaQuality,
+      options.capture,
     ),
     localTransport$,
     remoteTransports$: transports$,
@@ -133,29 +152,48 @@ export function createMatrixRTCClient(
       scope,
       client,
       jsSdkSession,
-      sessionTimings.syncDisconnectGracePeriodMs,
+      timings.syncDisconnectGracePeriodMs,
     ),
     createPublisher: (connection) =>
       new Publisher(
         connection.livekitRoom,
         localMedia,
+        desired,
         logger.getChild(
           `[Publisher ${connection.transport.livekit_service_url}]`,
         ),
       ),
-    joinMatrixRTC: (transport) =>
+    joinMatrixRTC: (transport, delayedLeave) =>
       joinJsSdkSession(jsSdkSession, ownMembershipIdentity, transport, {
         encryptMedia: keyProvider !== undefined,
         matrixRTCMode,
         sendNotificationType: options.sendNotificationType,
         applicationData: options.applicationData,
+        timings,
+        delayedLeave,
       }),
     membershipManagerError$: fromEvent(
       jsSdkSession,
       MatrixRTCSessionEvent.MembershipManagerError,
     ),
     matrixRTCSession: jsSdkSession,
-    cameraEnabled$: localMedia.cameraEnabled$,
+    delayId$: scope.behavior(
+      (
+        fromEvent(
+          jsSdkSession,
+          MembershipManagerEvent.DelayIdChanged,
+          // The re-emitted event carries the original emitter as the second argument
+        ) as Observable<[string | undefined, IMembershipManager]>
+      ).pipe(map(([delayId]) => delayId ?? null)),
+      jsSdkSession.delayId ?? null,
+    ),
+    client,
+    roomId: room.roomId,
+    ownMembershipIdentity,
+    matrixRTCMode,
+    timings,
+    desired,
+    screenShare: options.capture?.screenShare,
     logger,
   });
 
@@ -228,6 +266,9 @@ export function createMatrixRTCClient(
     status$,
     connected$: localMembership.connected$,
     reconnecting$: localMembership.reconnecting$,
+    disconnectReason$: localMembership.disconnectReason$,
+    setMicrophoneEnabled: localMembership.setMicrophoneEnabled,
+    setCameraEnabled: localMembership.setCameraEnabled,
     fatalError$: scope.behavior(localMembership.state$.pipe(map(fatalError))),
     localMember$,
     remoteMembers$,

sdk/src/session/Publisher.ts

@@ -15,14 +15,26 @@ import {
   Track,
 } from "livekit-client";
 import { type Logger } from "matrix-js-sdk/lib/logger";
-import { combineLatest, distinctUntilChanged, map, skip } from "rxjs";
+import {
+  type BehaviorSubject,
+  combineLatest,
+  distinctUntilChanged,
+  map,
+} from "rxjs";
 
 import { type Behavior } from "../reactive/Behavior";
 import { ObservableScope } from "../reactive/ObservableScope";
 import { type LocalMediaInputs } from "../api";
 
+/** What the host last asked for, kept across publishers. */
+export interface DesiredMedia {
+  microphone$: BehaviorSubject<boolean>;
+  camera$: BehaviorSubject<boolean>;
+}
+
 /**
- * Publishes the local media on one LiveKit room, following `LocalMediaInputs`.
+ * Publishes the local media on one LiveKit room, following the host's
+ * requests.
  *
  * LiveKit publishes a track the moment it is created, but a member must not be
  * heard before it has joined the MatrixRTC session, nor after it has left. The
@@ -38,6 +50,7 @@ export class Publisher {
   public constructor(
     room: LivekitRoom,
     private readonly inputs: LocalMediaInputs,
+    private readonly desired: DesiredMedia,
     private readonly logger: Logger,
   ) {
     this.room = room;
@@ -67,24 +80,47 @@ export class Publisher {
   }
 
   /**
-   * Creates the microphone and camera tracks the inputs ask for, and keeps
-   * them in step with the inputs from then on. Both are enabled in one call so
-   * that the browser asks for permission once. Safe to call more than once.
+   * Creates the microphone and camera tracks the host asked for. Both are
+   * enabled in one call so that the browser asks for permission once. Safe to
+   * call more than once.
    */
   public createAndSetupTracks(): void {
     if (this.tracksRequested) return;
     this.tracksRequested = true;
     const participant = this.room.localParticipant;
-    const audio = this.inputs.microphoneEnabled$.value;
-    const video = this.inputs.cameraEnabled$.value;
+    const audio = this.desired.microphone$.value;
+    const video = this.desired.camera$.value;
     // LiveKit resolves these once the track is published, which may block on
     // the connection; LocalTrackPublished is what tells us a track exists.
     if (audio && video) void participant.enableCameraAndMicrophone();
     else if (audio) void participant.setMicrophoneEnabled(true);
     else if (video) void participant.setCameraEnabled(true);
+  }
 
-    this.follow(this.inputs.microphoneEnabled$, Track.Source.Microphone);
-    this.follow(this.inputs.cameraEnabled$, Track.Source.Camera);
+  /**
+   * Enables or disables a source and reports what resulted: the request where
+   * it succeeded, the state LiveKit is left in where it did not (a denied
+   * permission, a missing device).
+   */
+  public async setEnabled(
+    source: Track.Source.Microphone | Track.Source.Camera,
+    enabled: boolean,
+  ): Promise<boolean> {
+    const participant = this.room.localParticipant;
+    try {
+      if (source === Track.Source.Microphone)
+        await participant.setMicrophoneEnabled(enabled);
+      else await participant.setCameraEnabled(enabled);
+      // Unmuting restarts the upstream; until the member has joined, it has
+      // to stay paused
+      if (enabled && !this.shouldPublish) await this.pauseUpstreams([source]);
+      return enabled;
+    } catch (e) {
+      this.logger.error(`Failed to set ${source} enabled=${enabled}`, e);
+      return source === Track.Source.Microphone
+        ? participant.isMicrophoneEnabled
+        : participant.isCameraEnabled;
+    }
   }
 
   public async startPublishing(): Promise<void> {
@@ -122,40 +158,18 @@ export class Publisher {
       this.pauseUpstreams([publication.source]).catch((e) => {
         this.logger.error("Failed to pause the upstream", e);
       });
-    // The input may have changed while the track was being created
-    const enabled =
+    // The host may have changed its mind while the track was being created
+    const desired =
       publication.source === Track.Source.Microphone
-        ? this.inputs.microphoneEnabled$.value
+        ? this.desired.microphone$.value
         : publication.source === Track.Source.Camera
-          ? this.inputs.cameraEnabled$.value
+          ? this.desired.camera$.value
           : undefined;
-    if (enabled === false) this.setEnabled(publication.source, false);
-  }
-
-  private follow(
-    enabled$: Behavior<boolean>,
-    source: Track.Source.Microphone | Track.Source.Camera,
-  ): void {
-    enabled$
-      .pipe(skip(1), distinctUntilChanged(), this.scope.bind())
-      .subscribe((enabled) => this.setEnabled(source, enabled));
-  }
-
-  private setEnabled(source: Track.Source, enabled: boolean): void {
-    const participant = this.room.localParticipant;
-    const toggle =
-      source === Track.Source.Microphone
-        ? participant.setMicrophoneEnabled(enabled)
-        : participant.setCameraEnabled(enabled);
-    toggle
-      .then(async () => {
-        // Unmuting restarts the upstream; until the member has joined, it
-        // has to stay paused
-        if (enabled && !this.shouldPublish) await this.pauseUpstreams([source]);
-      })
-      .catch((e) => {
-        this.logger.error(`Failed to set ${source} enabled=${enabled}`, e);
-      });
+    if (desired === false)
+      void this.setEnabled(
+        publication.source as Track.Source.Microphone | Track.Source.Camera,
+        false,
+      );
   }
 
   private async pauseUpstreams(sources: Track.Source[]): Promise<void> {
@@ -192,6 +206,7 @@ export class Publisher {
       });
     };
     sync("audioinput", this.inputs.audioInputDeviceId$);
+    sync("audiooutput", this.inputs.audioOutputDeviceId$);
     sync("videoinput", this.inputs.videoInputDeviceId$);
   }
 

sdk/src/session/joinJsSdkSession.ts

@@ -13,18 +13,22 @@ import {
 } from "matrix-js-sdk/lib/matrixrtc";
 import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
 
-import { defaultSessionTimings, MatrixRTCMode } from "../config";
+import {
+  type DelayedLeaveTimings,
+  MatrixRTCMode,
+  type SessionTimings,
+} from "../config";
 
 interface Options {
   encryptMedia: boolean;
   matrixRTCMode: MatrixRTCMode;
   sendNotificationType?: RTCNotificationType;
   applicationData?: Record<string, unknown>;
+  timings: SessionTimings;
+  /** The delayed leave timings to use: delegated to the SFU or not. */
+  delayedLeave: DelayedLeaveTimings;
 }
 
-/** The session's timings; a later slice makes them an input. */
-export const sessionTimings = defaultSessionTimings;
-
 /**
  * Sends the membership and starts the membership manager, which keeps it
  * alive and retries on failure.
@@ -39,33 +43,34 @@ export function joinJsSdkSession(
     matrixRTCMode,
     sendNotificationType,
     applicationData,
+    timings,
+    delayedLeave,
   }: Options,
 ): void {
   // The one piece of application data the js-sdk can put in a membership
   const intent = applicationData?.["m.call.intent"];
   const callIntent =
     typeof intent === "string" ? (intent as RTCCallIntent) : undefined;
-  const timings = sessionTimings.delayedLeave;
   // Give up on the network as soon as either the sync has been down for the
   // grace period or the delayed leave has probably been sent, whichever is
   // sooner
   const maxWaitMs = Math.min(
-    sessionTimings.syncDisconnectGracePeriodMs,
-    timings.delay_ms,
+    timings.syncDisconnectGracePeriodMs,
+    delayedLeave.delay_ms,
   );
   session.joinRTCSession(ownMembershipIdentity, [transport], {
     notificationType: sendNotificationType,
     callIntent,
     manageMediaKeys: encryptMedia,
-    delayedLeaveEventRestartMs: timings.restart_ms,
-    delayedLeaveEventDelayMs: timings.delay_ms,
-    delayedLeaveEventRestartLocalTimeoutMs: timings.restart_timeout_ms,
-    networkErrorRetryMs: sessionTimings.networkErrorRetryMs,
-    makeKeyDelay: sessionTimings.waitForKeyRotationMs,
-    membershipEventExpiryMs: sessionTimings.membershipEventExpiryMs,
-    keyRotationParticipantLimit: sessionTimings.keyRotationParticipantLimit,
+    delayedLeaveEventRestartMs: delayedLeave.restart_ms,
+    delayedLeaveEventDelayMs: delayedLeave.delay_ms,
+    delayedLeaveEventRestartLocalTimeoutMs: delayedLeave.restart_timeout_ms,
+    networkErrorRetryMs: timings.networkErrorRetryMs,
+    makeKeyDelay: timings.waitForKeyRotationMs,
+    membershipEventExpiryMs: timings.membershipEventExpiryMs,
+    keyRotationParticipantLimit: timings.keyRotationParticipantLimit,
     unstableSendStickyEvents: matrixRTCMode === MatrixRTCMode.Matrix_2_0,
     maximumNetworkErrorRetryCount:
-      Math.ceil(maxWaitMs / sessionTimings.networkErrorRetryMs) + 1,
+      Math.ceil(maxWaitMs / timings.networkErrorRetryMs) + 1,
   });
 }