Merge LocalMediaInputs into MatrixRTCClientOptions

The SDK now takes `publish` requests and audio output control directly in the main options object, removing the separate `LocalMediaInputs` type. Audio output device selection moves from a reactive behavior to an imperative method on `MatrixRTCClient`, giving hosts direct control over when devices switch.
2026-10-16 23:45:50 +00:00 · 2026-10-08 13:53:57 +02:00
parent 5b13e1da81
commit f1ea4cc5a2
10 changed files with 88 additions and 88 deletions
@@ -19,7 +19,6 @@ Call's own `CallViewModel` is built on it. It depends on nothing in Element Call

 ```ts
 import {
-  constant,
  createMatrixRTCClient,
  E2eeType,
  MatrixRTCMode,
@@ -31,13 +30,10 @@ const rtcClient = createMatrixRTCClient(
  scope,
  client, // a matrix-js-sdk MatrixClient, logged in and syncing
  room, // the matrix-js-sdk Room, from client.getRoom() once the join has synced
-  {
-    publish: [{ source: "microphone" }, { source: "camera" }],
-    audioOutputDeviceId$: constant(undefined),
-  },
  {
    encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
    matrixRTCMode: MatrixRTCMode.Compatibility,
+    publish: [{ source: "microphone" }, { source: "camera" }],
  },
 );
 rtcClient.join();
@@ -18,7 +18,7 @@ therefore `Element Call → sdk`, never the reverse. The migration from today's
 ```
 host application (Element Call, a whiteboard, watch-together, …)
   │
-   │  createMatrixRTCClient(scope, matrixClient, room, localMedia, options)
+   │  createMatrixRTCClient(scope, matrixClient, room, options)
   ▼
 ┌──────────────────────────────────────────────────────────────────────────────┐
 │ MatrixRTCClient                                                              │
@@ -135,7 +135,7 @@ it, and `leave()` also sends the leave.
 | media             | a `MemberMedia`: what a member sends, once it has arrived on its transport. `null` while waiting for media                                                                                            | Element Call's `MediaViewModel` is a view of it                   |
 | track             | a `MediaTrack`: one publication of a member, audio or video, with `source`, `kind`, `id` and the behaviors on it. `LocalAudioMediaTrack` and `LocalVideoMediaTrack` add the controls over our own     | LiveKit `TrackPublication` and `Track`, which never leave the SDK |
 | source            | a `MediaSource`: what a track is captured from, `microphone`, `camera`, `screenShare`, `screenShareAudio` or `unknown`. Picked out of `tracks$` with `trackBySource$`                                 | LiveKit `Track.Source`                                            |
-| publish request   | a `PublishRequest`: a source, where to capture it from and how to encode it. Given at the join in `LocalMediaInputs.publish`, or later to `publish` on the local member                               |                                                                   |
+| publish request   | a `PublishRequest`: a source, where to capture it from and how to encode it. Given at the join in `options.publish`, or later to `publish` on the local member                                        |                                                                   |
 | active            | `isActive$` on an audio track: the track carries sound, as the backend measures it. "Speaking" is a call's reading of it on the microphone track                                                      | LiveKit "speaking", `isSpeaking`                                  |
 | attach, detach    | handing a `<video>` or `<audio>` element to a track and taking it back. The SDK owns the stream and the observers on the element                                                                      | LiveKit `Track.attach`                                            |
 | transport         | where media is exchanged: a LiveKit service url today, described by `TransportMetadata`                                                                                                               | MSC4143 "focus"                                                   |
@@ -182,7 +182,7 @@ context, it is in the view model.

 ### Creating one

-```ts
+````ts
 import { type MatrixClient, type Room } from "matrix-js-sdk";
 import { type RTCNotificationType } from "matrix-js-sdk/lib/matrixrtc";

@@ -190,6 +190,8 @@ export interface MatrixRTCClientOptions {
  encryptionSystem: EncryptionSystem;
  /** Resolved by the host; the SDK reads neither config.json nor settings. */
  matrixRTCMode: MatrixRTCMode;
+  /** Published at the join; `publish` on the local member adds to it from then on. */
+  publish: PublishRequest[];
  /**
   * MSC4075 notification sent with the join. A parameter of the MatrixRTC
   * join itself, so it is here even though it is named after calls; reacting
@@ -236,18 +238,12 @@ export type PublishRequest =
      audio?: boolean;
      capture?: VideoCaptureSettings;
    };
+```

-/**
- * Device enumeration, permission prompts and the lobby preview stay with the
- * host.
- */
-export interface LocalMediaInputs {
-  /** Published at the join; `publish` on the local member adds to it from then on. */
-  publish: PublishRequest[];
-  /** Undefined where the host routes audio itself, or to leave the browser's choice. */
-  audioOutputDeviceId$: Behavior<string | undefined>;
-}
+Device enumeration, permission prompts and the lobby preview stay with the
+host: it hands over publish requests naming the devices it chose.

+```ts
 /**
 * Takes the whole MatrixClient, not a narrow pick, and finds the MatrixRTC
 * session itself. The js-sdk is the MatrixRTC implementation today; when the
@@ -263,10 +259,9 @@ export function createMatrixRTCClient(
  scope: ObservableScope,
  client: MatrixClient,
  room: Room,
-  localMedia: LocalMediaInputs,
  options: MatrixRTCClientOptions,
 ): MatrixRTCClient;
-```
+````

 ### The client

@@ -328,6 +323,13 @@ export interface MatrixRTCClient {
   */
  connectedTransports$: Behavior<TransportMetadata[]>;

+  /**
+   * Plays the members' audio on this device from now on. Rejects where the
+   * browser cannot switch to it. A host that routes audio itself, or leaves
+   * the browser's choice, never calls it.
+   */
+  setAudioOutputDeviceId(deviceId: string): Promise<void>;
+
  /**
   * Sends a short text to every member on the local transport, over a
   * reliable data channel beside the media. A packet holds about 15 KiB;
@@ -15,7 +15,6 @@ import { logger } from "matrix-js-sdk/lib/logger";
 import { combineLatest, type Observable, of, switchMap } from "rxjs";
 import {
  type AudioMediaTrack,
-  constant,
  createMatrixRTCClient,
  E2eeType,
  MatrixRTCMode,
@@ -70,19 +69,11 @@ async function start(
    const room = await joinRoom(client, roomIdOrAlias);

    const scope = new ObservableScope();
-    const rtcClient = createMatrixRTCClient(
-      scope,
-      client,
-      room,
-      {
-        publish: [{ source: "microphone" }, { source: "camera" }],
-        audioOutputDeviceId$: constant(undefined),
-      },
-      {
+    const rtcClient = createMatrixRTCClient(scope, client, room, {
      encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
      matrixRTCMode: MatrixRTCMode.Compatibility,
-      },
-    );
+      publish: [{ source: "microphone" }, { source: "camera" }],
+    });
    rtcClient.status$.pipe(scope.bind()).subscribe((s) => {
      status.textContent = s;
    });
@@ -25,7 +25,6 @@ import { filterBehavior, generateItems } from "./reactive/observable";
 import { mapScoped } from "./utils/mapScoped";
 import {
  type DataMessage,
-  type LocalMediaInputs,
  type LocalRTCMember,
  type MatrixRTCClient,
  type MatrixRTCClientOptions,
@@ -72,7 +71,6 @@ export function createMatrixRTCClient(
  scope: ObservableScope,
  client: MatrixClient,
  room: Room,
-  localMedia: LocalMediaInputs,
  options: MatrixRTCClientOptions,
 ): MatrixRTCClient {
  const logger = rootLogger.getChild("[MatrixRTCClient]");
@@ -105,7 +103,7 @@ export function createMatrixRTCClient(
  const backend = createBackend(scope, client, options, {
    roomId: room.roomId,
    ownMembershipIdentity,
-    localMedia,
+    publish: options.publish,
    encryptionSystem,
    mediaKeys$: mediaKeys$(jsSdkSession),
    timings,
@@ -261,6 +259,8 @@ export function createMatrixRTCClient(
    ),
    keyRotationSuppressed$: createKeyRotationSuppressed$(scope, jsSdkSession),
    connectedTransports$: transports.connected$,
+    setAudioOutputDeviceId: async (deviceId) =>
+      backend.setAudioOutputDeviceId(deviceId),
    sendData: async (topic, text) => backend.sendData(topic, text),
    data$,
  };
@@ -32,12 +32,14 @@ import {
 } from "./config";

 // ---------------------------------------------------------------------------
-// Session
+// MatrixRTCClient

 export interface MatrixRTCClientOptions {
  encryptionSystem: EncryptionSystem;
  /** Resolved by the host; the SDK reads neither config.json nor settings. */
  matrixRTCMode: MatrixRTCMode;
+  /** Published at the join; `publish` on the local member adds to it from then on. */
+  publish: PublishRequest[];
  /**
   * MSC4075 notification sent with the join. A parameter of the MatrixRTC
   * join itself, so it is here even though it is named after calls; reacting
@@ -104,13 +106,6 @@ export type PublishRequest =
      capture?: VideoCaptureSettings;
    };

-export interface LocalMediaInputs {
-  /** Published at the join; `publish` on the local member adds to it from then on. */
-  publish: PublishRequest[];
-  /** Undefined where the host routes audio itself, or to leave the browser's choice. */
-  audioOutputDeviceId$: Behavior<string | undefined>;
-}
-
 export type ConnectionStatus =
  | "waitingForTransport"
  | "connecting"
@@ -158,6 +153,13 @@ export interface MatrixRTCClient {
  /** Transports the session currently holds a live connection to. */
  connectedTransports$: Behavior<TransportMetadata[]>;

+  /**
+   * Plays the members' audio on this device from now on. Rejects where the
+   * browser cannot switch to it. A host that routes audio itself, or leaves
+   * the browser's choice, never calls it.
+   */
+  setAudioOutputDeviceId(deviceId: string): Promise<void>;
+
  /**
   * Sends a short text to every member on the local transport, over a
   * reliable data channel beside the media. A packet holds about 15 KiB;
@@ -19,7 +19,6 @@ import { type Observable } from "rxjs";

 import {
  type LocalAudioMediaTrack,
-  type LocalMediaInputs,
  type LocalMemberMedia,
  type LocalVideoMediaTrack,
  type MemberMedia,
@@ -55,7 +54,8 @@ export interface MediaKey {
 export interface MediaBackendContext {
  roomId: string;
  ownMembershipIdentity: CallMembershipIdentityParts;
-  localMedia: LocalMediaInputs;
+  /** What the local member publishes at the join. */
+  publish: PublishRequest[];
  encryptionSystem: EncryptionSystem;
  /**
   * Every media key the session knows: the current ones replayed on
@@ -147,6 +147,9 @@ export interface MediaBackend {
  /** Every connection the backend holds. For debugging and devtool purposes.*/
  readonly connections$: Behavior<BackendConnection[]>;

+  /** Plays the members' audio on this device, on the connections of now and later. */
+  setAudioOutputDeviceId(deviceId: string): Promise<void>;
+
  /**
   * A text channel beside the media: sent on the local transport, received on every one.
   * This could well be just another track kind. Not needed for element call. just for the sdk target.
@@ -23,7 +23,7 @@ import { type OpenIDClientParts, type SFUConfig } from "./openIDSFU";
 import { type Behavior } from "../../reactive/Behavior";
 import { Connection, type ConnectionOpts, ConnectionState } from "./Connection";
 import { type ObservableScope } from "../../reactive/ObservableScope";
-import { type LocalMediaInputs, type ResolvedTransport } from "../../api";
+import { type PublishRequest, type ResolvedTransport } from "../../api";
 import { type MediaQuality } from "../../config";
 import {
  audioCaptureOptions,
@@ -95,7 +95,8 @@ export class LivekitConnectionFactory implements ConnectionFactory {
  public constructor(
    private readonly client: OpenIDClientParts,
    private readonly roomId: string,
-    private readonly localMedia: LocalMediaInputs,
+    private readonly publish: PublishRequest[],
+    private readonly audioOutputDeviceId$: Behavior<string | undefined>,
    private readonly keyProvider: BaseKeyProvider | undefined,
    private readonly mediaQuality: MediaQuality | undefined,
  ) {}
@@ -116,7 +117,12 @@ export class LivekitConnectionFactory implements ConnectionFactory {
        scope,
        livekitRoomFactory: () =>
          new LivekitRoom(
-            roomOptions(this.localMedia, this.keyProvider, this.mediaQuality),
+            roomOptions(
+              this.publish,
+              this.audioOutputDeviceId$.value,
+              this.keyProvider,
+              this.mediaQuality,
+            ),
          ),
        ownMembershipIdentity,
      },
@@ -127,13 +133,14 @@ export class LivekitConnectionFactory implements ConnectionFactory {

 /** The initial requests become the room's defaults, so that one permission prompt covers both. */
 function roomOptions(
-  localMedia: LocalMediaInputs,
+  publish: PublishRequest[],
+  audioOutputDeviceId: string | undefined,
  keyProvider: BaseKeyProvider | undefined,
  mediaQuality: MediaQuality | undefined,
 ): RoomOptions {
  const base = buildLiveKitOptions(mediaQuality);
-  const microphone = initialRequest(localMedia, "microphone");
-  const camera = initialRequest(localMedia, "camera");
+  const microphone = initialRequest(publish, "microphone");
+  const camera = initialRequest(publish, "camera");
  return {
    ...base,
    videoCaptureDefaults: {
@@ -148,7 +155,7 @@ function roomOptions(
      ...base.audioCaptureDefaults,
      ...(microphone && audioCaptureOptions(microphone)),
    },
-    audioOutput: { deviceId: localMedia.audioOutputDeviceId$.value },
+    audioOutput: { deviceId: audioOutputDeviceId },
    // Every room needs a worker of its own: one gets confused by streams from
    // several rooms
    e2ee: keyProvider && { keyProvider, worker: new E2EEWorker() },
@@ -156,10 +163,10 @@ function roomOptions(
 }

 function initialRequest<S extends "microphone" | "camera">(
-  localMedia: LocalMediaInputs,
+  publish: PublishRequest[],
  source: S,
 ): PublishRequestFor<S> | undefined {
-  return localMedia.publish.find(
+  return publish.find(
    (request): request is PublishRequestFor<S> => request.source === source,
  );
 }
@@ -77,7 +77,7 @@ export function createLivekitBackend(
  {
    roomId,
    ownMembershipIdentity,
-    localMedia,
+    publish,
    encryptionSystem,
    mediaKeys$,
    timings,
@@ -89,13 +89,17 @@ export function createLivekitBackend(
  const keyProvider = createKeyProvider(encryptionSystem, mediaKeys$, logger);
  const transports = createTransportRegistry(scope);
  const localTransport$ = new ReplaySubject<LocalTransport>(1);
+  const audioOutputDeviceId$ = new BehaviorSubject<string | undefined>(
+    undefined,
+  );

  const connectionManager = createConnectionManager$({
    scope,
    connectionFactory: new LivekitConnectionFactory(
      client,
      roomId,
-      localMedia,
+      publish,
+      audioOutputDeviceId$,
      keyProvider,
      mediaQuality,
    ),
@@ -215,7 +219,8 @@ export function createLivekitBackend(
    scope,
    connectionManager,
    localTransport$,
-    localMedia,
+    publish,
+    audioOutputDeviceId$,
    encryptionSystem,
    logger,
  });
@@ -233,6 +238,7 @@ export function createLivekitBackend(
    local: localMedia_,
    mediaFor$,
    connections$,
+    setAudioOutputDeviceId: localMedia_.setAudioOutputDeviceId,
    sendData,
    data$,
  };
@@ -28,7 +28,6 @@ import {

 import {
  type LocalAudioMediaTrack,
-  type LocalMediaInputs,
  type LocalMemberMedia,
  type LocalVideoMediaTrack,
  type MediaSource,
@@ -54,7 +53,9 @@ interface Props {
  scope: ObservableScope;
  connectionManager: IConnectionManager;
  localTransport$: Observable<LocalTransport>;
-  localMedia: LocalMediaInputs;
+  publish: PublishRequest[];
+  /** Read by the connection factory for the rooms it creates from now on. */
+  audioOutputDeviceId$: BehaviorSubject<string | undefined>;
  encryptionSystem: EncryptionSystem;
  logger: Logger;
 }
@@ -62,6 +63,7 @@ interface Props {
 export interface LivekitLocalMedia extends LocalMediaBackend {
  /** The connection the local member publishes on; the data channel sends on it. */
  connection$: Behavior<Connection | null>;
+  setAudioOutputDeviceId(deviceId: string): Promise<void>;
 }

 /**
@@ -73,7 +75,8 @@ export function createLivekitLocalMedia({
  scope,
  connectionManager,
  localTransport$,
-  localMedia,
+  publish: initialRequests,
+  audioOutputDeviceId$,
  encryptionSystem,
  logger: parentLogger,
 }: Props): LivekitLocalMedia {
@@ -102,7 +105,7 @@ export function createLivekitLocalMedia({
  );

  const desired: DesiredMedia = new Map(
-    localMedia.publish.map((request) => [
+    initialRequests.map((request) => [
      request.source,
      { request, enabled: true },
    ]),
@@ -115,7 +118,6 @@ export function createLivekitLocalMedia({
    if (connection === null) return;
    const publisher = new Publisher(
      connection.livekitRoom,
-      localMedia,
      desired,
      logger.getChild(
        `[Publisher ${connection.transport.livekit_service_url}]`,
@@ -209,6 +211,11 @@ export function createLivekitLocalMedia({
    await publisher$.value?.unpublish(source);
  };

+  const setAudioOutputDeviceId = async (deviceId: string): Promise<void> => {
+    audioOutputDeviceId$.next(deviceId);
+    await publisher$.value?.setAudioOutputDevice(deviceId);
+  };
+
  return {
    connection$,
    connectionState$,
@@ -217,6 +224,7 @@ export function createLivekitLocalMedia({
    publishError$,
    publish,
    unpublish,
+    setAudioOutputDeviceId,
  };
 }

@@ -14,12 +14,7 @@ import {
 } from "livekit-client";
 import { type Logger } from "matrix-js-sdk/lib/logger";

-import { ObservableScope } from "../../reactive/ObservableScope";
-import {
-  type LocalMediaInputs,
-  type MediaSource,
-  type PublishRequest,
-} from "../../api";
+import { type MediaSource, type PublishRequest } from "../../api";
 import { livekitSources, mediaSources } from "./LivekitMediaTrack";
 import {
  audioCaptureOptions,
@@ -50,12 +45,10 @@ export class Publisher {
  public shouldPublish = false;
  /** Whether the initial requests have been published; `publish` is immediate from then on. */
  public started = false;
-  private readonly scope = new ObservableScope();
  private readonly room: LivekitRoom;

  public constructor(
    room: LivekitRoom,
-    private readonly inputs: LocalMediaInputs,
    private readonly desired: DesiredMedia,
    private readonly logger: Logger,
  ) {
@@ -63,7 +56,6 @@ export class Publisher {
    room.setE2EEEnabled(room.options.e2ee !== undefined)?.catch((e: Error) => {
      this.logger.error("Failed to enable E2EE on the room", e);
    });
-    this.followAudioOutput();
    this.onLocalTrackPublished = this.onLocalTrackPublished.bind(this);
    room.localParticipant.on(
      ParticipantEvent.LocalTrackPublished,
@@ -72,7 +64,6 @@ export class Publisher {
  }

  public async destroy(): Promise<void> {
-    this.scope.end();
    this.room.localParticipant.off(
      ParticipantEvent.LocalTrackPublished,
      this.onLocalTrackPublished,
@@ -216,19 +207,13 @@ export class Publisher {
    return [...this.room.localParticipant.trackPublications.values()];
  }

-  private followAudioOutput(): void {
-    this.inputs.audioOutputDeviceId$
-      .pipe(this.scope.bind())
-      .subscribe((deviceId) => {
+  /** A room that is not connected yet takes the device from its options instead. */
+  public async setAudioOutputDevice(deviceId: string): Promise<void> {
    if (
-          deviceId === undefined ||
      this.room.state !== LivekitConnectionState.Connected ||
      this.room.getActiveDevice("audiooutput") === deviceId
    )
      return;
-        this.room
-          .switchActiveDevice("audiooutput", deviceId)
-          .catch((e) => this.logger.error("Failed to switch audiooutput", e));
-      });
+    await this.room.switchActiveDevice("audiooutput", deviceId);
  }
 }