Merge LocalMediaInputs into MatrixRTCClientOptions

The SDK now takes `publish` requests and audio output control directly
in the main options object, removing the separate `LocalMediaInputs`
type. Audio output device selection moves from a reactive behavior to an
imperative method on `MatrixRTCClient`, giving hosts direct control over
when devices switch.

sdk/README.md

@@ -19,7 +19,6 @@ Call's own `CallViewModel` is built on it. It depends on nothing in Element Call
 
 ```ts
 import {
-  constant,
   createMatrixRTCClient,
   E2eeType,
   MatrixRTCMode,
@@ -31,13 +30,10 @@ const rtcClient = createMatrixRTCClient(
   scope,
   client, // a matrix-js-sdk MatrixClient, logged in and syncing
   room, // the matrix-js-sdk Room, from client.getRoom() once the join has synced
-  {
-    publish: [{ source: "microphone" }, { source: "camera" }],
-    audioOutputDeviceId$: constant(undefined),
-  },
   {
     encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
     matrixRTCMode: MatrixRTCMode.Compatibility,
+    publish: [{ source: "microphone" }, { source: "camera" }],
   },
 );
 rtcClient.join();

sdk/SdkArchitecture.md

@@ -18,7 +18,7 @@ therefore `Element Call → sdk`, never the reverse. The migration from today's
 ```
  host application (Element Call, a whiteboard, watch-together, …)
    │
-   │  createMatrixRTCClient(scope, matrixClient, room, localMedia, options)
+   │  createMatrixRTCClient(scope, matrixClient, room, options)
    ▼
 ┌──────────────────────────────────────────────────────────────────────────────┐
 │ MatrixRTCClient                                                              │
@@ -135,7 +135,7 @@ it, and `leave()` also sends the leave.
 | media             | a `MemberMedia`: what a member sends, once it has arrived on its transport. `null` while waiting for media                                                                                            | Element Call's `MediaViewModel` is a view of it                   |
 | track             | a `MediaTrack`: one publication of a member, audio or video, with `source`, `kind`, `id` and the behaviors on it. `LocalAudioMediaTrack` and `LocalVideoMediaTrack` add the controls over our own     | LiveKit `TrackPublication` and `Track`, which never leave the SDK |
 | source            | a `MediaSource`: what a track is captured from, `microphone`, `camera`, `screenShare`, `screenShareAudio` or `unknown`. Picked out of `tracks$` with `trackBySource$`                                 | LiveKit `Track.Source`                                            |
-| publish request   | a `PublishRequest`: a source, where to capture it from and how to encode it. Given at the join in `LocalMediaInputs.publish`, or later to `publish` on the local member                               |                                                                   |
+| publish request   | a `PublishRequest`: a source, where to capture it from and how to encode it. Given at the join in `options.publish`, or later to `publish` on the local member                                        |                                                                   |
 | active            | `isActive$` on an audio track: the track carries sound, as the backend measures it. "Speaking" is a call's reading of it on the microphone track                                                      | LiveKit "speaking", `isSpeaking`                                  |
 | attach, detach    | handing a `<video>` or `<audio>` element to a track and taking it back. The SDK owns the stream and the observers on the element                                                                      | LiveKit `Track.attach`                                            |
 | transport         | where media is exchanged: a LiveKit service url today, described by `TransportMetadata`                                                                                                               | MSC4143 "focus"                                                   |
@@ -182,7 +182,7 @@ context, it is in the view model.
 
 ### Creating one
 
-```ts
+````ts
 import { type MatrixClient, type Room } from "matrix-js-sdk";
 import { type RTCNotificationType } from "matrix-js-sdk/lib/matrixrtc";
 
@@ -190,6 +190,8 @@ export interface MatrixRTCClientOptions {
   encryptionSystem: EncryptionSystem;
   /** Resolved by the host; the SDK reads neither config.json nor settings. */
   matrixRTCMode: MatrixRTCMode;
+  /** Published at the join; `publish` on the local member adds to it from then on. */
+  publish: PublishRequest[];
   /**
    * MSC4075 notification sent with the join. A parameter of the MatrixRTC
    * join itself, so it is here even though it is named after calls; reacting
@@ -236,18 +238,12 @@ export type PublishRequest =
       audio?: boolean;
       capture?: VideoCaptureSettings;
     };
+```
 
-/**
+Device enumeration, permission prompts and the lobby preview stay with the
- * Device enumeration, permission prompts and the lobby preview stay with the
+host: it hands over publish requests naming the devices it chose.
- * host.
- */
-export interface LocalMediaInputs {
-  /** Published at the join; `publish` on the local member adds to it from then on. */
-  publish: PublishRequest[];
-  /** Undefined where the host routes audio itself, or to leave the browser's choice. */
-  audioOutputDeviceId$: Behavior<string | undefined>;
-}
 
+```ts
 /**
  * Takes the whole MatrixClient, not a narrow pick, and finds the MatrixRTC
  * session itself. The js-sdk is the MatrixRTC implementation today; when the
@@ -263,10 +259,9 @@ export function createMatrixRTCClient(
   scope: ObservableScope,
   client: MatrixClient,
   room: Room,
-  localMedia: LocalMediaInputs,
   options: MatrixRTCClientOptions,
 ): MatrixRTCClient;
-```
+````
 
 ### The client
 
@@ -328,6 +323,13 @@ export interface MatrixRTCClient {
    */
   connectedTransports$: Behavior<TransportMetadata[]>;
 
+  /**
+   * Plays the members' audio on this device from now on. Rejects where the
+   * browser cannot switch to it. A host that routes audio itself, or leaves
+   * the browser's choice, never calls it.
+   */
+  setAudioOutputDeviceId(deviceId: string): Promise<void>;
+
   /**
    * Sends a short text to every member on the local transport, over a
    * reliable data channel beside the media. A packet holds about 15 KiB;

sdk/dev/main.ts

@@ -15,7 +15,6 @@ import { logger } from "matrix-js-sdk/lib/logger";
 import { combineLatest, type Observable, of, switchMap } from "rxjs";
 import {
   type AudioMediaTrack,
-  constant,
   createMatrixRTCClient,
   E2eeType,
   MatrixRTCMode,
@@ -70,19 +69,11 @@ async function start(
     const room = await joinRoom(client, roomIdOrAlias);
 
     const scope = new ObservableScope();
-    const rtcClient = createMatrixRTCClient(
+    const rtcClient = createMatrixRTCClient(scope, client, room, {
-      scope,
+      encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
-      client,
+      matrixRTCMode: MatrixRTCMode.Compatibility,
-      room,
+      publish: [{ source: "microphone" }, { source: "camera" }],
-      {
+    });
-        publish: [{ source: "microphone" }, { source: "camera" }],
-        audioOutputDeviceId$: constant(undefined),
-      },
-      {
-        encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
-        matrixRTCMode: MatrixRTCMode.Compatibility,
-      },
-    );
     rtcClient.status$.pipe(scope.bind()).subscribe((s) => {
       status.textContent = s;
     });

sdk/src/MatrixRTCClient.ts

@@ -25,7 +25,6 @@ import { filterBehavior, generateItems } from "./reactive/observable";
 import { mapScoped } from "./utils/mapScoped";
 import {
   type DataMessage,
-  type LocalMediaInputs,
   type LocalRTCMember,
   type MatrixRTCClient,
   type MatrixRTCClientOptions,
@@ -72,7 +71,6 @@ export function createMatrixRTCClient(
   scope: ObservableScope,
   client: MatrixClient,
   room: Room,
-  localMedia: LocalMediaInputs,
   options: MatrixRTCClientOptions,
 ): MatrixRTCClient {
   const logger = rootLogger.getChild("[MatrixRTCClient]");
@@ -105,7 +103,7 @@ export function createMatrixRTCClient(
   const backend = createBackend(scope, client, options, {
     roomId: room.roomId,
     ownMembershipIdentity,
-    localMedia,
+    publish: options.publish,
     encryptionSystem,
     mediaKeys$: mediaKeys$(jsSdkSession),
     timings,
@@ -261,6 +259,8 @@ export function createMatrixRTCClient(
     ),
     keyRotationSuppressed$: createKeyRotationSuppressed$(scope, jsSdkSession),
     connectedTransports$: transports.connected$,
+    setAudioOutputDeviceId: async (deviceId) =>
+      backend.setAudioOutputDeviceId(deviceId),
     sendData: async (topic, text) => backend.sendData(topic, text),
     data$,
   };

sdk/src/api.ts

@@ -32,12 +32,14 @@ import {
 } from "./config";
 
 // ---------------------------------------------------------------------------
-// Session
+// MatrixRTCClient
 
 export interface MatrixRTCClientOptions {
   encryptionSystem: EncryptionSystem;
   /** Resolved by the host; the SDK reads neither config.json nor settings. */
   matrixRTCMode: MatrixRTCMode;
+  /** Published at the join; `publish` on the local member adds to it from then on. */
+  publish: PublishRequest[];
   /**
    * MSC4075 notification sent with the join. A parameter of the MatrixRTC
    * join itself, so it is here even though it is named after calls; reacting
@@ -104,13 +106,6 @@ export type PublishRequest =
       capture?: VideoCaptureSettings;
     };
 
-export interface LocalMediaInputs {
-  /** Published at the join; `publish` on the local member adds to it from then on. */
-  publish: PublishRequest[];
-  /** Undefined where the host routes audio itself, or to leave the browser's choice. */
-  audioOutputDeviceId$: Behavior<string | undefined>;
-}
-
 export type ConnectionStatus =
   | "waitingForTransport"
   | "connecting"
@@ -158,6 +153,13 @@ export interface MatrixRTCClient {
   /** Transports the session currently holds a live connection to. */
   connectedTransports$: Behavior<TransportMetadata[]>;
 
+  /**
+   * Plays the members' audio on this device from now on. Rejects where the
+   * browser cannot switch to it. A host that routes audio itself, or leaves
+   * the browser's choice, never calls it.
+   */
+  setAudioOutputDeviceId(deviceId: string): Promise<void>;
+
   /**
    * Sends a short text to every member on the local transport, over a
    * reliable data channel beside the media. A packet holds about 15 KiB;

sdk/src/media-backend/api.ts

@@ -19,7 +19,6 @@ import { type Observable } from "rxjs";
 
 import {
   type LocalAudioMediaTrack,
-  type LocalMediaInputs,
   type LocalMemberMedia,
   type LocalVideoMediaTrack,
   type MemberMedia,
@@ -55,7 +54,8 @@ export interface MediaKey {
 export interface MediaBackendContext {
   roomId: string;
   ownMembershipIdentity: CallMembershipIdentityParts;
-  localMedia: LocalMediaInputs;
+  /** What the local member publishes at the join. */
+  publish: PublishRequest[];
   encryptionSystem: EncryptionSystem;
   /**
    * Every media key the session knows: the current ones replayed on
@@ -147,6 +147,9 @@ export interface MediaBackend {
   /** Every connection the backend holds. For debugging and devtool purposes.*/
   readonly connections$: Behavior<BackendConnection[]>;
 
+  /** Plays the members' audio on this device, on the connections of now and later. */
+  setAudioOutputDeviceId(deviceId: string): Promise<void>;
+
   /**
    * A text channel beside the media: sent on the local transport, received on every one.
    * This could well be just another track kind. Not needed for element call. just for the sdk target.

sdk/src/media-backend/livekit/ConnectionFactory.ts

@@ -23,7 +23,7 @@ import { type OpenIDClientParts, type SFUConfig } from "./openIDSFU";
 import { type Behavior } from "../../reactive/Behavior";
 import { Connection, type ConnectionOpts, ConnectionState } from "./Connection";
 import { type ObservableScope } from "../../reactive/ObservableScope";
-import { type LocalMediaInputs, type ResolvedTransport } from "../../api";
+import { type PublishRequest, type ResolvedTransport } from "../../api";
 import { type MediaQuality } from "../../config";
 import {
   audioCaptureOptions,
@@ -95,7 +95,8 @@ export class LivekitConnectionFactory implements ConnectionFactory {
   public constructor(
     private readonly client: OpenIDClientParts,
     private readonly roomId: string,
-    private readonly localMedia: LocalMediaInputs,
+    private readonly publish: PublishRequest[],
+    private readonly audioOutputDeviceId$: Behavior<string | undefined>,
     private readonly keyProvider: BaseKeyProvider | undefined,
     private readonly mediaQuality: MediaQuality | undefined,
   ) {}
@@ -116,7 +117,12 @@ export class LivekitConnectionFactory implements ConnectionFactory {
         scope,
         livekitRoomFactory: () =>
           new LivekitRoom(
-            roomOptions(this.localMedia, this.keyProvider, this.mediaQuality),
+            roomOptions(
+              this.publish,
+              this.audioOutputDeviceId$.value,
+              this.keyProvider,
+              this.mediaQuality,
+            ),
           ),
         ownMembershipIdentity,
       },
@@ -127,13 +133,14 @@ export class LivekitConnectionFactory implements ConnectionFactory {
 
 /** The initial requests become the room's defaults, so that one permission prompt covers both. */
 function roomOptions(
-  localMedia: LocalMediaInputs,
+  publish: PublishRequest[],
+  audioOutputDeviceId: string | undefined,
   keyProvider: BaseKeyProvider | undefined,
   mediaQuality: MediaQuality | undefined,
 ): RoomOptions {
   const base = buildLiveKitOptions(mediaQuality);
-  const microphone = initialRequest(localMedia, "microphone");
+  const microphone = initialRequest(publish, "microphone");
-  const camera = initialRequest(localMedia, "camera");
+  const camera = initialRequest(publish, "camera");
   return {
     ...base,
     videoCaptureDefaults: {
@@ -148,7 +155,7 @@ function roomOptions(
       ...base.audioCaptureDefaults,
       ...(microphone && audioCaptureOptions(microphone)),
     },
-    audioOutput: { deviceId: localMedia.audioOutputDeviceId$.value },
+    audioOutput: { deviceId: audioOutputDeviceId },
     // Every room needs a worker of its own: one gets confused by streams from
     // several rooms
     e2ee: keyProvider && { keyProvider, worker: new E2EEWorker() },
@@ -156,10 +163,10 @@ function roomOptions(
 }
 
 function initialRequest<S extends "microphone" | "camera">(
-  localMedia: LocalMediaInputs,
+  publish: PublishRequest[],
   source: S,
 ): PublishRequestFor<S> | undefined {
-  return localMedia.publish.find(
+  return publish.find(
     (request): request is PublishRequestFor<S> => request.source === source,
   );
 }

sdk/src/media-backend/livekit/LivekitBackend.ts

@@ -77,7 +77,7 @@ export function createLivekitBackend(
   {
     roomId,
     ownMembershipIdentity,
-    localMedia,
+    publish,
     encryptionSystem,
     mediaKeys$,
     timings,
@@ -89,13 +89,17 @@ export function createLivekitBackend(
   const keyProvider = createKeyProvider(encryptionSystem, mediaKeys$, logger);
   const transports = createTransportRegistry(scope);
   const localTransport$ = new ReplaySubject<LocalTransport>(1);
+  const audioOutputDeviceId$ = new BehaviorSubject<string | undefined>(
+    undefined,
+  );
 
   const connectionManager = createConnectionManager$({
     scope,
     connectionFactory: new LivekitConnectionFactory(
       client,
       roomId,
-      localMedia,
+      publish,
+      audioOutputDeviceId$,
       keyProvider,
       mediaQuality,
     ),
@@ -215,7 +219,8 @@ export function createLivekitBackend(
     scope,
     connectionManager,
     localTransport$,
-    localMedia,
+    publish,
+    audioOutputDeviceId$,
     encryptionSystem,
     logger,
   });
@@ -233,6 +238,7 @@ export function createLivekitBackend(
     local: localMedia_,
     mediaFor$,
     connections$,
+    setAudioOutputDeviceId: localMedia_.setAudioOutputDeviceId,
     sendData,
     data$,
   };

sdk/src/media-backend/livekit/LocalMedia.ts

@@ -28,7 +28,6 @@ import {
 
 import {
   type LocalAudioMediaTrack,
-  type LocalMediaInputs,
   type LocalMemberMedia,
   type LocalVideoMediaTrack,
   type MediaSource,
@@ -54,7 +53,9 @@ interface Props {
   scope: ObservableScope;
   connectionManager: IConnectionManager;
   localTransport$: Observable<LocalTransport>;
-  localMedia: LocalMediaInputs;
+  publish: PublishRequest[];
+  /** Read by the connection factory for the rooms it creates from now on. */
+  audioOutputDeviceId$: BehaviorSubject<string | undefined>;
   encryptionSystem: EncryptionSystem;
   logger: Logger;
 }
@@ -62,6 +63,7 @@ interface Props {
 export interface LivekitLocalMedia extends LocalMediaBackend {
   /** The connection the local member publishes on; the data channel sends on it. */
   connection$: Behavior<Connection | null>;
+  setAudioOutputDeviceId(deviceId: string): Promise<void>;
 }
 
 /**
@@ -73,7 +75,8 @@ export function createLivekitLocalMedia({
   scope,
   connectionManager,
   localTransport$,
-  localMedia,
+  publish: initialRequests,
+  audioOutputDeviceId$,
   encryptionSystem,
   logger: parentLogger,
 }: Props): LivekitLocalMedia {
@@ -102,7 +105,7 @@ export function createLivekitLocalMedia({
   );
 
   const desired: DesiredMedia = new Map(
-    localMedia.publish.map((request) => [
+    initialRequests.map((request) => [
       request.source,
       { request, enabled: true },
     ]),
@@ -115,7 +118,6 @@ export function createLivekitLocalMedia({
     if (connection === null) return;
     const publisher = new Publisher(
       connection.livekitRoom,
-      localMedia,
       desired,
       logger.getChild(
         `[Publisher ${connection.transport.livekit_service_url}]`,
@@ -209,6 +211,11 @@ export function createLivekitLocalMedia({
     await publisher$.value?.unpublish(source);
   };
 
+  const setAudioOutputDeviceId = async (deviceId: string): Promise<void> => {
+    audioOutputDeviceId$.next(deviceId);
+    await publisher$.value?.setAudioOutputDevice(deviceId);
+  };
+
   return {
     connection$,
     connectionState$,
@@ -217,6 +224,7 @@ export function createLivekitLocalMedia({
     publishError$,
     publish,
     unpublish,
+    setAudioOutputDeviceId,
   };
 }
 

sdk/src/media-backend/livekit/Publisher.ts

@@ -14,12 +14,7 @@ import {
 } from "livekit-client";
 import { type Logger } from "matrix-js-sdk/lib/logger";
 
-import { ObservableScope } from "../../reactive/ObservableScope";
+import { type MediaSource, type PublishRequest } from "../../api";
-import {
-  type LocalMediaInputs,
-  type MediaSource,
-  type PublishRequest,
-} from "../../api";
 import { livekitSources, mediaSources } from "./LivekitMediaTrack";
 import {
   audioCaptureOptions,
@@ -50,12 +45,10 @@ export class Publisher {
   public shouldPublish = false;
   /** Whether the initial requests have been published; `publish` is immediate from then on. */
   public started = false;
-  private readonly scope = new ObservableScope();
   private readonly room: LivekitRoom;
 
   public constructor(
     room: LivekitRoom,
-    private readonly inputs: LocalMediaInputs,
     private readonly desired: DesiredMedia,
     private readonly logger: Logger,
   ) {
@@ -63,7 +56,6 @@ export class Publisher {
     room.setE2EEEnabled(room.options.e2ee !== undefined)?.catch((e: Error) => {
       this.logger.error("Failed to enable E2EE on the room", e);
     });
-    this.followAudioOutput();
     this.onLocalTrackPublished = this.onLocalTrackPublished.bind(this);
     room.localParticipant.on(
       ParticipantEvent.LocalTrackPublished,
@@ -72,7 +64,6 @@ export class Publisher {
   }
 
   public async destroy(): Promise<void> {
-    this.scope.end();
     this.room.localParticipant.off(
       ParticipantEvent.LocalTrackPublished,
       this.onLocalTrackPublished,
@@ -216,19 +207,13 @@ export class Publisher {
     return [...this.room.localParticipant.trackPublications.values()];
   }
 
-  private followAudioOutput(): void {
+  /** A room that is not connected yet takes the device from its options instead. */
-    this.inputs.audioOutputDeviceId$
+  public async setAudioOutputDevice(deviceId: string): Promise<void> {
-      .pipe(this.scope.bind())
+    if (
-      .subscribe((deviceId) => {
+      this.room.state !== LivekitConnectionState.Connected ||
-        if (
+      this.room.getActiveDevice("audiooutput") === deviceId
-          deviceId === undefined ||
+    )
-          this.room.state !== LivekitConnectionState.Connected ||
+      return;
-          this.room.getActiveDevice("audiooutput") === deviceId
+    await this.room.switchActiveDevice("audiooutput", deviceId);
-        )
-          return;
-        this.room
-          .switchActiveDevice("audiooutput", deviceId)
-          .catch((e) => this.logger.error("Failed to switch audiooutput", e));
-      });
   }
 }