Merge LocalMediaInputs into MatrixRTCClientOptions

The SDK now takes `publish` requests and audio output control directly
in the main options object, removing the separate `LocalMediaInputs`
type. Audio output device selection moves from a reactive behavior to an
imperative method on `MatrixRTCClient`, giving hosts direct control over
when devices switch.

sdk/src/MatrixRTCClient.ts

@@ -25,7 +25,6 @@ import { filterBehavior, generateItems } from "./reactive/observable";
 import { mapScoped } from "./utils/mapScoped";
 import {
   type DataMessage,
-  type LocalMediaInputs,
   type LocalRTCMember,
   type MatrixRTCClient,
   type MatrixRTCClientOptions,
@@ -72,7 +71,6 @@ export function createMatrixRTCClient(
   scope: ObservableScope,
   client: MatrixClient,
   room: Room,
-  localMedia: LocalMediaInputs,
   options: MatrixRTCClientOptions,
 ): MatrixRTCClient {
   const logger = rootLogger.getChild("[MatrixRTCClient]");
@@ -105,7 +103,7 @@ export function createMatrixRTCClient(
   const backend = createBackend(scope, client, options, {
     roomId: room.roomId,
     ownMembershipIdentity,
-    localMedia,
+    publish: options.publish,
     encryptionSystem,
     mediaKeys$: mediaKeys$(jsSdkSession),
     timings,
@@ -261,6 +259,8 @@ export function createMatrixRTCClient(
     ),
     keyRotationSuppressed$: createKeyRotationSuppressed$(scope, jsSdkSession),
     connectedTransports$: transports.connected$,
+    setAudioOutputDeviceId: async (deviceId) =>
+      backend.setAudioOutputDeviceId(deviceId),
     sendData: async (topic, text) => backend.sendData(topic, text),
     data$,
   };

sdk/src/api.ts

@@ -32,12 +32,14 @@ import {
 } from "./config";
 
 // ---------------------------------------------------------------------------
-// Session
+// MatrixRTCClient
 
 export interface MatrixRTCClientOptions {
   encryptionSystem: EncryptionSystem;
   /** Resolved by the host; the SDK reads neither config.json nor settings. */
   matrixRTCMode: MatrixRTCMode;
+  /** Published at the join; `publish` on the local member adds to it from then on. */
+  publish: PublishRequest[];
   /**
    * MSC4075 notification sent with the join. A parameter of the MatrixRTC
    * join itself, so it is here even though it is named after calls; reacting
@@ -104,13 +106,6 @@ export type PublishRequest =
       capture?: VideoCaptureSettings;
     };
 
-export interface LocalMediaInputs {
-  /** Published at the join; `publish` on the local member adds to it from then on. */
-  publish: PublishRequest[];
-  /** Undefined where the host routes audio itself, or to leave the browser's choice. */
-  audioOutputDeviceId$: Behavior<string | undefined>;
-}
-
 export type ConnectionStatus =
   | "waitingForTransport"
   | "connecting"
@@ -158,6 +153,13 @@ export interface MatrixRTCClient {
   /** Transports the session currently holds a live connection to. */
   connectedTransports$: Behavior<TransportMetadata[]>;
 
+  /**
+   * Plays the members' audio on this device from now on. Rejects where the
+   * browser cannot switch to it. A host that routes audio itself, or leaves
+   * the browser's choice, never calls it.
+   */
+  setAudioOutputDeviceId(deviceId: string): Promise<void>;
+
   /**
    * Sends a short text to every member on the local transport, over a
    * reliable data channel beside the media. A packet holds about 15 KiB;

sdk/src/media-backend/api.ts

@@ -19,7 +19,6 @@ import { type Observable } from "rxjs";
 
 import {
   type LocalAudioMediaTrack,
-  type LocalMediaInputs,
   type LocalMemberMedia,
   type LocalVideoMediaTrack,
   type MemberMedia,
@@ -55,7 +54,8 @@ export interface MediaKey {
 export interface MediaBackendContext {
   roomId: string;
   ownMembershipIdentity: CallMembershipIdentityParts;
-  localMedia: LocalMediaInputs;
+  /** What the local member publishes at the join. */
+  publish: PublishRequest[];
   encryptionSystem: EncryptionSystem;
   /**
    * Every media key the session knows: the current ones replayed on
@@ -147,6 +147,9 @@ export interface MediaBackend {
   /** Every connection the backend holds. For debugging and devtool purposes.*/
   readonly connections$: Behavior<BackendConnection[]>;
 
+  /** Plays the members' audio on this device, on the connections of now and later. */
+  setAudioOutputDeviceId(deviceId: string): Promise<void>;
+
   /**
    * A text channel beside the media: sent on the local transport, received on every one.
    * This could well be just another track kind. Not needed for element call. just for the sdk target.

sdk/src/media-backend/livekit/ConnectionFactory.ts

@@ -23,7 +23,7 @@ import { type OpenIDClientParts, type SFUConfig } from "./openIDSFU";
 import { type Behavior } from "../../reactive/Behavior";
 import { Connection, type ConnectionOpts, ConnectionState } from "./Connection";
 import { type ObservableScope } from "../../reactive/ObservableScope";
-import { type LocalMediaInputs, type ResolvedTransport } from "../../api";
+import { type PublishRequest, type ResolvedTransport } from "../../api";
 import { type MediaQuality } from "../../config";
 import {
   audioCaptureOptions,
@@ -95,7 +95,8 @@ export class LivekitConnectionFactory implements ConnectionFactory {
   public constructor(
     private readonly client: OpenIDClientParts,
     private readonly roomId: string,
-    private readonly localMedia: LocalMediaInputs,
+    private readonly publish: PublishRequest[],
+    private readonly audioOutputDeviceId$: Behavior<string | undefined>,
     private readonly keyProvider: BaseKeyProvider | undefined,
     private readonly mediaQuality: MediaQuality | undefined,
   ) {}
@@ -116,7 +117,12 @@ export class LivekitConnectionFactory implements ConnectionFactory {
         scope,
         livekitRoomFactory: () =>
           new LivekitRoom(
-            roomOptions(this.localMedia, this.keyProvider, this.mediaQuality),
+            roomOptions(
+              this.publish,
+              this.audioOutputDeviceId$.value,
+              this.keyProvider,
+              this.mediaQuality,
+            ),
           ),
         ownMembershipIdentity,
       },
@@ -127,13 +133,14 @@ export class LivekitConnectionFactory implements ConnectionFactory {
 
 /** The initial requests become the room's defaults, so that one permission prompt covers both. */
 function roomOptions(
-  localMedia: LocalMediaInputs,
+  publish: PublishRequest[],
+  audioOutputDeviceId: string | undefined,
   keyProvider: BaseKeyProvider | undefined,
   mediaQuality: MediaQuality | undefined,
 ): RoomOptions {
   const base = buildLiveKitOptions(mediaQuality);
-  const microphone = initialRequest(localMedia, "microphone");
-  const camera = initialRequest(localMedia, "camera");
+  const microphone = initialRequest(publish, "microphone");
+  const camera = initialRequest(publish, "camera");
   return {
     ...base,
     videoCaptureDefaults: {
@@ -148,7 +155,7 @@ function roomOptions(
       ...base.audioCaptureDefaults,
       ...(microphone && audioCaptureOptions(microphone)),
     },
-    audioOutput: { deviceId: localMedia.audioOutputDeviceId$.value },
+    audioOutput: { deviceId: audioOutputDeviceId },
     // Every room needs a worker of its own: one gets confused by streams from
     // several rooms
     e2ee: keyProvider && { keyProvider, worker: new E2EEWorker() },
@@ -156,10 +163,10 @@ function roomOptions(
 }
 
 function initialRequest<S extends "microphone" | "camera">(
-  localMedia: LocalMediaInputs,
+  publish: PublishRequest[],
   source: S,
 ): PublishRequestFor<S> | undefined {
-  return localMedia.publish.find(
+  return publish.find(
     (request): request is PublishRequestFor<S> => request.source === source,
   );
 }

sdk/src/media-backend/livekit/LivekitBackend.ts

@@ -77,7 +77,7 @@ export function createLivekitBackend(
   {
     roomId,
     ownMembershipIdentity,
-    localMedia,
+    publish,
     encryptionSystem,
     mediaKeys$,
     timings,
@@ -89,13 +89,17 @@ export function createLivekitBackend(
   const keyProvider = createKeyProvider(encryptionSystem, mediaKeys$, logger);
   const transports = createTransportRegistry(scope);
   const localTransport$ = new ReplaySubject<LocalTransport>(1);
+  const audioOutputDeviceId$ = new BehaviorSubject<string | undefined>(
+    undefined,
+  );
 
   const connectionManager = createConnectionManager$({
     scope,
     connectionFactory: new LivekitConnectionFactory(
       client,
       roomId,
-      localMedia,
+      publish,
+      audioOutputDeviceId$,
       keyProvider,
       mediaQuality,
     ),
@@ -215,7 +219,8 @@ export function createLivekitBackend(
     scope,
     connectionManager,
     localTransport$,
-    localMedia,
+    publish,
+    audioOutputDeviceId$,
     encryptionSystem,
     logger,
   });
@@ -233,6 +238,7 @@ export function createLivekitBackend(
     local: localMedia_,
     mediaFor$,
     connections$,
+    setAudioOutputDeviceId: localMedia_.setAudioOutputDeviceId,
     sendData,
     data$,
   };

sdk/src/media-backend/livekit/LocalMedia.ts

@@ -28,7 +28,6 @@ import {
 
 import {
   type LocalAudioMediaTrack,
-  type LocalMediaInputs,
   type LocalMemberMedia,
   type LocalVideoMediaTrack,
   type MediaSource,
@@ -54,7 +53,9 @@ interface Props {
   scope: ObservableScope;
   connectionManager: IConnectionManager;
   localTransport$: Observable<LocalTransport>;
-  localMedia: LocalMediaInputs;
+  publish: PublishRequest[];
+  /** Read by the connection factory for the rooms it creates from now on. */
+  audioOutputDeviceId$: BehaviorSubject<string | undefined>;
   encryptionSystem: EncryptionSystem;
   logger: Logger;
 }
@@ -62,6 +63,7 @@ interface Props {
 export interface LivekitLocalMedia extends LocalMediaBackend {
   /** The connection the local member publishes on; the data channel sends on it. */
   connection$: Behavior<Connection | null>;
+  setAudioOutputDeviceId(deviceId: string): Promise<void>;
 }
 
 /**
@@ -73,7 +75,8 @@ export function createLivekitLocalMedia({
   scope,
   connectionManager,
   localTransport$,
-  localMedia,
+  publish: initialRequests,
+  audioOutputDeviceId$,
   encryptionSystem,
   logger: parentLogger,
 }: Props): LivekitLocalMedia {
@@ -102,7 +105,7 @@ export function createLivekitLocalMedia({
   );
 
   const desired: DesiredMedia = new Map(
-    localMedia.publish.map((request) => [
+    initialRequests.map((request) => [
       request.source,
       { request, enabled: true },
     ]),
@@ -115,7 +118,6 @@ export function createLivekitLocalMedia({
     if (connection === null) return;
     const publisher = new Publisher(
       connection.livekitRoom,
-      localMedia,
       desired,
       logger.getChild(
         `[Publisher ${connection.transport.livekit_service_url}]`,
@@ -209,6 +211,11 @@ export function createLivekitLocalMedia({
     await publisher$.value?.unpublish(source);
   };
 
+  const setAudioOutputDeviceId = async (deviceId: string): Promise<void> => {
+    audioOutputDeviceId$.next(deviceId);
+    await publisher$.value?.setAudioOutputDevice(deviceId);
+  };
+
   return {
     connection$,
     connectionState$,
@@ -217,6 +224,7 @@ export function createLivekitLocalMedia({
     publishError$,
     publish,
     unpublish,
+    setAudioOutputDeviceId,
   };
 }
 

sdk/src/media-backend/livekit/Publisher.ts

@@ -14,12 +14,7 @@ import {
 } from "livekit-client";
 import { type Logger } from "matrix-js-sdk/lib/logger";
 
-import { ObservableScope } from "../../reactive/ObservableScope";
-import {
-  type LocalMediaInputs,
-  type MediaSource,
-  type PublishRequest,
-} from "../../api";
+import { type MediaSource, type PublishRequest } from "../../api";
 import { livekitSources, mediaSources } from "./LivekitMediaTrack";
 import {
   audioCaptureOptions,
@@ -50,12 +45,10 @@ export class Publisher {
   public shouldPublish = false;
   /** Whether the initial requests have been published; `publish` is immediate from then on. */
   public started = false;
-  private readonly scope = new ObservableScope();
   private readonly room: LivekitRoom;
 
   public constructor(
     room: LivekitRoom,
-    private readonly inputs: LocalMediaInputs,
     private readonly desired: DesiredMedia,
     private readonly logger: Logger,
   ) {
@@ -63,7 +56,6 @@ export class Publisher {
     room.setE2EEEnabled(room.options.e2ee !== undefined)?.catch((e: Error) => {
       this.logger.error("Failed to enable E2EE on the room", e);
     });
-    this.followAudioOutput();
     this.onLocalTrackPublished = this.onLocalTrackPublished.bind(this);
     room.localParticipant.on(
       ParticipantEvent.LocalTrackPublished,
@@ -72,7 +64,6 @@ export class Publisher {
   }
 
   public async destroy(): Promise<void> {
-    this.scope.end();
     this.room.localParticipant.off(
       ParticipantEvent.LocalTrackPublished,
       this.onLocalTrackPublished,
@@ -216,19 +207,13 @@ export class Publisher {
     return [...this.room.localParticipant.trackPublications.values()];
   }
 
-  private followAudioOutput(): void {
-    this.inputs.audioOutputDeviceId$
-      .pipe(this.scope.bind())
-      .subscribe((deviceId) => {
-        if (
-          deviceId === undefined ||
-          this.room.state !== LivekitConnectionState.Connected ||
-          this.room.getActiveDevice("audiooutput") === deviceId
-        )
-          return;
-        this.room
-          .switchActiveDevice("audiooutput", deviceId)
-          .catch((e) => this.logger.error("Failed to switch audiooutput", e));
-      });
+  /** A room that is not connected yet takes the device from its options instead. */
+  public async setAudioOutputDevice(deviceId: string): Promise<void> {
+    if (
+      this.room.state !== LivekitConnectionState.Connected ||
+      this.room.getActiveDevice("audiooutput") === deviceId
+    )
+      return;
+    await this.room.switchActiveDevice("audiooutput", deviceId);
   }
 }