Give the local tracks their controls

Our own tracks are LocalAudioMediaTrack and LocalVideoMediaTrack, with setEnabled and setDevice, and on video facingMode$, switchFacingMode and setProcessor. switchCamera on the local media goes, as do the device and processor behaviors in LocalMediaInputs: a host calls the track when its setting changes instead of the SDK watching behaviors it does not own. The input device ids and the processor stay as initial values for the tracks created at the join. setMicrophoneEnabled and setCameraEnabled stay for the case of no track yet.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

sdk/src/session/ConnectionFactory.ts

@@ -138,8 +138,8 @@ function roomOptions(
     ...base,
     videoCaptureDefaults: {
       ...base.videoCaptureDefaults,
-      deviceId: localMedia.videoInputDeviceId$.value,
-      processor: localMedia.videoProcessor$.value,
+      deviceId: localMedia.videoInputDeviceId,
+      processor: localMedia.videoProcessor,
       ...(camera?.resolution && { resolution: camera.resolution }),
     },
     publishDefaults: {
@@ -154,7 +154,7 @@ function roomOptions(
     },
     audioCaptureDefaults: {
       ...base.audioCaptureDefaults,
-      deviceId: localMedia.audioInputDeviceId$.value,
+      deviceId: localMedia.audioInputDeviceId,
       ...capture?.audio,
     },
     audioOutput: { deviceId: localMedia.audioOutputDeviceId$.value },

sdk/src/session/LocalMember.ts

@@ -46,7 +46,11 @@ import {
   type MatrixRTCMode,
   type SessionTimings,
 } from "../config";
-import { type DisconnectReason, type VideoCaptureSettings } from "../api";
+import {
+  type DisconnectReason,
+  type MediaSource,
+  type VideoCaptureSettings,
+} from "../api";
 import {
   FailToStartLivekitConnection,
   type MatrixRTCError,
@@ -58,6 +62,7 @@ import { type Connection, ConnectionState } from "./Connection";
 import { type IConnectionManager } from "./ConnectionManager";
 import { type LocalTransport } from "./LocalTransport";
 import { getSFUConfigWithOpenID } from "./openIDSFU";
+import { livekitSources } from "../media/LivekitMediaTrack";
 import { type DesiredMedia, type Publisher } from "./Publisher";
 
 export enum TransportState {
@@ -118,6 +123,8 @@ export interface LocalMembership {
   disconnectReason$: Behavior<DisconnectReason | null>;
   setMicrophoneEnabled: (enabled: boolean) => Promise<boolean>;
   setCameraEnabled: (enabled: boolean) => Promise<boolean>;
+  /** Mutes or unmutes one of our publications; the result is the state that resulted. */
+  setEnabled: (source: MediaSource, enabled: boolean) => Promise<boolean>;
   sharingScreen$: Behavior<boolean>;
   toggleScreenSharing: (() => void) | null;
   screenShareError$: Behavior<Error | null>;
@@ -392,22 +399,40 @@ export function createLocalMembership$({
     });
 
   const setEnabled = async (
-    source: Track.Source.Microphone | Track.Source.Camera,
+    source: MediaSource,
     enabled: boolean,
   ): Promise<boolean> => {
+    if (source !== "microphone" && source !== "camera")
+      return setPublicationEnabled(source, enabled);
     const desired$ =
-      source === Track.Source.Microphone
-        ? desired.microphone$
-        : desired.camera$;
+      source === "microphone" ? desired.microphone$ : desired.camera$;
     desired$.next(enabled);
     // Without a publisher the request waits for the tracks to be created
     const publisher = publisher$.value;
     if (publisher === null) return enabled;
-    const result = await publisher.setEnabled(source, enabled);
+    const result = await publisher.setEnabled(
+      source === "microphone" ? Track.Source.Microphone : Track.Source.Camera,
+      enabled,
+    );
     if (result !== enabled) desired$.next(result);
     return result;
   };
 
+  // A publication the publisher does not recreate on reconnection, so there
+  // is nothing to remember for it
+  const setPublicationEnabled = async (
+    source: MediaSource,
+    enabled: boolean,
+  ): Promise<boolean> => {
+    const publication = participant$.value?.getTrackPublication(
+      livekitSources[source],
+    );
+    if (!publication) return false;
+    if (enabled) await publication.unmute();
+    else await publication.mute();
+    return !publication.isMuted;
+  };
+
   const sharingScreen$ = scope.behavior(
     participant$.pipe(
       switchMap((p) => (p === null ? of(false) : observeSharingScreen$(p))),
@@ -451,10 +476,9 @@ export function createLocalMembership$({
     connected$,
     reconnecting$,
     disconnectReason$,
-    setMicrophoneEnabled: async (enabled) =>
-      setEnabled(Track.Source.Microphone, enabled),
-    setCameraEnabled: async (enabled) =>
-      setEnabled(Track.Source.Camera, enabled),
+    setMicrophoneEnabled: async (enabled) => setEnabled("microphone", enabled),
+    setCameraEnabled: async (enabled) => setEnabled("camera", enabled),
+    setEnabled,
     sharingScreen$,
     toggleScreenSharing,
     screenShareError$,

sdk/src/session/Members.ts

@@ -93,6 +93,7 @@ export function createLocalRTCMember(
           participant,
           room,
           context.encryptionSystem,
+          localMembership.setEnabled,
         ),
     ),
     sharingScreen$: localMembership.sharingScreen$,

sdk/src/session/Publisher.ts

@@ -5,24 +5,16 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import { observeParticipantMedia } from "@livekit/components-core";
 import {
   ConnectionState as LivekitConnectionState,
   type LocalTrackPublication,
-  LocalVideoTrack,
   ParticipantEvent,
   type Room as LivekitRoom,
   Track,
 } from "livekit-client";
 import { type Logger } from "matrix-js-sdk/lib/logger";
-import {
-  type BehaviorSubject,
-  combineLatest,
-  distinctUntilChanged,
-  map,
-} from "rxjs";
+import { type BehaviorSubject } from "rxjs";
 
-import { type Behavior } from "../reactive/Behavior";
 import { ObservableScope } from "../reactive/ObservableScope";
 import { type LocalMediaInputs } from "../api";
 
@@ -57,8 +49,7 @@ export class Publisher {
     room.setE2EEEnabled(room.options.e2ee !== undefined)?.catch((e: Error) => {
       this.logger.error("Failed to enable E2EE on the room", e);
     });
-    this.followVideoProcessor();
-    this.followDevices();
+    this.followAudioOutput();
     this.onLocalTrackPublished = this.onLocalTrackPublished.bind(this);
     room.localParticipant.on(
       ParticipantEvent.LocalTrackPublished,
@@ -188,54 +179,19 @@ export class Publisher {
     }
   }
 
-  private followDevices(): void {
-    const sync = (
-      kind: MediaDeviceKind,
-      deviceId$: Behavior<string | undefined>,
-    ): void => {
-      deviceId$.pipe(this.scope.bind()).subscribe((deviceId) => {
+  private followAudioOutput(): void {
+    this.inputs.audioOutputDeviceId$
+      .pipe(this.scope.bind())
+      .subscribe((deviceId) => {
         if (
           deviceId === undefined ||
           this.room.state !== LivekitConnectionState.Connected ||
-          this.room.getActiveDevice(kind) === deviceId
+          this.room.getActiveDevice("audiooutput") === deviceId
         )
           return;
         this.room
-          .switchActiveDevice(kind, deviceId)
-          .catch((e) => this.logger.error(`Failed to switch ${kind}`, e));
-      });
-    };
-    sync("audioinput", this.inputs.audioInputDeviceId$);
-    sync("audiooutput", this.inputs.audioOutputDeviceId$);
-    sync("videoinput", this.inputs.videoInputDeviceId$);
-  }
-
-  private followVideoProcessor(): void {
-    const participant = this.room.localParticipant;
-    const cameraTrack$ = observeParticipantMedia(participant).pipe(
-      map(() => {
-        const track = participant.getTrackPublication(
-          Track.Source.Camera,
-        )?.track;
-        return track instanceof LocalVideoTrack ? track : undefined;
-      }),
-      distinctUntilChanged(),
-    );
-    combineLatest([cameraTrack$, this.inputs.videoProcessor$])
-      .pipe(this.scope.bind())
-      .subscribe(([track, processor]) => {
-        if (!track) return;
-        if (processor && !track.getProcessor()) {
-          // A processor cannot be built on a track that has already ended
-          if (track.mediaStreamTrack.readyState === "ended") return;
-          track.setProcessor(processor).catch((e) => {
-            this.logger.warn("Failed to attach the video processor", e);
-          });
-        } else if (!processor && track.getProcessor()) {
-          track.stopProcessor().catch((e) => {
-            this.logger.warn("Failed to stop the video processor", e);
-          });
-        }
+          .switchActiveDevice("audiooutput", deviceId)
+          .catch((e) => this.logger.error("Failed to switch audiooutput", e));
       });
   }
 }