update EC based on sdk media refactor

No more call specifics in the sdk layer

sdk-target-based-on-call-view-model/main.ts

@@ -34,7 +34,6 @@ import {
   of,
   switchMap,
 } from "rxjs";
-import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
 import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
 import {
   type Behavior,
@@ -43,11 +42,13 @@ import {
   E2eeType,
   ObservableScope,
   type RTCMember,
+  type RTCMembership,
 } from "@element-hq/matrixrtc-sdk";
 
 import {
   callViewModelOptionsFromParams,
   createCallViewModel$,
+  initialPublishRequests,
 } from "../src/state/CallViewModel/CallViewModel";
 import { getUrlParams } from "../src/UrlParams";
 import { MuteStates } from "../src/state/MuteStates";
@@ -57,6 +58,7 @@ import { initializeWidget } from "../src/widget";
 import { createWidgetHostBridge } from "../src/HostBridge";
 import { observeElementSize$ } from "../src/utils/elementSize";
 import {
+  captureSettings,
   matrixRTCClientOptions,
   selectedDeviceId$,
 } from "../src/room/InCallView";
@@ -70,7 +72,7 @@ import {
 interface MatrixRTCSdkMember {
   /** @deprecated Always null: the SDK owns the transport connection. */
   connection: null;
-  membership: CallMembership;
+  membership: RTCMembership;
   /** @deprecated Always null: the SDK owns the transport participant. */
   participant: null;
 }
@@ -140,19 +142,16 @@ export async function createMatrixRTCSdk(
 
   // rtc client: the session, the transport and the media, as in the app
   const encryptionSystem = { kind: E2eeType.PER_PARTICIPANT } as const;
+  const capture = captureSettings();
   const rtcClient = createMatrixRTCClient(
     scope,
     client,
     room,
     {
-      microphoneEnabled: muteStates.audio.enabled$.value,
-      cameraEnabled: muteStates.video.enabled$.value,
-      audioInputDeviceId$: selectedDeviceId$(scope, mediaDevices.audioInput),
-      videoInputDeviceId$: selectedDeviceId$(scope, mediaDevices.videoInput),
+      publish: initialPublishRequests(muteStates, mediaDevices, capture),
       audioOutputDeviceId$: controlledAudioDevices
         ? constant(undefined)
         : selectedDeviceId$(scope, mediaDevices.audioOutput),
-      videoProcessor$: constant(undefined),
     },
     {
       ...matrixRTCClientOptions(urlParams, encryptionSystem),
@@ -171,6 +170,7 @@ export async function createMatrixRTCSdk(
     {
       ...callViewModelOptionsFromParams(urlParams),
       encryptionSystem,
+      capture,
       hostBridge,
       // The SDK owns its page, so the body is the space it has
       windowSize$: scope.behavior(observeElementSize$(document.body)),

src/room/InCallView.tsx

@@ -42,7 +42,9 @@ import { InviteButton } from "../button/InviteButton";
 import {
   type CallViewModel,
   callViewModelOptionsFromParams,
+  type CaptureSettings,
   createCallViewModel$,
+  initialPublishRequests,
 } from "../state/CallViewModel/CallViewModel.ts";
 import { Grid, type TileProps } from "../grid/Grid";
 import { SpotlightTile } from "../tile/SpotlightTile";
@@ -160,22 +162,21 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
     const { autoLeaveWhenOthersLeft, waitForCallPickup, sendNotificationType } =
       urlParams;
 
+    const capture = captureSettings();
     const rtcClient = createMatrixRTCClient(
       scope,
       props.client,
       props.matrixRoom,
       {
-        microphoneEnabled: props.muteStates.audio.enabled$.value,
-        cameraEnabled: props.muteStates.video.enabled$.value,
-        audioInputDeviceId$: selectedDeviceId$(scope, mediaDevices.audioInput),
-        videoInputDeviceId$: selectedDeviceId$(scope, mediaDevices.videoInput),
+        publish: initialPublishRequests(
+          props.muteStates,
+          mediaDevices,
+          capture,
+        ),
         // A host that routes audio itself leaves the browser's output alone
         audioOutputDeviceId$: urlParams.controlledAudioDevices
           ? constant(undefined)
           : selectedDeviceId$(scope, mediaDevices.audioOutput),
-        videoProcessor$: scope.behavior(
-          trackProcessorState$.pipe(map((state) => state.processor)),
-        ),
       },
       matrixRTCClientOptions(urlParams, props.e2eeSystem),
     );
@@ -188,6 +189,10 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
       {
         ...callViewModelOptionsFromParams(urlParams),
         encryptionSystem: props.e2eeSystem,
+        capture,
+        videoProcessor$: scope.behavior(
+          trackProcessorState$.pipe(map((state) => state.processor)),
+        ),
         hostBridge,
         autoLeaveWhenOthersLeft,
         waitForCallPickup: waitForCallPickup && sendNotificationType === "ring",
@@ -748,16 +753,12 @@ export function selectedDeviceId$(
 }
 
 /**
- * What the client needs from Element Call's configuration, settings and URL:
- * the SDK reads none of them itself. Shared with the legacy SDK bundle so that
- * a widget built from it behaves like the app.
+ * How the tracks are captured and encoded, from Element Call's settings and
+ * configuration. Shared with the legacy SDK bundle so that a widget built from
+ * it behaves like the app.
  */
-export function matrixRTCClientOptions(
-  urlParams: ReturnType<typeof useUrlParams>,
-  encryptionSystem: EncryptionSystem,
-): MatrixRTCClientOptions {
+export function captureSettings(): CaptureSettings {
   const config = Config.get();
-  const session = config.matrix_rtc_session;
   const cameraSettings = advancedCamera.getValue()
     ? {
         resolution: {
@@ -789,6 +790,28 @@ export function matrixRTCClientOptions(
           },
         }
       : undefined;
+  return {
+    audio: {
+      echoCancellation: echoCancellationSetting.getValue(),
+      noiseSuppression: noiseSuppressionSetting.getValue(),
+      autoGainControl: autoGainControlSetting.getValue(),
+    },
+    camera: cameraSettings,
+    screenShare: screenShareSettings,
+  };
+}
+
+/**
+ * What the client needs from Element Call's configuration, settings and URL:
+ * the SDK reads none of them itself. Shared with the legacy SDK bundle so that
+ * a widget built from it behaves like the app.
+ */
+export function matrixRTCClientOptions(
+  urlParams: ReturnType<typeof useUrlParams>,
+  encryptionSystem: EncryptionSystem,
+): MatrixRTCClientOptions {
+  const config = Config.get();
+  const session = config.matrix_rtc_session;
   return {
     encryptionSystem,
     // matrix_rtc_mode in config.json overrides the user's Developer Settings
@@ -811,15 +834,6 @@ export function matrixRTCClientOptions(
       delegatedDelayedLeave: session.delegated_delayed_leave,
     },
     mediaQuality: config.media_quality,
-    capture: {
-      audio: {
-        echoCancellation: echoCancellationSetting.getValue(),
-        noiseSuppression: noiseSuppressionSetting.getValue(),
-        autoGainControl: autoGainControlSetting.getValue(),
-      },
-      camera: cameraSettings,
-      screenShare: screenShareSettings,
-    },
     transportUrl: customLivekitUrl.value$.value ?? undefined,
     fallbackTransportUrl: config.livekit?.livekit_service_url,
   };

src/state/CallViewModel/CallViewModel.ts

@@ -7,19 +7,27 @@ Please see LICENSE in the repository root for full details.
 */
 
 import {
+  type AudioCaptureSettings,
   type Behavior,
   constant,
   type DisconnectReason,
   type EncryptionSystem,
   generateItems,
+  type LocalAudioMediaTrack,
+  type LocalMediaTrack,
   type LocalRTCMember,
+  type LocalVideoMediaTrack,
   type MatrixRTCClient,
   type ObservableScope,
   pauseWhen,
+  type PublishRequest,
   type RemoteRTCMember,
   type RTCMember,
+  trackBySource$,
   type TransportMetadata,
+  type VideoCaptureSettings,
 } from "@element-hq/matrixrtc-sdk";
+import { type Track, type TrackProcessor } from "livekit-client";
 import {
   KnownMembership,
   type Room as MatrixRoom,
@@ -73,8 +81,8 @@ import {
 } from "../../reactions";
 import { shallowEquals as shallowArrayEquals } from "../../utils/array";
 import { shallowEquals as shallowObjectEquals } from "../../utils/object";
-import { type MediaDevices } from "../MediaDevices";
-import { type MuteStates } from "../MuteStates";
+import { type MediaDevice, type MediaDevices } from "../MediaDevices";
+import { type Handler, type MuteStates } from "../MuteStates";
 import { HeaderStyle, type UrlParams } from "../../UrlParams";
 import { type HostBridge, nullHostBridge } from "../../HostBridge";
 import {
@@ -121,8 +129,59 @@ import {
 import { type GridTileViewModel } from "../TileViewModel.ts";
 import { type TransportLocator } from "../../livekit/auth";
 
+/** How a call captures and encodes the tracks it publishes. */
+export interface CaptureSettings {
+  audio?: AudioCaptureSettings;
+  camera?: VideoCaptureSettings;
+  screenShare?: VideoCaptureSettings;
+}
+
+/**
+ * What to publish for a microphone or camera: the device the user picked and
+ * the host's capture settings. The video processor is not in here; the call
+ * attaches it to the camera track once that exists.
+ */
+export function publishRequest(
+  source: "microphone" | "camera",
+  mediaDevices: MediaDevices,
+  capture: CaptureSettings,
+): PublishRequest {
+  return source === "microphone"
+    ? {
+        source,
+        deviceId: mediaDevices.audioInput.selected$.value?.id,
+        capture: capture.audio,
+      }
+    : {
+        source,
+        deviceId: mediaDevices.videoInput.selected$.value?.id,
+        capture: capture.camera,
+      };
+}
+
+/** What the client publishes at the join: the sources whose mute switch is on. */
+export function initialPublishRequests(
+  muteStates: MuteStates,
+  mediaDevices: MediaDevices,
+  capture: CaptureSettings,
+): PublishRequest[] {
+  const requests: PublishRequest[] = [];
+  if (muteStates.audio.enabled$.value)
+    requests.push(publishRequest("microphone", mediaDevices, capture));
+  if (muteStates.video.enabled$.value)
+    requests.push(publishRequest("camera", mediaDevices, capture));
+  return requests;
+}
+
 export interface CallViewModelOptions {
   encryptionSystem: EncryptionSystem;
+  /**
+   * How our tracks are captured and encoded; the browser's and LiveKit's
+   * defaults otherwise.
+   */
+  capture?: CaptureSettings;
+  /** Background blur and the like, applied to the camera as it changes. */
+  videoProcessor$?: Behavior<TrackProcessor<Track.Kind.Video> | undefined>;
   /**
    * The application hosting Element Call, which can ask it to hang up and wants
    * to know when the user joins or leaves. Defaults to no host.
@@ -420,15 +479,107 @@ export function createCallViewModel$(
     sentCallNotification$ = of(null),
   } = options;
 
-  // The mute switches drive the client, which reports back what the device
-  // allowed, so a denied permission flips the switch back
-  muteStates.audio.setHandler(rtcClient.setMicrophoneEnabled);
-  muteStates.video.setHandler(rtcClient.setCameraEnabled);
+  const { capture = {}, videoProcessor$ = constant(undefined) } = options;
+  const localTracks$ = scope.behavior<
+    (LocalAudioMediaTrack | LocalVideoMediaTrack)[]
+  >(
+    rtcClient.localMember$.pipe(
+      switchMap((member) => member?.media$ ?? of(null)),
+      switchMap((media) => media?.tracks$ ?? of([])),
+    ),
+  );
+  // Our own tracks, which carry the controls remote ones lack
+  const microphone$ = trackBySource$(
+    scope,
+    localTracks$,
+    "microphone",
+  ) as Behavior<LocalAudioMediaTrack | undefined>;
+  const camera$ = trackBySource$(scope, localTracks$, "camera") as Behavior<
+    LocalVideoMediaTrack | undefined
+  >;
+  const screenShare$ = trackBySource$(
+    scope,
+    localTracks$,
+    "screenShare",
+  ) as Behavior<LocalVideoMediaTrack | undefined>;
+
+  // The mute switches drive our tracks, which report back what the device
+  // allowed, so a denied permission flips the switch back. A source is muted
+  // and unmuted while its track exists and published when it does not.
+  const followMuteState = (
+    muteState: {
+      setHandler: (handler: Handler) => void;
+      enabled$: Behavior<boolean>;
+    },
+    track$: Behavior<LocalMediaTrack | undefined>,
+    source: "microphone" | "camera",
+  ): void => {
+    // What the switch asks for, ahead of what it shows: the switch follows
+    // only once the handler has resolved
+    let desired = muteState.enabled$.value;
+    muteState.setHandler(async (enabled) => {
+      desired = enabled;
+      const track = track$.value;
+      if (track !== undefined) {
+        desired = await track.setEnabled(enabled);
+        return desired;
+      }
+      if (!enabled) return false;
+      const member = rtcClient.localMember$.value;
+      if (member === null) return false;
+      try {
+        await member.publish(publishRequest(source, mediaDevices, capture));
+        return true;
+      } catch (e) {
+        logger.error(`Failed to publish the ${source}`, e);
+        desired = false;
+        return false;
+      }
+    });
+    // A track that arrives after its switch was flipped off catches up with it
+    track$.pipe(scope.bind()).subscribe((track) => {
+      if (track !== undefined && !desired)
+        track.setEnabled(false).catch((e) => {
+          logger.error(`Failed to mute the ${source}`, e);
+        });
+    });
+  };
+  followMuteState(muteStates.audio, microphone$, "microphone");
+  followMuteState(muteStates.video, camera$, "camera");
   scope.onEnd(() => {
     muteStates.audio.unsetHandler();
     muteStates.video.unsetHandler();
   });
 
+  // The device the user picked follows onto the track as it is made and as the
+  // choice changes; the SDK does not watch the settings itself
+  const followDevice = (
+    track$: Behavior<LocalMediaTrack | undefined>,
+    device: MediaDevice<unknown, { id: string }>,
+    source: "microphone" | "camera",
+  ): void => {
+    combineLatest([track$, device.selected$])
+      .pipe(scope.bind())
+      .subscribe(([track, selected]) => {
+        if (track === undefined || selected === undefined) return;
+        track.setDevice(selected.id).catch((e) => {
+          logger.error(`Failed to switch the ${source} device`, e);
+        });
+      });
+  };
+  followDevice(microphone$, mediaDevices.audioInput, "microphone");
+  followDevice(camera$, mediaDevices.videoInput, "camera");
+
+  // Attached once the camera track exists rather than carried in the publish
+  // request, so that a new track and a change of processor take the same path
+  combineLatest([camera$, videoProcessor$])
+    .pipe(scope.bind())
+    .subscribe(([camera, processor]) => {
+      camera?.setProcessor(processor).catch((e) => {
+        logger.error("Failed to set the video processor", e);
+      });
+    });
+
   const members$ = scope.behavior<RTCMember[]>(
     combineLatest(
       [rtcClient.localMember$, rtcClient.remoteMembers$],
@@ -1286,14 +1437,13 @@ export function createCallViewModel$(
   );
 
   /**
-   * Whether we are sharing our screen.
+   * Whether we are sharing our screen: one of our tracks is a screen share.
    */
   const sharingScreen$ = scope.behavior(
-    rtcClient.localMember$.pipe(
-      switchMap((member) => member?.sharingScreen$ ?? of(false)),
-    ),
+    screenShare$.pipe(map((track) => track !== undefined)),
   );
 
+  const screenShareError$ = new BehaviorSubject<Error | null>(null);
   /**
    * Callback to toggle screen sharing. If null, screen sharing is not possible:
    * the platform cannot capture a screen, or the host hides the button.
@@ -1302,13 +1452,28 @@ export function createCallViewModel$(
     options.toggleScreensharing ??
     (hideScreensharing || !("getDisplayMedia" in (navigator.mediaDevices ?? {}))
       ? null
-      : (): void => rtcClient.localMember$.value?.toggleScreenSharing?.());
-
-  const screenShareError$ = scope.behavior(
-    rtcClient.localMember$.pipe(
-      switchMap((member) => member?.screenShareError$ ?? of(null)),
-    ),
-  );
+      : (): void => {
+          const member = rtcClient.localMember$.value;
+          if (member === null) return;
+          const track = screenShare$.value;
+          (track === undefined
+            ? member.publish({
+                source: "screenShare",
+                capture: capture.screenShare,
+              })
+            : member.unpublish(track.id)
+          ).catch((e: unknown) => {
+            logger.error(
+              `Screen share ${track === undefined ? "start" : "stop"} failed`,
+              e,
+            );
+            // The user closing the picker is not an error worth showing
+            if (isPermissionDenied(e)) return;
+            screenShareError$.next(
+              e instanceof Error ? e : new Error(String(e)),
+            );
+          });
+        });
 
   // Tell the host and the analytics about the user's joins and leaves
   const join = (): void => {
@@ -1404,11 +1569,16 @@ export function createCallViewModel$(
     reconnecting$: rtcClient.reconnecting$,
     connected$: rtcClient.connected$,
     screenShareError$,
-    dismissScreenShareError: (): void =>
-      rtcClient.localMember$.value?.dismissScreenShareError(),
+    dismissScreenShareError: (): void => screenShareError$.next(null),
   };
 }
 
+/** Whether an error, or the browser error the SDK wrapped, is the user saying no. */
+function isPermissionDenied(e: unknown): boolean {
+  const cause = e instanceof Error && e.cause !== undefined ? e.cause : e;
+  return cause instanceof DOMException && cause.name === "NotAllowedError";
+}
+
 /** The members a call can be with: those in the room and those invited. */
 function roomMembers(room: MatrixRoom): RoomMember[] {
   return room

src/state/CallViewModel/CallViewModelTestUtils.ts

@@ -34,6 +34,7 @@ import {
   mockMatrixRoom,
   mockMatrixRoomMember,
   mockMatrixRTCClient,
+  mockAudioTrack,
   mockMediaDevices,
   mockVideoTrack,
   mockMemberMedia,
@@ -154,26 +155,34 @@ export function withCallViewModel(mode: MatrixRTCMode) {
       scope: ObservableScope,
       participant: Participant,
       local: boolean,
-    ): LocalMemberMedia =>
-      mockMemberMedia({
-        local,
-        speaking$: speaking.get(participant) ?? constant(false),
-        screenShareEnabled$: sharingScreen.get(participant) ?? constant(false),
-        camera$: scope.behavior(
+    ): LocalMemberMedia => {
+      // A microphone and a camera are always published; the screen share
+      // comes and goes
+      const microphone = mockAudioTrack({
+        isActive$: speaking.get(participant) ?? constant(false),
+      });
+      const camera = mockVideoTrack({
+        source: "camera",
+        muted$: scope.behavior(
           (videoEnabled.get(participant) ?? constant(false)).pipe(
-            map((enabled) =>
-              mockVideoTrack({ source: "camera", muted$: constant(!enabled) }),
-            ),
+            map((enabled) => !enabled),
           ),
         ),
-        screenShare$: scope.behavior(
+      });
+      const screenShare = mockVideoTrack({ source: "screenShare" });
+      return mockMemberMedia({
+        local,
+        tracks$: scope.behavior(
           (sharingScreen.get(participant) ?? constant(false)).pipe(
             map((sharing) =>
-              sharing ? mockVideoTrack({ source: "screenShare" }) : undefined,
+              sharing
+                ? [microphone, camera, screenShare]
+                : [microphone, camera],
             ),
           ),
         ),
       });
+    };
     const participantOf = (
       scope: ObservableScope,
       membership: CallMembership,

src/state/media/LocalUserMediaViewModel.ts

@@ -9,6 +9,7 @@ Please see LICENSE in the repository root for full details.
 import {
   type Behavior,
   type LocalMemberMedia,
+  type LocalVideoMediaTrack,
   type ObservableScope,
 } from "@element-hq/matrixrtc-sdk";
 import { map, of, switchMap } from "rxjs";
@@ -22,6 +23,7 @@ import {
 import { alwaysShowSelf } from "../../settings/settings";
 import { platform } from "../../Platform";
 import { type MediaDevices } from "../MediaDevices";
+import { memberTrack$ } from "./MemberMediaViewModel";
 
 export interface LocalUserMediaViewModel extends BaseUserMediaViewModel {
   local: true;
@@ -51,10 +53,14 @@ export function createLocalUserMedia(
   { mediaDevices, ...inputs }: LocalUserMediaInputs,
 ): LocalUserMediaViewModel {
   const baseUserMedia = createBaseUserMedia(scope, inputs);
+  // Our own camera track, which carries the controls a remote one lacks
+  const camera$ = memberTrack$<LocalVideoMediaTrack>(
+    scope,
+    inputs.media$,
+    "camera",
+  );
   const facingMode$ = scope.behavior(
-    baseUserMedia.video$.pipe(
-      switchMap((video) => video?.facingMode$ ?? of(undefined)),
-    ),
+    camera$.pipe(switchMap((camera) => camera?.facingMode$ ?? of(undefined))),
   );
 
   return {
@@ -76,8 +82,8 @@ export function createLocalUserMedia(
               facingMode === undefined
                 ? null
                 : (): void =>
-                    void inputs.media$.value
-                      ?.switchCamera()
+                    void camera$.value
+                      ?.switchFacingMode()
                       .then((deviceId) => {
                         // Inform the MediaDevices which camera was chosen
                         if (deviceId !== undefined)

src/state/media/MediaViewModel.test.ts

@@ -42,7 +42,7 @@ test("control a participant's volume", () => {
     rtcMembership,
     {},
     mockMemberMedia({
-      microphone$: constant(mockAudioTrack({ setVolume })),
+      tracks$: constant([mockAudioTrack({ setVolume })]),
     }),
   );
   withTestScheduler(({ expectObservable, schedule }) => {
@@ -93,9 +93,9 @@ test("control a participant's screen share volume", () => {
     rtcMembership,
     {},
     mockMemberMedia({
-      screenShareAudio$: constant(
+      tracks$: constant([
         mockAudioTrack({ source: "screenShareAudio", setVolume }),
-      ),
+      ]),
     }),
   );
   withTestScheduler(({ expectObservable, schedule }) => {
@@ -169,7 +169,7 @@ test("switch cameras", async () => {
   );
   // The SDK restarts the camera the other way round and says which device it
   // ended up on
-  const switchCamera = vi.fn(async (): Promise<string> => {
+  const switchFacingMode = vi.fn(async (): Promise<string> => {
     const back = facingMode$.value === "user";
     facingMode$.next(back ? "environment" : "user");
     return Promise.resolve(back ? "back camera" : "front camera");
@@ -180,8 +180,7 @@ test("switch cameras", async () => {
     {},
     mockMemberMedia({
       local: true,
-      camera$: constant(mockVideoTrack({ facingMode$ })),
-      switchCamera,
+      tracks$: constant([mockVideoTrack({ facingMode$, switchFacingMode })]),
     }),
     mockMediaDevices({
       videoInput: {
@@ -195,7 +194,7 @@ test("switch cameras", async () => {
 
   // Switch to back camera
   vm.switchCamera$.value!();
-  expect(switchCamera).toHaveBeenCalledTimes(1);
+  expect(switchFacingMode).toHaveBeenCalledTimes(1);
   await waitFor(() => {
     expect(selectVideoInput).toHaveBeenCalledWith("back camera");
   });
@@ -203,7 +202,7 @@ test("switch cameras", async () => {
 
   // Switch to front camera
   vm.switchCamera$.value!();
-  expect(switchCamera).toHaveBeenCalledTimes(2);
+  expect(switchFacingMode).toHaveBeenCalledTimes(2);
   await waitFor(() => {
     expect(selectVideoInput).toHaveBeenLastCalledWith("front camera");
   });
@@ -216,7 +215,7 @@ test("no camera switch where the facing mode is unknown", () => {
   const vm = mockLocalMedia(
     rtcMembership,
     {},
-    mockMemberMedia({ local: true, camera$: constant(mockVideoTrack()) }),
+    mockMemberMedia({ local: true, tracks$: constant([mockVideoTrack()]) }),
     mockMediaDevices({}),
   );
   expect(vm.switchCamera$.value).toBeNull();
@@ -242,8 +241,10 @@ test("audio and video follow the tracks' mute state", () => {
     rtcMembership,
     {},
     mockMemberMedia({
-      microphone$: constant(mockAudioTrack({ muted$: constant(false) })),
-      camera$: constant(mockVideoTrack({ muted$: constant(true) })),
+      tracks$: constant([
+        mockAudioTrack({ muted$: constant(false) }),
+        mockVideoTrack({ muted$: constant(true) }),
+      ]),
     }),
   );
   expect(vm.audioEnabled$.value).toBe(true);
@@ -259,13 +260,13 @@ test("user media polls stream stats only while the setting is on", () => {
       rtcMembership,
       {},
       mockMemberMedia({
-        microphone$: constant(
+        tracks$: constant([
           mockAudioTrack({
             stats$: cold("-s", {
               s: stats,
             }) as unknown as Behavior<MediaStreamStats>,
           }),
-        ),
+        ]),
       }),
     );
     schedule("-a-b", {

src/state/media/MemberMediaViewModel.ts

@@ -12,9 +12,11 @@ import {
   E2eeType,
   type EncryptionError,
   type EncryptionSystem,
+  type MediaSource,
   type MediaTrack,
   type MemberMedia,
   type ObservableScope,
+  trackBySource$,
   type VideoMediaTrack,
 } from "@element-hq/matrixrtc-sdk";
 import {
@@ -71,18 +73,19 @@ export interface MemberMediaInputs extends BaseMediaViewModel {
   encryptionSystem: EncryptionSystem;
 }
 
-/** The member's track for a source, undefined while there is none. */
+/**
+ * The member's first track of a source: `trackBySource$` over media that may
+ * not have arrived yet. `T` narrows to a local track where the media is ours.
+ */
 export function memberTrack$<T extends MediaTrack>(
+  scope: ObservableScope,
   media$: Behavior<MemberMedia | null>,
-  source: "microphone" | "camera" | "screenShare" | "screenShareAudio",
-): Observable<T | undefined> {
-  return media$.pipe(
-    switchMap((media) =>
-      media === null
-        ? of(undefined)
-        : (media[`${source}$`] as Behavior<T | undefined>),
-    ),
+  source: MediaSource,
+): Behavior<T | undefined> {
+  const tracks$ = scope.behavior<MediaTrack[]>(
+    media$.pipe(switchMap((media) => media?.tracks$ ?? of([]))),
   );
+  return trackBySource$(scope, tracks$, source) as Behavior<T | undefined>;
 }
 
 export function createMemberMedia(
@@ -96,12 +99,8 @@ export function createMemberMedia(
     ...inputs
   }: MemberMediaInputs,
 ): BaseMemberMediaViewModel {
-  const audio$ = scope.behavior(
-    memberTrack$<AudioMediaTrack>(media$, audioSource),
-  );
-  const video$ = scope.behavior(
-    memberTrack$<VideoMediaTrack>(media$, videoSource),
-  );
+  const audio$ = memberTrack$<AudioMediaTrack>(scope, media$, audioSource);
+  const video$ = memberTrack$<VideoMediaTrack>(scope, media$, videoSource);
   const unencrypted$ = (
     track$: Behavior<MediaTrack | undefined>,
   ): Observable<boolean> =>

src/state/media/RemoteScreenShareViewModel.ts

@@ -42,8 +42,10 @@ export function createRemoteScreenShare(
   scope: ObservableScope,
   { pretendToBeDisconnected$, ...inputs }: RemoteScreenShareInputs,
 ): RemoteScreenShareViewModel {
-  const audio$ = scope.behavior(
-    memberTrack$<AudioMediaTrack>(inputs.media$, "screenShareAudio"),
+  const audio$ = memberTrack$<AudioMediaTrack>(
+    scope,
+    inputs.media$,
+    "screenShareAudio",
   );
   return {
     ...createBaseScreenShare(scope, inputs),

src/state/media/RemoteUserMediaViewModel.ts

@@ -60,7 +60,7 @@ export function createRemoteUserMedia(
     ...createVolumeControls(scope, {
       pretendToBeDisconnected$,
       sink$: scope.behavior(
-        memberTrack$<AudioMediaTrack>(inputs.media$, "microphone").pipe(
+        memberTrack$<AudioMediaTrack>(scope, inputs.media$, "microphone").pipe(
           map((track) => (volume: number) => track?.setVolume(volume)),
         ),
       ),

src/state/media/UserMediaViewModel.ts

@@ -7,6 +7,7 @@ Please see LICENSE in the repository root for full details.
 */
 
 import {
+  type AudioMediaTrack,
   type Behavior,
   type MediaStreamStats,
   type MediaTrack,
@@ -86,7 +87,7 @@ export function createBaseUserMedia(
     source: "microphone" | "camera",
   ): Behavior<boolean> =>
     scope.behavior(
-      memberTrack$<MediaTrack>(media$, source).pipe(
+      memberTrack$<MediaTrack>(scope, media$, source).pipe(
         switchMap((track) =>
           track === undefined
             ? of(false)
@@ -101,7 +102,7 @@ export function createBaseUserMedia(
   ): Behavior<MediaStreamStats> =>
     scope.behavior(
       combineLatest([
-        memberTrack$<MediaTrack>(media$, source),
+        memberTrack$<MediaTrack>(scope, media$, source),
         showConnectionStats.value$,
       ]).pipe(
         switchMap(([track, show]) =>
@@ -117,9 +118,10 @@ export function createBaseUserMedia(
       videoSource: "camera",
     }),
     type: "user",
+    // What a call calls speaking is the microphone carrying sound
     speaking$: scope.behavior(
-      media$.pipe(
-        switchMap((media) => (media === null ? of(false) : media.speaking$)),
+      memberTrack$<AudioMediaTrack>(scope, media$, "microphone").pipe(
+        switchMap((track) => track?.isActive$ ?? of(false)),
       ),
     ),
     audioEnabled$: enabled$(scope, "microphone"),

src/state/media/WrappedUserMediaViewModel.ts

@@ -12,7 +12,7 @@ import {
   type MemberMedia,
   type ObservableScope,
 } from "@element-hq/matrixrtc-sdk";
-import { combineLatest, map, of, switchMap } from "rxjs";
+import { combineLatest, distinctUntilChanged, map, of } from "rxjs";
 
 import { observeSpeaker$ } from "./observeSpeaker.ts";
 import { type UserMediaViewModel } from "./UserMediaViewModel.ts";
@@ -27,6 +27,7 @@ import {
 } from "./RemoteUserMediaViewModel.ts";
 import { createLocalScreenShare } from "./LocalScreenShareViewModel.ts";
 import { createRemoteScreenShare } from "./RemoteScreenShareViewModel.ts";
+import { memberTrack$ } from "./MemberMediaViewModel.ts";
 
 /**
  * Sorting bins defining the order in which media tiles appear in the layout.
@@ -107,10 +108,9 @@ export function createWrappedUserMedia(
   const media$: Behavior<MemberMedia | null> = _media$;
 
   const screenShares$ = scope.behavior(
-    media$.pipe(
-      switchMap((media) =>
-        media === null ? of(false) : media.screenShareEnabled$,
-      ),
+    memberTrack$(scope, media$, "screenShare").pipe(
+      map((track) => track !== undefined),
+      distinctUntilChanged(),
       // Technically more than one screen share might be possible... our
       // MediaViewModels don't support it though since they look for a unique
       // track for the given source. So generateItems here is a bit overkill.

src/tracks/MemberAudioRenderer.tsx

@@ -8,6 +8,7 @@ Please see LICENSE in the repository root for full details.
 
 import {
   type AudioMediaTrack,
+  type MediaSource,
   type MemberMedia,
   type RemoteRTCMember,
 } from "@element-hq/matrixrtc-sdk";
@@ -123,33 +124,37 @@ const MemberAudio: FC<{
   const media = useBehavior(media$);
   return (
     media !== null && (
-      <>
-        <AudioTrack
-          track$={media.microphone$}
-          muted={muted}
-          plugins={plugins}
-        />
-        <AudioTrack
-          track$={media.screenShareAudio$}
-          muted={muted}
-          plugins={plugins}
-        />
-      </>
+      <MemberTracksAudio
+        tracks$={media.tracks$}
+        muted={muted}
+        plugins={plugins}
+      />
     )
   );
 };
 
-const AudioTrack: FC<{
-  track$: MemberMedia["microphone$"];
+/** The sources a call plays; anything published without one stays silent. */
+const playedSources = new Set<MediaSource>(["microphone", "screenShareAudio"]);
+
+const MemberTracksAudio: FC<{
+  tracks$: MemberMedia["tracks$"];
   muted?: boolean;
   plugins: AudioPlugins;
-}> = ({ track$, muted, plugins }) => {
-  const track = useBehavior(track$);
-  return (
-    track !== undefined && (
-      <TrackAudio track={track} muted={muted} plugins={plugins} />
+}> = ({ tracks$, muted, plugins }) => {
+  const tracks = useBehavior(tracks$);
+  return tracks
+    .filter(
+      (track): track is AudioMediaTrack =>
+        track.kind === "audio" && playedSources.has(track.source),
     )
-  );
+    .map((track) => (
+      <TrackAudio
+        key={track.id}
+        track={track}
+        muted={muted}
+        plugins={plugins}
+      />
+    ));
 };
 
 /**

src/utils/test.ts

@@ -59,18 +59,19 @@ import {
 import { Config } from "../config/Config";
 import { type MediaDevices } from "../state/MediaDevices";
 import {
-  type AudioMediaTrack,
   type Behavior,
   constant,
   type DisconnectReason,
+  type LocalAudioMediaTrack,
   type LocalMemberMedia,
   type LocalRTCMember,
+  type LocalVideoMediaTrack,
   type MatrixRTCClient,
   type MatrixRTCError,
   type MemberMedia,
+  type PublishRequest,
   type RemoteRTCMember,
   type TransportMetadata,
-  type VideoMediaTrack,
 } from "@element-hq/matrixrtc-sdk";
 import { ObservableScope } from "../state/ObservableScope";
 import { MuteStates } from "../state/MuteStates";
@@ -291,9 +292,12 @@ export function mockMatrixRoom(room: Partial<MatrixRoom>): MatrixRoom {
   return { ...mockEmitter(), ...room } as Partial<MatrixRoom> as MatrixRoom;
 }
 
-/** A track as the SDK hands it out, with spies where a test may look. */
-export type MockMediaTrack = Omit<AudioMediaTrack, "kind"> &
-  Omit<VideoMediaTrack, "kind"> & { kind: "audio" | "video" };
+/**
+ * A track as the SDK hands it out, with spies where a test may look. It
+ * carries the local controls too, so that one mock serves either side.
+ */
+export type MockMediaTrack = Omit<LocalAudioMediaTrack, "kind"> &
+  Omit<LocalVideoMediaTrack, "kind"> & { kind: "audio" | "video" };
 
 export function mockMediaTrack(
   track: Partial<MockMediaTrack> = {},
@@ -305,28 +309,33 @@ export function mockMediaTrack(
     muted$: constant(false),
     encrypted$: constant(true),
     stats$: constant(undefined),
+    isActive$: constant(false),
     attach: vi.fn(),
     detach: vi.fn(),
     setAudioContext: vi.fn(),
     setVolume: vi.fn(),
+    setEnabled: vi.fn(async (enabled: boolean) => Promise.resolve(enabled)),
+    setDevice: vi.fn(async () => Promise.resolve()),
     facingMode$: constant(undefined),
+    switchFacingMode: vi.fn(async () => Promise.resolve(undefined)),
+    setProcessor: vi.fn(async () => Promise.resolve()),
     ...track,
   };
 }
 
 export const mockVideoTrack = (
   track: Partial<MockMediaTrack> = {},
-): VideoMediaTrack =>
-  mockMediaTrack({ kind: "video", ...track }) as VideoMediaTrack;
+): LocalVideoMediaTrack =>
+  mockMediaTrack({ kind: "video", ...track }) as LocalVideoMediaTrack;
 
 export const mockAudioTrack = (
   track: Partial<MockMediaTrack> = {},
-): AudioMediaTrack =>
+): LocalAudioMediaTrack =>
   mockMediaTrack({
     kind: "audio",
     source: "microphone",
     ...track,
-  }) as AudioMediaTrack;
+  }) as LocalAudioMediaTrack;
 
 /** A member's media as the SDK hands it out, with nothing published unless given. */
 export function mockMemberMedia(
@@ -334,14 +343,8 @@ export function mockMemberMedia(
 ): LocalMemberMedia {
   return {
     local: false,
-    speaking$: constant(false),
-    screenShareEnabled$: constant(false),
-    microphone$: constant(undefined),
-    camera$: constant(undefined),
-    screenShare$: constant(undefined),
-    screenShareAudio$: constant(undefined),
+    tracks$: constant([]),
     encryptionError$: NEVER,
-    switchCamera: vi.fn().mockResolvedValue(undefined),
     ...media,
   } as LocalMemberMedia;
 }
@@ -463,10 +466,14 @@ export function mockRTCMember(
   return {
     ...base,
     local: true,
-    sharingScreen$: constant(false),
-    toggleScreenSharing: null,
-    screenShareError$: constant(null),
-    dismissScreenShareError: vi.fn(),
+    publish: vi.fn(async (request: PublishRequest) =>
+      Promise.resolve(
+        request.source === "microphone"
+          ? mockAudioTrack()
+          : mockVideoTrack({ source: request.source }),
+      ),
+    ),
+    unpublish: vi.fn(async () => Promise.resolve()),
   };
 }
 
@@ -504,12 +511,6 @@ export function mockMatrixRTCClient(
     connected$,
     reconnecting$,
     disconnectReason$,
-    setMicrophoneEnabled: vi.fn(async (enabled: boolean) =>
-      Promise.resolve(enabled),
-    ),
-    setCameraEnabled: vi.fn(async (enabled: boolean) =>
-      Promise.resolve(enabled),
-    ),
     fatalError$,
     localMember$,
     remoteMembers$,