From 52aaf84c3e447f552fe6c2032f214bd403588d3f Mon Sep 17 00:00:00 2001 From: "Timo K." Date: Sun, 4 Oct 2026 15:00:02 +0200 Subject: [PATCH] update EC based on sdk media refactor No more call specifics in the sdk layer --- sdk-target-based-on-call-view-model/main.ts | 14 +- src/room/InCallView.tsx | 62 ++++-- src/state/CallViewModel/CallViewModel.ts | 208 ++++++++++++++++-- .../CallViewModel/CallViewModelTestUtils.ts | 31 ++- src/state/media/LocalUserMediaViewModel.ts | 16 +- src/state/media/MediaViewModel.test.ts | 27 +-- src/state/media/MemberMediaViewModel.ts | 29 ++- src/state/media/RemoteScreenShareViewModel.ts | 6 +- src/state/media/RemoteUserMediaViewModel.ts | 2 +- src/state/media/UserMediaViewModel.ts | 10 +- src/state/media/WrappedUserMediaViewModel.ts | 10 +- src/tracks/MemberAudioRenderer.tsx | 45 ++-- src/utils/test.ts | 53 ++--- 13 files changed, 361 insertions(+), 152 deletions(-) diff --git a/sdk-target-based-on-call-view-model/main.ts b/sdk-target-based-on-call-view-model/main.ts index 26dad98ad..e0dc52a64 100644 --- a/sdk-target-based-on-call-view-model/main.ts +++ b/sdk-target-based-on-call-view-model/main.ts @@ -34,7 +34,6 @@ import { of, switchMap, } from "rxjs"; -import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { type Behavior, @@ -43,11 +42,13 @@ import { E2eeType, ObservableScope, type RTCMember, + type RTCMembership, } from "@element-hq/matrixrtc-sdk"; import { callViewModelOptionsFromParams, createCallViewModel$, + initialPublishRequests, } from "../src/state/CallViewModel/CallViewModel"; import { getUrlParams } from "../src/UrlParams"; import { MuteStates } from "../src/state/MuteStates"; @@ -57,6 +58,7 @@ import { initializeWidget } from "../src/widget"; import { createWidgetHostBridge } from "../src/HostBridge"; import { observeElementSize$ } from "../src/utils/elementSize"; import { + captureSettings, matrixRTCClientOptions, selectedDeviceId$, } from "../src/room/InCallView"; @@ -70,7 +72,7 @@ import { interface MatrixRTCSdkMember { /** @deprecated Always null: the SDK owns the transport connection. */ connection: null; - membership: CallMembership; + membership: RTCMembership; /** @deprecated Always null: the SDK owns the transport participant. */ participant: null; } @@ -140,19 +142,16 @@ export async function createMatrixRTCSdk( // rtc client: the session, the transport and the media, as in the app const encryptionSystem = { kind: E2eeType.PER_PARTICIPANT } as const; + const capture = captureSettings(); const rtcClient = createMatrixRTCClient( scope, client, room, { - microphoneEnabled: muteStates.audio.enabled$.value, - cameraEnabled: muteStates.video.enabled$.value, - audioInputDeviceId$: selectedDeviceId$(scope, mediaDevices.audioInput), - videoInputDeviceId$: selectedDeviceId$(scope, mediaDevices.videoInput), + publish: initialPublishRequests(muteStates, mediaDevices, capture), audioOutputDeviceId$: controlledAudioDevices ? constant(undefined) : selectedDeviceId$(scope, mediaDevices.audioOutput), - videoProcessor$: constant(undefined), }, { ...matrixRTCClientOptions(urlParams, encryptionSystem), @@ -171,6 +170,7 @@ export async function createMatrixRTCSdk( { ...callViewModelOptionsFromParams(urlParams), encryptionSystem, + capture, hostBridge, // The SDK owns its page, so the body is the space it has windowSize$: scope.behavior(observeElementSize$(document.body)), diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 761f93e71..b3f792459 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -42,7 +42,9 @@ import { InviteButton } from "../button/InviteButton"; import { type CallViewModel, callViewModelOptionsFromParams, + type CaptureSettings, createCallViewModel$, + initialPublishRequests, } from "../state/CallViewModel/CallViewModel.ts"; import { Grid, type TileProps } from "../grid/Grid"; import { SpotlightTile } from "../tile/SpotlightTile"; @@ -160,22 +162,21 @@ export const ActiveCall: FC = (props) => { const { autoLeaveWhenOthersLeft, waitForCallPickup, sendNotificationType } = urlParams; + const capture = captureSettings(); const rtcClient = createMatrixRTCClient( scope, props.client, props.matrixRoom, { - microphoneEnabled: props.muteStates.audio.enabled$.value, - cameraEnabled: props.muteStates.video.enabled$.value, - audioInputDeviceId$: selectedDeviceId$(scope, mediaDevices.audioInput), - videoInputDeviceId$: selectedDeviceId$(scope, mediaDevices.videoInput), + publish: initialPublishRequests( + props.muteStates, + mediaDevices, + capture, + ), // A host that routes audio itself leaves the browser's output alone audioOutputDeviceId$: urlParams.controlledAudioDevices ? constant(undefined) : selectedDeviceId$(scope, mediaDevices.audioOutput), - videoProcessor$: scope.behavior( - trackProcessorState$.pipe(map((state) => state.processor)), - ), }, matrixRTCClientOptions(urlParams, props.e2eeSystem), ); @@ -188,6 +189,10 @@ export const ActiveCall: FC = (props) => { { ...callViewModelOptionsFromParams(urlParams), encryptionSystem: props.e2eeSystem, + capture, + videoProcessor$: scope.behavior( + trackProcessorState$.pipe(map((state) => state.processor)), + ), hostBridge, autoLeaveWhenOthersLeft, waitForCallPickup: waitForCallPickup && sendNotificationType === "ring", @@ -748,16 +753,12 @@ export function selectedDeviceId$( } /** - * What the client needs from Element Call's configuration, settings and URL: - * the SDK reads none of them itself. Shared with the legacy SDK bundle so that - * a widget built from it behaves like the app. + * How the tracks are captured and encoded, from Element Call's settings and + * configuration. Shared with the legacy SDK bundle so that a widget built from + * it behaves like the app. */ -export function matrixRTCClientOptions( - urlParams: ReturnType, - encryptionSystem: EncryptionSystem, -): MatrixRTCClientOptions { +export function captureSettings(): CaptureSettings { const config = Config.get(); - const session = config.matrix_rtc_session; const cameraSettings = advancedCamera.getValue() ? { resolution: { @@ -789,6 +790,28 @@ export function matrixRTCClientOptions( }, } : undefined; + return { + audio: { + echoCancellation: echoCancellationSetting.getValue(), + noiseSuppression: noiseSuppressionSetting.getValue(), + autoGainControl: autoGainControlSetting.getValue(), + }, + camera: cameraSettings, + screenShare: screenShareSettings, + }; +} + +/** + * What the client needs from Element Call's configuration, settings and URL: + * the SDK reads none of them itself. Shared with the legacy SDK bundle so that + * a widget built from it behaves like the app. + */ +export function matrixRTCClientOptions( + urlParams: ReturnType, + encryptionSystem: EncryptionSystem, +): MatrixRTCClientOptions { + const config = Config.get(); + const session = config.matrix_rtc_session; return { encryptionSystem, // matrix_rtc_mode in config.json overrides the user's Developer Settings @@ -811,15 +834,6 @@ export function matrixRTCClientOptions( delegatedDelayedLeave: session.delegated_delayed_leave, }, mediaQuality: config.media_quality, - capture: { - audio: { - echoCancellation: echoCancellationSetting.getValue(), - noiseSuppression: noiseSuppressionSetting.getValue(), - autoGainControl: autoGainControlSetting.getValue(), - }, - camera: cameraSettings, - screenShare: screenShareSettings, - }, transportUrl: customLivekitUrl.value$.value ?? undefined, fallbackTransportUrl: config.livekit?.livekit_service_url, }; diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 53de80c08..0ba9ccb02 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -7,19 +7,27 @@ Please see LICENSE in the repository root for full details. */ import { + type AudioCaptureSettings, type Behavior, constant, type DisconnectReason, type EncryptionSystem, generateItems, + type LocalAudioMediaTrack, + type LocalMediaTrack, type LocalRTCMember, + type LocalVideoMediaTrack, type MatrixRTCClient, type ObservableScope, pauseWhen, + type PublishRequest, type RemoteRTCMember, type RTCMember, + trackBySource$, type TransportMetadata, + type VideoCaptureSettings, } from "@element-hq/matrixrtc-sdk"; +import { type Track, type TrackProcessor } from "livekit-client"; import { KnownMembership, type Room as MatrixRoom, @@ -73,8 +81,8 @@ import { } from "../../reactions"; import { shallowEquals as shallowArrayEquals } from "../../utils/array"; import { shallowEquals as shallowObjectEquals } from "../../utils/object"; -import { type MediaDevices } from "../MediaDevices"; -import { type MuteStates } from "../MuteStates"; +import { type MediaDevice, type MediaDevices } from "../MediaDevices"; +import { type Handler, type MuteStates } from "../MuteStates"; import { HeaderStyle, type UrlParams } from "../../UrlParams"; import { type HostBridge, nullHostBridge } from "../../HostBridge"; import { @@ -121,8 +129,59 @@ import { import { type GridTileViewModel } from "../TileViewModel.ts"; import { type TransportLocator } from "../../livekit/auth"; +/** How a call captures and encodes the tracks it publishes. */ +export interface CaptureSettings { + audio?: AudioCaptureSettings; + camera?: VideoCaptureSettings; + screenShare?: VideoCaptureSettings; +} + +/** + * What to publish for a microphone or camera: the device the user picked and + * the host's capture settings. The video processor is not in here; the call + * attaches it to the camera track once that exists. + */ +export function publishRequest( + source: "microphone" | "camera", + mediaDevices: MediaDevices, + capture: CaptureSettings, +): PublishRequest { + return source === "microphone" + ? { + source, + deviceId: mediaDevices.audioInput.selected$.value?.id, + capture: capture.audio, + } + : { + source, + deviceId: mediaDevices.videoInput.selected$.value?.id, + capture: capture.camera, + }; +} + +/** What the client publishes at the join: the sources whose mute switch is on. */ +export function initialPublishRequests( + muteStates: MuteStates, + mediaDevices: MediaDevices, + capture: CaptureSettings, +): PublishRequest[] { + const requests: PublishRequest[] = []; + if (muteStates.audio.enabled$.value) + requests.push(publishRequest("microphone", mediaDevices, capture)); + if (muteStates.video.enabled$.value) + requests.push(publishRequest("camera", mediaDevices, capture)); + return requests; +} + export interface CallViewModelOptions { encryptionSystem: EncryptionSystem; + /** + * How our tracks are captured and encoded; the browser's and LiveKit's + * defaults otherwise. + */ + capture?: CaptureSettings; + /** Background blur and the like, applied to the camera as it changes. */ + videoProcessor$?: Behavior | undefined>; /** * The application hosting Element Call, which can ask it to hang up and wants * to know when the user joins or leaves. Defaults to no host. @@ -420,15 +479,107 @@ export function createCallViewModel$( sentCallNotification$ = of(null), } = options; - // The mute switches drive the client, which reports back what the device - // allowed, so a denied permission flips the switch back - muteStates.audio.setHandler(rtcClient.setMicrophoneEnabled); - muteStates.video.setHandler(rtcClient.setCameraEnabled); + const { capture = {}, videoProcessor$ = constant(undefined) } = options; + const localTracks$ = scope.behavior< + (LocalAudioMediaTrack | LocalVideoMediaTrack)[] + >( + rtcClient.localMember$.pipe( + switchMap((member) => member?.media$ ?? of(null)), + switchMap((media) => media?.tracks$ ?? of([])), + ), + ); + // Our own tracks, which carry the controls remote ones lack + const microphone$ = trackBySource$( + scope, + localTracks$, + "microphone", + ) as Behavior; + const camera$ = trackBySource$(scope, localTracks$, "camera") as Behavior< + LocalVideoMediaTrack | undefined + >; + const screenShare$ = trackBySource$( + scope, + localTracks$, + "screenShare", + ) as Behavior; + + // The mute switches drive our tracks, which report back what the device + // allowed, so a denied permission flips the switch back. A source is muted + // and unmuted while its track exists and published when it does not. + const followMuteState = ( + muteState: { + setHandler: (handler: Handler) => void; + enabled$: Behavior; + }, + track$: Behavior, + source: "microphone" | "camera", + ): void => { + // What the switch asks for, ahead of what it shows: the switch follows + // only once the handler has resolved + let desired = muteState.enabled$.value; + muteState.setHandler(async (enabled) => { + desired = enabled; + const track = track$.value; + if (track !== undefined) { + desired = await track.setEnabled(enabled); + return desired; + } + if (!enabled) return false; + const member = rtcClient.localMember$.value; + if (member === null) return false; + try { + await member.publish(publishRequest(source, mediaDevices, capture)); + return true; + } catch (e) { + logger.error(`Failed to publish the ${source}`, e); + desired = false; + return false; + } + }); + // A track that arrives after its switch was flipped off catches up with it + track$.pipe(scope.bind()).subscribe((track) => { + if (track !== undefined && !desired) + track.setEnabled(false).catch((e) => { + logger.error(`Failed to mute the ${source}`, e); + }); + }); + }; + followMuteState(muteStates.audio, microphone$, "microphone"); + followMuteState(muteStates.video, camera$, "camera"); scope.onEnd(() => { muteStates.audio.unsetHandler(); muteStates.video.unsetHandler(); }); + // The device the user picked follows onto the track as it is made and as the + // choice changes; the SDK does not watch the settings itself + const followDevice = ( + track$: Behavior, + device: MediaDevice, + source: "microphone" | "camera", + ): void => { + combineLatest([track$, device.selected$]) + .pipe(scope.bind()) + .subscribe(([track, selected]) => { + if (track === undefined || selected === undefined) return; + track.setDevice(selected.id).catch((e) => { + logger.error(`Failed to switch the ${source} device`, e); + }); + }); + }; + followDevice(microphone$, mediaDevices.audioInput, "microphone"); + followDevice(camera$, mediaDevices.videoInput, "camera"); + + // Attached once the camera track exists rather than carried in the publish + // request, so that a new track and a change of processor take the same path + combineLatest([camera$, videoProcessor$]) + .pipe(scope.bind()) + .subscribe(([camera, processor]) => { + camera?.setProcessor(processor).catch((e) => { + logger.error("Failed to set the video processor", e); + }); + }); + const members$ = scope.behavior( combineLatest( [rtcClient.localMember$, rtcClient.remoteMembers$], @@ -1286,14 +1437,13 @@ export function createCallViewModel$( ); /** - * Whether we are sharing our screen. + * Whether we are sharing our screen: one of our tracks is a screen share. */ const sharingScreen$ = scope.behavior( - rtcClient.localMember$.pipe( - switchMap((member) => member?.sharingScreen$ ?? of(false)), - ), + screenShare$.pipe(map((track) => track !== undefined)), ); + const screenShareError$ = new BehaviorSubject(null); /** * Callback to toggle screen sharing. If null, screen sharing is not possible: * the platform cannot capture a screen, or the host hides the button. @@ -1302,13 +1452,28 @@ export function createCallViewModel$( options.toggleScreensharing ?? (hideScreensharing || !("getDisplayMedia" in (navigator.mediaDevices ?? {})) ? null - : (): void => rtcClient.localMember$.value?.toggleScreenSharing?.()); - - const screenShareError$ = scope.behavior( - rtcClient.localMember$.pipe( - switchMap((member) => member?.screenShareError$ ?? of(null)), - ), - ); + : (): void => { + const member = rtcClient.localMember$.value; + if (member === null) return; + const track = screenShare$.value; + (track === undefined + ? member.publish({ + source: "screenShare", + capture: capture.screenShare, + }) + : member.unpublish(track.id) + ).catch((e: unknown) => { + logger.error( + `Screen share ${track === undefined ? "start" : "stop"} failed`, + e, + ); + // The user closing the picker is not an error worth showing + if (isPermissionDenied(e)) return; + screenShareError$.next( + e instanceof Error ? e : new Error(String(e)), + ); + }); + }); // Tell the host and the analytics about the user's joins and leaves const join = (): void => { @@ -1404,11 +1569,16 @@ export function createCallViewModel$( reconnecting$: rtcClient.reconnecting$, connected$: rtcClient.connected$, screenShareError$, - dismissScreenShareError: (): void => - rtcClient.localMember$.value?.dismissScreenShareError(), + dismissScreenShareError: (): void => screenShareError$.next(null), }; } +/** Whether an error, or the browser error the SDK wrapped, is the user saying no. */ +function isPermissionDenied(e: unknown): boolean { + const cause = e instanceof Error && e.cause !== undefined ? e.cause : e; + return cause instanceof DOMException && cause.name === "NotAllowedError"; +} + /** The members a call can be with: those in the room and those invited. */ function roomMembers(room: MatrixRoom): RoomMember[] { return room diff --git a/src/state/CallViewModel/CallViewModelTestUtils.ts b/src/state/CallViewModel/CallViewModelTestUtils.ts index 8881c27a2..28778af2c 100644 --- a/src/state/CallViewModel/CallViewModelTestUtils.ts +++ b/src/state/CallViewModel/CallViewModelTestUtils.ts @@ -34,6 +34,7 @@ import { mockMatrixRoom, mockMatrixRoomMember, mockMatrixRTCClient, + mockAudioTrack, mockMediaDevices, mockVideoTrack, mockMemberMedia, @@ -154,26 +155,34 @@ export function withCallViewModel(mode: MatrixRTCMode) { scope: ObservableScope, participant: Participant, local: boolean, - ): LocalMemberMedia => - mockMemberMedia({ - local, - speaking$: speaking.get(participant) ?? constant(false), - screenShareEnabled$: sharingScreen.get(participant) ?? constant(false), - camera$: scope.behavior( + ): LocalMemberMedia => { + // A microphone and a camera are always published; the screen share + // comes and goes + const microphone = mockAudioTrack({ + isActive$: speaking.get(participant) ?? constant(false), + }); + const camera = mockVideoTrack({ + source: "camera", + muted$: scope.behavior( (videoEnabled.get(participant) ?? constant(false)).pipe( - map((enabled) => - mockVideoTrack({ source: "camera", muted$: constant(!enabled) }), - ), + map((enabled) => !enabled), ), ), - screenShare$: scope.behavior( + }); + const screenShare = mockVideoTrack({ source: "screenShare" }); + return mockMemberMedia({ + local, + tracks$: scope.behavior( (sharingScreen.get(participant) ?? constant(false)).pipe( map((sharing) => - sharing ? mockVideoTrack({ source: "screenShare" }) : undefined, + sharing + ? [microphone, camera, screenShare] + : [microphone, camera], ), ), ), }); + }; const participantOf = ( scope: ObservableScope, membership: CallMembership, diff --git a/src/state/media/LocalUserMediaViewModel.ts b/src/state/media/LocalUserMediaViewModel.ts index 289e091d3..3ec6be5e3 100644 --- a/src/state/media/LocalUserMediaViewModel.ts +++ b/src/state/media/LocalUserMediaViewModel.ts @@ -9,6 +9,7 @@ Please see LICENSE in the repository root for full details. import { type Behavior, type LocalMemberMedia, + type LocalVideoMediaTrack, type ObservableScope, } from "@element-hq/matrixrtc-sdk"; import { map, of, switchMap } from "rxjs"; @@ -22,6 +23,7 @@ import { import { alwaysShowSelf } from "../../settings/settings"; import { platform } from "../../Platform"; import { type MediaDevices } from "../MediaDevices"; +import { memberTrack$ } from "./MemberMediaViewModel"; export interface LocalUserMediaViewModel extends BaseUserMediaViewModel { local: true; @@ -51,10 +53,14 @@ export function createLocalUserMedia( { mediaDevices, ...inputs }: LocalUserMediaInputs, ): LocalUserMediaViewModel { const baseUserMedia = createBaseUserMedia(scope, inputs); + // Our own camera track, which carries the controls a remote one lacks + const camera$ = memberTrack$( + scope, + inputs.media$, + "camera", + ); const facingMode$ = scope.behavior( - baseUserMedia.video$.pipe( - switchMap((video) => video?.facingMode$ ?? of(undefined)), - ), + camera$.pipe(switchMap((camera) => camera?.facingMode$ ?? of(undefined))), ); return { @@ -76,8 +82,8 @@ export function createLocalUserMedia( facingMode === undefined ? null : (): void => - void inputs.media$.value - ?.switchCamera() + void camera$.value + ?.switchFacingMode() .then((deviceId) => { // Inform the MediaDevices which camera was chosen if (deviceId !== undefined) diff --git a/src/state/media/MediaViewModel.test.ts b/src/state/media/MediaViewModel.test.ts index eae61f378..067f484a3 100644 --- a/src/state/media/MediaViewModel.test.ts +++ b/src/state/media/MediaViewModel.test.ts @@ -42,7 +42,7 @@ test("control a participant's volume", () => { rtcMembership, {}, mockMemberMedia({ - microphone$: constant(mockAudioTrack({ setVolume })), + tracks$: constant([mockAudioTrack({ setVolume })]), }), ); withTestScheduler(({ expectObservable, schedule }) => { @@ -93,9 +93,9 @@ test("control a participant's screen share volume", () => { rtcMembership, {}, mockMemberMedia({ - screenShareAudio$: constant( + tracks$: constant([ mockAudioTrack({ source: "screenShareAudio", setVolume }), - ), + ]), }), ); withTestScheduler(({ expectObservable, schedule }) => { @@ -169,7 +169,7 @@ test("switch cameras", async () => { ); // The SDK restarts the camera the other way round and says which device it // ended up on - const switchCamera = vi.fn(async (): Promise => { + const switchFacingMode = vi.fn(async (): Promise => { const back = facingMode$.value === "user"; facingMode$.next(back ? "environment" : "user"); return Promise.resolve(back ? "back camera" : "front camera"); @@ -180,8 +180,7 @@ test("switch cameras", async () => { {}, mockMemberMedia({ local: true, - camera$: constant(mockVideoTrack({ facingMode$ })), - switchCamera, + tracks$: constant([mockVideoTrack({ facingMode$, switchFacingMode })]), }), mockMediaDevices({ videoInput: { @@ -195,7 +194,7 @@ test("switch cameras", async () => { // Switch to back camera vm.switchCamera$.value!(); - expect(switchCamera).toHaveBeenCalledTimes(1); + expect(switchFacingMode).toHaveBeenCalledTimes(1); await waitFor(() => { expect(selectVideoInput).toHaveBeenCalledWith("back camera"); }); @@ -203,7 +202,7 @@ test("switch cameras", async () => { // Switch to front camera vm.switchCamera$.value!(); - expect(switchCamera).toHaveBeenCalledTimes(2); + expect(switchFacingMode).toHaveBeenCalledTimes(2); await waitFor(() => { expect(selectVideoInput).toHaveBeenLastCalledWith("front camera"); }); @@ -216,7 +215,7 @@ test("no camera switch where the facing mode is unknown", () => { const vm = mockLocalMedia( rtcMembership, {}, - mockMemberMedia({ local: true, camera$: constant(mockVideoTrack()) }), + mockMemberMedia({ local: true, tracks$: constant([mockVideoTrack()]) }), mockMediaDevices({}), ); expect(vm.switchCamera$.value).toBeNull(); @@ -242,8 +241,10 @@ test("audio and video follow the tracks' mute state", () => { rtcMembership, {}, mockMemberMedia({ - microphone$: constant(mockAudioTrack({ muted$: constant(false) })), - camera$: constant(mockVideoTrack({ muted$: constant(true) })), + tracks$: constant([ + mockAudioTrack({ muted$: constant(false) }), + mockVideoTrack({ muted$: constant(true) }), + ]), }), ); expect(vm.audioEnabled$.value).toBe(true); @@ -259,13 +260,13 @@ test("user media polls stream stats only while the setting is on", () => { rtcMembership, {}, mockMemberMedia({ - microphone$: constant( + tracks$: constant([ mockAudioTrack({ stats$: cold("-s", { s: stats, }) as unknown as Behavior, }), - ), + ]), }), ); schedule("-a-b", { diff --git a/src/state/media/MemberMediaViewModel.ts b/src/state/media/MemberMediaViewModel.ts index cbf9a05cf..5e0210e75 100644 --- a/src/state/media/MemberMediaViewModel.ts +++ b/src/state/media/MemberMediaViewModel.ts @@ -12,9 +12,11 @@ import { E2eeType, type EncryptionError, type EncryptionSystem, + type MediaSource, type MediaTrack, type MemberMedia, type ObservableScope, + trackBySource$, type VideoMediaTrack, } from "@element-hq/matrixrtc-sdk"; import { @@ -71,18 +73,19 @@ export interface MemberMediaInputs extends BaseMediaViewModel { encryptionSystem: EncryptionSystem; } -/** The member's track for a source, undefined while there is none. */ +/** + * The member's first track of a source: `trackBySource$` over media that may + * not have arrived yet. `T` narrows to a local track where the media is ours. + */ export function memberTrack$( + scope: ObservableScope, media$: Behavior, - source: "microphone" | "camera" | "screenShare" | "screenShareAudio", -): Observable { - return media$.pipe( - switchMap((media) => - media === null - ? of(undefined) - : (media[`${source}$`] as Behavior), - ), + source: MediaSource, +): Behavior { + const tracks$ = scope.behavior( + media$.pipe(switchMap((media) => media?.tracks$ ?? of([]))), ); + return trackBySource$(scope, tracks$, source) as Behavior; } export function createMemberMedia( @@ -96,12 +99,8 @@ export function createMemberMedia( ...inputs }: MemberMediaInputs, ): BaseMemberMediaViewModel { - const audio$ = scope.behavior( - memberTrack$(media$, audioSource), - ); - const video$ = scope.behavior( - memberTrack$(media$, videoSource), - ); + const audio$ = memberTrack$(scope, media$, audioSource); + const video$ = memberTrack$(scope, media$, videoSource); const unencrypted$ = ( track$: Behavior, ): Observable => diff --git a/src/state/media/RemoteScreenShareViewModel.ts b/src/state/media/RemoteScreenShareViewModel.ts index b7b48e6a5..78e58b856 100644 --- a/src/state/media/RemoteScreenShareViewModel.ts +++ b/src/state/media/RemoteScreenShareViewModel.ts @@ -42,8 +42,10 @@ export function createRemoteScreenShare( scope: ObservableScope, { pretendToBeDisconnected$, ...inputs }: RemoteScreenShareInputs, ): RemoteScreenShareViewModel { - const audio$ = scope.behavior( - memberTrack$(inputs.media$, "screenShareAudio"), + const audio$ = memberTrack$( + scope, + inputs.media$, + "screenShareAudio", ); return { ...createBaseScreenShare(scope, inputs), diff --git a/src/state/media/RemoteUserMediaViewModel.ts b/src/state/media/RemoteUserMediaViewModel.ts index 71b215c83..239feef77 100644 --- a/src/state/media/RemoteUserMediaViewModel.ts +++ b/src/state/media/RemoteUserMediaViewModel.ts @@ -60,7 +60,7 @@ export function createRemoteUserMedia( ...createVolumeControls(scope, { pretendToBeDisconnected$, sink$: scope.behavior( - memberTrack$(inputs.media$, "microphone").pipe( + memberTrack$(scope, inputs.media$, "microphone").pipe( map((track) => (volume: number) => track?.setVolume(volume)), ), ), diff --git a/src/state/media/UserMediaViewModel.ts b/src/state/media/UserMediaViewModel.ts index 08c43a42a..a7743f9db 100644 --- a/src/state/media/UserMediaViewModel.ts +++ b/src/state/media/UserMediaViewModel.ts @@ -7,6 +7,7 @@ Please see LICENSE in the repository root for full details. */ import { + type AudioMediaTrack, type Behavior, type MediaStreamStats, type MediaTrack, @@ -86,7 +87,7 @@ export function createBaseUserMedia( source: "microphone" | "camera", ): Behavior => scope.behavior( - memberTrack$(media$, source).pipe( + memberTrack$(scope, media$, source).pipe( switchMap((track) => track === undefined ? of(false) @@ -101,7 +102,7 @@ export function createBaseUserMedia( ): Behavior => scope.behavior( combineLatest([ - memberTrack$(media$, source), + memberTrack$(scope, media$, source), showConnectionStats.value$, ]).pipe( switchMap(([track, show]) => @@ -117,9 +118,10 @@ export function createBaseUserMedia( videoSource: "camera", }), type: "user", + // What a call calls speaking is the microphone carrying sound speaking$: scope.behavior( - media$.pipe( - switchMap((media) => (media === null ? of(false) : media.speaking$)), + memberTrack$(scope, media$, "microphone").pipe( + switchMap((track) => track?.isActive$ ?? of(false)), ), ), audioEnabled$: enabled$(scope, "microphone"), diff --git a/src/state/media/WrappedUserMediaViewModel.ts b/src/state/media/WrappedUserMediaViewModel.ts index ed781ef3a..b67f1d0a9 100644 --- a/src/state/media/WrappedUserMediaViewModel.ts +++ b/src/state/media/WrappedUserMediaViewModel.ts @@ -12,7 +12,7 @@ import { type MemberMedia, type ObservableScope, } from "@element-hq/matrixrtc-sdk"; -import { combineLatest, map, of, switchMap } from "rxjs"; +import { combineLatest, distinctUntilChanged, map, of } from "rxjs"; import { observeSpeaker$ } from "./observeSpeaker.ts"; import { type UserMediaViewModel } from "./UserMediaViewModel.ts"; @@ -27,6 +27,7 @@ import { } from "./RemoteUserMediaViewModel.ts"; import { createLocalScreenShare } from "./LocalScreenShareViewModel.ts"; import { createRemoteScreenShare } from "./RemoteScreenShareViewModel.ts"; +import { memberTrack$ } from "./MemberMediaViewModel.ts"; /** * Sorting bins defining the order in which media tiles appear in the layout. @@ -107,10 +108,9 @@ export function createWrappedUserMedia( const media$: Behavior = _media$; const screenShares$ = scope.behavior( - media$.pipe( - switchMap((media) => - media === null ? of(false) : media.screenShareEnabled$, - ), + memberTrack$(scope, media$, "screenShare").pipe( + map((track) => track !== undefined), + distinctUntilChanged(), // Technically more than one screen share might be possible... our // MediaViewModels don't support it though since they look for a unique // track for the given source. So generateItems here is a bit overkill. diff --git a/src/tracks/MemberAudioRenderer.tsx b/src/tracks/MemberAudioRenderer.tsx index e67e4781b..0f079e0a6 100644 --- a/src/tracks/MemberAudioRenderer.tsx +++ b/src/tracks/MemberAudioRenderer.tsx @@ -8,6 +8,7 @@ Please see LICENSE in the repository root for full details. import { type AudioMediaTrack, + type MediaSource, type MemberMedia, type RemoteRTCMember, } from "@element-hq/matrixrtc-sdk"; @@ -123,33 +124,37 @@ const MemberAudio: FC<{ const media = useBehavior(media$); return ( media !== null && ( - <> - - - + ) ); }; -const AudioTrack: FC<{ - track$: MemberMedia["microphone$"]; +/** The sources a call plays; anything published without one stays silent. */ +const playedSources = new Set(["microphone", "screenShareAudio"]); + +const MemberTracksAudio: FC<{ + tracks$: MemberMedia["tracks$"]; muted?: boolean; plugins: AudioPlugins; -}> = ({ track$, muted, plugins }) => { - const track = useBehavior(track$); - return ( - track !== undefined && ( - +}> = ({ tracks$, muted, plugins }) => { + const tracks = useBehavior(tracks$); + return tracks + .filter( + (track): track is AudioMediaTrack => + track.kind === "audio" && playedSources.has(track.source), ) - ); + .map((track) => ( + + )); }; /** diff --git a/src/utils/test.ts b/src/utils/test.ts index 79355aa5b..aa8acbbcb 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -59,18 +59,19 @@ import { import { Config } from "../config/Config"; import { type MediaDevices } from "../state/MediaDevices"; import { - type AudioMediaTrack, type Behavior, constant, type DisconnectReason, + type LocalAudioMediaTrack, type LocalMemberMedia, type LocalRTCMember, + type LocalVideoMediaTrack, type MatrixRTCClient, type MatrixRTCError, type MemberMedia, + type PublishRequest, type RemoteRTCMember, type TransportMetadata, - type VideoMediaTrack, } from "@element-hq/matrixrtc-sdk"; import { ObservableScope } from "../state/ObservableScope"; import { MuteStates } from "../state/MuteStates"; @@ -291,9 +292,12 @@ export function mockMatrixRoom(room: Partial): MatrixRoom { return { ...mockEmitter(), ...room } as Partial as MatrixRoom; } -/** A track as the SDK hands it out, with spies where a test may look. */ -export type MockMediaTrack = Omit & - Omit & { kind: "audio" | "video" }; +/** + * A track as the SDK hands it out, with spies where a test may look. It + * carries the local controls too, so that one mock serves either side. + */ +export type MockMediaTrack = Omit & + Omit & { kind: "audio" | "video" }; export function mockMediaTrack( track: Partial = {}, @@ -305,28 +309,33 @@ export function mockMediaTrack( muted$: constant(false), encrypted$: constant(true), stats$: constant(undefined), + isActive$: constant(false), attach: vi.fn(), detach: vi.fn(), setAudioContext: vi.fn(), setVolume: vi.fn(), + setEnabled: vi.fn(async (enabled: boolean) => Promise.resolve(enabled)), + setDevice: vi.fn(async () => Promise.resolve()), facingMode$: constant(undefined), + switchFacingMode: vi.fn(async () => Promise.resolve(undefined)), + setProcessor: vi.fn(async () => Promise.resolve()), ...track, }; } export const mockVideoTrack = ( track: Partial = {}, -): VideoMediaTrack => - mockMediaTrack({ kind: "video", ...track }) as VideoMediaTrack; +): LocalVideoMediaTrack => + mockMediaTrack({ kind: "video", ...track }) as LocalVideoMediaTrack; export const mockAudioTrack = ( track: Partial = {}, -): AudioMediaTrack => +): LocalAudioMediaTrack => mockMediaTrack({ kind: "audio", source: "microphone", ...track, - }) as AudioMediaTrack; + }) as LocalAudioMediaTrack; /** A member's media as the SDK hands it out, with nothing published unless given. */ export function mockMemberMedia( @@ -334,14 +343,8 @@ export function mockMemberMedia( ): LocalMemberMedia { return { local: false, - speaking$: constant(false), - screenShareEnabled$: constant(false), - microphone$: constant(undefined), - camera$: constant(undefined), - screenShare$: constant(undefined), - screenShareAudio$: constant(undefined), + tracks$: constant([]), encryptionError$: NEVER, - switchCamera: vi.fn().mockResolvedValue(undefined), ...media, } as LocalMemberMedia; } @@ -463,10 +466,14 @@ export function mockRTCMember( return { ...base, local: true, - sharingScreen$: constant(false), - toggleScreenSharing: null, - screenShareError$: constant(null), - dismissScreenShareError: vi.fn(), + publish: vi.fn(async (request: PublishRequest) => + Promise.resolve( + request.source === "microphone" + ? mockAudioTrack() + : mockVideoTrack({ source: request.source }), + ), + ), + unpublish: vi.fn(async () => Promise.resolve()), }; } @@ -504,12 +511,6 @@ export function mockMatrixRTCClient( connected$, reconnecting$, disconnectReason$, - setMicrophoneEnabled: vi.fn(async (enabled: boolean) => - Promise.resolve(enabled), - ), - setCameraEnabled: vi.fn(async (enabled: boolean) => - Promise.resolve(enabled), - ), fatalError$, localMember$, remoteMembers$,