diff --git a/sdk-target-based-on-call-view-model/main.ts b/sdk-target-based-on-call-view-model/main.ts index e0dc52a64..a44ab2f66 100644 --- a/sdk-target-based-on-call-view-model/main.ts +++ b/sdk-target-based-on-call-view-model/main.ts @@ -37,7 +37,6 @@ import { import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { type Behavior, - constant, createMatrixRTCClient, E2eeType, ObservableScope, @@ -60,7 +59,6 @@ import { observeElementSize$ } from "../src/utils/elementSize"; import { captureSettings, matrixRTCClientOptions, - selectedDeviceId$, } from "../src/room/InCallView"; /** @@ -143,22 +141,12 @@ export async function createMatrixRTCSdk( // rtc client: the session, the transport and the media, as in the app const encryptionSystem = { kind: E2eeType.PER_PARTICIPANT } as const; const capture = captureSettings(); - const rtcClient = createMatrixRTCClient( - scope, - client, - room, - { - publish: initialPublishRequests(muteStates, mediaDevices, capture), - audioOutputDeviceId$: controlledAudioDevices - ? constant(undefined) - : selectedDeviceId$(scope, mediaDevices.audioOutput), - }, - { - ...matrixRTCClientOptions(urlParams, encryptionSystem), - application, - slot: id, - }, - ); + const rtcClient = createMatrixRTCClient(scope, client, room, { + ...matrixRTCClientOptions(urlParams, encryptionSystem), + publish: initialPublishRequests(muteStates, mediaDevices, capture), + application, + slot: id, + }); // call view model: the host bridge's hang-up handling and the leave flow const callViewModel = createCallViewModel$( diff --git a/src/room/InCallView.test.tsx b/src/room/InCallView.test.tsx index 7f2111615..e4fc1d8bd 100644 --- a/src/room/InCallView.test.tsx +++ b/src/room/InCallView.test.tsx @@ -27,7 +27,6 @@ import { mockRtcMembership, type MockRTCSession, mockMatrixRTCClient, - mockMemberMedia, mockRTCMember, } from "../utils/test"; import { E2eeType } from "../e2ee/e2eeType"; @@ -103,13 +102,13 @@ beforeEach(() => { localMember$: constant( mockRTCMember(true, { membership: localRtcMember, - media$: constant(mockMemberMedia({ local: true })), + tracks$: constant([]), }), ), remoteMembers$: constant([ mockRTCMember(false, { membership: aliceRtcMember, - media$: constant(mockMemberMedia()), + tracks$: constant([]), }), ]), }), diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index b3f792459..742fd04c3 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -72,7 +72,6 @@ import { createMatrixRTCClient, type MatrixRTCClientOptions, constant, - type Behavior, } from "@element-hq/matrixrtc-sdk"; import { map } from "rxjs"; import { @@ -94,7 +93,6 @@ import { screenShareResolution, } from "../settings/settings"; import { Config } from "../config/Config"; -import { type MediaDevice } from "../state/MediaDevices"; import { createSentCallNotification$ } from "../state/CallViewModel/CallNotificationLifecycle.ts"; import { ReactionsReader } from "../reactions/ReactionsReader"; import { MemberAudioRenderer } from "../tracks/MemberAudioRenderer.tsx"; @@ -168,17 +166,13 @@ export const ActiveCall: FC = (props) => { props.client, props.matrixRoom, { + ...matrixRTCClientOptions(urlParams, props.e2eeSystem), publish: initialPublishRequests( props.muteStates, mediaDevices, capture, ), - // A host that routes audio itself leaves the browser's output alone - audioOutputDeviceId$: urlParams.controlledAudioDevices - ? constant(undefined) - : selectedDeviceId$(scope, mediaDevices.audioOutput), }, - matrixRTCClientOptions(urlParams, props.e2eeSystem), ); const vm = createCallViewModel$( scope, @@ -744,14 +738,6 @@ export const InCallView: FC = ({ ); }; -/** The id of the device the user picked, for the client to capture or play on. */ -export function selectedDeviceId$( - scope: ObservableScope, - device: MediaDevice, -): Behavior { - return scope.behavior(device.selected$.pipe(map((d) => d?.id))); -} - /** * How the tracks are captured and encoded, from Element Call's settings and * configuration. Shared with the legacy SDK bundle so that a widget built from @@ -803,13 +789,14 @@ export function captureSettings(): CaptureSettings { /** * What the client needs from Element Call's configuration, settings and URL: - * the SDK reads none of them itself. Shared with the legacy SDK bundle so that - * a widget built from it behaves like the app. + * the SDK reads none of them itself. What to publish depends on the mute + * switches and the devices, so the caller adds that. Shared with the legacy + * SDK bundle so that a widget built from it behaves like the app. */ export function matrixRTCClientOptions( urlParams: ReturnType, encryptionSystem: EncryptionSystem, -): MatrixRTCClientOptions { +): Omit { const config = Config.get(); const session = config.matrix_rtc_session; return { diff --git a/src/state/CallViewModel/CallViewModel.test.ts b/src/state/CallViewModel/CallViewModel.test.ts index 6f659ac31..b3072befb 100644 --- a/src/state/CallViewModel/CallViewModel.test.ts +++ b/src/state/CallViewModel/CallViewModel.test.ts @@ -8,6 +8,7 @@ Please see LICENSE in the repository root for full details. import { test, vi, onTestFinished, it, describe, expect } from "vitest"; import { + BehaviorSubject, combineLatest, debounceTime, distinctUntilChanged, @@ -26,6 +27,7 @@ import { deepCompare } from "matrix-js-sdk/lib/utils"; import { type Layout } from "../layout-types.ts"; import { mockMatrixRoomMember, + mockMediaDevices, mockRemoteParticipant, withTestScheduler, mockRtcMembership, @@ -46,7 +48,10 @@ import { localRtcMember, localRtcMemberDevice2, } from "../../utils/test-fixtures.ts"; -import { MediaDevices } from "../MediaDevices.ts"; +import { + MediaDevices, + type SelectedAudioOutputDevice, +} from "../MediaDevices.ts"; import { getValue } from "../../utils/observable.ts"; import { type Behavior, constant } from "../Behavior.ts"; import { @@ -236,6 +241,51 @@ const modes = [[MatrixRTCMode.Compatibility], [MatrixRTCMode.Matrix_2_0]]; describe.each(modes)("CallViewModel (%s mode)", (mode) => { const withCallViewModel = withCallViewModelInMode(mode); + /** Media devices whose audio output the test selects. */ + const withAudioOutput = (): { + mediaDevices: MediaDevices; + selected$: BehaviorSubject; + } => { + const selected$ = new BehaviorSubject< + SelectedAudioOutputDevice | undefined + >(undefined); + const mediaDevices = mockMediaDevices({ + audioOutput: { + available$: constant(new Map()), + selected$, + select: vi.fn(), + }, + }); + return { mediaDevices, selected$ }; + }; + + test("the selected audio output is handed to the client as it changes", () => { + const { mediaDevices, selected$ } = withAudioOutput(); + withCallViewModel({ mediaDevices }, (_vm, _session, { rtcClient }) => { + // Nothing selected yet, so the browser's choice stands + expect(rtcClient.setAudioOutputDeviceId).not.toHaveBeenCalled(); + selected$.next({ id: "speaker", virtualEarpiece: false }); + expect(rtcClient.setAudioOutputDeviceId).toHaveBeenCalledWith("speaker"); + selected$.next({ id: "headphones", virtualEarpiece: false }); + expect(rtcClient.setAudioOutputDeviceId).toHaveBeenLastCalledWith( + "headphones", + ); + expect(rtcClient.setAudioOutputDeviceId).toHaveBeenCalledTimes(2); + }); + }); + + test("a host that controls the audio devices keeps the output from the client", () => { + const { mediaDevices, selected$ } = withAudioOutput(); + withCallViewModel( + { mediaDevices }, + (_vm, _session, { rtcClient }) => { + selected$.next({ id: "earpiece", virtualEarpiece: true }); + expect(rtcClient.setAudioOutputDeviceId).not.toHaveBeenCalled(); + }, + { controlledAudioDevices: true }, + ); + }); + test("participants are retained during a focus switch", () => { withTestScheduler(({ behavior, expectObservable }) => { // Participants disappear on frame 2 and come back on frame 3 diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 0ba9ccb02..e494cba26 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -22,7 +22,6 @@ import { pauseWhen, type PublishRequest, type RemoteRTCMember, - type RTCMember, trackBySource$, type TransportMetadata, type VideoCaptureSettings, @@ -193,6 +192,12 @@ export interface CallViewModelOptions { showControls?: boolean; /** Whether to hide the screen-sharing button. Defaults to false. */ hideScreensharing?: boolean; + /** + * Whether the host routes the audio itself, outside the browser, so that + * the selected output device is never handed to the client. Defaults to + * false. + */ + controlledAudioDevices?: boolean; autoLeaveWhenOthersLeft?: boolean; /** * If the call is started in a way where we want it to behave like a telephone usecase @@ -232,11 +237,15 @@ export interface CallViewModelOptions { */ export function callViewModelOptionsFromParams( params: UrlParams, -): Pick { +): Pick< + CallViewModelOptions, + "header" | "showControls" | "hideScreensharing" | "controlledAudioDevices" +> { return { header: params.header, showControls: params.showControls, hideScreensharing: params.hideScreensharing, + controlledAudioDevices: params.controlledAudioDevices, }; } @@ -484,8 +493,8 @@ export function createCallViewModel$( (LocalAudioMediaTrack | LocalVideoMediaTrack)[] >( rtcClient.localMember$.pipe( - switchMap((member) => member?.media$ ?? of(null)), - switchMap((media) => media?.tracks$ ?? of([])), + switchMap((member) => member?.tracks$ ?? of(null)), + map((tracks) => tracks ?? []), ), ); // Our own tracks, which carry the controls remote ones lack @@ -570,6 +579,19 @@ export function createCallViewModel$( followDevice(microphone$, mediaDevices.audioInput, "microphone"); followDevice(camera$, mediaDevices.videoInput, "camera"); + // The output device follows the selection onto the client the same way. A + // host that routes audio itself picks the output outside the browser, so the + // client is left alone + if (!options.controlledAudioDevices) + mediaDevices.audioOutput.selected$ + .pipe(scope.bind()) + .subscribe((selected) => { + if (selected === undefined) return; + rtcClient.setAudioOutputDeviceId(selected.id).catch((e) => { + logger.error("Failed to switch the audio output device", e); + }); + }); + // Attached once the camera track exists rather than carried in the publish // request, so that a new track and a change of processor take the same path combineLatest([camera$, videoProcessor$]) @@ -580,7 +602,7 @@ export function createCallViewModel$( }); }); - const members$ = scope.behavior( + const members$ = scope.behavior<(LocalRTCMember | RemoteRTCMember)[]>( combineLatest( [rtcClient.localMember$, rtcClient.remoteMembers$], (local, remote) => (local === null ? remote : [local, ...remote]), @@ -651,9 +673,7 @@ export function createCallViewModel$( id: `${mediaId}:${dup}`, userId: member.userId, rtcBackendIdentity: member.id, - ...(member.local - ? { local: true, media$: (member as LocalRTCMember).media$ } - : { local: false, media$: member.media$ }), + member, encryptionSystem: options.encryptionSystem, focusUrl$: scope.behavior( member.transport$.pipe(map((transport) => transport?.id)), diff --git a/src/state/CallViewModel/CallViewModelTestUtils.ts b/src/state/CallViewModel/CallViewModelTestUtils.ts index 28778af2c..73032a779 100644 --- a/src/state/CallViewModel/CallViewModelTestUtils.ts +++ b/src/state/CallViewModel/CallViewModelTestUtils.ts @@ -10,8 +10,8 @@ import { type Behavior, constant, E2eeType, - type LocalMemberMedia, type LocalRTCMember, + type MatrixRTCClient, type MatrixRTCMode, type ObservableScope, type RemoteRTCMember, @@ -37,7 +37,7 @@ import { mockAudioTrack, mockMediaDevices, mockVideoTrack, - mockMemberMedia, + type MockTracks, mockMuteStates, MockRTCSession, mockRTCMember, @@ -112,6 +112,8 @@ export function withCallViewModel(mode: MatrixRTCMode) { rtcSession: MockRTCSession, subjects: { raisedHands$: BehaviorSubject>; + /** The client the view model was built on, with spies for its calls. */ + rtcClient: MatrixRTCClient; }, setSyncState: (value: SyncState) => void, ) => void, @@ -151,11 +153,10 @@ export function withCallViewModel(mode: MatrixRTCMode) { ); const scope = testScope(); - const mediaOf = ( + const tracksOf = ( scope: ObservableScope, participant: Participant, - local: boolean, - ): LocalMemberMedia => { + ): Behavior => { // A microphone and a camera are always published; the screen share // comes and goes const microphone = mockAudioTrack({ @@ -170,18 +171,13 @@ export function withCallViewModel(mode: MatrixRTCMode) { ), }); const screenShare = mockVideoTrack({ source: "screenShare" }); - return mockMemberMedia({ - local, - tracks$: scope.behavior( - (sharingScreen.get(participant) ?? constant(false)).pipe( - map((sharing) => - sharing - ? [microphone, camera, screenShare] - : [microphone, camera], - ), + return scope.behavior( + (sharingScreen.get(participant) ?? constant(false)).pipe( + map((sharing) => + sharing ? [microphone, camera, screenShare] : [microphone, camera], ), ), - }); + ); }; const participantOf = ( scope: ObservableScope, @@ -204,12 +200,12 @@ export function withCallViewModel(mode: MatrixRTCMode) { membership, roomMember: roomMembers.find((m) => m.userId === membership.userId), transportUrl: "http://my-default-service-url.com", - media$: scope.behavior( + tracks$: scope.behavior( participantOf(scope, membership).pipe( - map((participant) => + switchMap((participant) => participant === undefined - ? null - : mediaOf(scope, participant, false), + ? of(null) + : tracksOf(scope, participant), ), ), ), @@ -239,7 +235,7 @@ export function withCallViewModel(mode: MatrixRTCMode) { membership: localRtcMember, roomMember: carol, transportUrl: "http://my-default-service-url.com", - media$: constant(mediaOf(scope, localParticipant, true)), + tracks$: tracksOf(scope, localParticipant), }) : null, ), @@ -283,6 +279,6 @@ export function withCallViewModel(mode: MatrixRTCMode) { ); void mode; - continuation(vm, rtcSession, { raisedHands$: raisedHands$ }, setSyncState); + continuation(vm, rtcSession, { raisedHands$, rtcClient }, setSyncState); }; } diff --git a/src/state/media/LocalUserMediaViewModel.ts b/src/state/media/LocalUserMediaViewModel.ts index 3ec6be5e3..f78b247b3 100644 --- a/src/state/media/LocalUserMediaViewModel.ts +++ b/src/state/media/LocalUserMediaViewModel.ts @@ -8,9 +8,10 @@ Please see LICENSE in the repository root for full details. import { type Behavior, - type LocalMemberMedia, + type LocalRTCMember, type LocalVideoMediaTrack, type ObservableScope, + trackBySource$, } from "@element-hq/matrixrtc-sdk"; import { map, of, switchMap } from "rxjs"; import { logger } from "matrix-js-sdk/lib/logger"; @@ -23,7 +24,6 @@ import { import { alwaysShowSelf } from "../../settings/settings"; import { platform } from "../../Platform"; import { type MediaDevices } from "../MediaDevices"; -import { memberTrack$ } from "./MemberMediaViewModel"; export interface LocalUserMediaViewModel extends BaseUserMediaViewModel { local: true; @@ -42,9 +42,9 @@ export interface LocalUserMediaViewModel extends BaseUserMediaViewModel { export interface LocalUserMediaInputs extends Omit< BaseUserMediaInputs, - "media$" + "member" > { - media$: Behavior; + member: Pick; mediaDevices: MediaDevices; } @@ -54,11 +54,11 @@ export function createLocalUserMedia( ): LocalUserMediaViewModel { const baseUserMedia = createBaseUserMedia(scope, inputs); // Our own camera track, which carries the controls a remote one lacks - const camera$ = memberTrack$( + const camera$ = trackBySource$( scope, - inputs.media$, + inputs.member.tracks$, "camera", - ); + ) as Behavior; const facingMode$ = scope.behavior( camera$.pipe(switchMap((camera) => camera?.facingMode$ ?? of(undefined))), ); diff --git a/src/state/media/MediaViewModel.test.ts b/src/state/media/MediaViewModel.test.ts index 067f484a3..2abe2e9f3 100644 --- a/src/state/media/MediaViewModel.test.ts +++ b/src/state/media/MediaViewModel.test.ts @@ -18,14 +18,15 @@ import { mockAudioTrack, mockLocalMedia, mockMediaDevices, - mockMemberMedia, mockRemoteMedia, mockRemoteScreenShare, mockRtcMembership, + type MockTracks, mockVideoTrack, withTestScheduler, } from "../../utils/test"; import { showConnectionStats } from "../../settings/settings"; +import { EncryptionStatus } from "./MemberMediaViewModel"; const platformMock = vi.hoisted(() => vi.fn(() => "desktop")); vi.mock("../../Platform", () => ({ @@ -38,13 +39,9 @@ const rtcMembership = mockRtcMembership("@alice:example.org", "AAAA"); test("control a participant's volume", () => { const setVolume = vi.fn(); - const vm = mockRemoteMedia( - rtcMembership, - {}, - mockMemberMedia({ - tracks$: constant([mockAudioTrack({ setVolume })]), - }), - ); + const vm = mockRemoteMedia(rtcMembership, {}, [ + mockAudioTrack({ setVolume }), + ]); withTestScheduler(({ expectObservable, schedule }) => { schedule("-ab---c---d|", { a() { @@ -89,15 +86,9 @@ test("control a participant's volume", () => { test("control a participant's screen share volume", () => { const setVolume = vi.fn(); - const vm = mockRemoteScreenShare( - rtcMembership, - {}, - mockMemberMedia({ - tracks$: constant([ - mockAudioTrack({ source: "screenShareAudio", setVolume }), - ]), - }), - ); + const vm = mockRemoteScreenShare(rtcMembership, {}, [ + mockAudioTrack({ source: "screenShareAudio", setVolume }), + ]); withTestScheduler(({ expectObservable, schedule }) => { schedule("-ab---c---d|", { a() { @@ -137,23 +128,13 @@ test("control a participant's screen share volume", () => { }); test("local media remembers whether it should always be shown", () => { - const vm1 = mockLocalMedia( - rtcMembership, - {}, - mockMemberMedia({ local: true }), - mockMediaDevices({}), - ); + const vm1 = mockLocalMedia(rtcMembership, {}, [], mockMediaDevices({})); withTestScheduler(({ expectObservable, schedule }) => { schedule("-a|", { a: () => vm1.setAlwaysShow(false) }); expectObservable(vm1.alwaysShow$).toBe("ab", { a: true, b: false }); }); // Next local media should start out *not* always shown - const vm2 = mockLocalMedia( - rtcMembership, - {}, - mockMemberMedia({ local: true }), - mockMediaDevices({}), - ); + const vm2 = mockLocalMedia(rtcMembership, {}, [], mockMediaDevices({})); withTestScheduler(({ expectObservable, schedule }) => { schedule("-a|", { a: () => vm2.setAlwaysShow(true) }); expectObservable(vm2.alwaysShow$).toBe("ab", { a: false, b: true }); @@ -178,10 +159,7 @@ test("switch cameras", async () => { const vm = mockLocalMedia( rtcMembership, {}, - mockMemberMedia({ - local: true, - tracks$: constant([mockVideoTrack({ facingMode$, switchFacingMode })]), - }), + [mockVideoTrack({ facingMode$, switchFacingMode })], mockMediaDevices({ videoInput: { available$: constant(new Map()), @@ -215,7 +193,7 @@ test("no camera switch where the facing mode is unknown", () => { const vm = mockLocalMedia( rtcMembership, {}, - mockMemberMedia({ local: true, tracks$: constant([mockVideoTrack()]) }), + [mockVideoTrack()], mockMediaDevices({}), ); expect(vm.switchCamera$.value).toBeNull(); @@ -227,26 +205,34 @@ test("remote media is in waiting state while its media has not arrived", () => { }); test("remote media is not in waiting state once its media is there", () => { - const vm = mockRemoteMedia(rtcMembership, {}, mockMemberMedia()); + const vm = mockRemoteMedia(rtcMembership, {}, []); expect(vm.waitingForMedia$.value).toBe(false); }); +test("remote media stops waiting once its tracks arrive, however few", () => { + const tracks$ = new BehaviorSubject(null); + const vm = mockRemoteMedia(rtcMembership, {}, tracks$); + expect(vm.waitingForMedia$.value).toBe(true); + expect(vm.encryptionStatus$.value).toBe(EncryptionStatus.Connecting); + tracks$.next([]); + expect(vm.waitingForMedia$.value).toBe(false); +}); + +test("local media never reports an encryption problem", () => { + const vm = mockLocalMedia(rtcMembership, {}, [], mockMediaDevices({})); + expect(vm.encryptionStatus$.value).toBe(EncryptionStatus.Okay); +}); + test("remote media is not in waiting state when user does not intend to publish anywhere", () => { const vm = mockRemoteMedia(rtcMembership, {}, null, {}); expect(vm.waitingForMedia$.value).toBe(false); }); test("audio and video follow the tracks' mute state", () => { - const vm = mockRemoteMedia( - rtcMembership, - {}, - mockMemberMedia({ - tracks$: constant([ - mockAudioTrack({ muted$: constant(false) }), - mockVideoTrack({ muted$: constant(true) }), - ]), - }), - ); + const vm = mockRemoteMedia(rtcMembership, {}, [ + mockAudioTrack({ muted$: constant(false) }), + mockVideoTrack({ muted$: constant(true) }), + ]); expect(vm.audioEnabled$.value).toBe(true); expect(vm.videoEnabled$.value).toBe(false); }); @@ -256,19 +242,13 @@ test("user media polls stream stats only while the setting is on", () => { withTestScheduler(({ cold, expectObservable, schedule }) => { const stats = { type: "inbound-rtp" } as RTCInboundRtpStreamStats; // A behavior that only runs its source while subscribed, as the SDK's does - const vm = mockRemoteMedia( - rtcMembership, - {}, - mockMemberMedia({ - tracks$: constant([ - mockAudioTrack({ - stats$: cold("-s", { - s: stats, - }) as unknown as Behavior, - }), - ]), + const vm = mockRemoteMedia(rtcMembership, {}, [ + mockAudioTrack({ + stats$: cold("-s", { + s: stats, + }) as unknown as Behavior, }), - ); + ]); schedule("-a-b", { a() { showConnectionStats.setValue(true); diff --git a/src/state/media/MemberMediaViewModel.ts b/src/state/media/MemberMediaViewModel.ts index 5e0210e75..5ba969104 100644 --- a/src/state/media/MemberMediaViewModel.ts +++ b/src/state/media/MemberMediaViewModel.ts @@ -7,15 +7,13 @@ Please see LICENSE in the repository root for full details. */ import { - type AudioMediaTrack, type Behavior, E2eeType, type EncryptionError, type EncryptionSystem, - type MediaSource, type MediaTrack, - type MemberMedia, type ObservableScope, + type RTCMember, trackBySource$, type VideoMediaTrack, } from "@element-hq/matrixrtc-sdk"; @@ -65,33 +63,24 @@ export interface BaseMemberMediaViewModel extends BaseMediaViewModel { encryptionStatus$: Behavior; } +/** The member whose media this is, as far as the media view models read it. */ +export type MediaMember = Pick< + RTCMember, + "local" | "tracks$" | "encryptionError$" +>; + export interface MemberMediaInputs extends BaseMediaViewModel { - media$: Behavior; + member: MediaMember; audioSource: "microphone" | "screenShareAudio"; videoSource: "camera" | "screenShare"; focusUrl$: Behavior; encryptionSystem: EncryptionSystem; } -/** - * The member's first track of a source: `trackBySource$` over media that may - * not have arrived yet. `T` narrows to a local track where the media is ours. - */ -export function memberTrack$( - scope: ObservableScope, - media$: Behavior, - source: MediaSource, -): Behavior { - const tracks$ = scope.behavior( - media$.pipe(switchMap((media) => media?.tracks$ ?? of([]))), - ); - return trackBySource$(scope, tracks$, source) as Behavior; -} - export function createMemberMedia( scope: ObservableScope, { - media$, + member, audioSource, videoSource, transport$, @@ -99,8 +88,8 @@ export function createMemberMedia( ...inputs }: MemberMediaInputs, ): BaseMemberMediaViewModel { - const audio$ = memberTrack$(scope, media$, audioSource); - const video$ = memberTrack$(scope, media$, videoSource); + const audio$ = trackBySource$(scope, member.tracks$, audioSource); + const video$ = trackBySource$(scope, member.tracks$, videoSource); const unencrypted$ = ( track$: Behavior, ): Observable => @@ -123,16 +112,16 @@ export function createMemberMedia( ), ), encryptionStatus$: scope.behavior( - media$.pipe( - switchMap((media): Observable => { - if (media === null) return of(EncryptionStatus.Connecting); - if (media.local || encryptionSystem.kind === E2eeType.NONE) + member.tracks$.pipe( + switchMap((tracks): Observable => { + if (tracks === null) return of(EncryptionStatus.Connecting); + if (member.local || encryptionSystem.kind === E2eeType.NONE) return of(EncryptionStatus.Okay); const perParticipant = encryptionSystem.kind === E2eeType.PER_PARTICIPANT; return combineLatest([ - encryptionError$(media, "MissingKey"), - encryptionError$(media, "InvalidKey"), + encryptionError$(member.encryptionError$, "MissingKey"), + encryptionError$(member.encryptionError$, "InvalidKey"), receivingOkay$(audio$), receivingOkay$(video$), ]).pipe( @@ -156,10 +145,10 @@ export function createMemberMedia( } function encryptionError$( - media: MemberMedia, + errors$: Observable, criteria: EncryptionError, ): Observable { - return media.encryptionError$.pipe( + return errors$.pipe( map((error) => error === criteria), distinctUntilChanged(), throttleTime(1000), // Throttle to avoid spamming the UI diff --git a/src/state/media/RemoteScreenShareViewModel.ts b/src/state/media/RemoteScreenShareViewModel.ts index 78e58b856..de92a8160 100644 --- a/src/state/media/RemoteScreenShareViewModel.ts +++ b/src/state/media/RemoteScreenShareViewModel.ts @@ -7,9 +7,9 @@ Please see LICENSE in the repository root for full details. */ import { - type AudioMediaTrack, type Behavior, type ObservableScope, + trackBySource$, } from "@element-hq/matrixrtc-sdk"; import { map } from "rxjs"; @@ -19,7 +19,6 @@ import { createBaseScreenShare, } from "./ScreenShareViewModel"; import { createVolumeControls, type VolumeControls } from "../VolumeControls"; -import { memberTrack$ } from "./MemberMediaViewModel"; export interface RemoteScreenShareViewModel extends BaseScreenShareViewModel, VolumeControls { @@ -42,9 +41,9 @@ export function createRemoteScreenShare( scope: ObservableScope, { pretendToBeDisconnected$, ...inputs }: RemoteScreenShareInputs, ): RemoteScreenShareViewModel { - const audio$ = memberTrack$( + const audio$ = trackBySource$( scope, - inputs.media$, + inputs.member.tracks$, "screenShareAudio", ); return { diff --git a/src/state/media/RemoteUserMediaViewModel.ts b/src/state/media/RemoteUserMediaViewModel.ts index 239feef77..79a4bd391 100644 --- a/src/state/media/RemoteUserMediaViewModel.ts +++ b/src/state/media/RemoteUserMediaViewModel.ts @@ -7,9 +7,9 @@ Please see LICENSE in the repository root for full details. */ import { - type AudioMediaTrack, type Behavior, type ObservableScope, + trackBySource$, } from "@element-hq/matrixrtc-sdk"; import { combineLatest, map, of, switchMap } from "rxjs"; import { logger } from "matrix-js-sdk/lib/logger"; @@ -20,7 +20,6 @@ import { type BaseUserMediaViewModel, createBaseUserMedia, } from "./UserMediaViewModel"; -import { memberTrack$ } from "./MemberMediaViewModel"; export interface RemoteUserMediaViewModel extends BaseUserMediaViewModel, VolumeControls { @@ -44,11 +43,11 @@ export function createRemoteUserMedia( const waitingForMedia$ = scope.behavior( combineLatest( - [inputs.focusUrl$, inputs.media$], + [inputs.focusUrl$, inputs.member.tracks$], // Without a transport the user is not attempting to publish anywhere // and so we shouldn't expect media. (They might be a subscribe-only bot // for example.) - (focusUrl, media) => focusUrl !== undefined && media === null, + (focusUrl, tracks) => focusUrl !== undefined && tracks === null, ), ); waitingForMedia$.pipe(scope.bind()).subscribe((waiting) => { @@ -60,7 +59,7 @@ export function createRemoteUserMedia( ...createVolumeControls(scope, { pretendToBeDisconnected$, sink$: scope.behavior( - memberTrack$(scope, inputs.media$, "microphone").pipe( + trackBySource$(scope, inputs.member.tracks$, "microphone").pipe( map((track) => (volume: number) => track?.setVolume(volume)), ), ), diff --git a/src/state/media/UserMediaViewModel.ts b/src/state/media/UserMediaViewModel.ts index a7743f9db..6be71593e 100644 --- a/src/state/media/UserMediaViewModel.ts +++ b/src/state/media/UserMediaViewModel.ts @@ -7,11 +7,10 @@ Please see LICENSE in the repository root for full details. */ import { - type AudioMediaTrack, type Behavior, type MediaStreamStats, - type MediaTrack, type ObservableScope, + trackBySource$, } from "@element-hq/matrixrtc-sdk"; import { BehaviorSubject, @@ -27,7 +26,6 @@ import { type LocalUserMediaViewModel } from "./LocalUserMediaViewModel"; import { createMemberMedia, type MemberMediaInputs, - memberTrack$, type BaseMemberMediaViewModel, } from "./MemberMediaViewModel"; import { type RemoteUserMediaViewModel } from "./RemoteUserMediaViewModel"; @@ -79,7 +77,7 @@ export function createBaseUserMedia( ...inputs }: BaseUserMediaInputs, ): BaseUserMediaViewModel { - const { media$ } = inputs; + const { tracks$ } = inputs.member; const toggleCropVideo$ = new Subject(); const videoAspectRatio$ = new BehaviorSubject(NaN); const enabled$ = ( @@ -87,7 +85,7 @@ export function createBaseUserMedia( source: "microphone" | "camera", ): Behavior => scope.behavior( - memberTrack$(scope, media$, source).pipe( + trackBySource$(scope, tracks$, source).pipe( switchMap((track) => track === undefined ? of(false) @@ -102,7 +100,7 @@ export function createBaseUserMedia( ): Behavior => scope.behavior( combineLatest([ - memberTrack$(scope, media$, source), + trackBySource$(scope, tracks$, source), showConnectionStats.value$, ]).pipe( switchMap(([track, show]) => @@ -120,7 +118,7 @@ export function createBaseUserMedia( type: "user", // What a call calls speaking is the microphone carrying sound speaking$: scope.behavior( - memberTrack$(scope, media$, "microphone").pipe( + trackBySource$(scope, tracks$, "microphone").pipe( switchMap((track) => track?.isActive$ ?? of(false)), ), ), diff --git a/src/state/media/WrappedUserMediaViewModel.ts b/src/state/media/WrappedUserMediaViewModel.ts index b67f1d0a9..5414eff99 100644 --- a/src/state/media/WrappedUserMediaViewModel.ts +++ b/src/state/media/WrappedUserMediaViewModel.ts @@ -8,9 +8,10 @@ Please see LICENSE in the repository root for full details. import { type Behavior, generateItems, - type LocalMemberMedia, - type MemberMedia, + type LocalRTCMember, type ObservableScope, + type RemoteRTCMember, + trackBySource$, } from "@element-hq/matrixrtc-sdk"; import { combineLatest, distinctUntilChanged, map, of } from "rxjs"; @@ -27,7 +28,6 @@ import { } from "./RemoteUserMediaViewModel.ts"; import { createLocalScreenShare } from "./LocalScreenShareViewModel.ts"; import { createRemoteScreenShare } from "./RemoteScreenShareViewModel.ts"; -import { memberTrack$ } from "./MemberMediaViewModel.ts"; /** * Sorting bins defining the order in which media tiles appear in the layout. @@ -81,34 +81,28 @@ export type WrappedUserMediaViewModel = UserMediaViewModel & { type WrappedUserMediaInputs = Omit< LocalUserMediaInputs & RemoteUserMediaInputs, - "media$" -> & - ( - | { local: true; media$: Behavior } - | { local: false; media$: Behavior } - ); + "member" +> & { member: LocalRTCMember | RemoteRTCMember }; export function createWrappedUserMedia( scope: ObservableScope, - { mediaDevices, pretendToBeDisconnected$, ...rest }: WrappedUserMediaInputs, + { + member, + mediaDevices, + pretendToBeDisconnected$, + ...inputs + }: WrappedUserMediaInputs, ): WrappedUserMediaViewModel { - const { local, media$: _media$, ...inputs } = rest; - const userMedia = rest.local - ? createLocalUserMedia(scope, { - media$: rest.media$, - mediaDevices, - ...inputs, - }) + const userMedia = member.local + ? createLocalUserMedia(scope, { member, mediaDevices, ...inputs }) : createRemoteUserMedia(scope, { - media$: rest.media$, + member, pretendToBeDisconnected$, ...inputs, }); - // TypeScript needs this widening of the type to happen in a separate statement - const media$: Behavior = _media$; const screenShares$ = scope.behavior( - memberTrack$(scope, media$, "screenShare").pipe( + trackBySource$(scope, member.tracks$, "screenShare").pipe( map((track) => track !== undefined), distinctUntilChanged(), // Technically more than one screen share might be possible... our @@ -121,12 +115,12 @@ export function createWrappedUserMedia( }, (scope, _data$, key) => { const id = `${inputs.id}:${key}`; - return local - ? createLocalScreenShare(scope, { ...inputs, id, media$ }) + return member.local + ? createLocalScreenShare(scope, { ...inputs, id, member }) : createRemoteScreenShare(scope, { ...inputs, id, - media$, + member, pretendToBeDisconnected$, }); }, diff --git a/src/tile/GridTile.test.tsx b/src/tile/GridTile.test.tsx index 5d6fb81ae..f7867c247 100644 --- a/src/tile/GridTile.test.tsx +++ b/src/tile/GridTile.test.tsx @@ -17,7 +17,6 @@ import { mockRemoteMedia, mockLocalMedia, mockMediaDevices, - mockMemberMedia, } from "../utils/test"; import { GridTileViewModel } from "../state/TileViewModel"; import { ReactionsSenderProvider } from "../reactions/useReactionsSender"; @@ -62,7 +61,7 @@ test("GridTile displays remote media", async () => { rawDisplayName: "Alice", getMxcAvatarUrl: () => "mxc://adfsg", }, - mockMemberMedia(), + [], ); const { container } = render( @@ -92,7 +91,7 @@ test("GridTile displays local media", async () => { rawDisplayName: "Alice", getMxcAvatarUrl: () => "mxc://adfsg", }, - mockMemberMedia({ local: true }), + [], mockMediaDevices({}), ); diff --git a/src/tile/SpotlightTile.test.tsx b/src/tile/SpotlightTile.test.tsx index 6cb51dc06..3adc62da2 100644 --- a/src/tile/SpotlightTile.test.tsx +++ b/src/tile/SpotlightTile.test.tsx @@ -19,7 +19,6 @@ import { mockLocalMedia, mockRemoteMedia, mockRemoteScreenShare, - mockMemberMedia, } from "../utils/test"; import { SpotlightTileViewModel } from "../state/TileViewModel"; import { constant } from "../state/Behavior"; @@ -41,7 +40,7 @@ test("SpotlightTile is accessible", async () => { rawDisplayName: "Alice", getMxcAvatarUrl: () => "mxc://adfsg", }, - mockMemberMedia(), + [], ); const vm2 = mockLocalMedia( @@ -50,7 +49,7 @@ test("SpotlightTile is accessible", async () => { rawDisplayName: "Bob", getMxcAvatarUrl: () => "mxc://dlskf", }, - mockMemberMedia({ local: true }), + [], mockMediaDevices({}), ); @@ -98,7 +97,7 @@ test("Screen share volume UI is shown when screen share has audio", async () => const vm = mockRemoteScreenShare( mockRtcMembership("@alice:example.org", "AAAA"), {}, - mockMemberMedia(), + [], ); vi.spyOn(vm, "audioEnabled$", "get").mockReturnValue(constant(true)); @@ -136,7 +135,7 @@ test("Screen share volume UI is hidden when screen share has no audio", async () const vm = mockRemoteScreenShare( mockRtcMembership("@alice:example.org", "AAAA"), {}, - mockMemberMedia(), + [], ); vi.spyOn(vm, "audioEnabled$", "get").mockReturnValue(constant(false)); diff --git a/src/tracks/MemberAudioRenderer.test.tsx b/src/tracks/MemberAudioRenderer.test.tsx new file mode 100644 index 000000000..0da961b8b --- /dev/null +++ b/src/tracks/MemberAudioRenderer.test.tsx @@ -0,0 +1,101 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { afterEach, beforeEach, expect, test, vi } from "vitest"; +import { act, render, type RenderResult } from "@testing-library/react"; +import { type Behavior, type RemoteRTCMember } from "@element-hq/matrixrtc-sdk"; +import { BehaviorSubject } from "rxjs"; + +import { MediaDevicesContext } from "../MediaDevicesContext"; +import { MemberAudioRenderer } from "./MemberAudioRenderer"; +import { + mockAudioTrack, + mockMediaDevices, + mockRTCMember, + mockRtcMembership, + type MockTracks, + mockVideoTrack, +} from "../utils/test"; + +/** The renderer creates one for the earpiece graph; jsdom has none. */ +class FakeAudioContext { + public createGain(): unknown { + return { gain: { value: 1 } }; + } + public createStereoPanner(): unknown { + return { pan: { value: 0 } }; + } + public async close(): Promise {} +} + +beforeEach(() => { + vi.stubGlobal("AudioContext", FakeAudioContext); +}); + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +function renderMembers( + members: { id: string; tracks$: Behavior }[], +): RenderResult { + const rtcMembers: RemoteRTCMember[] = members.map(({ id, tracks$ }) => + mockRTCMember(false, { + membership: mockRtcMembership(`@${id}:example.org`, "DEVICE"), + tracks$, + }), + ); + return render( + + + , + ); +} + +test("plays nothing for a member whose media has not arrived", () => { + const { queryAllByTestId } = renderMembers([ + { id: "alice", tracks$: new BehaviorSubject(null) }, + ]); + expect(queryAllByTestId("audio")).toHaveLength(0); +}); + +test("plays the microphone and the screen share audio, and nothing else", () => { + const microphone = mockAudioTrack({ id: "mic" }); + const screenShareAudio = mockAudioTrack({ + id: "share-audio", + source: "screenShareAudio", + }); + const camera = mockVideoTrack({ id: "camera" }); + const unknownAudio = mockAudioTrack({ id: "unknown", source: "unknown" }); + const { queryAllByTestId } = renderMembers([ + { + id: "alice", + tracks$: new BehaviorSubject([ + microphone, + camera, + screenShareAudio, + unknownAudio, + ]), + }, + ]); + expect(queryAllByTestId("audio")).toHaveLength(2); + expect(microphone.attach).toHaveBeenCalledTimes(1); + expect(screenShareAudio.attach).toHaveBeenCalledTimes(1); + expect(camera.attach).not.toHaveBeenCalled(); + expect(unknownAudio.attach).not.toHaveBeenCalled(); +}); + +test("starts playing once the member's tracks arrive", () => { + const tracks$ = new BehaviorSubject(null); + const { queryAllByTestId } = renderMembers([{ id: "alice", tracks$ }]); + expect(queryAllByTestId("audio")).toHaveLength(0); + + const microphone = mockAudioTrack({ id: "mic" }); + act(() => tracks$.next([microphone])); + expect(queryAllByTestId("audio")).toHaveLength(1); + expect(microphone.attach).toHaveBeenCalledTimes(1); +}); diff --git a/src/tracks/MemberAudioRenderer.tsx b/src/tracks/MemberAudioRenderer.tsx index 0f079e0a6..fe1b687df 100644 --- a/src/tracks/MemberAudioRenderer.tsx +++ b/src/tracks/MemberAudioRenderer.tsx @@ -9,7 +9,6 @@ Please see LICENSE in the repository root for full details. import { type AudioMediaTrack, type MediaSource, - type MemberMedia, type RemoteRTCMember, } from "@element-hq/matrixrtc-sdk"; import { type FC, useEffect, useMemo, useState } from "react"; @@ -102,7 +101,7 @@ export const MemberAudioRenderer: FC = ({ {members.map((member) => ( @@ -116,32 +115,16 @@ interface AudioPlugins { nodes: AudioNode[]; } -const MemberAudio: FC<{ - media$: RemoteRTCMember["media$"]; - muted?: boolean; - plugins: AudioPlugins; -}> = ({ media$, muted, plugins }) => { - const media = useBehavior(media$); - return ( - media !== null && ( - - ) - ); -}; - /** The sources a call plays; anything published without one stays silent. */ const playedSources = new Set(["microphone", "screenShareAudio"]); -const MemberTracksAudio: FC<{ - tracks$: MemberMedia["tracks$"]; +/** One member's audio tracks; nothing while its media has not arrived. */ +const MemberAudio: FC<{ + tracks$: RemoteRTCMember["tracks$"]; muted?: boolean; plugins: AudioPlugins; }> = ({ tracks$, muted, plugins }) => { - const tracks = useBehavior(tracks$); + const tracks = useBehavior(tracks$) ?? []; return tracks .filter( (track): track is AudioMediaTrack => diff --git a/src/utils/test-viewmodel.ts b/src/utils/test-viewmodel.ts index d8e270fd1..7972a207a 100644 --- a/src/utils/test-viewmodel.ts +++ b/src/utils/test-viewmodel.ts @@ -28,7 +28,6 @@ import { mockMatrixRoom, mockMatrixRTCClient, mockMediaDevices, - mockMemberMedia, mockMuteStates, MockRTCSession, mockRTCMember, @@ -172,7 +171,7 @@ export function getBasicCallViewModelEnvironment( (m) => m.userId === localRtcMember.userId, ), transportUrl: "https://example.com", - media$: constant(mockMemberMedia({ local: true })), + tracks$: constant([]), }) : null, ), @@ -188,7 +187,7 @@ export function getBasicCallViewModelEnvironment( membership, roomMember: members.find((m) => m.userId === membership.userId), transportUrl: "https://example.com", - media$: constant(mockMemberMedia()), + tracks$: constant([]), }), ), ), diff --git a/src/utils/test.ts b/src/utils/test.ts index aa8acbbcb..eff7919f2 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -59,19 +59,20 @@ import { import { Config } from "../config/Config"; import { type MediaDevices } from "../state/MediaDevices"; import { + type AudioMediaTrack, type Behavior, constant, type DisconnectReason, + type EncryptionError, type LocalAudioMediaTrack, - type LocalMemberMedia, type LocalRTCMember, type LocalVideoMediaTrack, type MatrixRTCClient, type MatrixRTCError, - type MemberMedia, type PublishRequest, type RemoteRTCMember, type TransportMetadata, + type VideoMediaTrack, } from "@element-hq/matrixrtc-sdk"; import { ObservableScope } from "../state/ObservableScope"; import { MuteStates } from "../state/MuteStates"; @@ -337,17 +338,8 @@ export const mockAudioTrack = ( ...track, }) as LocalAudioMediaTrack; -/** A member's media as the SDK hands it out, with nothing published unless given. */ -export function mockMemberMedia( - media: Partial> & { local?: boolean } = {}, -): LocalMemberMedia { - return { - local: false, - tracks$: constant([]), - encryptionError$: NEVER, - ...media, - } as LocalMemberMedia; -} +/** The tracks a member has, or null while its media has not arrived. */ +export type MockTracks = (AudioMediaTrack | VideoMediaTrack)[] | null; /** Whoever publishes media, known by identity only. */ export function mockRemoteParticipant( @@ -363,7 +355,7 @@ export function mockRemoteParticipant( export function mockLocalMedia( rtcMember: CallMembership, roomMember: Partial, - media: LocalMemberMedia | null, + tracks: MockTracks, mediaDevices: MediaDevices, ): LocalUserMediaViewModel { const member = mockMatrixRoomMember(rtcMember, roomMember); @@ -371,7 +363,11 @@ export function mockLocalMedia( id: "local", userId: member.userId, rtcBackendIdentity: rtcMember.rtcBackendIdentity, - media$: constant(media), + member: mockRTCMember(true, { + membership: rtcMember, + roomMember, + tracks$: constant(tracks), + }), encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, focusUrl$: constant("https://rtc-example.org"), mediaDevices, @@ -385,7 +381,8 @@ export function mockLocalMedia( export function mockRemoteMedia( rtcMember: CallMembership, roomMember: Partial, - media: MemberMedia | null, + /** The tracks, or a behavior of them for a test that has them arrive. */ + tracks: MockTracks | Behavior, { focusUrl }: { focusUrl?: string } = { focusUrl: "https://rtc-example.org" }, ): RemoteUserMediaViewModel { const member = mockMatrixRoomMember(rtcMember, roomMember); @@ -393,7 +390,12 @@ export function mockRemoteMedia( id: "remote", userId: member.userId, rtcBackendIdentity: rtcMember.rtcBackendIdentity, - media$: constant(media), + member: mockRTCMember(false, { + membership: rtcMember, + roomMember, + tracks$: + tracks === null || Array.isArray(tracks) ? constant(tracks) : tracks, + }), encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, focusUrl$: constant(focusUrl), pretendToBeDisconnected$: constant(false), @@ -407,13 +409,17 @@ export function mockRemoteMedia( export function mockRemoteScreenShare( rtcMember: CallMembership, roomMember: Partial, - media: MemberMedia | null, + tracks: MockTracks, ): RemoteScreenShareViewModel { const member = mockMatrixRoomMember(rtcMember, roomMember); return createRemoteScreenShare(testScope(), { id: "screenshare", userId: member.userId, - media$: constant(media), + member: mockRTCMember(false, { + membership: rtcMember, + roomMember, + tracks$: constant(tracks), + }), encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, focusUrl$: constant("https://rtc-example.org"), pretendToBeDisconnected$: constant(false), @@ -425,7 +431,9 @@ export function mockRemoteScreenShare( export interface MockMemberInputs { membership: CallMembership; roomMember?: Partial; - media$?: Behavior; + /** Null, the default, is a member whose media has not arrived. */ + tracks$?: Behavior; + encryptionError$?: Observable; transportUrl?: string | undefined; } @@ -440,7 +448,13 @@ export function mockRTCMember( ): RemoteRTCMember; export function mockRTCMember( local: boolean, - { membership, roomMember, media$, transportUrl }: MockMemberInputs, + { + membership, + roomMember, + tracks$ = constant(null), + encryptionError$ = NEVER, + transportUrl, + }: MockMemberInputs, ): LocalRTCMember | RemoteRTCMember { const member = mockMatrixRoomMember(membership, roomMember); const transport: TransportMetadata | undefined = @@ -460,12 +474,16 @@ export function mockRTCMember( displayName$: constant(member.rawDisplayName ?? membership.userId), avatarUrl$: constant(member.getMxcAvatarUrl()), transport$: constant(transport), - media$: media$ ?? constant(null), + encryptionError$, }; - if (!local) return { ...base, local: false }; + if (!local) return { ...base, local: false, tracks$ }; return { ...base, local: true, + // The mock tracks carry the local controls, whichever side they stand in for + tracks$: tracks$ as Behavior< + (LocalAudioMediaTrack | LocalVideoMediaTrack)[] | null + >, publish: vi.fn(async (request: PublishRequest) => Promise.resolve( request.source === "microphone" @@ -522,6 +540,7 @@ export function mockMatrixRTCClient( ), keyRotationSuppressed$, connectedTransports$, + setAudioOutputDeviceId: vi.fn(async () => Promise.resolve()), sendData: vi.fn(async () => Promise.resolve()), data$: NEVER, };