diff --git a/src/room/InCallView.test.tsx b/src/room/InCallView.test.tsx index e4fc1d8bd..209fcc113 100644 --- a/src/room/InCallView.test.tsx +++ b/src/room/InCallView.test.tsx @@ -26,7 +26,7 @@ import { mockMuteStates, mockRtcMembership, type MockRTCSession, - mockMatrixRTCClient, + mockRTCSlot, mockRTCMember, } from "../utils/test"; import { E2eeType } from "../e2ee/e2eeType"; @@ -69,11 +69,11 @@ vi.mock("../tile/GridTile"); vi.mock("../tile/SpotlightTile"); vi.mock("../e2ee/sharedKeyManagement"); vi.mock("../tracks/MemberAudioRenderer"); -// ActiveCall builds the client itself; the view is what these tests are about -const createMatrixRTCClientMock = vi.hoisted(() => vi.fn()); +// ActiveCall builds the slot itself; the view is what these tests are about +const createRTCSlotMock = vi.hoisted(() => vi.fn()); vi.mock("@element-hq/matrixrtc-sdk", async (importOriginal) => ({ ...(await importOriginal()), - createMatrixRTCClient: createMatrixRTCClientMock, + createRTCSlot: createRTCSlotMock, })); vi.mock("react-use-measure", () => ({ default: (): [() => void, object] => [(): void => {}, {}], @@ -97,8 +97,8 @@ let useRoomEncryptionSystemMock: MockedFunction; beforeEach(() => { vi.clearAllMocks(); - createMatrixRTCClientMock.mockImplementation((scope: ObservableScope) => - mockMatrixRTCClient(scope, { + createRTCSlotMock.mockImplementation((scope: ObservableScope) => + mockRTCSlot(scope, { localMember$: constant( mockRTCMember(true, { membership: localRtcMember, diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 742fd04c3..ff8b3c3a5 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -69,8 +69,9 @@ import { ReactionsAudioRenderer } from "./ReactionAudioRenderer"; import { ReactionsOverlay } from "./ReactionsOverlay"; import { CallEventAudioRenderer } from "./CallEventAudioRenderer"; import { - createMatrixRTCClient, - type MatrixRTCClientOptions, + createRTCSlot, + type RTCParticipationOptions, + type RTCSlotOptions, constant, } from "@element-hq/matrixrtc-sdk"; import { map } from "rxjs"; @@ -161,22 +162,20 @@ export const ActiveCall: FC = (props) => { urlParams; const capture = captureSettings(); - const rtcClient = createMatrixRTCClient( + const slot = createRTCSlot( scope, props.client, props.matrixRoom, - { - ...matrixRTCClientOptions(urlParams, props.e2eeSystem), - publish: initialPublishRequests( - props.muteStates, - mediaDevices, - capture, - ), - }, + rtcSlotOptions(props.e2eeSystem), ); + const rtcParticipation = slot.join({ + ...rtcParticipationOptions(urlParams), + publish: initialPublishRequests(props.muteStates, mediaDevices, capture), + }); + const vm = createCallViewModel$( scope, - rtcClient, + rtcParticipation, props.matrixRoom, mediaDevices, props.muteStates, @@ -199,8 +198,6 @@ export const ActiveCall: FC = (props) => { reactionsReader.raisedHands$, reactionsReader.reactions$, ); - // TODO move this somewhere else once we use the callViewModel in the lobby as well! - vm.join(); setVm(vm); vm.leave$.pipe(scope.bind()).subscribe(props.onLeft); @@ -793,20 +790,26 @@ export function captureSettings(): CaptureSettings { * switches and the devices, so the caller adds that. Shared with the legacy * SDK bundle so that a widget built from it behaves like the app. */ -export function matrixRTCClientOptions( - urlParams: ReturnType, +export function rtcSlotOptions( encryptionSystem: EncryptionSystem, -): Omit { - const config = Config.get(); - const session = config.matrix_rtc_session; +): RTCSlotOptions { return { encryptionSystem, // matrix_rtc_mode in config.json overrides the user's Developer Settings // choice. We merely sample the current mode here, so the user would need // to manually rejoin to switch to a different one. matrixRTCMode: - (config.matrix_rtc_mode as MatrixRTCMode | undefined) ?? + (Config.get().matrix_rtc_mode as MatrixRTCMode | undefined) ?? matrixRTCModeSetting.value$.value, + }; +} + +export function rtcParticipationOptions( + urlParams: ReturnType, +): Omit { + const config = Config.get(); + const session = config.matrix_rtc_session; + return { sendNotificationType: urlParams.sendNotificationType, applicationData: urlParams.callIntent ? { "m.call.intent": urlParams.callIntent } diff --git a/src/state/CallViewModel/CallViewModel.test.ts b/src/state/CallViewModel/CallViewModel.test.ts index b3072befb..ce8d57c3f 100644 --- a/src/state/CallViewModel/CallViewModel.test.ts +++ b/src/state/CallViewModel/CallViewModel.test.ts @@ -261,16 +261,18 @@ describe.each(modes)("CallViewModel (%s mode)", (mode) => { test("the selected audio output is handed to the client as it changes", () => { const { mediaDevices, selected$ } = withAudioOutput(); - withCallViewModel({ mediaDevices }, (_vm, _session, { rtcClient }) => { + withCallViewModel({ mediaDevices }, (_vm, _session, { rtcParticipation: participation }) => { // Nothing selected yet, so the browser's choice stands - expect(rtcClient.setAudioOutputDeviceId).not.toHaveBeenCalled(); + expect(participation.setAudioOutputDeviceId).not.toHaveBeenCalled(); selected$.next({ id: "speaker", virtualEarpiece: false }); - expect(rtcClient.setAudioOutputDeviceId).toHaveBeenCalledWith("speaker"); + expect(participation.setAudioOutputDeviceId).toHaveBeenCalledWith( + "speaker", + ); selected$.next({ id: "headphones", virtualEarpiece: false }); - expect(rtcClient.setAudioOutputDeviceId).toHaveBeenLastCalledWith( + expect(participation.setAudioOutputDeviceId).toHaveBeenLastCalledWith( "headphones", ); - expect(rtcClient.setAudioOutputDeviceId).toHaveBeenCalledTimes(2); + expect(participation.setAudioOutputDeviceId).toHaveBeenCalledTimes(2); }); }); @@ -278,9 +280,9 @@ describe.each(modes)("CallViewModel (%s mode)", (mode) => { const { mediaDevices, selected$ } = withAudioOutput(); withCallViewModel( { mediaDevices }, - (_vm, _session, { rtcClient }) => { + (_vm, _session, { rtcParticipation: participation }) => { selected$.next({ id: "earpiece", virtualEarpiece: true }); - expect(rtcClient.setAudioOutputDeviceId).not.toHaveBeenCalled(); + expect(participation.setAudioOutputDeviceId).not.toHaveBeenCalled(); }, { controlledAudioDevices: true }, ); diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index e494cba26..7cc2c7cf1 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -17,7 +17,7 @@ import { type LocalMediaTrack, type LocalRTCMember, type LocalVideoMediaTrack, - type MatrixRTCClient, + type RTCParticipation, type ObservableScope, pauseWhen, type PublishRequest, @@ -158,7 +158,7 @@ export function publishRequest( }; } -/** What the client publishes at the join: the sources whose mute switch is on. */ +/** What the participation publishes at the join: the sources whose mute switch is on. */ export function initialPublishRequests( muteStates: MuteStates, mediaDevices: MediaDevices, @@ -295,12 +295,7 @@ export interface CallViewModel { /** Call to initiate hangup. Use in conbination with reconnection state track the async hangup process. */ hangup: () => void; - // joining - join: () => void; - - /** - * calls requestDisconnect. The async leave state can than be observed via connected$ - */ + /** Leaves the participation. The async leave can then be observed via connected$. */ leave: () => void; // screen sharing /** @@ -457,12 +452,12 @@ export interface CallViewModel { * UI (may eventually be expanded to cover the lobby and feedback screens in the * future). * - * The session itself — memberships, transports, media — is the client's; this - * adds everything that makes it a call. + * The slot and the participation — memberships, transports, media — are the + * SDK's; this adds everything that makes it a call. */ export function createCallViewModel$( scope: ObservableScope, - rtcClient: MatrixRTCClient, + rtcParticipation: RTCParticipation, // A call is permanently tied to a single Matrix room matrixRoom: MatrixRoom, mediaDevices: MediaDevices, @@ -492,7 +487,7 @@ export function createCallViewModel$( const localTracks$ = scope.behavior< (LocalAudioMediaTrack | LocalVideoMediaTrack)[] >( - rtcClient.localMember$.pipe( + rtcParticipation.localMember$.pipe( switchMap((member) => member?.tracks$ ?? of(null)), map((tracks) => tracks ?? []), ), @@ -534,10 +529,10 @@ export function createCallViewModel$( return desired; } if (!enabled) return false; - const member = rtcClient.localMember$.value; - if (member === null) return false; try { - await member.publish(publishRequest(source, mediaDevices, capture)); + await rtcParticipation.publish( + publishRequest(source, mediaDevices, capture), + ); return true; } catch (e) { logger.error(`Failed to publish the ${source}`, e); @@ -587,7 +582,7 @@ export function createCallViewModel$( .pipe(scope.bind()) .subscribe((selected) => { if (selected === undefined) return; - rtcClient.setAudioOutputDeviceId(selected.id).catch((e) => { + rtcParticipation.setAudioOutputDeviceId(selected.id).catch((e) => { logger.error("Failed to switch the audio output device", e); }); }); @@ -604,7 +599,7 @@ export function createCallViewModel$( const members$ = scope.behavior<(LocalRTCMember | RemoteRTCMember)[]>( combineLatest( - [rtcClient.localMember$, rtcClient.remoteMembers$], + [rtcParticipation.localMember$, rtcParticipation.remoteMembers$], (local, remote) => (local === null ? remote : [local, ...remote]), ), ); @@ -633,7 +628,7 @@ export function createCallViewModel$( }); const handsRaised$ = scope.behavior( - handsRaisedSubject$.pipe(pauseWhen(rtcClient.reconnecting$)), + handsRaisedSubject$.pipe(pauseWhen(rtcParticipation.reconnecting$)), ); const reactions$ = scope.behavior( @@ -646,7 +641,7 @@ export function createCallViewModel$( ]), ), ), - pauseWhen(rtcClient.reconnecting$), + pauseWhen(rtcParticipation.reconnecting$), ), ); @@ -663,7 +658,10 @@ export function createCallViewModel$( for (const member of members) { const mediaId = `${member.userId}:${member.deviceId}`; for (let dup = 0; dup < 1 + duplicateTiles; dup++) { - yield { keys: [dup, mediaId, member.id], data: member }; + yield { + keys: [dup, mediaId, member.rtcBackendIdentity], + data: member, + }; } } }, @@ -672,14 +670,14 @@ export function createCallViewModel$( return createWrappedUserMedia(scope, { id: `${mediaId}:${dup}`, userId: member.userId, - rtcBackendIdentity: member.id, + rtcBackendIdentity: member.rtcBackendIdentity, member, encryptionSystem: options.encryptionSystem, focusUrl$: scope.behavior( member.transport$.pipe(map((transport) => transport?.id)), ), mediaDevices, - pretendToBeDisconnected$: rtcClient.reconnecting$, + pretendToBeDisconnected$: rtcParticipation.reconnecting$, displayName$: member.displayName$, mxcAvatarUrl$: member.avatarUrl$, handRaised$: scope.behavior( @@ -761,9 +759,12 @@ export function createCallViewModel$( * - There can be multiple participants for one Matrix user if they join from * multiple devices. */ - const participantCount$ = rtcClient.memberCount$; + // One per membership, so one user may count several times + const participantCount$ = scope.behavior( + rtcParticipation.slot.members$.pipe(map((members) => members.length)), + ); - const keyRotationSuppressed$ = rtcClient.keyRotationSuppressed$; + const keyRotationSuppressed$ = rtcParticipation.keyRotationSuppressed$; const leaveSoundEffect$ = userMedia$.pipe( pairwise(), @@ -1473,15 +1474,13 @@ export function createCallViewModel$( (hideScreensharing || !("getDisplayMedia" in (navigator.mediaDevices ?? {})) ? null : (): void => { - const member = rtcClient.localMember$.value; - if (member === null) return; const track = screenShare$.value; (track === undefined - ? member.publish({ + ? rtcParticipation.publish({ source: "screenShare", capture: capture.screenShare, }) - : member.unpublish(track.id) + : rtcParticipation.unpublish("screenShare") ).catch((e: unknown) => { logger.error( `Screen share ${track === undefined ? "start" : "stop"} failed`, @@ -1495,24 +1494,22 @@ export function createCallViewModel$( }); }); - // Tell the host and the analytics about the user's joins and leaves - const join = (): void => { - PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date()); - PosthogAnalytics.instance.eventCallStarted.track(matrixRoom.roomId); - rtcClient.join(); - hostBridge.notifyJoined().catch((e) => { - logger.error("Failed to notify the host that we joined", e); - }); - }; + // Tell the host and the analytics about the user's joins and leaves. The + // participation is joined by the time the view model is built on it + PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date()); + PosthogAnalytics.instance.eventCallStarted.track(matrixRoom.roomId); + hostBridge.notifyJoined().catch((e) => { + logger.error("Failed to notify the host that we joined", e); + }); const leave = (): void => { - rtcClient.leave(); + rtcParticipation.leave(); hostBridge.notifyHungUp().catch((e) => { logger.error("Failed to notify the host that we hung up", e); }); }; let reconnectStart: { time: number; reason: DisconnectReason } | null = null; - rtcClient.disconnectReason$ + rtcParticipation.disconnectReason$ .pipe(distinctUntilChanged(), pairwise(), scope.bind()) .subscribe(([prev, reason]) => { if (reason !== null) { @@ -1538,7 +1535,6 @@ export function createCallViewModel$( ringingStatusLocation: header === HeaderStyle.AppBar ? "app_bar" : "tile", leave$: leave$, hangup: (): void => userHangup$.next(), - join, leave, toggleScreenSharing: toggleScreenSharing, sharingScreen$: sharingScreen$, @@ -1549,15 +1545,15 @@ export function createCallViewModel$( unhoverScreen: (): void => screenUnhover$.next(), fatalError$: scope.behavior( - rtcClient.fatalError$.pipe( + rtcParticipation.fatalError$.pipe( map((error) => error && fromMatrixRTCError(error)), ), ), participantCount$: participantCount$, keyRotationSuppressed$: keyRotationSuppressed$, - localMember$: rtcClient.localMember$, - remoteMembers$: rtcClient.remoteMembers$, - connectedTransports$: rtcClient.connectedTransports$, + localMember$: rtcParticipation.localMember$, + remoteMembers$: rtcParticipation.remoteMembers$, + connectedTransports$: rtcParticipation.connectedTransports$, handsRaised$: handsRaised$, reactions$: reactions$, joinSoundEffect$: joinSoundEffect$, @@ -1586,8 +1582,8 @@ export function createCallViewModel$( overflowing$, earpieceMode$: earpieceMode$, audioOutputSwitcher$: audioOutputSwitcher$, - reconnecting$: rtcClient.reconnecting$, - connected$: rtcClient.connected$, + reconnecting$: rtcParticipation.reconnecting$, + connected$: rtcParticipation.connected$, screenShareError$, dismissScreenShareError: (): void => screenShareError$.next(null), }; diff --git a/src/state/CallViewModel/CallViewModelTestUtils.ts b/src/state/CallViewModel/CallViewModelTestUtils.ts index 73032a779..d41119514 100644 --- a/src/state/CallViewModel/CallViewModelTestUtils.ts +++ b/src/state/CallViewModel/CallViewModelTestUtils.ts @@ -11,7 +11,7 @@ import { constant, E2eeType, type LocalRTCMember, - type MatrixRTCClient, + type RTCParticipation, type MatrixRTCMode, type ObservableScope, type RemoteRTCMember, @@ -33,7 +33,7 @@ import { mockConfig, mockMatrixRoom, mockMatrixRoomMember, - mockMatrixRTCClient, + mockRTCParticipation, mockAudioTrack, mockMediaDevices, mockVideoTrack, @@ -112,8 +112,8 @@ export function withCallViewModel(mode: MatrixRTCMode) { rtcSession: MockRTCSession, subjects: { raisedHands$: BehaviorSubject>; - /** The client the view model was built on, with spies for its calls. */ - rtcClient: MatrixRTCClient; + /** The participation the view model was built on, with spies for its calls. */ + rtcParticipation: RTCParticipation; }, setSyncState: (value: SyncState) => void, ) => void, @@ -241,7 +241,7 @@ export function withCallViewModel(mode: MatrixRTCMode) { ), ), ); - const rtcClient = mockMatrixRTCClient(scope, { + const rtcParticipation = mockRTCParticipation(scope, { localMember$, remoteMembers$, connected$, @@ -260,7 +260,7 @@ export function withCallViewModel(mode: MatrixRTCMode) { const vm = createCallViewModel$( scope, - rtcClient, + rtcParticipation, room, mediaDevices, muteStates, @@ -279,6 +279,6 @@ export function withCallViewModel(mode: MatrixRTCMode) { ); void mode; - continuation(vm, rtcSession, { raisedHands$, rtcClient }, setSyncState); + continuation(vm, rtcSession, { raisedHands$, rtcParticipation }, setSyncState); }; } diff --git a/src/state/media/MemberMediaViewModel.ts b/src/state/media/MemberMediaViewModel.ts index 5ba969104..bdff672d5 100644 --- a/src/state/media/MemberMediaViewModel.ts +++ b/src/state/media/MemberMediaViewModel.ts @@ -13,6 +13,7 @@ import { type EncryptionSystem, type MediaTrack, type ObservableScope, + type MemberMedia, type RTCMember, trackBySource$, type VideoMediaTrack, @@ -64,10 +65,7 @@ export interface BaseMemberMediaViewModel extends BaseMediaViewModel { } /** The member whose media this is, as far as the media view models read it. */ -export type MediaMember = Pick< - RTCMember, - "local" | "tracks$" | "encryptionError$" ->; +export type MediaMember = Pick & MemberMedia; export interface MemberMediaInputs extends BaseMediaViewModel { member: MediaMember; diff --git a/src/tracks/MemberAudioRenderer.tsx b/src/tracks/MemberAudioRenderer.tsx index fe1b687df..dd9beb14b 100644 --- a/src/tracks/MemberAudioRenderer.tsx +++ b/src/tracks/MemberAudioRenderer.tsx @@ -100,7 +100,7 @@ export const MemberAudioRenderer: FC = ({
{members.map((member) => ( @@ -196,7 +196,7 @@ export function getBasicCallViewModelEnvironment( }); const vm = createCallViewModel$( scope, - rtcClient, + rtcParticipation, matrixRoom, mediaDevices, muteStates, diff --git a/src/utils/test.ts b/src/utils/test.ts index eff7919f2..2592351e5 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -5,6 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ import { + BehaviorSubject, combineLatest, map, NEVER, @@ -67,7 +68,9 @@ import { type LocalAudioMediaTrack, type LocalRTCMember, type LocalVideoMediaTrack, - type MatrixRTCClient, + type RTCMember, + type RTCParticipation, + type RTCSlot, type MatrixRTCError, type PublishRequest, type RemoteRTCMember, @@ -467,13 +470,14 @@ export function mockRTCMember( resolved$: constant(undefined), }; const base = { - id: membership.rtcBackendIdentity, + rtcBackendIdentity: membership.rtcBackendIdentity, userId: membership.userId, deviceId: membership.deviceId, - membership$: constant(membership), + memberId: membership.memberId, displayName$: constant(member.rawDisplayName ?? membership.userId), avatarUrl$: constant(member.getMxcAvatarUrl()), transport$: constant(transport), + applicationData$: constant(membership.applicationData), encryptionError$, }; if (!local) return { ...base, local: false, tracks$ }; @@ -484,18 +488,10 @@ export function mockRTCMember( tracks$: tracks$ as Behavior< (LocalAudioMediaTrack | LocalVideoMediaTrack)[] | null >, - publish: vi.fn(async (request: PublishRequest) => - Promise.resolve( - request.source === "microphone" - ? mockAudioTrack() - : mockVideoTrack({ source: request.source }), - ), - ), - unpublish: vi.fn(async () => Promise.resolve()), }; } -export interface MockClientInputs { +export interface MockParticipationInputs { localMember$?: Behavior; remoteMembers$?: Behavior; connected$?: Behavior; @@ -506,8 +502,11 @@ export interface MockClientInputs { connectedTransports$?: Behavior; } -/** A client that does nothing but hold the state a test hands it. */ -export function mockMatrixRTCClient( +/** + * A participation that does nothing but hold the state a test hands it, in a + * slot whose members are the same ones without their media. + */ +export function mockRTCParticipation( scope: ObservableScope, { localMember$ = constant(null), @@ -518,10 +517,25 @@ export function mockMatrixRTCClient( fatalError$ = constant(null), keyRotationSuppressed$ = constant(false), connectedTransports$ = constant([]), - }: MockClientInputs = {}, -): MatrixRTCClient { - return { - join: vi.fn(), + }: MockParticipationInputs = {}, +): RTCParticipation { + const members$ = scope.behavior( + combineLatest([localMember$, remoteMembers$], (local, remote) => + local === null ? remote : [local, ...remote], + ), + ); + const participation$ = new BehaviorSubject(null); + const slot: RTCSlot = { + roomId: "!room:example.org", + application: "m.call", + id: "ROOM", + status$: constant("open"), + members$, + participation$, + join: vi.fn(() => participation$.value!), + }; + const participation: RTCParticipation = { + slot, leave: vi.fn(), status$: scope.behavior( connected$.pipe(map((c) => (c ? "connected" : "connecting"))), @@ -532,18 +546,30 @@ export function mockMatrixRTCClient( fatalError$, localMember$, remoteMembers$, - memberCount$: scope.behavior( - combineLatest( - [localMember$, remoteMembers$], - (local, remote) => remote.length + (local === null ? 0 : 1), + publish: vi.fn(async (request: PublishRequest) => + Promise.resolve( + request.source === "microphone" + ? mockAudioTrack() + : mockVideoTrack({ source: request.source }), ), ), + unpublish: vi.fn(async () => Promise.resolve()), keyRotationSuppressed$, connectedTransports$, setAudioOutputDeviceId: vi.fn(async () => Promise.resolve()), sendData: vi.fn(async () => Promise.resolve()), data$: NEVER, }; + participation$.next(participation); + return participation; +} + +/** The slot of a `mockRTCParticipation`, whose `join()` returns that participation. */ +export function mockRTCSlot( + scope: ObservableScope, + inputs: MockParticipationInputs = {}, +): RTCSlot { + return mockRTCParticipation(scope, inputs).slot; } export function mockConfig(