diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 1f410adb..10b86770 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -59,7 +59,8 @@ import { type MatrixInfo } from "./VideoPreview"; import { InviteButton } from "../button/InviteButton"; import { LayoutToggle } from "./LayoutToggle"; import { - CallViewModel, + type CallViewModel, + createCallViewModel$, type GridMode, } from "../state/CallViewModel/CallViewModel.ts"; import { Grid, type TileProps } from "../grid/Grid"; @@ -128,7 +129,7 @@ export const ActiveCall: FC = (props) => { const reactionsReader = new ReactionsReader(scope, props.rtcSession); const { autoLeaveWhenOthersLeft, waitForCallPickup, sendNotificationType } = urlParams; - const vm = new CallViewModel( + const vm = createCallViewModel$( scope, props.rtcSession, props.matrixRoom, diff --git a/src/state/CallViewModel/CallViewModel.test.ts b/src/state/CallViewModel/CallViewModel.test.ts index eb8d438d..9e28687e 100644 --- a/src/state/CallViewModel/CallViewModel.test.ts +++ b/src/state/CallViewModel/CallViewModel.test.ts @@ -37,7 +37,7 @@ import { import { deepCompare } from "matrix-js-sdk/lib/utils"; import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery"; -import { CallViewModel } from "./CallViewModel"; +import { createCallViewModel$ } from "./CallViewModel"; import { type Layout } from "../layout-types.ts"; import { mockLocalParticipant, @@ -277,7 +277,7 @@ describe("CallViewModel", () => { vi.spyOn(AutoDiscovery, "getRawClientConfig").mockResolvedValue({}); - const callVM = new CallViewModel( + const callVM = createCallViewModel$( testScope(), fakeRtcSession.asMockedSession(), matrixRoom, diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index f1332286..62e4ae44 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -172,55 +172,56 @@ type AudioLivekitItem = { }; /** - * A view model providing all the application logic needed to show the in-call - * UI (may eventually be expanded to cover the lobby and feedback screens in the - * future). + * The return of createCallViewModel$ + * This interface represents the root snapshot for the call view. Snapshots in EC with rxjs behave like snapshot trees. + * They are a list of observables and objects containing observables to allow for a very granular update mechanism. + * + * This allows to have one huge call view model that represents the entire view without a unnecessary amount of updates. + * + * (Mocking this interface should allow building a full view in all states.) */ -// Throughout this class and related code we must distinguish between MatrixRTC -// state and LiveKit state. We use the common terminology of room "members", RTC -// "memberships", and LiveKit "participants". -export class CallViewModel { +export interface CallViewModel { // lifecycle - public autoLeave$: Observable; + autoLeave$: Observable; // TODO if we are in "unknown" state we need a loading rendering (or empty screen) // Otherwise it looks like we already connected and only than the ringing starts which is weird. - public callPickupState$: Behavior< + callPickupState$: Behavior< "unknown" | "ringing" | "timeout" | "decline" | "success" | null >; - public leave$: Observable<"user" | AutoLeaveReason>; + leave$: Observable<"user" | AutoLeaveReason>; /** Call to initiate hangup. Use in conbination with connectino state track the async hangup process. */ - public hangup: () => void; + hangup: () => void; // joining - public join: () => LocalMemberConnectionState; + join: () => LocalMemberConnectionState; // screen sharing /** * Callback to toggle screen sharing. If null, screen sharing is not possible. */ - public toggleScreenSharing: (() => void) | null; + toggleScreenSharing: (() => void) | null; /** * Whether we are sharing our screen. */ - public sharingScreen$: Behavior; + sharingScreen$: Behavior; // UI interactions /** * Callback for when the user taps the call view. */ - public tapScreen: () => void; + tapScreen: () => void; /** * Callback for when the user taps the call's controls. */ - public tapControls: () => void; + tapControls: () => void; /** * Callback for when the user hovers over the call view. */ - public hoverScreen: () => void; + hoverScreen: () => void; /** * Callback for when the user stops hovering over the call view. */ - public unhoverScreen: () => void; + unhoverScreen: () => void; // errors /** @@ -228,7 +229,7 @@ export class CallViewModel { * This is a fatal error that prevents the call from being created/joined. * Should render a blocking error screen. */ - public configError$: Behavior; + configError$: Behavior; // participants and counts /** @@ -238,15 +239,15 @@ export class CallViewModel { * - There can be multiple participants for one Matrix user if they join from * multiple devices. */ - public participantCount$: Behavior; + participantCount$: Behavior; /** Participants sorted by livekit room so they can be used in the audio rendering */ - public audioParticipants$: Behavior; + audioParticipants$: Behavior; /** List of participants raising their hand */ - public handsRaised$: Behavior>; + handsRaised$: Behavior>; /** List of reactions. Keys are: membership.membershipId (currently predefined as: `${membershipEvent.userId}:${membershipEvent.deviceId}`)*/ - public reactions$: Behavior>; + reactions$: Behavior>; - public ringOverlay$: Behavior; // sounds and events - public joinSoundEffect$: Observable; - public leaveSoundEffect$: Observable; + joinSoundEffect$: Observable; + leaveSoundEffect$: Observable; /** * Emits an event every time a new hand is raised in * the call. */ - public newHandRaised$: Observable<{ value: number; playSounds: boolean }>; + newHandRaised$: Observable<{ value: number; playSounds: boolean }>; /** * Emits an event every time a new screenshare is started in * the call. */ - public newScreenShare$: Observable<{ value: number; playSounds: boolean }>; + newScreenShare$: Observable<{ value: number; playSounds: boolean }>; /** * Emits an array of reactions that should be played. */ - public audibleReactions$: Observable; + audibleReactions$: Observable; /** * Emits an array of reactions that should be visible on the screen. */ // DISCUSSION move this into a reaction file - public visibleReactions$: Behavior< + visibleReactions$: Behavior< { sender: string; emoji: string; startX: number }[] >; @@ -282,43 +283,43 @@ export class CallViewModel { /** * The general shape of the window. */ - public windowMode$: Behavior; - public spotlightExpanded$: Behavior; - public toggleSpotlightExpanded$: Behavior<(() => void) | null>; - public gridMode$: Behavior; - public setGridMode: (value: GridMode) => void; + windowMode$: Behavior; + spotlightExpanded$: Behavior; + toggleSpotlightExpanded$: Behavior<(() => void) | null>; + gridMode$: Behavior; + setGridMode: (value: GridMode) => void; // media view models and layout - public grid$: Behavior; - public spotlight$: Behavior; - public pip$: Behavior; + grid$: Behavior; + spotlight$: Behavior; + pip$: Behavior; /** * The layout of tiles in the call interface. */ - public layout$: Behavior; + layout$: Behavior; /** * The current generation of the tile store, exposed for debugging purposes. */ - public tileStoreGeneration$: Behavior; - public showSpotlightIndicators$: Behavior; - public showSpeakingIndicators$: Behavior; + tileStoreGeneration$: Behavior; + showSpotlightIndicators$: Behavior; + showSpeakingIndicators$: Behavior; // header/footer visibility - public showHeader$: Behavior; - public showFooter$: Behavior; + showHeader$: Behavior; + showFooter$: Behavior; // audio routing /** * Whether audio is currently being output through the earpiece. */ - public earpieceMode$: Behavior; + earpieceMode$: Behavior; /** * Callback to toggle between the earpiece and the loudspeaker. * * This will be `null` in case the target does not exist in the list * of available audio outputs. */ - public audioOutputSwitcher$: Behavior<{ + audioOutputSwitcher$: Behavior<{ targetOutput: "earpiece" | "speaker"; switch: () => void; } | null>; @@ -333,1160 +334,1145 @@ export class CallViewModel { // down, for example, and we want to avoid making people worry that the app is // in a split-brained state. // DISCUSSION own membership manager ALSO this probably can be simplifis - public reconnecting$: Behavior; + reconnecting$: Behavior; +} +/** + * A view model providing all the application logic needed to show the in-call + * UI (may eventually be expanded to cover the lobby and feedback screens in the + * future). + */ +// Throughout this class and related code we must distinguish between MatrixRTC +// state and LiveKit state. We use the common terminology of room "members", RTC +// "memberships", and LiveKit "participants". +export function createCallViewModel$( + scope: ObservableScope, + // A call is permanently tied to a single Matrix room + matrixRTCSession: MatrixRTCSession, + matrixRoom: MatrixRoom, + mediaDevices: MediaDevices, + muteStates: MuteStates, + options: CallViewModelOptions, + handsRaisedSubject$: Observable>, + reactionsSubject$: Observable>, + trackProcessorState$: Behavior, +): CallViewModel { + const userId = matrixRoom.client.getUserId()!; + const deviceId = matrixRoom.client.getDeviceId()!; - // THIS has to be the last public field declaration - public constructor( - scope: ObservableScope, - // A call is permanently tied to a single Matrix room - matrixRTCSession: MatrixRTCSession, - matrixRoom: MatrixRoom, - mediaDevices: MediaDevices, - muteStates: MuteStates, - options: CallViewModelOptions, - handsRaisedSubject$: Observable>, - reactionsSubject$: Observable>, - trackProcessorState$: Behavior, - ) { - const userId = matrixRoom.client.getUserId()!; - const deviceId = matrixRoom.client.getDeviceId()!; + const livekitKeyProvider = getE2eeKeyProvider( + options.encryptionSystem, + matrixRTCSession, + ); - const livekitKeyProvider = getE2eeKeyProvider( - options.encryptionSystem, - matrixRTCSession, - ); + // Each hbar seperates a block of input variables required for the CallViewModel to function. + // The outputs of this block is written under the hbar. + // + // For mocking purposes it is recommended to only mock the functions creating those outputs. + // All other fields are just temp computations for the mentioned output. + // The class does not need anything except the values underneath the bar. + // The creation of the values under the bar are all tested independently and testing the callViewModel Should + // not test their cretation. Call view model only needs: + // - memberships$ via createMemberships$ + // - localMembership via createLocalMembership$ + // - callLifecycle via createCallNotificationLifecycle$ + // - matrixMemberMetadataStore via createMatrixMemberMetadata$ - // Each hbar seperates a block of input variables required for the CallViewModel to function. - // The outputs of this block is written under the hbar. - // - // For mocking purposes it is recommended to only mock the functions creating those outputs. - // All other fields are just temp computations for the mentioned output. - // The class does not need anything except the values underneath the bar. - // The creation of the values under the bar are all tested independently and testing the callViewModel Should - // not test their cretation. Call view model only needs: - // - memberships$ via createMemberships$ - // - localMembership via createLocalMembership$ - // - callLifecycle via createCallNotificationLifecycle$ - // - matrixMemberMetadataStore via createMatrixMemberMetadata$ + // ------------------------------------------------------------------------ + // memberships$ + const memberships$ = createMemberships$(scope, matrixRTCSession); - // ------------------------------------------------------------------------ - // memberships$ - const memberships$ = createMemberships$(scope, matrixRTCSession); + // ------------------------------------------------------------------------ + // matrixLivekitMembers$ AND localMembership - // ------------------------------------------------------------------------ - // matrixLivekitMembers$ AND localMembership + const membershipsAndTransports = membershipsAndTransports$( + scope, + memberships$, + ); - const membershipsAndTransports = membershipsAndTransports$( - scope, - memberships$, - ); + const localTransport$ = createLocalTransport$({ + scope: scope, + memberships$: memberships$, + client: matrixRoom.client, + roomId: matrixRoom.roomId, + useOldestMember$: scope.behavior( + matrixRTCMode.value$.pipe(map((v) => v === MatrixRTCMode.Legacy)), + ), + }); - const localTransport$ = createLocalTransport$({ - scope: scope, - memberships$: memberships$, - client: matrixRoom.client, - roomId: matrixRoom.roomId, - useOldestMember$: scope.behavior( - matrixRTCMode.value$.pipe(map((v) => v === MatrixRTCMode.Legacy)), + const connectionFactory = new ECConnectionFactory( + matrixRoom.client, + mediaDevices, + trackProcessorState$, + livekitKeyProvider, + getUrlParams().controlledAudioDevices, + options.livekitRoomFactory, + ); + + const connectionManager = createConnectionManager$({ + scope: scope, + connectionFactory: connectionFactory, + inputTransports$: scope.behavior( + combineLatest( + [localTransport$, membershipsAndTransports.transports$], + (localTransport, transports) => { + const localTransportAsArray = localTransport ? [localTransport] : []; + return transports.mapInner((transports) => [ + ...localTransportAsArray, + ...transports, + ]); + }, ), - }); + ), + logger: logger, + }); - const connectionFactory = new ECConnectionFactory( - matrixRoom.client, - mediaDevices, - trackProcessorState$, - livekitKeyProvider, - getUrlParams().controlledAudioDevices, - options.livekitRoomFactory, - ); + const matrixLivekitMembers$ = createMatrixLivekitMembers$({ + scope: scope, + membershipsWithTransport$: + membershipsAndTransports.membershipsWithTransport$, + connectionManager: connectionManager, + }); - const connectionManager = createConnectionManager$({ - scope: scope, - connectionFactory: connectionFactory, - inputTransports$: scope.behavior( - combineLatest( - [localTransport$, membershipsAndTransports.transports$], - (localTransport, transports) => { - const localTransportAsArray = localTransport - ? [localTransport] - : []; - return transports.mapInner((transports) => [ - ...localTransportAsArray, - ...transports, - ]); - }, - ), + const connectOptions$ = scope.behavior( + matrixRTCMode.value$.pipe( + map((mode) => ({ + encryptMedia: livekitKeyProvider !== undefined, + // TODO. This might need to get called again on each cahnge of matrixRTCMode... + matrixRTCMode: mode, + })), + ), + ); + + const localMembership = createLocalMembership$({ + scope: scope, + muteStates: muteStates, + mediaDevices: mediaDevices, + connectionManager: connectionManager, + matrixRTCSession: matrixRTCSession, + matrixRoom: matrixRoom, + localTransport$: localTransport$, + trackProcessorState$: trackProcessorState$, + widget, + options: connectOptions$, + logger: logger.getChild(`[${Date.now()}]`), + }); + + const localRtcMembership$ = scope.behavior( + memberships$.pipe( + map( + (memberships) => + memberships.value.find( + (membership) => + membership.userId === userId && membership.deviceId === deviceId, + ) ?? null, ), - logger: logger, - }); + ), + ); - const matrixLivekitMembers$ = createMatrixLivekitMembers$({ - scope: scope, - membershipsWithTransport$: - membershipsAndTransports.membershipsWithTransport$, - connectionManager: connectionManager, - }); + const localMatrixLivekitMemberUninitialized = { + membership$: localRtcMembership$, + participant$: localMembership.participant$, + connection$: localMembership.connection$, + userId: userId, + }; - const connectOptions$ = scope.behavior( - matrixRTCMode.value$.pipe( - map((mode) => ({ - encryptMedia: livekitKeyProvider !== undefined, - // TODO. This might need to get called again on each cahnge of matrixRTCMode... - matrixRTCMode: mode, - })), - ), - ); - - const localMembership = createLocalMembership$({ - scope: scope, - muteStates: muteStates, - mediaDevices: mediaDevices, - connectionManager: connectionManager, - matrixRTCSession: matrixRTCSession, - matrixRoom: matrixRoom, - localTransport$: localTransport$, - trackProcessorState$: trackProcessorState$, - widget, - options: connectOptions$, - logger: logger.getChild(`[${Date.now()}]`), - }); - - const localRtcMembership$ = scope.behavior( - memberships$.pipe( - map( - (memberships) => - memberships.value.find( - (membership) => - membership.userId === userId && - membership.deviceId === deviceId, - ) ?? null, - ), - ), - ); - - const localMatrixLivekitMemberUninitialized = { - membership$: localRtcMembership$, - participant$: localMembership.participant$, - connection$: localMembership.connection$, - userId: userId, - }; - - const localMatrixLivekitMember$: Behavior = - scope.behavior( - localRtcMembership$.pipe( - switchMap((membership) => { - if (!membership) return of(null); - return of( - // casting is save here since we know that localRtcMembership$ is !== null since we reached this case. - localMatrixLivekitMemberUninitialized as MatrixLivekitMember, - ); - }), - ), - ); - - // ------------------------------------------------------------------------ - // callLifecycle - - // TODO if we are in "unknown" state we need a loading rendering (or empty screen) - // Otherwise it looks like we already connected and only than the ringing starts which is weird. - const { callPickupState$, autoLeave$ } = createCallNotificationLifecycle$({ - scope: scope, - memberships$: memberships$, - sentCallNotification$: createSentCallNotification$( - scope, - matrixRTCSession, - ), - receivedDecline$: createReceivedDecline$(matrixRoom), - options: options, - localUser: { userId: userId, deviceId: deviceId }, - }); - - // ------------------------------------------------------------------------ - // matrixMemberMetadataStore - - const matrixRoomMembers$ = createRoomMembers$(scope, matrixRoom); - const matrixMemberMetadataStore = createMatrixMemberMetadata$( - scope, - scope.behavior(memberships$.pipe(map((mems) => mems.value))), - matrixRoomMembers$, - ); - - const dmMember$ = createDMMember$(scope, matrixRoomMembers$, matrixRoom); - const noUserToCallInRoom$ = scope.behavior( - matrixRoomMembers$.pipe( - map( - (roomMembersMap) => - roomMembersMap.size === 1 && - roomMembersMap.get(userId) !== undefined, - ), - ), - ); - - const ringOverlay$ = scope.behavior( - combineLatest([noUserToCallInRoom$, dmMember$, callPickupState$]).pipe( - map(([noUserToCallInRoom, dmMember, callPickupState]) => { - // No overlay if not in ringing state - if (callPickupState !== "ringing" || noUserToCallInRoom) return null; - - const name = dmMember ? dmMember.rawDisplayName : matrixRoom.name; - const id = dmMember ? dmMember.userId : matrixRoom.roomId; - const text = dmMember - ? `Waiting for ${name} to join…` - : "Waiting for other participants…"; - const avatarMxc = dmMember - ? (dmMember.getMxcAvatarUrl?.() ?? undefined) - : (matrixRoom.getMxcAvatarUrl() ?? undefined); - return { - name: name ?? id, - idForAvatar: id, - text, - avatarMxc, - }; - }), - ), - ); - - // CODESMELL? - // This is functionally the same Observable as leave$, except here it's - // hoisted to the top of the class. This enables the cyclic dependency between - // leave$ -> autoLeave$ -> callPickupState$ -> livekitConnectionState$ -> - // localConnection$ -> transports$ -> joined$ -> leave$. - const leaveHoisted$ = new Subject< - "user" | "timeout" | "decline" | "allOthersLeft" - >(); - - /** - * Whether various media/event sources should pretend to be disconnected from - * all network input, even if their connection still technically works. - */ - // We do this when the app is in the 'reconnecting' state, because it might be - // that the LiveKit connection is still functional while the homeserver is - // down, for example, and we want to avoid making people worry that the app is - // in a split-brained state. - // DISCUSSION own membership manager ALSO this probably can be simplifis - const reconnecting$ = localMembership.reconnecting$; - const pretendToBeDisconnected$ = reconnecting$; - - const audioParticipants$ = scope.behavior( - matrixLivekitMembers$.pipe( - switchMap((membersWithEpoch) => { - const members = membersWithEpoch.value; - const a$ = combineLatest( - members.map((member) => - combineLatest([member.connection$, member.participant$]).pipe( - map(([connection, participant]) => { - // do not render audio for local participant - if (!connection || !participant || participant.isLocal) - return null; - const livekitRoom = connection.livekitRoom; - const url = connection.transport.livekit_service_url; - - return { - url, - livekitRoom, - participant: participant.identity, - }; - }), - ), - ), + const localMatrixLivekitMember$: Behavior = + scope.behavior( + localRtcMembership$.pipe( + switchMap((membership) => { + if (!membership) return of(null); + return of( + // casting is save here since we know that localRtcMembership$ is !== null since we reached this case. + localMatrixLivekitMemberUninitialized as MatrixLivekitMember, ); - return a$; }), - map((members) => - members.reduce((acc, curr) => { - if (!curr) return acc; - - const existing = acc.find((item) => item.url === curr.url); - if (existing) { - existing.participants.push(curr.participant); - } else { - acc.push({ - livekitRoom: curr.livekitRoom, - participants: [curr.participant], - url: curr.url, - }); - } - return acc; - }, []), - ), ), - [], ); - const handsRaised$ = scope.behavior( - handsRaisedSubject$.pipe(pauseWhen(pretendToBeDisconnected$)), - ); + // ------------------------------------------------------------------------ + // callLifecycle - const reactions$ = scope.behavior( - reactionsSubject$.pipe( - map((v) => - Object.fromEntries( - Object.entries(v).map(([a, { reactionOption }]) => [ - a, - reactionOption, - ]), + // TODO if we are in "unknown" state we need a loading rendering (or empty screen) + // Otherwise it looks like we already connected and only than the ringing starts which is weird. + const { callPickupState$, autoLeave$ } = createCallNotificationLifecycle$({ + scope: scope, + memberships$: memberships$, + sentCallNotification$: createSentCallNotification$(scope, matrixRTCSession), + receivedDecline$: createReceivedDecline$(matrixRoom), + options: options, + localUser: { userId: userId, deviceId: deviceId }, + }); + + // ------------------------------------------------------------------------ + // matrixMemberMetadataStore + + const matrixRoomMembers$ = createRoomMembers$(scope, matrixRoom); + const matrixMemberMetadataStore = createMatrixMemberMetadata$( + scope, + scope.behavior(memberships$.pipe(map((mems) => mems.value))), + matrixRoomMembers$, + ); + + const dmMember$ = createDMMember$(scope, matrixRoomMembers$, matrixRoom); + const noUserToCallInRoom$ = scope.behavior( + matrixRoomMembers$.pipe( + map( + (roomMembersMap) => + roomMembersMap.size === 1 && roomMembersMap.get(userId) !== undefined, + ), + ), + ); + + const ringOverlay$ = scope.behavior( + combineLatest([noUserToCallInRoom$, dmMember$, callPickupState$]).pipe( + map(([noUserToCallInRoom, dmMember, callPickupState]) => { + // No overlay if not in ringing state + if (callPickupState !== "ringing" || noUserToCallInRoom) return null; + + const name = dmMember ? dmMember.rawDisplayName : matrixRoom.name; + const id = dmMember ? dmMember.userId : matrixRoom.roomId; + const text = dmMember + ? `Waiting for ${name} to join…` + : "Waiting for other participants…"; + const avatarMxc = dmMember + ? (dmMember.getMxcAvatarUrl?.() ?? undefined) + : (matrixRoom.getMxcAvatarUrl() ?? undefined); + return { + name: name ?? id, + idForAvatar: id, + text, + avatarMxc, + }; + }), + ), + ); + + // CODESMELL? + // This is functionally the same Observable as leave$, except here it's + // hoisted to the top of the class. This enables the cyclic dependency between + // leave$ -> autoLeave$ -> callPickupState$ -> livekitConnectionState$ -> + // localConnection$ -> transports$ -> joined$ -> leave$. + const leaveHoisted$ = new Subject< + "user" | "timeout" | "decline" | "allOthersLeft" + >(); + + /** + * Whether various media/event sources should pretend to be disconnected from + * all network input, even if their connection still technically works. + */ + // We do this when the app is in the 'reconnecting' state, because it might be + // that the LiveKit connection is still functional while the homeserver is + // down, for example, and we want to avoid making people worry that the app is + // in a split-brained state. + // DISCUSSION own membership manager ALSO this probably can be simplifis + const reconnecting$ = localMembership.reconnecting$; + const pretendToBeDisconnected$ = reconnecting$; + + const audioParticipants$ = scope.behavior( + matrixLivekitMembers$.pipe( + switchMap((membersWithEpoch) => { + const members = membersWithEpoch.value; + const a$ = combineLatest( + members.map((member) => + combineLatest([member.connection$, member.participant$]).pipe( + map(([connection, participant]) => { + // do not render audio for local participant + if (!connection || !participant || participant.isLocal) + return null; + const livekitRoom = connection.livekitRoom; + const url = connection.transport.livekit_service_url; + + return { + url, + livekitRoom, + participant: participant.identity, + }; + }), + ), ), - ), - pauseWhen(pretendToBeDisconnected$), - ), - ); + ); + return a$; + }), + map((members) => + members.reduce((acc, curr) => { + if (!curr) return acc; - /** - * List of user media (camera feeds) that we want tiles for. - */ - // TODO this also needs the local participant to be added. - const userMedia$ = scope.behavior( - combineLatest([ - localMatrixLivekitMember$, - matrixLivekitMembers$, - duplicateTiles.value$, - ]).pipe( - // Generate a collection of MediaItems from the list of expected (whether - // present or missing) LiveKit participants. - generateItems( - function* ([ - localMatrixLivekitMember, - { value: matrixLivekitMembers }, - duplicateTiles, - ]) { - let localParticipantId = undefined; - // add local member if available - if (localMatrixLivekitMember) { - const { userId, participant$, connection$, membership$ } = - localMatrixLivekitMember; - localParticipantId = `${userId}:${membership$.value.deviceId}`; // should be membership$.value.membershipID which is not optional - // const participantId = membership$.value.membershipID; - if (localParticipantId) { - for (let dup = 0; dup < 1 + duplicateTiles; dup++) { - yield { - keys: [ - dup, - localParticipantId, - userId, - participant$, - connection$, - ], - data: undefined, - }; - } - } - } - // add remote members that are available - for (const { - userId, - participant$, - connection$, - membership$, - } of matrixLivekitMembers) { - const participantId = `${userId}:${membership$.value.deviceId}`; - if (participantId === localParticipantId) continue; - // const participantId = membership$.value?.identity; + const existing = acc.find((item) => item.url === curr.url); + if (existing) { + existing.participants.push(curr.participant); + } else { + acc.push({ + livekitRoom: curr.livekitRoom, + participants: [curr.participant], + url: curr.url, + }); + } + return acc; + }, []), + ), + ), + [], + ); + + const handsRaised$ = scope.behavior( + handsRaisedSubject$.pipe(pauseWhen(pretendToBeDisconnected$)), + ); + + const reactions$ = scope.behavior( + reactionsSubject$.pipe( + map((v) => + Object.fromEntries( + Object.entries(v).map(([a, { reactionOption }]) => [ + a, + reactionOption, + ]), + ), + ), + pauseWhen(pretendToBeDisconnected$), + ), + ); + + /** + * List of user media (camera feeds) that we want tiles for. + */ + // TODO this also needs the local participant to be added. + const userMedia$ = scope.behavior( + combineLatest([ + localMatrixLivekitMember$, + matrixLivekitMembers$, + duplicateTiles.value$, + ]).pipe( + // Generate a collection of MediaItems from the list of expected (whether + // present or missing) LiveKit participants. + generateItems( + function* ([ + localMatrixLivekitMember, + { value: matrixLivekitMembers }, + duplicateTiles, + ]) { + let localParticipantId = undefined; + // add local member if available + if (localMatrixLivekitMember) { + const { userId, participant$, connection$, membership$ } = + localMatrixLivekitMember; + localParticipantId = `${userId}:${membership$.value.deviceId}`; // should be membership$.value.membershipID which is not optional + // const participantId = membership$.value.membershipID; + if (localParticipantId) { for (let dup = 0; dup < 1 + duplicateTiles; dup++) { yield { - keys: [dup, participantId, userId, participant$, connection$], + keys: [ + dup, + localParticipantId, + userId, + participant$, + connection$, + ], data: undefined, }; } } - }, - ( - scope, - _data$, - dup, - participantId, + } + // add remote members that are available + for (const { userId, participant$, connection$, - ) => { - const livekitRoom$ = scope.behavior( - connection$.pipe(map((c) => c?.livekitRoom)), - ); - const focusUrl$ = scope.behavior( - connection$.pipe(map((c) => c?.transport.livekit_service_url)), - ); - const displayName$ = scope.behavior( - matrixMemberMetadataStore - .createDisplayNameBehavior$(userId) - .pipe(map((name) => name ?? userId)), - ); + membership$, + } of matrixLivekitMembers) { + const participantId = `${userId}:${membership$.value.deviceId}`; + if (participantId === localParticipantId) continue; + // const participantId = membership$.value?.identity; + for (let dup = 0; dup < 1 + duplicateTiles; dup++) { + yield { + keys: [dup, participantId, userId, participant$, connection$], + data: undefined, + }; + } + } + }, + ( + scope, + _data$, + dup, + participantId, + userId, + participant$, + connection$, + ) => { + const livekitRoom$ = scope.behavior( + connection$.pipe(map((c) => c?.livekitRoom)), + ); + const focusUrl$ = scope.behavior( + connection$.pipe(map((c) => c?.transport.livekit_service_url)), + ); + const displayName$ = scope.behavior( + matrixMemberMetadataStore + .createDisplayNameBehavior$(userId) + .pipe(map((name) => name ?? userId)), + ); - return new UserMedia( - scope, - `${participantId}:${dup}`, - userId, - participant$, - options.encryptionSystem, - livekitRoom$, - focusUrl$, - mediaDevices, - pretendToBeDisconnected$, - displayName$, - matrixMemberMetadataStore.createAvatarUrlBehavior$(userId), - handsRaised$.pipe(map((v) => v[participantId]?.time ?? null)), - reactions$.pipe(map((v) => v[participantId] ?? undefined)), - ); - }, - ), + return new UserMedia( + scope, + `${participantId}:${dup}`, + userId, + participant$, + options.encryptionSystem, + livekitRoom$, + focusUrl$, + mediaDevices, + pretendToBeDisconnected$, + displayName$, + matrixMemberMetadataStore.createAvatarUrlBehavior$(userId), + handsRaised$.pipe(map((v) => v[participantId]?.time ?? null)), + reactions$.pipe(map((v) => v[participantId] ?? undefined)), + ); + }, ), + ), + ); + + /** + * List of all media items (user media and screen share media) that we want + * tiles for. + */ + const mediaItems$ = scope.behavior( + userMedia$.pipe( + switchMap((userMedia) => + userMedia.length === 0 + ? of([]) + : combineLatest( + userMedia.map((m) => m.screenShares$), + (...screenShares) => [...userMedia, ...screenShares.flat(1)], + ), + ), + ), + ); + + /** + * List of MediaItems that we want to display, that are of type ScreenShare + */ + const screenShares$ = scope.behavior( + mediaItems$.pipe( + map((mediaItems) => + mediaItems.filter((m): m is ScreenShare => m instanceof ScreenShare), + ), + ), + ); + + const joinSoundEffect$ = userMedia$.pipe( + pairwise(), + filter( + ([prev, current]) => + current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && + current.length > prev.length, + ), + map(() => {}), + throttleTime(THROTTLE_SOUND_EFFECT_MS), + ); + + /** + * The number of participants currently in the call. + * + * - Each participant has a corresponding MatrixRTC membership state event + * - There can be multiple participants for one Matrix user if they join from + * multiple devices. + */ + const participantCount$ = scope.behavior( + matrixLivekitMembers$.pipe(map((ms) => ms.value.length)), + ); + + const leaveSoundEffect$ = combineLatest([callPickupState$, userMedia$]).pipe( + // Until the call is successful, do not play a leave sound. + // If callPickupState$ is null, then we always play the sound as it will not conflict with a decline sound. + skipWhile(([c]) => c !== null && c !== "success"), + map(([, userMedia]) => userMedia), + pairwise(), + filter( + ([prev, current]) => + current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && + current.length < prev.length, + ), + map(() => {}), + throttleTime(THROTTLE_SOUND_EFFECT_MS), + ); + + const userHangup$ = new Subject(); + + const widgetHangup$ = + widget === null + ? NEVER + : ( + fromEvent( + widget.lazyActions, + ElementWidgetActions.HangupCall, + ) as Observable> + ).pipe( + tap((ev) => { + widget!.api.transport.reply(ev.detail, {}); + }), + ); + + const leave$: Observable<"user" | "timeout" | "decline" | "allOthersLeft"> = + merge( + autoLeave$, + merge(userHangup$, widgetHangup$).pipe(map(() => "user" as const)), + ).pipe( + scope.share, + tap((reason) => leaveHoisted$.next(reason)), ); - /** - * List of all media items (user media and screen share media) that we want - * tiles for. - */ - const mediaItems$ = scope.behavior( - userMedia$.pipe( - switchMap((userMedia) => - userMedia.length === 0 - ? of([]) - : combineLatest( - userMedia.map((m) => m.screenShares$), - (...screenShares) => [...userMedia, ...screenShares.flat(1)], + const spotlightSpeaker$ = scope.behavior( + userMedia$.pipe( + switchMap((mediaItems) => + mediaItems.length === 0 + ? of([]) + : combineLatest( + mediaItems.map((m) => + m.vm.speaking$.pipe(map((s) => [m, s] as const)), ), - ), + ), ), - ); - - /** - * List of MediaItems that we want to display, that are of type ScreenShare - */ - const screenShares$ = scope.behavior( - mediaItems$.pipe( - map((mediaItems) => - mediaItems.filter((m): m is ScreenShare => m instanceof ScreenShare), - ), + scan<(readonly [UserMedia, boolean])[], UserMedia | undefined, null>( + (prev, mediaItems) => { + // Only remote users that are still in the call should be sticky + const [stickyMedia, stickySpeaking] = + (!prev?.vm.local && mediaItems.find(([m]) => m === prev)) || []; + // Decide who to spotlight: + // If the previous speaker is still speaking, stick with them rather + // than switching eagerly to someone else + return stickySpeaking + ? stickyMedia! + : // Otherwise, select any remote user who is speaking + (mediaItems.find(([m, s]) => !m.vm.local && s)?.[0] ?? + // Otherwise, stick with the person who was last speaking + stickyMedia ?? + // Otherwise, spotlight an arbitrary remote user + mediaItems.find(([m]) => !m.vm.local)?.[0] ?? + // Otherwise, spotlight the local user + mediaItems.find(([m]) => m.vm.local)?.[0]); + }, + null, ), - ); + map((speaker) => speaker?.vm ?? null), + ), + ); - const joinSoundEffect$ = userMedia$.pipe( - pairwise(), - filter( - ([prev, current]) => - current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && - current.length > prev.length, - ), - map(() => {}), - throttleTime(THROTTLE_SOUND_EFFECT_MS), - ); + const grid$ = scope.behavior( + userMedia$.pipe( + switchMap((mediaItems) => { + const bins = mediaItems.map((m) => + m.bin$.pipe(map((bin) => [m, bin] as const)), + ); + // Sort the media by bin order and generate a tile for each one + return bins.length === 0 + ? of([]) + : combineLatest(bins, (...bins) => + bins.sort(([, bin1], [, bin2]) => bin1 - bin2).map(([m]) => m.vm), + ); + }), + distinctUntilChanged(shallowEquals), + ), + ); - /** - * The number of participants currently in the call. - * - * - Each participant has a corresponding MatrixRTC membership state event - * - There can be multiple participants for one Matrix user if they join from - * multiple devices. - */ - const participantCount$ = scope.behavior( - matrixLivekitMembers$.pipe(map((ms) => ms.value.length)), - ); + const spotlight$ = scope.behavior( + screenShares$.pipe( + switchMap((screenShares) => { + if (screenShares.length > 0) { + return of(screenShares.map((m) => m.vm)); + } - const leaveSoundEffect$ = combineLatest([ - callPickupState$, - userMedia$, + return spotlightSpeaker$.pipe( + map((speaker) => (speaker ? [speaker] : [])), + ); + }), + distinctUntilChanged(shallowEquals), + ), + ); + + const pip$ = scope.behavior( + combineLatest([ + // TODO This also needs epoch logic to dedupe the screenshares and mediaItems emits + screenShares$, + spotlightSpeaker$, + mediaItems$, ]).pipe( - // Until the call is successful, do not play a leave sound. - // If callPickupState$ is null, then we always play the sound as it will not conflict with a decline sound. - skipWhile(([c]) => c !== null && c !== "success"), - map(([, userMedia]) => userMedia), - pairwise(), - filter( - ([prev, current]) => - current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && - current.length < prev.length, - ), - map(() => {}), - throttleTime(THROTTLE_SOUND_EFFECT_MS), - ); + switchMap(([screenShares, spotlight, mediaItems]) => { + if (screenShares.length > 0) { + return spotlightSpeaker$; + } + if (!spotlight || spotlight.local) { + return of(null); + } - const userHangup$ = new Subject(); + const localUserMedia = mediaItems.find( + (m) => m.vm instanceof LocalUserMediaViewModel, + ) as UserMedia | undefined; - const widgetHangup$ = - widget === null - ? NEVER - : ( - fromEvent( - widget.lazyActions, - ElementWidgetActions.HangupCall, - ) as Observable> - ).pipe( - tap((ev) => { - widget!.api.transport.reply(ev.detail, {}); - }), - ); + const localUserMediaViewModel = localUserMedia?.vm as + | LocalUserMediaViewModel + | undefined; - const leave$: Observable<"user" | "timeout" | "decline" | "allOthersLeft"> = - merge( - autoLeave$, - merge(userHangup$, widgetHangup$).pipe(map(() => "user" as const)), - ).pipe( - scope.share, - tap((reason) => leaveHoisted$.next(reason)), - ); + if (!localUserMediaViewModel) { + return of(null); + } + return localUserMediaViewModel.alwaysShow$.pipe( + map((alwaysShow) => { + if (alwaysShow) { + return localUserMediaViewModel; + } - const spotlightSpeaker$ = scope.behavior( - userMedia$.pipe( - switchMap((mediaItems) => - mediaItems.length === 0 - ? of([]) - : combineLatest( - mediaItems.map((m) => - m.vm.speaking$.pipe(map((s) => [m, s] as const)), + return null; + }), + ); + }), + ), + ); + + const hasRemoteScreenShares$: Observable = spotlight$.pipe( + map((spotlight) => + spotlight.some((vm) => !vm.local && vm instanceof ScreenShareViewModel), + ), + distinctUntilChanged(), + ); + + const pipEnabled$ = scope.behavior(setPipEnabled$, false); + + const naturalWindowMode$ = scope.behavior( + fromEvent(window, "resize").pipe( + map(() => { + const height = window.innerHeight; + const width = window.innerWidth; + if (height <= 400 && width <= 340) return "pip"; + // Our layouts for flat windows are better at adapting to a small width + // than our layouts for narrow windows are at adapting to a small height, + // so we give "flat" precedence here + if (height <= 600) return "flat"; + if (width <= 600) return "narrow"; + return "normal"; + }), + ), + "normal", + ); + + /** + * The general shape of the window. + */ + const windowMode$ = scope.behavior( + pipEnabled$.pipe( + switchMap((pip) => (pip ? of("pip") : naturalWindowMode$)), + ), + ); + + const spotlightExpandedToggle$ = new Subject(); + const spotlightExpanded$ = scope.behavior( + spotlightExpandedToggle$.pipe(accumulate(false, (expanded) => !expanded)), + ); + + const gridModeUserSelection$ = new Subject(); + /** + * The layout mode of the media tile grid. + */ + const gridMode$ = + // If the user hasn't selected spotlight and somebody starts screen sharing, + // automatically switch to spotlight mode and reset when screen sharing ends + scope.behavior( + gridModeUserSelection$.pipe( + switchMap((userSelection) => + (userSelection === "spotlight" + ? EMPTY + : combineLatest([hasRemoteScreenShares$, windowMode$]).pipe( + skip(userSelection === null ? 0 : 1), + map( + ([hasScreenShares, windowMode]): GridMode => + hasScreenShares || windowMode === "flat" + ? "spotlight" + : "grid", ), - ), + ) + ).pipe(startWith(userSelection ?? "grid")), ), - scan<(readonly [UserMedia, boolean])[], UserMedia | undefined, null>( - (prev, mediaItems) => { - // Only remote users that are still in the call should be sticky - const [stickyMedia, stickySpeaking] = - (!prev?.vm.local && mediaItems.find(([m]) => m === prev)) || []; - // Decide who to spotlight: - // If the previous speaker is still speaking, stick with them rather - // than switching eagerly to someone else - return stickySpeaking - ? stickyMedia! - : // Otherwise, select any remote user who is speaking - (mediaItems.find(([m, s]) => !m.vm.local && s)?.[0] ?? - // Otherwise, stick with the person who was last speaking - stickyMedia ?? - // Otherwise, spotlight an arbitrary remote user - mediaItems.find(([m]) => !m.vm.local)?.[0] ?? - // Otherwise, spotlight the local user - mediaItems.find(([m]) => m.vm.local)?.[0]); - }, - null, - ), - map((speaker) => speaker?.vm ?? null), ), + "grid", ); - const grid$ = scope.behavior( - userMedia$.pipe( - switchMap((mediaItems) => { - const bins = mediaItems.map((m) => - m.bin$.pipe(map((bin) => [m, bin] as const)), - ); - // Sort the media by bin order and generate a tile for each one - return bins.length === 0 - ? of([]) - : combineLatest(bins, (...bins) => - bins - .sort(([, bin1], [, bin2]) => bin1 - bin2) - .map(([m]) => m.vm), - ); - }), - distinctUntilChanged(shallowEquals), - ), - ); + const setGridMode = (value: GridMode): void => { + gridModeUserSelection$.next(value); + }; - const spotlight$ = scope.behavior( - screenShares$.pipe( - switchMap((screenShares) => { - if (screenShares.length > 0) { - return of(screenShares.map((m) => m.vm)); - } + const gridLayoutMedia$: Observable = combineLatest( + [grid$, spotlight$], + (grid, spotlight) => ({ + type: "grid", + spotlight: spotlight.some((vm) => vm instanceof ScreenShareViewModel) + ? spotlight + : undefined, + grid, + }), + ); - return spotlightSpeaker$.pipe( - map((speaker) => (speaker ? [speaker] : [])), - ); - }), - distinctUntilChanged(shallowEquals), - ), - ); + const spotlightLandscapeLayoutMedia$: Observable = + combineLatest([grid$, spotlight$], (grid, spotlight) => ({ + type: "spotlight-landscape", + spotlight, + grid, + })); - const pip$ = scope.behavior( - combineLatest([ - // TODO This also needs epoch logic to dedupe the screenshares and mediaItems emits - screenShares$, - spotlightSpeaker$, - mediaItems$, - ]).pipe( - switchMap(([screenShares, spotlight, mediaItems]) => { - if (screenShares.length > 0) { - return spotlightSpeaker$; - } - if (!spotlight || spotlight.local) { - return of(null); - } + const spotlightPortraitLayoutMedia$: Observable = + combineLatest([grid$, spotlight$], (grid, spotlight) => ({ + type: "spotlight-portrait", + spotlight, + grid, + })); - const localUserMedia = mediaItems.find( - (m) => m.vm instanceof LocalUserMediaViewModel, - ) as UserMedia | undefined; + const spotlightExpandedLayoutMedia$: Observable = + combineLatest([spotlight$, pip$], (spotlight, pip) => ({ + type: "spotlight-expanded", + spotlight, + pip: pip ?? undefined, + })); - const localUserMediaViewModel = localUserMedia?.vm as - | LocalUserMediaViewModel - | undefined; + const oneOnOneLayoutMedia$: Observable = + mediaItems$.pipe( + map((mediaItems) => { + if (mediaItems.length !== 2) return null; + const local = mediaItems.find((vm) => vm.vm.local)?.vm as + | LocalUserMediaViewModel + | undefined; + const remote = mediaItems.find((vm) => !vm.vm.local)?.vm as + | RemoteUserMediaViewModel + | undefined; + // There might not be a remote tile if there are screen shares, or if + // only the local user is in the call and they're using the duplicate + // tiles option + if (!remote || !local) return null; - if (!localUserMediaViewModel) { - return of(null); - } - return localUserMediaViewModel.alwaysShow$.pipe( - map((alwaysShow) => { - if (alwaysShow) { - return localUserMediaViewModel; - } - - return null; - }), - ); - }), - ), - ); - - const hasRemoteScreenShares$: Observable = spotlight$.pipe( - map((spotlight) => - spotlight.some((vm) => !vm.local && vm instanceof ScreenShareViewModel), - ), - distinctUntilChanged(), - ); - - const pipEnabled$ = scope.behavior(setPipEnabled$, false); - - const naturalWindowMode$ = scope.behavior( - fromEvent(window, "resize").pipe( - map(() => { - const height = window.innerHeight; - const width = window.innerWidth; - if (height <= 400 && width <= 340) return "pip"; - // Our layouts for flat windows are better at adapting to a small width - // than our layouts for narrow windows are at adapting to a small height, - // so we give "flat" precedence here - if (height <= 600) return "flat"; - if (width <= 600) return "narrow"; - return "normal"; - }), - ), - "normal", - ); - - /** - * The general shape of the window. - */ - const windowMode$ = scope.behavior( - pipEnabled$.pipe( - switchMap((pip) => (pip ? of("pip") : naturalWindowMode$)), - ), - ); - - const spotlightExpandedToggle$ = new Subject(); - const spotlightExpanded$ = scope.behavior( - spotlightExpandedToggle$.pipe(accumulate(false, (expanded) => !expanded)), - ); - - const gridModeUserSelection$ = new Subject(); - /** - * The layout mode of the media tile grid. - */ - const gridMode$ = - // If the user hasn't selected spotlight and somebody starts screen sharing, - // automatically switch to spotlight mode and reset when screen sharing ends - scope.behavior( - gridModeUserSelection$.pipe( - switchMap((userSelection) => - (userSelection === "spotlight" - ? EMPTY - : combineLatest([hasRemoteScreenShares$, windowMode$]).pipe( - skip(userSelection === null ? 0 : 1), - map( - ([hasScreenShares, windowMode]): GridMode => - hasScreenShares || windowMode === "flat" - ? "spotlight" - : "grid", - ), - ) - ).pipe(startWith(userSelection ?? "grid")), - ), - ), - "grid", - ); - - const setGridMode = (value: GridMode): void => { - gridModeUserSelection$.next(value); - }; - - const gridLayoutMedia$: Observable = combineLatest( - [grid$, spotlight$], - (grid, spotlight) => ({ - type: "grid", - spotlight: spotlight.some((vm) => vm instanceof ScreenShareViewModel) - ? spotlight - : undefined, - grid, + return { type: "one-on-one", local, remote }; }), ); - const spotlightLandscapeLayoutMedia$: Observable = - combineLatest([grid$, spotlight$], (grid, spotlight) => ({ - type: "spotlight-landscape", - spotlight, - grid, - })); + const pipLayoutMedia$: Observable = spotlight$.pipe( + map((spotlight) => ({ type: "pip", spotlight })), + ); - const spotlightPortraitLayoutMedia$: Observable = - combineLatest([grid$, spotlight$], (grid, spotlight) => ({ - type: "spotlight-portrait", - spotlight, - grid, - })); + /** + * The media to be used to produce a layout. + */ + const layoutMedia$ = scope.behavior( + windowMode$.pipe( + switchMap((windowMode) => { + switch (windowMode) { + case "normal": + return gridMode$.pipe( + switchMap((gridMode) => { + switch (gridMode) { + case "grid": + return oneOnOneLayoutMedia$.pipe( + switchMap((oneOnOne) => + oneOnOne === null ? gridLayoutMedia$ : of(oneOnOne), + ), + ); + case "spotlight": + return spotlightExpanded$.pipe( + switchMap((expanded) => + expanded + ? spotlightExpandedLayoutMedia$ + : spotlightLandscapeLayoutMedia$, + ), + ); + } + }), + ); + case "narrow": + return oneOnOneLayoutMedia$.pipe( + switchMap((oneOnOne) => + oneOnOne === null + ? combineLatest([grid$, spotlight$], (grid, spotlight) => + grid.length > smallMobileCallThreshold || + spotlight.some((vm) => vm instanceof ScreenShareViewModel) + ? spotlightPortraitLayoutMedia$ + : gridLayoutMedia$, + ).pipe(switchAll()) + : // The expanded spotlight layout makes for a better one-on-one + // experience in narrow windows + spotlightExpandedLayoutMedia$, + ), + ); + case "flat": + return gridMode$.pipe( + switchMap((gridMode) => { + switch (gridMode) { + case "grid": + // Yes, grid mode actually gets you a "spotlight" layout in + // this window mode. + return spotlightLandscapeLayoutMedia$; + case "spotlight": + return spotlightExpandedLayoutMedia$; + } + }), + ); + case "pip": + return pipLayoutMedia$; + } + }), + ), + ); - const spotlightExpandedLayoutMedia$: Observable = - combineLatest([spotlight$, pip$], (spotlight, pip) => ({ - type: "spotlight-expanded", - spotlight, - pip: pip ?? undefined, - })); + // There is a cyclical dependency here: the layout algorithms want to know + // which tiles are on screen, but to know which tiles are on screen we have to + // first render a layout. To deal with this we assume initially that no tiles + // are visible, and loop the data back into the layouts with a Subject. + const visibleTiles$ = new Subject(); + const setVisibleTiles = (value: number): void => visibleTiles$.next(value); - const oneOnOneLayoutMedia$: Observable = - mediaItems$.pipe( - map((mediaItems) => { - if (mediaItems.length !== 2) return null; - const local = mediaItems.find((vm) => vm.vm.local)?.vm as - | LocalUserMediaViewModel - | undefined; - const remote = mediaItems.find((vm) => !vm.vm.local)?.vm as - | RemoteUserMediaViewModel - | undefined; - // There might not be a remote tile if there are screen shares, or if - // only the local user is in the call and they're using the duplicate - // tiles option - if (!remote || !local) return null; - - return { type: "one-on-one", local, remote }; - }), - ); - - const pipLayoutMedia$: Observable = spotlight$.pipe( - map((spotlight) => ({ type: "pip", spotlight })), - ); - - /** - * The media to be used to produce a layout. - */ - const layoutMedia$ = scope.behavior( - windowMode$.pipe( - switchMap((windowMode) => { - switch (windowMode) { - case "normal": - return gridMode$.pipe( - switchMap((gridMode) => { - switch (gridMode) { - case "grid": - return oneOnOneLayoutMedia$.pipe( - switchMap((oneOnOne) => - oneOnOne === null ? gridLayoutMedia$ : of(oneOnOne), - ), - ); - case "spotlight": - return spotlightExpanded$.pipe( - switchMap((expanded) => - expanded - ? spotlightExpandedLayoutMedia$ - : spotlightLandscapeLayoutMedia$, - ), - ); - } - }), - ); - case "narrow": - return oneOnOneLayoutMedia$.pipe( - switchMap((oneOnOne) => - oneOnOne === null - ? combineLatest([grid$, spotlight$], (grid, spotlight) => - grid.length > smallMobileCallThreshold || - spotlight.some( - (vm) => vm instanceof ScreenShareViewModel, - ) - ? spotlightPortraitLayoutMedia$ - : gridLayoutMedia$, - ).pipe(switchAll()) - : // The expanded spotlight layout makes for a better one-on-one - // experience in narrow windows - spotlightExpandedLayoutMedia$, - ), - ); - case "flat": - return gridMode$.pipe( - switchMap((gridMode) => { - switch (gridMode) { - case "grid": - // Yes, grid mode actually gets you a "spotlight" layout in - // this window mode. - return spotlightLandscapeLayoutMedia$; - case "spotlight": - return spotlightExpandedLayoutMedia$; - } - }), - ); - case "pip": - return pipLayoutMedia$; - } - }), - ), - ); - - // There is a cyclical dependency here: the layout algorithms want to know - // which tiles are on screen, but to know which tiles are on screen we have to - // first render a layout. To deal with this we assume initially that no tiles - // are visible, and loop the data back into the layouts with a Subject. - const visibleTiles$ = new Subject(); - const setVisibleTiles = (value: number): void => visibleTiles$.next(value); - - const layoutInternals$ = scope.behavior< - LayoutScanState & { layout: Layout } - >( - combineLatest([ - layoutMedia$, - visibleTiles$.pipe(startWith(0), distinctUntilChanged()), - ]).pipe( - scan< - [LayoutMedia, number], - LayoutScanState & { layout: Layout }, - LayoutScanState - >( - ({ tiles: prevTiles }, [media, visibleTiles]) => { - let layout: Layout; - let newTiles: TileStore; - switch (media.type) { - case "grid": - case "spotlight-landscape": - case "spotlight-portrait": - [layout, newTiles] = gridLikeLayout( - media, - visibleTiles, - setVisibleTiles, - prevTiles, - ); - break; - case "spotlight-expanded": - [layout, newTiles] = spotlightExpandedLayout(media, prevTiles); - break; - case "one-on-one": - [layout, newTiles] = oneOnOneLayout(media, prevTiles); - break; - case "pip": - [layout, newTiles] = pipLayout(media, prevTiles); - break; - } - - return { layout, tiles: newTiles }; - }, - { layout: null, tiles: TileStore.empty() }, - ), - ), - ); - - /** - * The layout of tiles in the call interface. - */ - const layout$ = scope.behavior( - layoutInternals$.pipe(map(({ layout }) => layout)), - ); - - /** - * The current generation of the tile store, exposed for debugging purposes. - */ - const tileStoreGeneration$ = scope.behavior( - layoutInternals$.pipe(map(({ tiles }) => tiles.generation)), - ); - - const showSpotlightIndicators$ = scope.behavior( - layout$.pipe(map((l) => l.type !== "grid")), - ); - - const showSpeakingIndicators$ = scope.behavior( - layout$.pipe( - switchMap((l) => { - switch (l.type) { + const layoutInternals$ = scope.behavior( + combineLatest([ + layoutMedia$, + visibleTiles$.pipe(startWith(0), distinctUntilChanged()), + ]).pipe( + scan< + [LayoutMedia, number], + LayoutScanState & { layout: Layout }, + LayoutScanState + >( + ({ tiles: prevTiles }, [media, visibleTiles]) => { + let layout: Layout; + let newTiles: TileStore; + switch (media.type) { + case "grid": case "spotlight-landscape": case "spotlight-portrait": - // If the spotlight is showing the active speaker, we can do without - // speaking indicators as they're a redundant visual cue. But if - // screen sharing feeds are in the spotlight we still need them. - return l.spotlight.media$.pipe( - map((models: MediaViewModel[]) => - models.some((m) => m instanceof ScreenShareViewModel), - ), + [layout, newTiles] = gridLikeLayout( + media, + visibleTiles, + setVisibleTiles, + prevTiles, ); - // In expanded spotlight layout, the active speaker is always shown in - // the picture-in-picture tile so there is no need for speaking - // indicators. And in one-on-one layout there's no question as to who is - // speaking. + break; case "spotlight-expanded": + [layout, newTiles] = spotlightExpandedLayout(media, prevTiles); + break; case "one-on-one": - return of(false); - default: - return of(true); - } - }), - ), - ); - - const toggleSpotlightExpanded$ = scope.behavior<(() => void) | null>( - windowMode$.pipe( - switchMap((mode) => - mode === "normal" - ? layout$.pipe( - map( - (l) => - l.type === "spotlight-landscape" || - l.type === "spotlight-expanded", - ), - ) - : of(false), - ), - distinctUntilChanged(), - map((enabled) => - enabled ? (): void => spotlightExpandedToggle$.next() : null, - ), - ), - ); - - const screenTap$ = new Subject(); - const controlsTap$ = new Subject(); - const screenHover$ = new Subject(); - const screenUnhover$ = new Subject(); - - const showHeader$ = scope.behavior( - windowMode$.pipe(map((mode) => mode !== "pip" && mode !== "flat")), - ); - - const showFooter$ = scope.behavior( - windowMode$.pipe( - switchMap((mode) => { - switch (mode) { + [layout, newTiles] = oneOnOneLayout(media, prevTiles); + break; case "pip": - return of(false); - case "normal": - case "narrow": - return of(true); - case "flat": - // Sadly Firefox has some layering glitches that prevent the footer - // from appearing properly. They happen less often if we never hide - // the footer. - if (isFirefox()) return of(true); - // Show/hide the footer in response to interactions - return merge( - screenTap$.pipe(map(() => "tap screen" as const)), - controlsTap$.pipe(map(() => "tap controls" as const)), - screenHover$.pipe(map(() => "hover" as const)), - ).pipe( - switchScan((state, interaction) => { - switch (interaction) { - case "tap screen": - return state - ? // Toggle visibility on tap - of(false) - : // Hide after a timeout - timer(showFooterMs).pipe( - map(() => false), - startWith(true), - ); - case "tap controls": - // The user is interacting with things, so reset the timeout - return timer(showFooterMs).pipe( - map(() => false), - startWith(true), - ); - case "hover": - // Show on hover and hide after a timeout - return race( - timer(showFooterMs), - screenUnhover$.pipe(take(1)), - ).pipe( - map(() => false), - startWith(true), - ); - } - }, false), - startWith(false), - ); + [layout, newTiles] = pipLayout(media, prevTiles); + break; } - }), - ), - ); - /** - * Whether audio is currently being output through the earpiece. - */ - const earpieceMode$ = scope.behavior( - combineLatest( - [ - mediaDevices.audioOutput.available$, - mediaDevices.audioOutput.selected$, - ], - (available, selected) => - selected !== undefined && - available.get(selected.id)?.type === "earpiece", - ), - ); - - /** - * Callback to toggle between the earpiece and the loudspeaker. - * - * This will be `null` in case the target does not exist in the list - * of available audio outputs. - */ - const audioOutputSwitcher$ = scope.behavior<{ - targetOutput: "earpiece" | "speaker"; - switch: () => void; - } | null>( - combineLatest( - [ - mediaDevices.audioOutput.available$, - mediaDevices.audioOutput.selected$, - ], - (available, selected) => { - const selectionType = selected && available.get(selected.id)?.type; - - // If we are in any output mode other than speaker switch to speaker. - const newSelectionType: "earpiece" | "speaker" = - selectionType === "speaker" ? "earpiece" : "speaker"; - const newSelection = [...available].find( - ([, d]) => d.type === newSelectionType, - ); - if (newSelection === undefined) return null; - - const [id] = newSelection; - return { - targetOutput: newSelectionType, - switch: (): void => mediaDevices.audioOutput.select(id), - }; + return { layout, tiles: newTiles }; }, + { layout: null, tiles: TileStore.empty() }, ), - ); + ), + ); - /** - * Emits an array of reactions that should be visible on the screen. - */ - // DISCUSSION move this into a reaction file - // const {visibleReactions$, audibleReactions$} = reactionsObservables$(showReactionSetting$, ) - const visibleReactions$ = scope.behavior( - showReactions.value$.pipe( - switchMap((show) => (show ? reactions$ : of({}))), - scan< - Record, - { sender: string; emoji: string; startX: number }[] - >((acc, latest) => { - const newSet: { sender: string; emoji: string; startX: number }[] = - []; - for (const [sender, reaction] of Object.entries(latest)) { - const startX = - acc.find((v) => v.sender === sender && v.emoji)?.startX ?? - Math.ceil(Math.random() * 80) + 10; - newSet.push({ sender, emoji: reaction.emoji, startX }); - } - return newSet; - }, []), + /** + * The layout of tiles in the call interface. + */ + const layout$ = scope.behavior( + layoutInternals$.pipe(map(({ layout }) => layout)), + ); + + /** + * The current generation of the tile store, exposed for debugging purposes. + */ + const tileStoreGeneration$ = scope.behavior( + layoutInternals$.pipe(map(({ tiles }) => tiles.generation)), + ); + + const showSpotlightIndicators$ = scope.behavior( + layout$.pipe(map((l) => l.type !== "grid")), + ); + + const showSpeakingIndicators$ = scope.behavior( + layout$.pipe( + switchMap((l) => { + switch (l.type) { + case "spotlight-landscape": + case "spotlight-portrait": + // If the spotlight is showing the active speaker, we can do without + // speaking indicators as they're a redundant visual cue. But if + // screen sharing feeds are in the spotlight we still need them. + return l.spotlight.media$.pipe( + map((models: MediaViewModel[]) => + models.some((m) => m instanceof ScreenShareViewModel), + ), + ); + // In expanded spotlight layout, the active speaker is always shown in + // the picture-in-picture tile so there is no need for speaking + // indicators. And in one-on-one layout there's no question as to who is + // speaking. + case "spotlight-expanded": + case "one-on-one": + return of(false); + default: + return of(true); + } + }), + ), + ); + + const toggleSpotlightExpanded$ = scope.behavior<(() => void) | null>( + windowMode$.pipe( + switchMap((mode) => + mode === "normal" + ? layout$.pipe( + map( + (l) => + l.type === "spotlight-landscape" || + l.type === "spotlight-expanded", + ), + ) + : of(false), ), - ); - - /** - * Emits an array of reactions that should be played. - */ - const audibleReactions$ = playReactionsSound.value$.pipe( - switchMap((show) => - show ? reactions$ : of>({}), + distinctUntilChanged(), + map((enabled) => + enabled ? (): void => spotlightExpandedToggle$.next() : null, ), - map((reactions) => Object.values(reactions).map((v) => v.name)), - scan( - (acc, latest) => { - return { - playing: latest.filter( - (v) => acc.playing.includes(v) || acc.newSounds.includes(v), - ), - newSounds: latest.filter( - (v) => !acc.playing.includes(v) && !acc.newSounds.includes(v), - ), - }; - }, - { playing: [], newSounds: [] }, - ), - map((v) => v.newSounds), - ); + ), + ); - const newHandRaised$ = handsRaised$.pipe( - map((v) => Object.keys(v).length), - scan( - (acc, newValue) => ({ - value: newValue, - playSounds: newValue > acc.value, - }), - { value: 0, playSounds: false }, - ), - filter((v) => v.playSounds), - ); + const screenTap$ = new Subject(); + const controlsTap$ = new Subject(); + const screenHover$ = new Subject(); + const screenUnhover$ = new Subject(); - const newScreenShare$ = screenShares$.pipe( - map((v) => v.length), - scan( - (acc, newValue) => ({ - value: newValue, - playSounds: newValue > acc.value, - }), - { value: 0, playSounds: false }, - ), - filter((v) => v.playSounds), - ); + const showHeader$ = scope.behavior( + windowMode$.pipe(map((mode) => mode !== "pip" && mode !== "flat")), + ); - /** - * Whether we are sharing our screen. - */ - // reassigned here to make it publicly accessible - const sharingScreen$ = localMembership.sharingScreen$; + const showFooter$ = scope.behavior( + windowMode$.pipe( + switchMap((mode) => { + switch (mode) { + case "pip": + return of(false); + case "normal": + case "narrow": + return of(true); + case "flat": + // Sadly Firefox has some layering glitches that prevent the footer + // from appearing properly. They happen less often if we never hide + // the footer. + if (isFirefox()) return of(true); + // Show/hide the footer in response to interactions + return merge( + screenTap$.pipe(map(() => "tap screen" as const)), + controlsTap$.pipe(map(() => "tap controls" as const)), + screenHover$.pipe(map(() => "hover" as const)), + ).pipe( + switchScan((state, interaction) => { + switch (interaction) { + case "tap screen": + return state + ? // Toggle visibility on tap + of(false) + : // Hide after a timeout + timer(showFooterMs).pipe( + map(() => false), + startWith(true), + ); + case "tap controls": + // The user is interacting with things, so reset the timeout + return timer(showFooterMs).pipe( + map(() => false), + startWith(true), + ); + case "hover": + // Show on hover and hide after a timeout + return race( + timer(showFooterMs), + screenUnhover$.pipe(take(1)), + ).pipe( + map(() => false), + startWith(true), + ); + } + }, false), + startWith(false), + ); + } + }), + ), + ); - /** - * Callback to toggle screen sharing. If null, screen sharing is not possible. - */ - // reassigned here to make it publicly accessible - const toggleScreenSharing = localMembership.toggleScreenSharing; + /** + * Whether audio is currently being output through the earpiece. + */ + const earpieceMode$ = scope.behavior( + combineLatest( + [mediaDevices.audioOutput.available$, mediaDevices.audioOutput.selected$], + (available, selected) => + selected !== undefined && + available.get(selected.id)?.type === "earpiece", + ), + ); - const join = localMembership.requestConnect; - join(); // TODO-MULTI-SFU: Use this view model for the lobby as well, and only call this once 'join' is clicked? + /** + * Callback to toggle between the earpiece and the loudspeaker. + * + * This will be `null` in case the target does not exist in the list + * of available audio outputs. + */ + const audioOutputSwitcher$ = scope.behavior<{ + targetOutput: "earpiece" | "speaker"; + switch: () => void; + } | null>( + combineLatest( + [mediaDevices.audioOutput.available$, mediaDevices.audioOutput.selected$], + (available, selected) => { + const selectionType = selected && available.get(selected.id)?.type; - this.autoLeave$ = autoLeave$; - this.callPickupState$ = callPickupState$; - this.ringOverlay$ = ringOverlay$; - this.leave$ = leave$; - this.hangup = (): void => userHangup$.next(); - this.join = join; - this.toggleScreenSharing = toggleScreenSharing; - this.sharingScreen$ = sharingScreen$; + // If we are in any output mode other than speaker switch to speaker. + const newSelectionType: "earpiece" | "speaker" = + selectionType === "speaker" ? "earpiece" : "speaker"; + const newSelection = [...available].find( + ([, d]) => d.type === newSelectionType, + ); + if (newSelection === undefined) return null; - this.tapScreen = (): void => screenTap$.next(); - this.tapControls = (): void => controlsTap$.next(); - this.hoverScreen = (): void => screenHover$.next(); - this.unhoverScreen = (): void => screenUnhover$.next(); + const [id] = newSelection; + return { + targetOutput: newSelectionType, + switch: (): void => mediaDevices.audioOutput.select(id), + }; + }, + ), + ); - this.configError$ = localMembership.configError$; - this.participantCount$ = participantCount$; - this.audioParticipants$ = audioParticipants$; + /** + * Emits an array of reactions that should be visible on the screen. + */ + // DISCUSSION move this into a reaction file + // const {visibleReactions$, audibleReactions$} = reactionsObservables$(showReactionSetting$, ) + const visibleReactions$ = scope.behavior( + showReactions.value$.pipe( + switchMap((show) => (show ? reactions$ : of({}))), + scan< + Record, + { sender: string; emoji: string; startX: number }[] + >((acc, latest) => { + const newSet: { sender: string; emoji: string; startX: number }[] = []; + for (const [sender, reaction] of Object.entries(latest)) { + const startX = + acc.find((v) => v.sender === sender && v.emoji)?.startX ?? + Math.ceil(Math.random() * 80) + 10; + newSet.push({ sender, emoji: reaction.emoji, startX }); + } + return newSet; + }, []), + ), + ); - this.handsRaised$ = handsRaised$; - this.reactions$ = reactions$; - this.joinSoundEffect$ = joinSoundEffect$; - this.leaveSoundEffect$ = leaveSoundEffect$; - this.newHandRaised$ = newHandRaised$; - this.newScreenShare$ = newScreenShare$; - this.audibleReactions$ = audibleReactions$; - this.visibleReactions$ = visibleReactions$; + /** + * Emits an array of reactions that should be played. + */ + const audibleReactions$ = playReactionsSound.value$.pipe( + switchMap((show) => + show ? reactions$ : of>({}), + ), + map((reactions) => Object.values(reactions).map((v) => v.name)), + scan( + (acc, latest) => { + return { + playing: latest.filter( + (v) => acc.playing.includes(v) || acc.newSounds.includes(v), + ), + newSounds: latest.filter( + (v) => !acc.playing.includes(v) && !acc.newSounds.includes(v), + ), + }; + }, + { playing: [], newSounds: [] }, + ), + map((v) => v.newSounds), + ); - this.windowMode$ = windowMode$; - this.spotlightExpanded$ = spotlightExpanded$; - this.toggleSpotlightExpanded$ = toggleSpotlightExpanded$; - this.gridMode$ = gridMode$; - this.setGridMode = setGridMode; - this.grid$ = grid$; - this.spotlight$ = spotlight$; - this.pip$ = pip$; - this.layout$ = layout$; - this.tileStoreGeneration$ = tileStoreGeneration$; - this.showSpotlightIndicators$ = showSpotlightIndicators$; - this.showSpeakingIndicators$ = showSpeakingIndicators$; - this.showHeader$ = showHeader$; - this.showFooter$ = showFooter$; - this.earpieceMode$ = earpieceMode$; - this.audioOutputSwitcher$ = audioOutputSwitcher$; - this.reconnecting$ = reconnecting$; - } + const newHandRaised$ = handsRaised$.pipe( + map((v) => Object.keys(v).length), + scan( + (acc, newValue) => ({ + value: newValue, + playSounds: newValue > acc.value, + }), + { value: 0, playSounds: false }, + ), + filter((v) => v.playSounds), + ); + + const newScreenShare$ = screenShares$.pipe( + map((v) => v.length), + scan( + (acc, newValue) => ({ + value: newValue, + playSounds: newValue > acc.value, + }), + { value: 0, playSounds: false }, + ), + filter((v) => v.playSounds), + ); + + /** + * Whether we are sharing our screen. + */ + // reassigned here to make it publicly accessible + const sharingScreen$ = localMembership.sharingScreen$; + + /** + * Callback to toggle screen sharing. If null, screen sharing is not possible. + */ + // reassigned here to make it publicly accessible + const toggleScreenSharing = localMembership.toggleScreenSharing; + + const join = localMembership.requestConnect; + // TODO-MULTI-SFU: Use this view model for the lobby as well, and only call this once 'join' is clicked? + join(); + return { + autoLeave$: autoLeave$, + callPickupState$: callPickupState$, + ringOverlay$: ringOverlay$, + leave$: leave$, + hangup: (): void => userHangup$.next(), + join: join, + toggleScreenSharing: toggleScreenSharing, + sharingScreen$: sharingScreen$, + + tapScreen: (): void => screenTap$.next(), + tapControls: (): void => controlsTap$.next(), + hoverScreen: (): void => screenHover$.next(), + unhoverScreen: (): void => screenUnhover$.next(), + + configError$: localMembership.configError$, + participantCount$: participantCount$, + audioParticipants$: audioParticipants$, + + handsRaised$: handsRaised$, + reactions$: reactions$, + joinSoundEffect$: joinSoundEffect$, + leaveSoundEffect$: leaveSoundEffect$, + newHandRaised$: newHandRaised$, + newScreenShare$: newScreenShare$, + audibleReactions$: audibleReactions$, + visibleReactions$: visibleReactions$, + + windowMode$: windowMode$, + spotlightExpanded$: spotlightExpanded$, + toggleSpotlightExpanded$: toggleSpotlightExpanded$, + gridMode$: gridMode$, + setGridMode: setGridMode, + grid$: grid$, + spotlight$: spotlight$, + pip$: pip$, + layout$: layout$, + tileStoreGeneration$: tileStoreGeneration$, + showSpotlightIndicators$: showSpotlightIndicators$, + showSpeakingIndicators$: showSpeakingIndicators$, + showHeader$: showHeader$, + showFooter$: showFooter$, + earpieceMode$: earpieceMode$, + audioOutputSwitcher$: audioOutputSwitcher$, + reconnecting$: reconnecting$, + }; } // TODO-MULTI-SFU // Setup and update the keyProvider which was create by `createRoom` was a thing before. Now we never update if the E2EEsystem changes // do we need this? diff --git a/src/state/CallViewModel/CallViewModelTestUtils.ts b/src/state/CallViewModel/CallViewModelTestUtils.ts index fefc57a0..f86921c5 100644 --- a/src/state/CallViewModel/CallViewModelTestUtils.ts +++ b/src/state/CallViewModel/CallViewModelTestUtils.ts @@ -24,7 +24,11 @@ import * as ComponentsCore from "@livekit/components-core"; import type { CallMembership } from "matrix-js-sdk/lib/matrixrtc"; import { E2eeType } from "../../e2ee/e2eeType"; import { type RaisedHandInfo, type ReactionInfo } from "../../reactions"; -import { CallViewModel, type CallViewModelOptions } from "./CallViewModel"; +import { + type CallViewModel, + createCallViewModel$, + type CallViewModelOptions, +} from "./CallViewModel"; import { mockConfig, mockLivekitRoom, @@ -154,7 +158,7 @@ export function withCallViewModel( const raisedHands$ = new BehaviorSubject>({}); const reactions$ = new BehaviorSubject>({}); - const vm = new CallViewModel( + const vm = createCallViewModel$( testScope(), rtcSession.asMockedSession(), room, diff --git a/src/utils/test-viewmodel.ts b/src/utils/test-viewmodel.ts index 990c43f8..98c45d86 100644 --- a/src/utils/test-viewmodel.ts +++ b/src/utils/test-viewmodel.ts @@ -20,7 +20,8 @@ import { ConnectionState, type Room as LivekitRoom } from "livekit-client"; import { E2eeType } from "../e2ee/e2eeType"; import { - CallViewModel, + type CallViewModel, + createCallViewModel$, type CallViewModelOptions, } from "../state/CallViewModel/CallViewModel"; import { @@ -145,7 +146,7 @@ export function getBasicCallViewModelEnvironment( // const remoteParticipants$ = of([aliceParticipant]); - const vm = new CallViewModel( + const vm = createCallViewModel$( testScope(), rtcSession.asMockedSession(), matrixRoom,