From 348f0dbfa81ebcc08ff532ee65f8312024bdaac9 Mon Sep 17 00:00:00 2001 From: Timo K Date: Mon, 17 Nov 2025 11:37:58 +0100 Subject: [PATCH 1/5] CallViewModel structure change. Clear input and outputs. --- src/state/CallViewModel/CallViewModel.ts | 2100 +++++++++-------- .../CallViewModel/CallViewModelFunction.ts | 1400 +++++++++++ src/state/CallViewModel/CallViewModelOld.ts | 1335 +++++++++++ 3 files changed, 3830 insertions(+), 1005 deletions(-) create mode 100644 src/state/CallViewModel/CallViewModelFunction.ts create mode 100644 src/state/CallViewModel/CallViewModelOld.ts diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index ff4a6269..ff2ed0cb 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -12,7 +12,7 @@ import { type Room as LivekitRoom, type RoomOptions, } from "livekit-client"; -import { type Room as MatrixRoom } from "matrix-js-sdk"; +import { type RoomMember, type Room as MatrixRoom } from "matrix-js-sdk"; import { combineLatest, distinctUntilChanged, @@ -106,6 +106,7 @@ import { type MatrixLivekitMember, } from "./remoteMembers/MatrixLivekitMembers.ts"; import { + type AutoLeaveReason, createCallNotificationLifecycle$, createReceivedDecline$, createSentCallNotification$, @@ -165,6 +166,11 @@ type AudioLivekitItem = { participants: string[]; url: string; }; + +type JoinReturn = ReturnType< + ReturnType["requestConnect"] +>; + /** * A view model providing all the application logic needed to show the in-call * UI (may eventually be expanded to cover the lobby and feedback screens in the @@ -174,463 +180,562 @@ type AudioLivekitItem = { // state and LiveKit state. We use the common terminology of room "members", RTC // "memberships", and LiveKit "participants". export class CallViewModel { - private readonly userId = this.matrixRoom.client.getUserId()!; - private readonly deviceId = this.matrixRoom.client.getDeviceId()!; + // lifecycle + public autoLeave$: Observable; + public callPickupState$: Behavior< + "unknown" | "ringing" | "timeout" | "decline" | "success" | null + >; + public leave$: Observable<"user" | AutoLeaveReason>; + public hangup: () => void; - private readonly livekitKeyProvider = getE2eeKeyProvider( - this.options.encryptionSystem, - this.matrixRTCSession, - ); + // joining + public join: () => JoinReturn; - // Each hbar seperates a block of input variables required for the CallViewModel to function. - // The outputs of this block is written under the hbar. - // - // For mocking purposes it is recommended to only mock the functions creating those outputs. - // All other fields are just temp computations for the mentioned output. - // The class does not need anything except the values underneath the bar. - // The creation of the values under the bar are all tested independently and testing the callViewModel Should - // not test their cretation. Call view model only needs: - // - memberships$ via createMemberships$ - // - localMembership via createLocalMembership$ - // - callLifecycle via createCallNotificationLifecycle$ - // - matrixMemberMetadataStore via createMatrixMemberMetadata$ + // screen sharing + public toggleScreenSharing: (() => void) | null; + public sharingScreen$: Behavior; - // ------------------------------------------------------------------------ - // memberships$ - private memberships$ = createMemberships$(this.scope, this.matrixRTCSession); + // UI interactions + public tapScreen: () => void; + public tapControls: () => void; + public hoverScreen: () => void; + public unhoverScreen: () => void; - // ------------------------------------------------------------------------ - // matrixLivekitMembers$ AND localMembership + // errors + public configError$: Behavior; - private membershipsAndTransports = membershipsAndTransports$( - this.scope, - this.memberships$, - ); + // participants and counts + public participantCount$: Behavior; + public audioParticipants$: Behavior; + public handsRaised$: Behavior>; + public reactions$: Behavior>; + public isOneOnOneWith$: Behavior | null>; + public localUserIsAlone$: Behavior; + // sounds and events + public joinSoundEffect$: Observable; + public leaveSoundEffect$: Observable; + public newHandRaised$: Observable<{ value: number; playSounds: boolean }>; + public newScreenShare$: Observable<{ value: number; playSounds: boolean }>; + public audibleReactions$: Observable; + public visibleReactions$: Behavior< + { sender: string; emoji: string; startX: number }[] + >; - private localTransport$ = createLocalTransport$({ - scope: this.scope, - memberships$: this.memberships$, - client: this.matrixRoom.client, - roomId: this.matrixRoom.roomId, - useOldestMember$: this.scope.behavior( - matrixRTCMode.value$.pipe(map((v) => v === MatrixRTCMode.Legacy)), - ), - }); + // window/layout + public windowMode$: Behavior; + public spotlightExpanded$: Behavior; + public toggleSpotlightExpanded$: Behavior<(() => void) | null>; + public gridMode$: Behavior; + public setGridMode: (value: GridMode) => void; - private connectionFactory = new ECConnectionFactory( - this.matrixRoom.client, - this.mediaDevices, - this.trackProcessorState$, - this.livekitKeyProvider, - getUrlParams().controlledAudioDevices, - this.options.livekitRoomFactory, - ); + // media view models and layout + public grid$: Behavior; + public spotlight$: Behavior; + public pip$: Behavior; + public layout$: Behavior; + public tileStoreGeneration$: Behavior; + public showSpotlightIndicators$: Behavior; + public showSpeakingIndicators$: Behavior; - private connectionManager = createConnectionManager$({ - scope: this.scope, - connectionFactory: this.connectionFactory, - inputTransports$: this.scope.behavior( - combineLatest( - [this.localTransport$, this.membershipsAndTransports.transports$], - (localTransport, transports) => { - const localTransportAsArray = localTransport ? [localTransport] : []; - return transports.mapInner((transports) => [ - ...localTransportAsArray, - ...transports, - ]); - }, + // header/footer visibility + public showHeader$: Behavior; + public showFooter$: Behavior; + + // audio routing + public earpieceMode$: Behavior; + public audioOutputSwitcher$: Behavior<{ + targetOutput: "earpiece" | "speaker"; + switch: () => void; + } | null>; + + // connection state + public reconnecting$: Behavior; + + // THIS has to be the last public field declaration + public constructor( + scope: ObservableScope, + // A call is permanently tied to a single Matrix room + matrixRTCSession: MatrixRTCSession, + matrixRoom: MatrixRoom, + mediaDevices: MediaDevices, + muteStates: MuteStates, + options: CallViewModelOptions, + handsRaisedSubject$: Observable>, + reactionsSubject$: Observable>, + trackProcessorState$: Behavior, + ) { + const userId = matrixRoom.client.getUserId()!; + const deviceId = matrixRoom.client.getDeviceId()!; + + const livekitKeyProvider = getE2eeKeyProvider( + options.encryptionSystem, + matrixRTCSession, + ); + + // Each hbar seperates a block of input variables required for the CallViewModel to function. + // The outputs of this block is written under the hbar. + // + // For mocking purposes it is recommended to only mock the functions creating those outputs. + // All other fields are just temp computations for the mentioned output. + // The class does not need anything except the values underneath the bar. + // The creation of the values under the bar are all tested independently and testing the callViewModel Should + // not test their cretation. Call view model only needs: + // - memberships$ via createMemberships$ + // - localMembership via createLocalMembership$ + // - callLifecycle via createCallNotificationLifecycle$ + // - matrixMemberMetadataStore via createMatrixMemberMetadata$ + + // ------------------------------------------------------------------------ + // memberships$ + const memberships$ = createMemberships$(scope, matrixRTCSession); + + // ------------------------------------------------------------------------ + // matrixLivekitMembers$ AND localMembership + + const membershipsAndTransports = membershipsAndTransports$( + scope, + memberships$, + ); + + const localTransport$ = createLocalTransport$({ + scope: scope, + memberships$: memberships$, + client: matrixRoom.client, + roomId: matrixRoom.roomId, + useOldestMember$: scope.behavior( + matrixRTCMode.value$.pipe(map((v) => v === MatrixRTCMode.Legacy)), ), - ), - logger: logger, - }); + }); - private matrixLivekitMembers$ = createMatrixLivekitMembers$({ - scope: this.scope, - membershipsWithTransport$: - this.membershipsAndTransports.membershipsWithTransport$, - connectionManager: this.connectionManager, - }); + const connectionFactory = new ECConnectionFactory( + matrixRoom.client, + mediaDevices, + trackProcessorState$, + livekitKeyProvider, + getUrlParams().controlledAudioDevices, + options.livekitRoomFactory, + ); - private connectOptions$ = this.scope.behavior( - matrixRTCMode.value$.pipe( - map((mode) => ({ - encryptMedia: this.livekitKeyProvider !== undefined, - // TODO. This might need to get called again on each cahnge of matrixRTCMode... - matrixRTCMode: mode, - })), - ), - ); - - private localMembership = createLocalMembership$({ - scope: this.scope, - muteStates: this.muteStates, - mediaDevices: this.mediaDevices, - connectionManager: this.connectionManager, - matrixRTCSession: this.matrixRTCSession, - matrixRoom: this.matrixRoom, - localTransport$: this.localTransport$, - trackProcessorState$: this.trackProcessorState$, - widget, - options: this.connectOptions$, - logger: logger.getChild(`[${Date.now()}]`), - }); - - private localRtcMembership$ = this.scope.behavior( - this.memberships$.pipe( - map( - (memberships) => - memberships.value.find( - (membership) => - membership.userId === this.userId && - membership.deviceId === this.deviceId, - ) ?? null, + const connectionManager = createConnectionManager$({ + scope: scope, + connectionFactory: connectionFactory, + inputTransports$: scope.behavior( + combineLatest( + [localTransport$, membershipsAndTransports.transports$], + (localTransport, transports) => { + const localTransportAsArray = localTransport + ? [localTransport] + : []; + return transports.mapInner((transports) => [ + ...localTransportAsArray, + ...transports, + ]); + }, + ), ), - ), - ); + logger: logger, + }); - private localMatrixLivekitMemberUninitialized = { - membership$: this.localRtcMembership$, - participant$: this.localMembership.participant$, - connection$: this.localMembership.connection$, - userId: this.userId, - }; + const matrixLivekitMembers$ = createMatrixLivekitMembers$({ + scope: scope, + membershipsWithTransport$: + membershipsAndTransports.membershipsWithTransport$, + connectionManager: connectionManager, + }); - private localMatrixLivekitMember$: Behavior = - this.scope.behavior( - this.localRtcMembership$.pipe( - switchMap((membership) => { - if (!membership) return of(null); - return of( - // casting is save here since we know that localRtcMembership$ is !== null since we reached this case. - this.localMatrixLivekitMemberUninitialized as MatrixLivekitMember, + const connectOptions$ = scope.behavior( + matrixRTCMode.value$.pipe( + map((mode) => ({ + encryptMedia: livekitKeyProvider !== undefined, + // TODO. This might need to get called again on each cahnge of matrixRTCMode... + matrixRTCMode: mode, + })), + ), + ); + + const localMembership = createLocalMembership$({ + scope: scope, + muteStates: muteStates, + mediaDevices: mediaDevices, + connectionManager: connectionManager, + matrixRTCSession: matrixRTCSession, + matrixRoom: matrixRoom, + localTransport$: localTransport$, + trackProcessorState$: trackProcessorState$, + widget, + options: connectOptions$, + logger: logger.getChild(`[${Date.now()}]`), + }); + + const localRtcMembership$ = scope.behavior( + memberships$.pipe( + map( + (memberships) => + memberships.value.find( + (membership) => + membership.userId === userId && + membership.deviceId === deviceId, + ) ?? null, + ), + ), + ); + + const localMatrixLivekitMemberUninitialized = { + membership$: localRtcMembership$, + participant$: localMembership.participant$, + connection$: localMembership.connection$, + userId: userId, + }; + + const localMatrixLivekitMember$: Behavior = + scope.behavior( + localRtcMembership$.pipe( + switchMap((membership) => { + if (!membership) return of(null); + return of( + // casting is save here since we know that localRtcMembership$ is !== null since we reached this case. + localMatrixLivekitMemberUninitialized as MatrixLivekitMember, + ); + }), + ), + ); + + // ------------------------------------------------------------------------ + // callLifecycle + + const callLifecycle = createCallNotificationLifecycle$({ + scope: scope, + memberships$: memberships$, + sentCallNotification$: createSentCallNotification$( + scope, + matrixRTCSession, + ), + receivedDecline$: createReceivedDecline$(matrixRoom), + options: options, + localUser: { userId: userId, deviceId: deviceId }, + }); + + // ------------------------------------------------------------------------ + // matrixMemberMetadataStore + + const matrixRoomMembers$ = createRoomMembers$(scope, matrixRoom); + const matrixMemberMetadataStore = createMatrixMemberMetadata$( + scope, + scope.behavior(memberships$.pipe(map((mems) => mems.value))), + matrixRoomMembers$, + ); + + /** + * Returns the Member {userId, getMxcAvatarUrl, rawDisplayName} of the other user in the call, if it's a one-on-one call. + */ + const isOneOnOneWith$ = scope.behavior( + matrixRoomMembers$.pipe( + map((roomMembersMap) => { + const otherMembers = Array.from(roomMembersMap.values()).filter( + (member) => member.userId !== userId, ); + return otherMembers.length === 1 ? otherMembers[0] : null; }), ), ); - // ------------------------------------------------------------------------ - // callLifecycle - - private callLifecycle = createCallNotificationLifecycle$({ - scope: this.scope, - memberships$: this.memberships$, - sentCallNotification$: createSentCallNotification$( - this.scope, - this.matrixRTCSession, - ), - receivedDecline$: createReceivedDecline$(this.matrixRoom), - options: this.options, - localUser: { userId: this.userId, deviceId: this.deviceId }, - }); - public autoLeave$ = this.callLifecycle.autoLeave$; - - // ------------------------------------------------------------------------ - // matrixMemberMetadataStore - - private matrixMemberMetadataStore = createMatrixMemberMetadata$( - this.scope, - this.scope.behavior(this.memberships$.pipe(map((mems) => mems.value))), - createRoomMembers$(this.scope, this.matrixRoom), - ); - - /** - * If there is a configuration error with the call (e.g. misconfigured E2EE). - * This is a fatal error that prevents the call from being created/joined. - * Should render a blocking error screen. - */ - public get configError$(): Behavior { - return this.localMembership.configError$; - } - - public join = this.localMembership.requestConnect; - - // CODESMELL? - // This is functionally the same Observable as leave$, except here it's - // hoisted to the top of the class. This enables the cyclic dependency between - // leave$ -> autoLeave$ -> callPickupState$ -> livekitConnectionState$ -> - // localConnection$ -> transports$ -> joined$ -> leave$. - private readonly leaveHoisted$ = new Subject< - "user" | "timeout" | "decline" | "allOthersLeft" - >(); - - /** - * Whether various media/event sources should pretend to be disconnected from - * all network input, even if their connection still technically works. - */ - // We do this when the app is in the 'reconnecting' state, because it might be - // that the LiveKit connection is still functional while the homeserver is - // down, for example, and we want to avoid making people worry that the app is - // in a split-brained state. - // DISCUSSION own membership manager ALSO this probably can be simplifis - public reconnecting$ = this.localMembership.reconnecting$; - private readonly pretendToBeDisconnected$ = this.reconnecting$; - - public readonly audioParticipants$ = this.scope.behavior( - this.matrixLivekitMembers$.pipe( - switchMap((membersWithEpoch) => { - const members = membersWithEpoch.value; - const a$ = combineLatest( - members.map((member) => - combineLatest([member.connection$, member.participant$]).pipe( - map(([connection, participant]) => { - // do not render audio for local participant - if (!connection || !participant || participant.isLocal) - return null; - const livekitRoom = connection.livekitRoom; - const url = connection.transport.livekit_service_url; - - return { url, livekitRoom, participant: participant.identity }; - }), - ), - ), - ); - return a$; - }), - map((members) => - members.reduce((acc, curr) => { - if (!curr) return acc; - - const existing = acc.find((item) => item.url === curr.url); - if (existing) { - existing.participants.push(curr.participant); - } else { - acc.push({ - livekitRoom: curr.livekitRoom, - participants: [curr.participant], - url: curr.url, - }); - } - return acc; - }, []), - ), - ), - [], - ); - - public readonly handsRaised$ = this.scope.behavior( - this.handsRaisedSubject$.pipe(pauseWhen(this.pretendToBeDisconnected$)), - ); - - public readonly reactions$ = this.scope.behavior( - this.reactionsSubject$.pipe( - map((v) => - Object.fromEntries( - Object.entries(v).map(([a, { reactionOption }]) => [ - a, - reactionOption, - ]), + const localUserIsAlone$ = scope.behavior( + matrixRoomMembers$.pipe( + map( + (roomMembersMap) => + roomMembersMap.size === 1 && + roomMembersMap.get(userId) !== undefined, ), ), - pauseWhen(this.pretendToBeDisconnected$), - ), - ); + ); - /** - * List of user media (camera feeds) that we want tiles for. - */ - // TODO this also needs the local participant to be added. - private readonly userMedia$ = this.scope.behavior( - combineLatest([ - this.localMatrixLivekitMember$, - this.matrixLivekitMembers$, - duplicateTiles.value$, - ]).pipe( - // Generate a collection of MediaItems from the list of expected (whether - // present or missing) LiveKit participants. - generateItems( - function* ([ - localMatrixLivekitMember, - { value: matrixLivekitMembers }, - duplicateTiles, - ]) { - let localParticipantId = undefined; - // add local member if available - if (localMatrixLivekitMember) { - const { userId, participant$, connection$, membership$ } = - localMatrixLivekitMember; - localParticipantId = `${userId}:${membership$.value.deviceId}`; // should be membership$.value.membershipID which is not optional - // const participantId = membership$.value.membershipID; - if (localParticipantId) { + // CODESMELL? + // This is functionally the same Observable as leave$, except here it's + // hoisted to the top of the class. This enables the cyclic dependency between + // leave$ -> autoLeave$ -> callPickupState$ -> livekitConnectionState$ -> + // localConnection$ -> transports$ -> joined$ -> leave$. + const leaveHoisted$ = new Subject< + "user" | "timeout" | "decline" | "allOthersLeft" + >(); + + /** + * Whether various media/event sources should pretend to be disconnected from + * all network input, even if their connection still technically works. + */ + // We do this when the app is in the 'reconnecting' state, because it might be + // that the LiveKit connection is still functional while the homeserver is + // down, for example, and we want to avoid making people worry that the app is + // in a split-brained state. + // DISCUSSION own membership manager ALSO this probably can be simplifis + /*PUBLIC*/ const reconnecting$ = localMembership.reconnecting$; + const pretendToBeDisconnected$ = reconnecting$; + + /*PUBLIC*/ const audioParticipants$ = scope.behavior( + matrixLivekitMembers$.pipe( + switchMap((membersWithEpoch) => { + const members = membersWithEpoch.value; + const a$ = combineLatest( + members.map((member) => + combineLatest([member.connection$, member.participant$]).pipe( + map(([connection, participant]) => { + // do not render audio for local participant + if (!connection || !participant || participant.isLocal) + return null; + const livekitRoom = connection.livekitRoom; + const url = connection.transport.livekit_service_url; + + return { + url, + livekitRoom, + participant: participant.identity, + }; + }), + ), + ), + ); + return a$; + }), + map((members) => + members.reduce((acc, curr) => { + if (!curr) return acc; + + const existing = acc.find((item) => item.url === curr.url); + if (existing) { + existing.participants.push(curr.participant); + } else { + acc.push({ + livekitRoom: curr.livekitRoom, + participants: [curr.participant], + url: curr.url, + }); + } + return acc; + }, []), + ), + ), + [], + ); + + /*PUBLIC*/ const handsRaised$ = scope.behavior( + handsRaisedSubject$.pipe(pauseWhen(pretendToBeDisconnected$)), + ); + + /*PUBLIC*/ const reactions$ = scope.behavior( + reactionsSubject$.pipe( + map((v) => + Object.fromEntries( + Object.entries(v).map(([a, { reactionOption }]) => [ + a, + reactionOption, + ]), + ), + ), + pauseWhen(pretendToBeDisconnected$), + ), + ); + + /** + * List of user media (camera feeds) that we want tiles for. + */ + // TODO this also needs the local participant to be added. + const userMedia$ = scope.behavior( + combineLatest([ + localMatrixLivekitMember$, + matrixLivekitMembers$, + duplicateTiles.value$, + ]).pipe( + // Generate a collection of MediaItems from the list of expected (whether + // present or missing) LiveKit participants. + generateItems( + function* ([ + localMatrixLivekitMember, + { value: matrixLivekitMembers }, + duplicateTiles, + ]) { + let localParticipantId = undefined; + // add local member if available + if (localMatrixLivekitMember) { + const { userId, participant$, connection$, membership$ } = + localMatrixLivekitMember; + localParticipantId = `${userId}:${membership$.value.deviceId}`; // should be membership$.value.membershipID which is not optional + // const participantId = membership$.value.membershipID; + if (localParticipantId) { + for (let dup = 0; dup < 1 + duplicateTiles; dup++) { + yield { + keys: [ + dup, + localParticipantId, + userId, + participant$, + connection$, + ], + data: undefined, + }; + } + } + } + // add remote members that are available + for (const { + userId, + participant$, + connection$, + membership$, + } of matrixLivekitMembers) { + const participantId = `${userId}:${membership$.value.deviceId}`; + if (participantId === localParticipantId) continue; + // const participantId = membership$.value?.identity; for (let dup = 0; dup < 1 + duplicateTiles; dup++) { yield { - keys: [ - dup, - localParticipantId, - userId, - participant$, - connection$, - ], + keys: [dup, participantId, userId, participant$, connection$], data: undefined, }; } } - } - // add remote members that are available - for (const { + }, + ( + scope, + _data$, + dup, + participantId, userId, participant$, connection$, - membership$, - } of matrixLivekitMembers) { - const participantId = `${userId}:${membership$.value.deviceId}`; - if (participantId === localParticipantId) continue; - // const participantId = membership$.value?.identity; - for (let dup = 0; dup < 1 + duplicateTiles; dup++) { - yield { - keys: [dup, participantId, userId, participant$, connection$], - data: undefined, - }; - } - } - }, - ( - scope, - _data$, - dup, - participantId, - userId, - participant$, - connection$, - ) => { - const livekitRoom$ = scope.behavior( - connection$.pipe(map((c) => c?.livekitRoom)), - ); - const focusUrl$ = scope.behavior( - connection$.pipe(map((c) => c?.transport.livekit_service_url)), - ); - const displayName$ = scope.behavior( - this.matrixMemberMetadataStore - .createDisplayNameBehavior$(userId) - .pipe(map((name) => name ?? userId)), - ); + ) => { + const livekitRoom$ = scope.behavior( + connection$.pipe(map((c) => c?.livekitRoom)), + ); + const focusUrl$ = scope.behavior( + connection$.pipe(map((c) => c?.transport.livekit_service_url)), + ); + const displayName$ = scope.behavior( + matrixMemberMetadataStore + .createDisplayNameBehavior$(userId) + .pipe(map((name) => name ?? userId)), + ); - return new UserMedia( - scope, - `${participantId}:${dup}`, - userId, - participant$, - this.options.encryptionSystem, - livekitRoom$, - focusUrl$, - this.mediaDevices, - this.pretendToBeDisconnected$, - displayName$, - this.matrixMemberMetadataStore.createAvatarUrlBehavior$(userId), - this.handsRaised$.pipe(map((v) => v[participantId]?.time ?? null)), - this.reactions$.pipe(map((v) => v[participantId] ?? undefined)), - ); - }, + return new UserMedia( + scope, + `${participantId}:${dup}`, + userId, + participant$, + options.encryptionSystem, + livekitRoom$, + focusUrl$, + mediaDevices, + pretendToBeDisconnected$, + displayName$, + matrixMemberMetadataStore.createAvatarUrlBehavior$(userId), + handsRaised$.pipe(map((v) => v[participantId]?.time ?? null)), + reactions$.pipe(map((v) => v[participantId] ?? undefined)), + ); + }, + ), ), - ), - ); + ); - /** - * List of all media items (user media and screen share media) that we want - * tiles for. - */ - private readonly mediaItems$ = this.scope.behavior( - this.userMedia$.pipe( - switchMap((userMedia) => - userMedia.length === 0 - ? of([]) - : combineLatest( - userMedia.map((m) => m.screenShares$), - (...screenShares) => [...userMedia, ...screenShares.flat(1)], - ), + /** + * List of all media items (user media and screen share media) that we want + * tiles for. + */ + const mediaItems$ = scope.behavior( + userMedia$.pipe( + switchMap((userMedia) => + userMedia.length === 0 + ? of([]) + : combineLatest( + userMedia.map((m) => m.screenShares$), + (...screenShares) => [...userMedia, ...screenShares.flat(1)], + ), + ), ), - ), - ); + ); - /** - * List of MediaItems that we want to display, that are of type ScreenShare - */ - private readonly screenShares$ = this.scope.behavior( - this.mediaItems$.pipe( - map((mediaItems) => - mediaItems.filter((m): m is ScreenShare => m instanceof ScreenShare), + /** + * List of MediaItems that we want to display, that are of type ScreenShare + */ + const screenShares$ = scope.behavior( + mediaItems$.pipe( + map((mediaItems) => + mediaItems.filter((m): m is ScreenShare => m instanceof ScreenShare), + ), ), - ), - ); + ); - public readonly joinSoundEffect$ = this.userMedia$.pipe( - pairwise(), - filter( - ([prev, current]) => - current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && - current.length > prev.length, - ), - map(() => {}), - throttleTime(THROTTLE_SOUND_EFFECT_MS), - ); + /*PUBLIC*/ const joinSoundEffect$ = userMedia$.pipe( + pairwise(), + filter( + ([prev, current]) => + current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && + current.length > prev.length, + ), + map(() => {}), + throttleTime(THROTTLE_SOUND_EFFECT_MS), + ); - /** - * The number of participants currently in the call. - * - * - Each participant has a corresponding MatrixRTC membership state event - * - There can be multiple participants for one Matrix user if they join from - * multiple devices. - */ - public readonly participantCount$ = this.scope.behavior( - this.matrixLivekitMembers$.pipe(map((ms) => ms.value.length)), - ); + /** + * The number of participants currently in the call. + * + * - Each participant has a corresponding MatrixRTC membership state event + * - There can be multiple participants for one Matrix user if they join from + * multiple devices. + */ + /*PUBLIC*/ const participantCount$ = scope.behavior( + matrixLivekitMembers$.pipe(map((ms) => ms.value.length)), + ); - // only public to expose to the view. - // TODO if we are in "unknown" state we need a loading rendering (or empty screen) - // Otherwise it looks like we already connected and only than the ringing starts which is weird. - public readonly callPickupState$ = this.callLifecycle.callPickupState$; + // only public to expose to the view. + // TODO if we are in "unknown" state we need a loading rendering (or empty screen) + // Otherwise it looks like we already connected and only than the ringing starts which is weird. + /*PUBLIC*/ const callPickupState$ = callLifecycle.callPickupState$; - public readonly leaveSoundEffect$ = combineLatest([ - this.callLifecycle.callPickupState$, - this.userMedia$, - ]).pipe( - // Until the call is successful, do not play a leave sound. - // If callPickupState$ is null, then we always play the sound as it will not conflict with a decline sound. - skipWhile(([c]) => c !== null && c !== "success"), - map(([, userMedia]) => userMedia), - pairwise(), - filter( - ([prev, current]) => - current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && - current.length < prev.length, - ), - map(() => {}), - throttleTime(THROTTLE_SOUND_EFFECT_MS), - ); + /*PUBLIC*/ const leaveSoundEffect$ = combineLatest([ + callLifecycle.callPickupState$, + userMedia$, + ]).pipe( + // Until the call is successful, do not play a leave sound. + // If callPickupState$ is null, then we always play the sound as it will not conflict with a decline sound. + skipWhile(([c]) => c !== null && c !== "success"), + map(([, userMedia]) => userMedia), + pairwise(), + filter( + ([prev, current]) => + current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && + current.length < prev.length, + ), + map(() => {}), + throttleTime(THROTTLE_SOUND_EFFECT_MS), + ); - private readonly userHangup$ = new Subject(); - public hangup(): void { - this.userHangup$.next(); - } + const userHangup$ = new Subject(); - private readonly widgetHangup$ = - widget === null - ? NEVER - : ( - fromEvent( - widget.lazyActions, - ElementWidgetActions.HangupCall, - ) as Observable> - ).pipe( - tap((ev) => { - widget!.api.transport.reply(ev.detail, {}); - }), - ); + const widgetHangup$ = + widget === null + ? NEVER + : ( + fromEvent( + widget.lazyActions, + ElementWidgetActions.HangupCall, + ) as Observable> + ).pipe( + tap((ev) => { + widget!.api.transport.reply(ev.detail, {}); + }), + ); - public readonly leave$: Observable< - "user" | "timeout" | "decline" | "allOthersLeft" - > = merge( - this.callLifecycle.autoLeave$, - merge(this.userHangup$, this.widgetHangup$).pipe( - map(() => "user" as const), - ), - ).pipe( - this.scope.share, - tap((reason) => this.leaveHoisted$.next(reason)), - ); + const leave$: Observable<"user" | "timeout" | "decline" | "allOthersLeft"> = + merge( + callLifecycle.autoLeave$, + merge(userHangup$, widgetHangup$).pipe(map(() => "user" as const)), + ).pipe( + scope.share, + tap((reason) => leaveHoisted$.next(reason)), + ); - private readonly spotlightSpeaker$ = - this.scope.behavior( - this.userMedia$.pipe( + const spotlightSpeaker$ = scope.behavior( + userMedia$.pipe( switchMap((mediaItems) => mediaItems.length === 0 ? of([]) @@ -665,653 +770,638 @@ export class CallViewModel { ), ); - private readonly grid$ = this.scope.behavior( - this.userMedia$.pipe( - switchMap((mediaItems) => { - const bins = mediaItems.map((m) => - m.bin$.pipe(map((bin) => [m, bin] as const)), - ); - // Sort the media by bin order and generate a tile for each one - return bins.length === 0 - ? of([]) - : combineLatest(bins, (...bins) => - bins.sort(([, bin1], [, bin2]) => bin1 - bin2).map(([m]) => m.vm), - ); - }), - distinctUntilChanged(shallowEquals), - ), - ); + const grid$ = scope.behavior( + userMedia$.pipe( + switchMap((mediaItems) => { + const bins = mediaItems.map((m) => + m.bin$.pipe(map((bin) => [m, bin] as const)), + ); + // Sort the media by bin order and generate a tile for each one + return bins.length === 0 + ? of([]) + : combineLatest(bins, (...bins) => + bins + .sort(([, bin1], [, bin2]) => bin1 - bin2) + .map(([m]) => m.vm), + ); + }), + distinctUntilChanged(shallowEquals), + ), + ); - private readonly spotlight$ = this.scope.behavior( - this.screenShares$.pipe( - switchMap((screenShares) => { - if (screenShares.length > 0) { - return of(screenShares.map((m) => m.vm)); - } + const spotlight$ = scope.behavior( + screenShares$.pipe( + switchMap((screenShares) => { + if (screenShares.length > 0) { + return of(screenShares.map((m) => m.vm)); + } - return this.spotlightSpeaker$.pipe( - map((speaker) => (speaker ? [speaker] : [])), - ); - }), - distinctUntilChanged(shallowEquals), - ), - ); + return spotlightSpeaker$.pipe( + map((speaker) => (speaker ? [speaker] : [])), + ); + }), + distinctUntilChanged(shallowEquals), + ), + ); - private readonly pip$ = this.scope.behavior( - combineLatest([ - // TODO This also needs epoch logic to dedupe the screenshares and mediaItems emits - this.screenShares$, - this.spotlightSpeaker$, - this.mediaItems$, - ]).pipe( - switchMap(([screenShares, spotlight, mediaItems]) => { - if (screenShares.length > 0) { - return this.spotlightSpeaker$; - } - if (!spotlight || spotlight.local) { - return of(null); - } + const pip$ = scope.behavior( + combineLatest([ + // TODO This also needs epoch logic to dedupe the screenshares and mediaItems emits + screenShares$, + spotlightSpeaker$, + mediaItems$, + ]).pipe( + switchMap(([screenShares, spotlight, mediaItems]) => { + if (screenShares.length > 0) { + return spotlightSpeaker$; + } + if (!spotlight || spotlight.local) { + return of(null); + } - const localUserMedia = mediaItems.find( - (m) => m.vm instanceof LocalUserMediaViewModel, - ) as UserMedia | undefined; + const localUserMedia = mediaItems.find( + (m) => m.vm instanceof LocalUserMediaViewModel, + ) as UserMedia | undefined; - const localUserMediaViewModel = localUserMedia?.vm as - | LocalUserMediaViewModel - | undefined; + const localUserMediaViewModel = localUserMedia?.vm as + | LocalUserMediaViewModel + | undefined; - if (!localUserMediaViewModel) { - return of(null); - } - return localUserMediaViewModel.alwaysShow$.pipe( - map((alwaysShow) => { - if (alwaysShow) { - return localUserMediaViewModel; - } + if (!localUserMediaViewModel) { + return of(null); + } + return localUserMediaViewModel.alwaysShow$.pipe( + map((alwaysShow) => { + if (alwaysShow) { + return localUserMediaViewModel; + } - return null; - }), - ); - }), - ), - ); + return null; + }), + ); + }), + ), + ); - private readonly hasRemoteScreenShares$: Observable = - this.spotlight$.pipe( + const hasRemoteScreenShares$: Observable = spotlight$.pipe( map((spotlight) => spotlight.some((vm) => !vm.local && vm instanceof ScreenShareViewModel), ), distinctUntilChanged(), ); - private readonly pipEnabled$ = this.scope.behavior(setPipEnabled$, false); + const pipEnabled$ = scope.behavior(setPipEnabled$, false); - private readonly naturalWindowMode$ = this.scope.behavior( - fromEvent(window, "resize").pipe( - map(() => { - const height = window.innerHeight; - const width = window.innerWidth; - if (height <= 400 && width <= 340) return "pip"; - // Our layouts for flat windows are better at adapting to a small width - // than our layouts for narrow windows are at adapting to a small height, - // so we give "flat" precedence here - if (height <= 600) return "flat"; - if (width <= 600) return "narrow"; - return "normal"; - }), - ), - "normal", - ); - - /** - * The general shape of the window. - */ - public readonly windowMode$ = this.scope.behavior( - this.pipEnabled$.pipe( - switchMap((pip) => - pip ? of("pip") : this.naturalWindowMode$, + const naturalWindowMode$ = scope.behavior( + fromEvent(window, "resize").pipe( + map(() => { + const height = window.innerHeight; + const width = window.innerWidth; + if (height <= 400 && width <= 340) return "pip"; + // Our layouts for flat windows are better at adapting to a small width + // than our layouts for narrow windows are at adapting to a small height, + // so we give "flat" precedence here + if (height <= 600) return "flat"; + if (width <= 600) return "narrow"; + return "normal"; + }), ), - ), - ); + "normal", + ); - private readonly spotlightExpandedToggle$ = new Subject(); - public readonly spotlightExpanded$ = this.scope.behavior( - this.spotlightExpandedToggle$.pipe( - accumulate(false, (expanded) => !expanded), - ), - ); + /** + * The general shape of the window. + */ + /*PUBLIC*/ const windowMode$ = scope.behavior( + pipEnabled$.pipe( + switchMap((pip) => (pip ? of("pip") : naturalWindowMode$)), + ), + ); - private readonly gridModeUserSelection$ = new Subject(); - /** - * The layout mode of the media tile grid. - */ - public readonly gridMode$ = - // If the user hasn't selected spotlight and somebody starts screen sharing, - // automatically switch to spotlight mode and reset when screen sharing ends - this.scope.behavior( - this.gridModeUserSelection$.pipe( - switchMap((userSelection) => - (userSelection === "spotlight" - ? EMPTY - : combineLatest([ - this.hasRemoteScreenShares$, - this.windowMode$, - ]).pipe( - skip(userSelection === null ? 0 : 1), - map( - ([hasScreenShares, windowMode]): GridMode => - hasScreenShares || windowMode === "flat" - ? "spotlight" - : "grid", - ), - ) - ).pipe(startWith(userSelection ?? "grid")), + const spotlightExpandedToggle$ = new Subject(); + /*PUBLIC*/ const spotlightExpanded$ = scope.behavior( + spotlightExpandedToggle$.pipe(accumulate(false, (expanded) => !expanded)), + ); + + const gridModeUserSelection$ = new Subject(); + /** + * The layout mode of the media tile grid. + */ + /*PUBLIC*/ const gridMode$ = + // If the user hasn't selected spotlight and somebody starts screen sharing, + // automatically switch to spotlight mode and reset when screen sharing ends + scope.behavior( + gridModeUserSelection$.pipe( + switchMap((userSelection) => + (userSelection === "spotlight" + ? EMPTY + : combineLatest([hasRemoteScreenShares$, windowMode$]).pipe( + skip(userSelection === null ? 0 : 1), + map( + ([hasScreenShares, windowMode]): GridMode => + hasScreenShares || windowMode === "flat" + ? "spotlight" + : "grid", + ), + ) + ).pipe(startWith(userSelection ?? "grid")), + ), ), - ), - "grid", - ); + "grid", + ); - public setGridMode(value: GridMode): void { - this.gridModeUserSelection$.next(value); - } + /*PUBLIC*/ const setGridMode = (value: GridMode): void => { + gridModeUserSelection$.next(value); + }; - private readonly gridLayoutMedia$: Observable = - combineLatest([this.grid$, this.spotlight$], (grid, spotlight) => ({ - type: "grid", - spotlight: spotlight.some((vm) => vm instanceof ScreenShareViewModel) - ? spotlight - : undefined, - grid, - })); - - private readonly spotlightLandscapeLayoutMedia$: Observable = - combineLatest([this.grid$, this.spotlight$], (grid, spotlight) => ({ - type: "spotlight-landscape", - spotlight, - grid, - })); - - private readonly spotlightPortraitLayoutMedia$: Observable = - combineLatest([this.grid$, this.spotlight$], (grid, spotlight) => ({ - type: "spotlight-portrait", - spotlight, - grid, - })); - - private readonly spotlightExpandedLayoutMedia$: Observable = - combineLatest([this.spotlight$, this.pip$], (spotlight, pip) => ({ - type: "spotlight-expanded", - spotlight, - pip: pip ?? undefined, - })); - - private readonly oneOnOneLayoutMedia$: Observable = - this.mediaItems$.pipe( - map((mediaItems) => { - if (mediaItems.length !== 2) return null; - const local = mediaItems.find((vm) => vm.vm.local)?.vm as - | LocalUserMediaViewModel - | undefined; - const remote = mediaItems.find((vm) => !vm.vm.local)?.vm as - | RemoteUserMediaViewModel - | undefined; - // There might not be a remote tile if there are screen shares, or if - // only the local user is in the call and they're using the duplicate - // tiles option - if (!remote || !local) return null; - - return { type: "one-on-one", local, remote }; + const gridLayoutMedia$: Observable = combineLatest( + [grid$, spotlight$], + (grid, spotlight) => ({ + type: "grid", + spotlight: spotlight.some((vm) => vm instanceof ScreenShareViewModel) + ? spotlight + : undefined, + grid, }), ); - private readonly pipLayoutMedia$: Observable = - this.spotlight$.pipe(map((spotlight) => ({ type: "pip", spotlight }))); + const spotlightLandscapeLayoutMedia$: Observable = + combineLatest([grid$, spotlight$], (grid, spotlight) => ({ + type: "spotlight-landscape", + spotlight, + grid, + })); - /** - * The media to be used to produce a layout. - */ - private readonly layoutMedia$ = this.scope.behavior( - this.windowMode$.pipe( - switchMap((windowMode) => { - switch (windowMode) { - case "normal": - return this.gridMode$.pipe( - switchMap((gridMode) => { - switch (gridMode) { - case "grid": - return this.oneOnOneLayoutMedia$.pipe( - switchMap((oneOnOne) => - oneOnOne === null - ? this.gridLayoutMedia$ - : of(oneOnOne), - ), - ); - case "spotlight": - return this.spotlightExpanded$.pipe( - switchMap((expanded) => - expanded - ? this.spotlightExpandedLayoutMedia$ - : this.spotlightLandscapeLayoutMedia$, - ), - ); - } - }), - ); - case "narrow": - return this.oneOnOneLayoutMedia$.pipe( - switchMap((oneOnOne) => - oneOnOne === null - ? combineLatest( - [this.grid$, this.spotlight$], - (grid, spotlight) => + const spotlightPortraitLayoutMedia$: Observable = + combineLatest([grid$, spotlight$], (grid, spotlight) => ({ + type: "spotlight-portrait", + spotlight, + grid, + })); + + const spotlightExpandedLayoutMedia$: Observable = + combineLatest([spotlight$, pip$], (spotlight, pip) => ({ + type: "spotlight-expanded", + spotlight, + pip: pip ?? undefined, + })); + + const oneOnOneLayoutMedia$: Observable = + mediaItems$.pipe( + map((mediaItems) => { + if (mediaItems.length !== 2) return null; + const local = mediaItems.find((vm) => vm.vm.local)?.vm as + | LocalUserMediaViewModel + | undefined; + const remote = mediaItems.find((vm) => !vm.vm.local)?.vm as + | RemoteUserMediaViewModel + | undefined; + // There might not be a remote tile if there are screen shares, or if + // only the local user is in the call and they're using the duplicate + // tiles option + if (!remote || !local) return null; + + return { type: "one-on-one", local, remote }; + }), + ); + + const pipLayoutMedia$: Observable = spotlight$.pipe( + map((spotlight) => ({ type: "pip", spotlight })), + ); + + /** + * The media to be used to produce a layout. + */ + const layoutMedia$ = scope.behavior( + windowMode$.pipe( + switchMap((windowMode) => { + switch (windowMode) { + case "normal": + return gridMode$.pipe( + switchMap((gridMode) => { + switch (gridMode) { + case "grid": + return oneOnOneLayoutMedia$.pipe( + switchMap((oneOnOne) => + oneOnOne === null ? gridLayoutMedia$ : of(oneOnOne), + ), + ); + case "spotlight": + return spotlightExpanded$.pipe( + switchMap((expanded) => + expanded + ? spotlightExpandedLayoutMedia$ + : spotlightLandscapeLayoutMedia$, + ), + ); + } + }), + ); + case "narrow": + return oneOnOneLayoutMedia$.pipe( + switchMap((oneOnOne) => + oneOnOne === null + ? combineLatest([grid$, spotlight$], (grid, spotlight) => grid.length > smallMobileCallThreshold || spotlight.some( (vm) => vm instanceof ScreenShareViewModel, ) - ? this.spotlightPortraitLayoutMedia$ - : this.gridLayoutMedia$, - ).pipe(switchAll()) - : // The expanded spotlight layout makes for a better one-on-one - // experience in narrow windows - this.spotlightExpandedLayoutMedia$, - ), - ); - case "flat": - return this.gridMode$.pipe( - switchMap((gridMode) => { - switch (gridMode) { - case "grid": - // Yes, grid mode actually gets you a "spotlight" layout in - // this window mode. - return this.spotlightLandscapeLayoutMedia$; - case "spotlight": - return this.spotlightExpandedLayoutMedia$; - } - }), - ); - case "pip": - return this.pipLayoutMedia$; - } - }), - ), - ); + ? spotlightPortraitLayoutMedia$ + : gridLayoutMedia$, + ).pipe(switchAll()) + : // The expanded spotlight layout makes for a better one-on-one + // experience in narrow windows + spotlightExpandedLayoutMedia$, + ), + ); + case "flat": + return gridMode$.pipe( + switchMap((gridMode) => { + switch (gridMode) { + case "grid": + // Yes, grid mode actually gets you a "spotlight" layout in + // this window mode. + return spotlightLandscapeLayoutMedia$; + case "spotlight": + return spotlightExpandedLayoutMedia$; + } + }), + ); + case "pip": + return pipLayoutMedia$; + } + }), + ), + ); - // There is a cyclical dependency here: the layout algorithms want to know - // which tiles are on screen, but to know which tiles are on screen we have to - // first render a layout. To deal with this we assume initially that no tiles - // are visible, and loop the data back into the layouts with a Subject. - private readonly visibleTiles$ = new Subject(); - private readonly setVisibleTiles = (value: number): void => - this.visibleTiles$.next(value); + // There is a cyclical dependency here: the layout algorithms want to know + // which tiles are on screen, but to know which tiles are on screen we have to + // first render a layout. To deal with this we assume initially that no tiles + // are visible, and loop the data back into the layouts with a Subject. + const visibleTiles$ = new Subject(); + const setVisibleTiles = (value: number): void => visibleTiles$.next(value); - private readonly layoutInternals$ = this.scope.behavior< - LayoutScanState & { layout: Layout } - >( - combineLatest([ - this.layoutMedia$, - this.visibleTiles$.pipe(startWith(0), distinctUntilChanged()), - ]).pipe( - scan< - [LayoutMedia, number], - LayoutScanState & { layout: Layout }, - LayoutScanState - >( - ({ tiles: prevTiles }, [media, visibleTiles]) => { - let layout: Layout; - let newTiles: TileStore; - switch (media.type) { - case "grid": + const layoutInternals$ = scope.behavior< + LayoutScanState & { layout: Layout } + >( + combineLatest([ + layoutMedia$, + visibleTiles$.pipe(startWith(0), distinctUntilChanged()), + ]).pipe( + scan< + [LayoutMedia, number], + LayoutScanState & { layout: Layout }, + LayoutScanState + >( + ({ tiles: prevTiles }, [media, visibleTiles]) => { + let layout: Layout; + let newTiles: TileStore; + switch (media.type) { + case "grid": + case "spotlight-landscape": + case "spotlight-portrait": + [layout, newTiles] = gridLikeLayout( + media, + visibleTiles, + setVisibleTiles, + prevTiles, + ); + break; + case "spotlight-expanded": + [layout, newTiles] = spotlightExpandedLayout(media, prevTiles); + break; + case "one-on-one": + [layout, newTiles] = oneOnOneLayout(media, prevTiles); + break; + case "pip": + [layout, newTiles] = pipLayout(media, prevTiles); + break; + } + + return { layout, tiles: newTiles }; + }, + { layout: null, tiles: TileStore.empty() }, + ), + ), + ); + + /** + * The layout of tiles in the call interface. + */ + /*PUBLIC*/ const layout$ = scope.behavior( + layoutInternals$.pipe(map(({ layout }) => layout)), + ); + + /** + * The current generation of the tile store, exposed for debugging purposes. + */ + /*PUBLIC*/ const tileStoreGeneration$ = scope.behavior( + layoutInternals$.pipe(map(({ tiles }) => tiles.generation)), + ); + + /*PUBLIC*/ const showSpotlightIndicators$ = scope.behavior( + layout$.pipe(map((l) => l.type !== "grid")), + ); + + /*PUBLIC*/ const showSpeakingIndicators$ = scope.behavior( + layout$.pipe( + switchMap((l) => { + switch (l.type) { case "spotlight-landscape": case "spotlight-portrait": - [layout, newTiles] = gridLikeLayout( - media, - visibleTiles, - this.setVisibleTiles, - prevTiles, + // If the spotlight is showing the active speaker, we can do without + // speaking indicators as they're a redundant visual cue. But if + // screen sharing feeds are in the spotlight we still need them. + return l.spotlight.media$.pipe( + map((models: MediaViewModel[]) => + models.some((m) => m instanceof ScreenShareViewModel), + ), ); - break; + // In expanded spotlight layout, the active speaker is always shown in + // the picture-in-picture tile so there is no need for speaking + // indicators. And in one-on-one layout there's no question as to who is + // speaking. case "spotlight-expanded": - [layout, newTiles] = spotlightExpandedLayout(media, prevTiles); - break; case "one-on-one": - [layout, newTiles] = oneOnOneLayout(media, prevTiles); - break; - case "pip": - [layout, newTiles] = pipLayout(media, prevTiles); - break; + return of(false); + default: + return of(true); } + }), + ), + ); - return { layout, tiles: newTiles }; + /*PUBLIC*/ const toggleSpotlightExpanded$ = scope.behavior< + (() => void) | null + >( + windowMode$.pipe( + switchMap((mode) => + mode === "normal" + ? layout$.pipe( + map( + (l) => + l.type === "spotlight-landscape" || + l.type === "spotlight-expanded", + ), + ) + : of(false), + ), + distinctUntilChanged(), + map((enabled) => + enabled ? (): void => spotlightExpandedToggle$.next() : null, + ), + ), + ); + + const screenTap$ = new Subject(); + const controlsTap$ = new Subject(); + const screenHover$ = new Subject(); + const screenUnhover$ = new Subject(); + + /*PUBLIC*/ const showHeader$ = scope.behavior( + windowMode$.pipe(map((mode) => mode !== "pip" && mode !== "flat")), + ); + + /*PUBLIC*/ const showFooter$ = scope.behavior( + windowMode$.pipe( + switchMap((mode) => { + switch (mode) { + case "pip": + return of(false); + case "normal": + case "narrow": + return of(true); + case "flat": + // Sadly Firefox has some layering glitches that prevent the footer + // from appearing properly. They happen less often if we never hide + // the footer. + if (isFirefox()) return of(true); + // Show/hide the footer in response to interactions + return merge( + screenTap$.pipe(map(() => "tap screen" as const)), + controlsTap$.pipe(map(() => "tap controls" as const)), + screenHover$.pipe(map(() => "hover" as const)), + ).pipe( + switchScan((state, interaction) => { + switch (interaction) { + case "tap screen": + return state + ? // Toggle visibility on tap + of(false) + : // Hide after a timeout + timer(showFooterMs).pipe( + map(() => false), + startWith(true), + ); + case "tap controls": + // The user is interacting with things, so reset the timeout + return timer(showFooterMs).pipe( + map(() => false), + startWith(true), + ); + case "hover": + // Show on hover and hide after a timeout + return race( + timer(showFooterMs), + screenUnhover$.pipe(take(1)), + ).pipe( + map(() => false), + startWith(true), + ); + } + }, false), + startWith(false), + ); + } + }), + ), + ); + + /** + * Whether audio is currently being output through the earpiece. + */ + /*PUBLIC*/ const earpieceMode$ = scope.behavior( + combineLatest( + [ + mediaDevices.audioOutput.available$, + mediaDevices.audioOutput.selected$, + ], + (available, selected) => + selected !== undefined && + available.get(selected.id)?.type === "earpiece", + ), + ); + + /** + * Callback to toggle between the earpiece and the loudspeaker. + * + * This will be `null` in case the target does not exist in the list + * of available audio outputs. + */ + /*PUBLIC*/ const audioOutputSwitcher$ = scope.behavior<{ + targetOutput: "earpiece" | "speaker"; + switch: () => void; + } | null>( + combineLatest( + [ + mediaDevices.audioOutput.available$, + mediaDevices.audioOutput.selected$, + ], + (available, selected) => { + const selectionType = selected && available.get(selected.id)?.type; + + // If we are in any output mode other than speaker switch to speaker. + const newSelectionType: "earpiece" | "speaker" = + selectionType === "speaker" ? "earpiece" : "speaker"; + const newSelection = [...available].find( + ([, d]) => d.type === newSelectionType, + ); + if (newSelection === undefined) return null; + + const [id] = newSelection; + return { + targetOutput: newSelectionType, + switch: (): void => mediaDevices.audioOutput.select(id), + }; }, - { layout: null, tiles: TileStore.empty() }, ), - ), - ); + ); - /** - * The layout of tiles in the call interface. - */ - public readonly layout$ = this.scope.behavior( - this.layoutInternals$.pipe(map(({ layout }) => layout)), - ); - - /** - * The current generation of the tile store, exposed for debugging purposes. - */ - public readonly tileStoreGeneration$ = this.scope.behavior( - this.layoutInternals$.pipe(map(({ tiles }) => tiles.generation)), - ); - - public showSpotlightIndicators$ = this.scope.behavior( - this.layout$.pipe(map((l) => l.type !== "grid")), - ); - - public showSpeakingIndicators$ = this.scope.behavior( - this.layout$.pipe( - switchMap((l) => { - switch (l.type) { - case "spotlight-landscape": - case "spotlight-portrait": - // If the spotlight is showing the active speaker, we can do without - // speaking indicators as they're a redundant visual cue. But if - // screen sharing feeds are in the spotlight we still need them. - return l.spotlight.media$.pipe( - map((models: MediaViewModel[]) => - models.some((m) => m instanceof ScreenShareViewModel), - ), - ); - // In expanded spotlight layout, the active speaker is always shown in - // the picture-in-picture tile so there is no need for speaking - // indicators. And in one-on-one layout there's no question as to who is - // speaking. - case "spotlight-expanded": - case "one-on-one": - return of(false); - default: - return of(true); - } - }), - ), - ); - - public readonly toggleSpotlightExpanded$ = this.scope.behavior< - (() => void) | null - >( - this.windowMode$.pipe( - switchMap((mode) => - mode === "normal" - ? this.layout$.pipe( - map( - (l) => - l.type === "spotlight-landscape" || - l.type === "spotlight-expanded", - ), - ) - : of(false), + /** + * Emits an array of reactions that should be visible on the screen. + */ + // DISCUSSION move this into a reaction file + // const {visibleReactions$, audibleReactions$} = reactionsObservables$(showReactionSetting$, ) + const visibleReactions$ = scope.behavior( + showReactions.value$.pipe( + switchMap((show) => (show ? reactions$ : of({}))), + scan< + Record, + { sender: string; emoji: string; startX: number }[] + >((acc, latest) => { + const newSet: { sender: string; emoji: string; startX: number }[] = + []; + for (const [sender, reaction] of Object.entries(latest)) { + const startX = + acc.find((v) => v.sender === sender && v.emoji)?.startX ?? + Math.ceil(Math.random() * 80) + 10; + newSet.push({ sender, emoji: reaction.emoji, startX }); + } + return newSet; + }, []), ), - distinctUntilChanged(), - map((enabled) => - enabled ? (): void => this.spotlightExpandedToggle$.next() : null, + ); + + /** + * Emits an array of reactions that should be played. + */ + const audibleReactions$ = playReactionsSound.value$.pipe( + switchMap((show) => + show ? reactions$ : of>({}), ), - ), - ); + map((reactions) => Object.values(reactions).map((v) => v.name)), + scan( + (acc, latest) => { + return { + playing: latest.filter( + (v) => acc.playing.includes(v) || acc.newSounds.includes(v), + ), + newSounds: latest.filter( + (v) => !acc.playing.includes(v) && !acc.newSounds.includes(v), + ), + }; + }, + { playing: [], newSounds: [] }, + ), + map((v) => v.newSounds), + ); - private readonly screenTap$ = new Subject(); - private readonly controlsTap$ = new Subject(); - private readonly screenHover$ = new Subject(); - private readonly screenUnhover$ = new Subject(); + const newHandRaised$ = handsRaised$.pipe( + map((v) => Object.keys(v).length), + scan( + (acc, newValue) => ({ + value: newValue, + playSounds: newValue > acc.value, + }), + { value: 0, playSounds: false }, + ), + filter((v) => v.playSounds), + ); - /** - * Callback for when the user taps the call view. - */ - public tapScreen(): void { - this.screenTap$.next(); - } + const newScreenShare$ = screenShares$.pipe( + map((v) => v.length), + scan( + (acc, newValue) => ({ + value: newValue, + playSounds: newValue > acc.value, + }), + { value: 0, playSounds: false }, + ), + filter((v) => v.playSounds), + ); - /** - * Callback for when the user taps the call's controls. - */ - public tapControls(): void { - this.controlsTap$.next(); - } + /** + * Whether we are sharing our screen. + */ + // reassigned here to make it publicly accessible + const sharingScreen$ = localMembership.sharingScreen$; - /** - * Callback for when the user hovers over the call view. - */ - public hoverScreen(): void { - this.screenHover$.next(); - } + /** + * Callback to toggle screen sharing. If null, screen sharing is not possible. + */ + // reassigned here to make it publicly accessible + const toggleScreenSharing = localMembership.toggleScreenSharing; - /** - * Callback for when the user stops hovering over the call view. - */ - public unhoverScreen(): void { - this.screenUnhover$.next(); - } + const join = localMembership.requestConnect; + join(); // TODO-MULTI-SFU: Use this view model for the lobby as well, and only call this once 'join' is clicked? - public readonly showHeader$ = this.scope.behavior( - this.windowMode$.pipe(map((mode) => mode !== "pip" && mode !== "flat")), - ); + this.autoLeave$ = callLifecycle.autoLeave$; + this.callPickupState$ = callPickupState$; + this.leave$ = leave$; + this.hangup = (): void => userHangup$.next(); + this.join = join; + this.toggleScreenSharing = toggleScreenSharing; + this.sharingScreen$ = sharingScreen$; - public readonly showFooter$ = this.scope.behavior( - this.windowMode$.pipe( - switchMap((mode) => { - switch (mode) { - case "pip": - return of(false); - case "normal": - case "narrow": - return of(true); - case "flat": - // Sadly Firefox has some layering glitches that prevent the footer - // from appearing properly. They happen less often if we never hide - // the footer. - if (isFirefox()) return of(true); - // Show/hide the footer in response to interactions - return merge( - this.screenTap$.pipe(map(() => "tap screen" as const)), - this.controlsTap$.pipe(map(() => "tap controls" as const)), - this.screenHover$.pipe(map(() => "hover" as const)), - ).pipe( - switchScan((state, interaction) => { - switch (interaction) { - case "tap screen": - return state - ? // Toggle visibility on tap - of(false) - : // Hide after a timeout - timer(showFooterMs).pipe( - map(() => false), - startWith(true), - ); - case "tap controls": - // The user is interacting with things, so reset the timeout - return timer(showFooterMs).pipe( - map(() => false), - startWith(true), - ); - case "hover": - // Show on hover and hide after a timeout - return race( - timer(showFooterMs), - this.screenUnhover$.pipe(take(1)), - ).pipe( - map(() => false), - startWith(true), - ); - } - }, false), - startWith(false), - ); - } - }), - ), - ); + this.tapScreen = (): void => screenTap$.next(); + this.tapControls = (): void => controlsTap$.next(); + this.hoverScreen = (): void => screenHover$.next(); + this.unhoverScreen = (): void => screenUnhover$.next(); - /** - * Whether audio is currently being output through the earpiece. - */ - public readonly earpieceMode$ = this.scope.behavior( - combineLatest( - [ - this.mediaDevices.audioOutput.available$, - this.mediaDevices.audioOutput.selected$, - ], - (available, selected) => - selected !== undefined && - available.get(selected.id)?.type === "earpiece", - ), - ); + this.configError$ = localMembership.configError$; + this.participantCount$ = participantCount$; + this.audioParticipants$ = audioParticipants$; + this.isOneOnOneWith$ = isOneOnOneWith$; + this.localUserIsAlone$ = localUserIsAlone$; - /** - * Callback to toggle between the earpiece and the loudspeaker. - * - * This will be `null` in case the target does not exist in the list - * of available audio outputs. - */ - public readonly audioOutputSwitcher$ = this.scope.behavior<{ - targetOutput: "earpiece" | "speaker"; - switch: () => void; - } | null>( - combineLatest( - [ - this.mediaDevices.audioOutput.available$, - this.mediaDevices.audioOutput.selected$, - ], - (available, selected) => { - const selectionType = selected && available.get(selected.id)?.type; + this.handsRaised$ = handsRaised$; + this.reactions$ = reactions$; + this.joinSoundEffect$ = joinSoundEffect$; + this.leaveSoundEffect$ = leaveSoundEffect$; + this.newHandRaised$ = newHandRaised$; + this.newScreenShare$ = newScreenShare$; + this.audibleReactions$ = audibleReactions$; + this.visibleReactions$ = visibleReactions$; - // If we are in any output mode other than speaker switch to speaker. - const newSelectionType: "earpiece" | "speaker" = - selectionType === "speaker" ? "earpiece" : "speaker"; - const newSelection = [...available].find( - ([, d]) => d.type === newSelectionType, - ); - if (newSelection === undefined) return null; - - const [id] = newSelection; - return { - targetOutput: newSelectionType, - switch: (): void => this.mediaDevices.audioOutput.select(id), - }; - }, - ), - ); - - /** - * Emits an array of reactions that should be visible on the screen. - */ - // DISCUSSION move this into a reaction file - // const {visibleReactions$, audibleReactions$} = reactionsObservables$(showReactionSetting$, ) - public readonly visibleReactions$ = this.scope.behavior( - showReactions.value$.pipe( - switchMap((show) => (show ? this.reactions$ : of({}))), - scan< - Record, - { sender: string; emoji: string; startX: number }[] - >((acc, latest) => { - const newSet: { sender: string; emoji: string; startX: number }[] = []; - for (const [sender, reaction] of Object.entries(latest)) { - const startX = - acc.find((v) => v.sender === sender && v.emoji)?.startX ?? - Math.ceil(Math.random() * 80) + 10; - newSet.push({ sender, emoji: reaction.emoji, startX }); - } - return newSet; - }, []), - ), - ); - - /** - * Emits an array of reactions that should be played. - */ - public readonly audibleReactions$ = playReactionsSound.value$.pipe( - switchMap((show) => - show ? this.reactions$ : of>({}), - ), - map((reactions) => Object.values(reactions).map((v) => v.name)), - scan( - (acc, latest) => { - return { - playing: latest.filter( - (v) => acc.playing.includes(v) || acc.newSounds.includes(v), - ), - newSounds: latest.filter( - (v) => !acc.playing.includes(v) && !acc.newSounds.includes(v), - ), - }; - }, - { playing: [], newSounds: [] }, - ), - map((v) => v.newSounds), - ); - - /** - * Emits an event every time a new hand is raised in - * the call. - */ - public readonly newHandRaised$ = this.handsRaised$.pipe( - map((v) => Object.keys(v).length), - scan( - (acc, newValue) => ({ - value: newValue, - playSounds: newValue > acc.value, - }), - { value: 0, playSounds: false }, - ), - filter((v) => v.playSounds), - ); - - /** - * Emits an event every time a new screenshare is started in - * the call. - */ - public readonly newScreenShare$ = this.screenShares$.pipe( - map((v) => v.length), - scan( - (acc, newValue) => ({ - value: newValue, - playSounds: newValue > acc.value, - }), - { value: 0, playSounds: false }, - ), - filter((v) => v.playSounds), - ); - - /** - * Whether we are sharing our screen. - */ - // reassigned here to make it publicly accessible - public readonly sharingScreen$ = this.localMembership.sharingScreen$; - - /** - * Callback to toggle screen sharing. If null, screen sharing is not possible. - */ - // reassigned here to make it publicly accessible - public readonly toggleScreenSharing = - this.localMembership.toggleScreenSharing; - - public constructor( - private readonly scope: ObservableScope, - // A call is permanently tied to a single Matrix room - private readonly matrixRTCSession: MatrixRTCSession, - private readonly matrixRoom: MatrixRoom, - private readonly mediaDevices: MediaDevices, - private readonly muteStates: MuteStates, - private readonly options: CallViewModelOptions, - private readonly handsRaisedSubject$: Observable< - Record - >, - private readonly reactionsSubject$: Observable< - Record - >, - private readonly trackProcessorState$: Behavior, - ) { - // Join automatically - this.join(); // TODO-MULTI-SFU: Use this view model for the lobby as well, and only call this once 'join' is clicked? + this.windowMode$ = windowMode$; + this.spotlightExpanded$ = spotlightExpanded$; + this.toggleSpotlightExpanded$ = toggleSpotlightExpanded$; + this.gridMode$ = gridMode$; + this.setGridMode = setGridMode; + this.grid$ = grid$; + this.spotlight$ = spotlight$; + this.pip$ = pip$; + this.layout$ = layout$; + this.tileStoreGeneration$ = tileStoreGeneration$; + this.showSpotlightIndicators$ = showSpotlightIndicators$; + this.showSpeakingIndicators$ = showSpeakingIndicators$; + this.showHeader$ = showHeader$; + this.showFooter$ = showFooter$; + this.earpieceMode$ = earpieceMode$; + this.audioOutputSwitcher$ = audioOutputSwitcher$; + this.reconnecting$ = reconnecting$; } } - // TODO-MULTI-SFU // Setup and update the keyProvider which was create by `createRoom` was a thing before. Now we never update if the E2EEsystem changes // do we need this? diff --git a/src/state/CallViewModel/CallViewModelFunction.ts b/src/state/CallViewModel/CallViewModelFunction.ts new file mode 100644 index 00000000..20e37395 --- /dev/null +++ b/src/state/CallViewModel/CallViewModelFunction.ts @@ -0,0 +1,1400 @@ +/* +Copyright 2023, 2024, 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + type BaseKeyProvider, + type ConnectionState, + ExternalE2EEKeyProvider, + type Room as LivekitRoom, + type RoomOptions, +} from "livekit-client"; +import { type RoomMember, type Room as MatrixRoom } from "matrix-js-sdk"; +import { + combineLatest, + distinctUntilChanged, + EMPTY, + filter, + fromEvent, + map, + merge, + NEVER, + type Observable, + of, + pairwise, + race, + scan, + skip, + skipWhile, + startWith, + Subject, + switchAll, + switchMap, + switchScan, + take, + tap, + throttleTime, + timer, +} from "rxjs"; +import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; +import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; +import { type IWidgetApiRequest } from "matrix-widget-api"; + +import { + LocalUserMediaViewModel, + type MediaViewModel, + type RemoteUserMediaViewModel, + ScreenShareViewModel, + type UserMediaViewModel, +} from "../MediaViewModel"; +import { accumulate, generateItems, pauseWhen } from "../../utils/observable"; +import { + duplicateTiles, + MatrixRTCMode, + matrixRTCMode, + playReactionsSound, + showReactions, +} from "../../settings/settings"; +import { isFirefox } from "../../Platform"; +import { setPipEnabled$ } from "../../controls"; +import { TileStore } from "../TileStore"; +import { gridLikeLayout } from "../GridLikeLayout"; +import { spotlightExpandedLayout } from "../SpotlightExpandedLayout"; +import { oneOnOneLayout } from "../OneOnOneLayout"; +import { pipLayout } from "../PipLayout"; +import { type EncryptionSystem } from "../../e2ee/sharedKeyManagement"; +import { + type RaisedHandInfo, + type ReactionInfo, + type ReactionOption, +} from "../../reactions"; +import { shallowEquals } from "../../utils/array"; +import { type MediaDevices } from "../MediaDevices"; +import { type Behavior } from "../Behavior"; +import { E2eeType } from "../../e2ee/e2eeType"; +import { MatrixKeyProvider } from "../../e2ee/matrixKeyProvider"; +import { type MuteStates } from "../MuteStates"; +import { getUrlParams } from "../../UrlParams"; +import { type ProcessorState } from "../../livekit/TrackProcessorContext"; +import { ElementWidgetActions, widget } from "../../widget"; +import { UserMedia } from "../UserMedia.ts"; +import { ScreenShare } from "../ScreenShare.ts"; +import { + type GridLayoutMedia, + type Layout, + type LayoutMedia, + type OneOnOneLayoutMedia, + type SpotlightExpandedLayoutMedia, + type SpotlightLandscapeLayoutMedia, + type SpotlightPortraitLayoutMedia, +} from "../layout-types.ts"; +import { type ElementCallError } from "../../utils/errors.ts"; +import { type ObservableScope } from "../ObservableScope.ts"; +import { createLocalMembership$ } from "./localMember/LocalMembership.ts"; +import { createLocalTransport$ } from "./localMember/LocalTransport.ts"; +import { + createMemberships$, + membershipsAndTransports$, +} from "../SessionBehaviors.ts"; +import { ECConnectionFactory } from "./remoteMembers/ConnectionFactory.ts"; +import { createConnectionManager$ } from "./remoteMembers/ConnectionManager.ts"; +import { + createMatrixLivekitMembers$, + type MatrixLivekitMember, +} from "./remoteMembers/MatrixLivekitMembers.ts"; +import { + type AutoLeaveReason, + createCallNotificationLifecycle$, + createReceivedDecline$, + createSentCallNotification$, +} from "./CallNotificationLifecycle.ts"; +import { + createMatrixMemberMetadata$, + createRoomMembers$, +} from "./remoteMembers/MatrixMemberMetadata.ts"; + +const logger = rootLogger.getChild("[CallViewModel]"); +//TODO +// Larger rename +// member,membership -> rtcMember +// participant -> livekitParticipant +// matrixLivekitItem -> callMember +// js-sdk +// callMembership -> rtcMembership +export interface CallViewModelOptions { + encryptionSystem: EncryptionSystem; + autoLeaveWhenOthersLeft?: boolean; + /** + * If the call is started in a way where we want it to behave like a telephone usecase + * If we sent a notification event, we want the ui to show a ringing state + */ + waitForCallPickup?: boolean; + /** Optional factory to create LiveKit rooms, mainly for testing purposes. */ + livekitRoomFactory?: (options?: RoomOptions) => LivekitRoom; + /** Optional behavior overriding the local connection state, mainly for testing purposes. */ + connectionState$?: Behavior; +} + +// Do not play any sounds if the participant count has exceeded this +// number. +export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8; +export const THROTTLE_SOUND_EFFECT_MS = 500; + +// This is the number of participants that we think constitutes a "small" call +// on mobile. No spotlight tile should be shown below this threshold. +const smallMobileCallThreshold = 3; + +// How long the footer should be shown for when hovering over or interacting +// with the interface +const showFooterMs = 4000; + +export type GridMode = "grid" | "spotlight"; + +export type WindowMode = "normal" | "narrow" | "flat" | "pip"; + +interface LayoutScanState { + layout: Layout | null; + tiles: TileStore; +} + +type MediaItem = UserMedia | ScreenShare; +type AudioLivekitItem = { + livekitRoom: LivekitRoom; + participants: string[]; + url: string; +}; + +type JoinReturn = ReturnType< + ReturnType["requestConnect"] +>; + +/** + * A view model providing all the application logic needed to show the in-call + * UI (may eventually be expanded to cover the lobby and feedback screens in the + * future). + */ +// Throughout this class and related code we must distinguish between MatrixRTC +// state and LiveKit state. We use the common terminology of room "members", RTC +// "memberships", and LiveKit "participants". +export function createCallViewModel$( + scope: ObservableScope, + // A call is permanently tied to a single Matrix room + matrixRTCSession: MatrixRTCSession, + matrixRoom: MatrixRoom, + mediaDevices: MediaDevices, + muteStates: MuteStates, + options: CallViewModelOptions, + handsRaisedSubject$: Observable>, + reactionsSubject$: Observable>, + trackProcessorState$: Behavior, +): { + // lifecycle + autoLeave$: Observable; + callPickupState$: Behavior< + "unknown" | "ringing" | "timeout" | "decline" | "success" | null + >; + leave$: Observable<"user" | AutoLeaveReason>; + hangup: () => void; + + // joining + join: () => JoinReturn; + + // screen sharing + toggleScreenSharing: (() => void) | null; + sharingScreen$: Behavior; + + // UI interactions + tapScreen: () => void; + tapControls: () => void; + hoverScreen: () => void; + unhoverScreen: () => void; + + // errors + configError$: Behavior; + + // participants and counts + participantCount$: Behavior; + audioParticipants$: Behavior; + handsRaised$: Behavior>; + reactions$: Behavior>; + isOneOnOneWith$: Behavior | null>; + localUserIsAlone$: Behavior; + // sounds and events + joinSoundEffect$: Observable; + leaveSoundEffect$: Observable; + newHandRaised$: Observable<{ value: number; playSounds: boolean }>; + newScreenShare$: Observable<{ value: number; playSounds: boolean }>; + audibleReactions$: Observable; + visibleReactions$: Behavior< + { sender: string; emoji: string; startX: number }[] + >; + + // window/layout + windowMode$: Behavior; + spotlightExpanded$: Behavior; + toggleSpotlightExpanded$: Behavior<(() => void) | null>; + gridMode$: Behavior; + setGridMode: (value: GridMode) => void; + + // media view models and layout + grid$: Behavior; + spotlight$: Behavior; + pip$: Behavior; + layout$: Behavior; + tileStoreGeneration$: Behavior; + showSpotlightIndicators$: Behavior; + showSpeakingIndicators$: Behavior; + + // header/footer visibility + showHeader$: Behavior; + showFooter$: Behavior; + + // audio routing + earpieceMode$: Behavior; + audioOutputSwitcher$: Behavior<{ + targetOutput: "earpiece" | "speaker"; + switch: () => void; + } | null>; + + // connection state + reconnecting$: Behavior; +} { + const userId = matrixRoom.client.getUserId()!; + const deviceId = matrixRoom.client.getDeviceId()!; + + const livekitKeyProvider = getE2eeKeyProvider( + options.encryptionSystem, + matrixRTCSession, + ); + + // Each hbar seperates a block of input variables required for the CallViewModel to function. + // The outputs of this block is written under the hbar. + // + // For mocking purposes it is recommended to only mock the functions creating those outputs. + // All other fields are just temp computations for the mentioned output. + // The class does not need anything except the values underneath the bar. + // The creation of the values under the bar are all tested independently and testing the callViewModel Should + // not test their cretation. Call view model only needs: + // - memberships$ via createMemberships$ + // - localMembership via createLocalMembership$ + // - callLifecycle via createCallNotificationLifecycle$ + // - matrixMemberMetadataStore via createMatrixMemberMetadata$ + + // ------------------------------------------------------------------------ + // memberships$ + const memberships$ = createMemberships$(scope, matrixRTCSession); + + // ------------------------------------------------------------------------ + // matrixLivekitMembers$ AND localMembership + + const membershipsAndTransports = membershipsAndTransports$( + scope, + memberships$, + ); + + const localTransport$ = createLocalTransport$({ + scope: scope, + memberships$: memberships$, + client: matrixRoom.client, + roomId: matrixRoom.roomId, + useOldestMember$: scope.behavior( + matrixRTCMode.value$.pipe(map((v) => v === MatrixRTCMode.Legacy)), + ), + }); + + const connectionFactory = new ECConnectionFactory( + matrixRoom.client, + mediaDevices, + trackProcessorState$, + livekitKeyProvider, + getUrlParams().controlledAudioDevices, + options.livekitRoomFactory, + ); + + const connectionManager = createConnectionManager$({ + scope: scope, + connectionFactory: connectionFactory, + inputTransports$: scope.behavior( + combineLatest( + [localTransport$, membershipsAndTransports.transports$], + (localTransport, transports) => { + const localTransportAsArray = localTransport ? [localTransport] : []; + return transports.mapInner((transports) => [ + ...localTransportAsArray, + ...transports, + ]); + }, + ), + ), + logger: logger, + }); + + const matrixLivekitMembers$ = createMatrixLivekitMembers$({ + scope: scope, + membershipsWithTransport$: + membershipsAndTransports.membershipsWithTransport$, + connectionManager: connectionManager, + }); + + const connectOptions$ = scope.behavior( + matrixRTCMode.value$.pipe( + map((mode) => ({ + encryptMedia: livekitKeyProvider !== undefined, + // TODO. This might need to get called again on each cahnge of matrixRTCMode... + matrixRTCMode: mode, + })), + ), + ); + + const localMembership = createLocalMembership$({ + scope: scope, + muteStates: muteStates, + mediaDevices: mediaDevices, + connectionManager: connectionManager, + matrixRTCSession: matrixRTCSession, + matrixRoom: matrixRoom, + localTransport$: localTransport$, + trackProcessorState$: trackProcessorState$, + widget, + options: connectOptions$, + logger: logger.getChild(`[${Date.now()}]`), + }); + + const localRtcMembership$ = scope.behavior( + memberships$.pipe( + map( + (memberships) => + memberships.value.find( + (membership) => + membership.userId === userId && membership.deviceId === deviceId, + ) ?? null, + ), + ), + ); + + const localMatrixLivekitMemberUninitialized = { + membership$: localRtcMembership$, + participant$: localMembership.participant$, + connection$: localMembership.connection$, + userId: userId, + }; + + const localMatrixLivekitMember$: Behavior = + scope.behavior( + localRtcMembership$.pipe( + switchMap((membership) => { + if (!membership) return of(null); + return of( + // casting is save here since we know that localRtcMembership$ is !== null since we reached this case. + localMatrixLivekitMemberUninitialized as MatrixLivekitMember, + ); + }), + ), + ); + + // ------------------------------------------------------------------------ + // callLifecycle + + const callLifecycle = createCallNotificationLifecycle$({ + scope: scope, + memberships$: memberships$, + sentCallNotification$: createSentCallNotification$(scope, matrixRTCSession), + receivedDecline$: createReceivedDecline$(matrixRoom), + options: options, + localUser: { userId: userId, deviceId: deviceId }, + }); + + // ------------------------------------------------------------------------ + // matrixMemberMetadataStore + + const matrixRoomMembers$ = createRoomMembers$(scope, matrixRoom); + const matrixMemberMetadataStore = createMatrixMemberMetadata$( + scope, + scope.behavior(memberships$.pipe(map((mems) => mems.value))), + matrixRoomMembers$, + ); + + /** + * Returns the Member {userId, getMxcAvatarUrl, rawDisplayName} of the other user in the call, if it's a one-on-one call. + */ + const isOneOnOneWith$ = scope.behavior( + matrixRoomMembers$.pipe( + map((roomMembersMap) => { + const otherMembers = Array.from(roomMembersMap.values()).filter( + (member) => member.userId !== userId, + ); + return otherMembers.length === 1 ? otherMembers[0] : null; + }), + ), + ); + + const localUserIsAlone$ = scope.behavior( + matrixRoomMembers$.pipe( + map( + (roomMembersMap) => + roomMembersMap.size === 1 && roomMembersMap.get(userId) !== undefined, + ), + ), + ); + + // CODESMELL? + // This is functionally the same Observable as leave$, except here it's + // hoisted to the top of the class. This enables the cyclic dependency between + // leave$ -> autoLeave$ -> callPickupState$ -> livekitConnectionState$ -> + // localConnection$ -> transports$ -> joined$ -> leave$. + const leaveHoisted$ = new Subject< + "user" | "timeout" | "decline" | "allOthersLeft" + >(); + + /** + * Whether various media/event sources should pretend to be disconnected from + * all network input, even if their connection still technically works. + */ + // We do this when the app is in the 'reconnecting' state, because it might be + // that the LiveKit connection is still functional while the homeserver is + // down, for example, and we want to avoid making people worry that the app is + // in a split-brained state. + // DISCUSSION own membership manager ALSO this probably can be simplifis + /*PUBLIC*/ const reconnecting$ = localMembership.reconnecting$; + const pretendToBeDisconnected$ = reconnecting$; + + /*PUBLIC*/ const audioParticipants$ = scope.behavior( + matrixLivekitMembers$.pipe( + switchMap((membersWithEpoch) => { + const members = membersWithEpoch.value; + const a$ = combineLatest( + members.map((member) => + combineLatest([member.connection$, member.participant$]).pipe( + map(([connection, participant]) => { + // do not render audio for local participant + if (!connection || !participant || participant.isLocal) + return null; + const livekitRoom = connection.livekitRoom; + const url = connection.transport.livekit_service_url; + + return { url, livekitRoom, participant: participant.identity }; + }), + ), + ), + ); + return a$; + }), + map((members) => + members.reduce((acc, curr) => { + if (!curr) return acc; + + const existing = acc.find((item) => item.url === curr.url); + if (existing) { + existing.participants.push(curr.participant); + } else { + acc.push({ + livekitRoom: curr.livekitRoom, + participants: [curr.participant], + url: curr.url, + }); + } + return acc; + }, []), + ), + ), + [], + ); + + /*PUBLIC*/ const handsRaised$ = scope.behavior( + handsRaisedSubject$.pipe(pauseWhen(pretendToBeDisconnected$)), + ); + + /*PUBLIC*/ const reactions$ = scope.behavior( + reactionsSubject$.pipe( + map((v) => + Object.fromEntries( + Object.entries(v).map(([a, { reactionOption }]) => [ + a, + reactionOption, + ]), + ), + ), + pauseWhen(pretendToBeDisconnected$), + ), + ); + + /** + * List of user media (camera feeds) that we want tiles for. + */ + // TODO this also needs the local participant to be added. + const userMedia$ = scope.behavior( + combineLatest([ + localMatrixLivekitMember$, + matrixLivekitMembers$, + duplicateTiles.value$, + ]).pipe( + // Generate a collection of MediaItems from the list of expected (whether + // present or missing) LiveKit participants. + generateItems( + function* ([ + localMatrixLivekitMember, + { value: matrixLivekitMembers }, + duplicateTiles, + ]) { + let localParticipantId = undefined; + // add local member if available + if (localMatrixLivekitMember) { + const { userId, participant$, connection$, membership$ } = + localMatrixLivekitMember; + localParticipantId = `${userId}:${membership$.value.deviceId}`; // should be membership$.value.membershipID which is not optional + // const participantId = membership$.value.membershipID; + if (localParticipantId) { + for (let dup = 0; dup < 1 + duplicateTiles; dup++) { + yield { + keys: [ + dup, + localParticipantId, + userId, + participant$, + connection$, + ], + data: undefined, + }; + } + } + } + // add remote members that are available + for (const { + userId, + participant$, + connection$, + membership$, + } of matrixLivekitMembers) { + const participantId = `${userId}:${membership$.value.deviceId}`; + if (participantId === localParticipantId) continue; + // const participantId = membership$.value?.identity; + for (let dup = 0; dup < 1 + duplicateTiles; dup++) { + yield { + keys: [dup, participantId, userId, participant$, connection$], + data: undefined, + }; + } + } + }, + ( + scope, + _data$, + dup, + participantId, + userId, + participant$, + connection$, + ) => { + const livekitRoom$ = scope.behavior( + connection$.pipe(map((c) => c?.livekitRoom)), + ); + const focusUrl$ = scope.behavior( + connection$.pipe(map((c) => c?.transport.livekit_service_url)), + ); + const displayName$ = scope.behavior( + matrixMemberMetadataStore + .createDisplayNameBehavior$(userId) + .pipe(map((name) => name ?? userId)), + ); + + return new UserMedia( + scope, + `${participantId}:${dup}`, + userId, + participant$, + options.encryptionSystem, + livekitRoom$, + focusUrl$, + mediaDevices, + pretendToBeDisconnected$, + displayName$, + matrixMemberMetadataStore.createAvatarUrlBehavior$(userId), + handsRaised$.pipe(map((v) => v[participantId]?.time ?? null)), + reactions$.pipe(map((v) => v[participantId] ?? undefined)), + ); + }, + ), + ), + ); + + /** + * List of all media items (user media and screen share media) that we want + * tiles for. + */ + const mediaItems$ = scope.behavior( + userMedia$.pipe( + switchMap((userMedia) => + userMedia.length === 0 + ? of([]) + : combineLatest( + userMedia.map((m) => m.screenShares$), + (...screenShares) => [...userMedia, ...screenShares.flat(1)], + ), + ), + ), + ); + + /** + * List of MediaItems that we want to display, that are of type ScreenShare + */ + const screenShares$ = scope.behavior( + mediaItems$.pipe( + map((mediaItems) => + mediaItems.filter((m): m is ScreenShare => m instanceof ScreenShare), + ), + ), + ); + + /*PUBLIC*/ const joinSoundEffect$ = userMedia$.pipe( + pairwise(), + filter( + ([prev, current]) => + current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && + current.length > prev.length, + ), + map(() => {}), + throttleTime(THROTTLE_SOUND_EFFECT_MS), + ); + + /** + * The number of participants currently in the call. + * + * - Each participant has a corresponding MatrixRTC membership state event + * - There can be multiple participants for one Matrix user if they join from + * multiple devices. + */ + /*PUBLIC*/ const participantCount$ = scope.behavior( + matrixLivekitMembers$.pipe(map((ms) => ms.value.length)), + ); + + // only public to expose to the view. + // TODO if we are in "unknown" state we need a loading rendering (or empty screen) + // Otherwise it looks like we already connected and only than the ringing starts which is weird. + /*PUBLIC*/ const callPickupState$ = callLifecycle.callPickupState$; + + /*PUBLIC*/ const leaveSoundEffect$ = combineLatest([ + callLifecycle.callPickupState$, + userMedia$, + ]).pipe( + // Until the call is successful, do not play a leave sound. + // If callPickupState$ is null, then we always play the sound as it will not conflict with a decline sound. + skipWhile(([c]) => c !== null && c !== "success"), + map(([, userMedia]) => userMedia), + pairwise(), + filter( + ([prev, current]) => + current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && + current.length < prev.length, + ), + map(() => {}), + throttleTime(THROTTLE_SOUND_EFFECT_MS), + ); + + const userHangup$ = new Subject(); + + const widgetHangup$ = + widget === null + ? NEVER + : ( + fromEvent( + widget.lazyActions, + ElementWidgetActions.HangupCall, + ) as Observable> + ).pipe( + tap((ev) => { + widget!.api.transport.reply(ev.detail, {}); + }), + ); + + const leave$: Observable<"user" | "timeout" | "decline" | "allOthersLeft"> = + merge( + callLifecycle.autoLeave$, + merge(userHangup$, widgetHangup$).pipe(map(() => "user" as const)), + ).pipe( + scope.share, + tap((reason) => leaveHoisted$.next(reason)), + ); + + const spotlightSpeaker$ = scope.behavior( + userMedia$.pipe( + switchMap((mediaItems) => + mediaItems.length === 0 + ? of([]) + : combineLatest( + mediaItems.map((m) => + m.vm.speaking$.pipe(map((s) => [m, s] as const)), + ), + ), + ), + scan<(readonly [UserMedia, boolean])[], UserMedia | undefined, null>( + (prev, mediaItems) => { + // Only remote users that are still in the call should be sticky + const [stickyMedia, stickySpeaking] = + (!prev?.vm.local && mediaItems.find(([m]) => m === prev)) || []; + // Decide who to spotlight: + // If the previous speaker is still speaking, stick with them rather + // than switching eagerly to someone else + return stickySpeaking + ? stickyMedia! + : // Otherwise, select any remote user who is speaking + (mediaItems.find(([m, s]) => !m.vm.local && s)?.[0] ?? + // Otherwise, stick with the person who was last speaking + stickyMedia ?? + // Otherwise, spotlight an arbitrary remote user + mediaItems.find(([m]) => !m.vm.local)?.[0] ?? + // Otherwise, spotlight the local user + mediaItems.find(([m]) => m.vm.local)?.[0]); + }, + null, + ), + map((speaker) => speaker?.vm ?? null), + ), + ); + + const grid$ = scope.behavior( + userMedia$.pipe( + switchMap((mediaItems) => { + const bins = mediaItems.map((m) => + m.bin$.pipe(map((bin) => [m, bin] as const)), + ); + // Sort the media by bin order and generate a tile for each one + return bins.length === 0 + ? of([]) + : combineLatest(bins, (...bins) => + bins.sort(([, bin1], [, bin2]) => bin1 - bin2).map(([m]) => m.vm), + ); + }), + distinctUntilChanged(shallowEquals), + ), + ); + + const spotlight$ = scope.behavior( + screenShares$.pipe( + switchMap((screenShares) => { + if (screenShares.length > 0) { + return of(screenShares.map((m) => m.vm)); + } + + return spotlightSpeaker$.pipe( + map((speaker) => (speaker ? [speaker] : [])), + ); + }), + distinctUntilChanged(shallowEquals), + ), + ); + + const pip$ = scope.behavior( + combineLatest([ + // TODO This also needs epoch logic to dedupe the screenshares and mediaItems emits + screenShares$, + spotlightSpeaker$, + mediaItems$, + ]).pipe( + switchMap(([screenShares, spotlight, mediaItems]) => { + if (screenShares.length > 0) { + return spotlightSpeaker$; + } + if (!spotlight || spotlight.local) { + return of(null); + } + + const localUserMedia = mediaItems.find( + (m) => m.vm instanceof LocalUserMediaViewModel, + ) as UserMedia | undefined; + + const localUserMediaViewModel = localUserMedia?.vm as + | LocalUserMediaViewModel + | undefined; + + if (!localUserMediaViewModel) { + return of(null); + } + return localUserMediaViewModel.alwaysShow$.pipe( + map((alwaysShow) => { + if (alwaysShow) { + return localUserMediaViewModel; + } + + return null; + }), + ); + }), + ), + ); + + const hasRemoteScreenShares$: Observable = spotlight$.pipe( + map((spotlight) => + spotlight.some((vm) => !vm.local && vm instanceof ScreenShareViewModel), + ), + distinctUntilChanged(), + ); + + const pipEnabled$ = scope.behavior(setPipEnabled$, false); + + const naturalWindowMode$ = scope.behavior( + fromEvent(window, "resize").pipe( + map(() => { + const height = window.innerHeight; + const width = window.innerWidth; + if (height <= 400 && width <= 340) return "pip"; + // Our layouts for flat windows are better at adapting to a small width + // than our layouts for narrow windows are at adapting to a small height, + // so we give "flat" precedence here + if (height <= 600) return "flat"; + if (width <= 600) return "narrow"; + return "normal"; + }), + ), + "normal", + ); + + /** + * The general shape of the window. + */ + /*PUBLIC*/ const windowMode$ = scope.behavior( + pipEnabled$.pipe( + switchMap((pip) => (pip ? of("pip") : naturalWindowMode$)), + ), + ); + + const spotlightExpandedToggle$ = new Subject(); + /*PUBLIC*/ const spotlightExpanded$ = scope.behavior( + spotlightExpandedToggle$.pipe(accumulate(false, (expanded) => !expanded)), + ); + + const gridModeUserSelection$ = new Subject(); + /** + * The layout mode of the media tile grid. + */ + /*PUBLIC*/ const gridMode$ = + // If the user hasn't selected spotlight and somebody starts screen sharing, + // automatically switch to spotlight mode and reset when screen sharing ends + scope.behavior( + gridModeUserSelection$.pipe( + switchMap((userSelection) => + (userSelection === "spotlight" + ? EMPTY + : combineLatest([hasRemoteScreenShares$, windowMode$]).pipe( + skip(userSelection === null ? 0 : 1), + map( + ([hasScreenShares, windowMode]): GridMode => + hasScreenShares || windowMode === "flat" + ? "spotlight" + : "grid", + ), + ) + ).pipe(startWith(userSelection ?? "grid")), + ), + ), + "grid", + ); + + /*PUBLIC*/ const setGridMode = (value: GridMode): void => { + gridModeUserSelection$.next(value); + }; + + const gridLayoutMedia$: Observable = combineLatest( + [grid$, spotlight$], + (grid, spotlight) => ({ + type: "grid", + spotlight: spotlight.some((vm) => vm instanceof ScreenShareViewModel) + ? spotlight + : undefined, + grid, + }), + ); + + const spotlightLandscapeLayoutMedia$: Observable = + combineLatest([grid$, spotlight$], (grid, spotlight) => ({ + type: "spotlight-landscape", + spotlight, + grid, + })); + + const spotlightPortraitLayoutMedia$: Observable = + combineLatest([grid$, spotlight$], (grid, spotlight) => ({ + type: "spotlight-portrait", + spotlight, + grid, + })); + + const spotlightExpandedLayoutMedia$: Observable = + combineLatest([spotlight$, pip$], (spotlight, pip) => ({ + type: "spotlight-expanded", + spotlight, + pip: pip ?? undefined, + })); + + const oneOnOneLayoutMedia$: Observable = + mediaItems$.pipe( + map((mediaItems) => { + if (mediaItems.length !== 2) return null; + const local = mediaItems.find((vm) => vm.vm.local)?.vm as + | LocalUserMediaViewModel + | undefined; + const remote = mediaItems.find((vm) => !vm.vm.local)?.vm as + | RemoteUserMediaViewModel + | undefined; + // There might not be a remote tile if there are screen shares, or if + // only the local user is in the call and they're using the duplicate + // tiles option + if (!remote || !local) return null; + + return { type: "one-on-one", local, remote }; + }), + ); + + const pipLayoutMedia$: Observable = spotlight$.pipe( + map((spotlight) => ({ type: "pip", spotlight })), + ); + + /** + * The media to be used to produce a layout. + */ + const layoutMedia$ = scope.behavior( + windowMode$.pipe( + switchMap((windowMode) => { + switch (windowMode) { + case "normal": + return gridMode$.pipe( + switchMap((gridMode) => { + switch (gridMode) { + case "grid": + return oneOnOneLayoutMedia$.pipe( + switchMap((oneOnOne) => + oneOnOne === null ? gridLayoutMedia$ : of(oneOnOne), + ), + ); + case "spotlight": + return spotlightExpanded$.pipe( + switchMap((expanded) => + expanded + ? spotlightExpandedLayoutMedia$ + : spotlightLandscapeLayoutMedia$, + ), + ); + } + }), + ); + case "narrow": + return oneOnOneLayoutMedia$.pipe( + switchMap((oneOnOne) => + oneOnOne === null + ? combineLatest([grid$, spotlight$], (grid, spotlight) => + grid.length > smallMobileCallThreshold || + spotlight.some((vm) => vm instanceof ScreenShareViewModel) + ? spotlightPortraitLayoutMedia$ + : gridLayoutMedia$, + ).pipe(switchAll()) + : // The expanded spotlight layout makes for a better one-on-one + // experience in narrow windows + spotlightExpandedLayoutMedia$, + ), + ); + case "flat": + return gridMode$.pipe( + switchMap((gridMode) => { + switch (gridMode) { + case "grid": + // Yes, grid mode actually gets you a "spotlight" layout in + // this window mode. + return spotlightLandscapeLayoutMedia$; + case "spotlight": + return spotlightExpandedLayoutMedia$; + } + }), + ); + case "pip": + return pipLayoutMedia$; + } + }), + ), + ); + + // There is a cyclical dependency here: the layout algorithms want to know + // which tiles are on screen, but to know which tiles are on screen we have to + // first render a layout. To deal with this we assume initially that no tiles + // are visible, and loop the data back into the layouts with a Subject. + const visibleTiles$ = new Subject(); + const setVisibleTiles = (value: number): void => visibleTiles$.next(value); + + const layoutInternals$ = scope.behavior( + combineLatest([ + layoutMedia$, + visibleTiles$.pipe(startWith(0), distinctUntilChanged()), + ]).pipe( + scan< + [LayoutMedia, number], + LayoutScanState & { layout: Layout }, + LayoutScanState + >( + ({ tiles: prevTiles }, [media, visibleTiles]) => { + let layout: Layout; + let newTiles: TileStore; + switch (media.type) { + case "grid": + case "spotlight-landscape": + case "spotlight-portrait": + [layout, newTiles] = gridLikeLayout( + media, + visibleTiles, + setVisibleTiles, + prevTiles, + ); + break; + case "spotlight-expanded": + [layout, newTiles] = spotlightExpandedLayout(media, prevTiles); + break; + case "one-on-one": + [layout, newTiles] = oneOnOneLayout(media, prevTiles); + break; + case "pip": + [layout, newTiles] = pipLayout(media, prevTiles); + break; + } + + return { layout, tiles: newTiles }; + }, + { layout: null, tiles: TileStore.empty() }, + ), + ), + ); + + /** + * The layout of tiles in the call interface. + */ + /*PUBLIC*/ const layout$ = scope.behavior( + layoutInternals$.pipe(map(({ layout }) => layout)), + ); + + /** + * The current generation of the tile store, exposed for debugging purposes. + */ + /*PUBLIC*/ const tileStoreGeneration$ = scope.behavior( + layoutInternals$.pipe(map(({ tiles }) => tiles.generation)), + ); + + /*PUBLIC*/ const showSpotlightIndicators$ = scope.behavior( + layout$.pipe(map((l) => l.type !== "grid")), + ); + + /*PUBLIC*/ const showSpeakingIndicators$ = scope.behavior( + layout$.pipe( + switchMap((l) => { + switch (l.type) { + case "spotlight-landscape": + case "spotlight-portrait": + // If the spotlight is showing the active speaker, we can do without + // speaking indicators as they're a redundant visual cue. But if + // screen sharing feeds are in the spotlight we still need them. + return l.spotlight.media$.pipe( + map((models: MediaViewModel[]) => + models.some((m) => m instanceof ScreenShareViewModel), + ), + ); + // In expanded spotlight layout, the active speaker is always shown in + // the picture-in-picture tile so there is no need for speaking + // indicators. And in one-on-one layout there's no question as to who is + // speaking. + case "spotlight-expanded": + case "one-on-one": + return of(false); + default: + return of(true); + } + }), + ), + ); + + /*PUBLIC*/ const toggleSpotlightExpanded$ = scope.behavior< + (() => void) | null + >( + windowMode$.pipe( + switchMap((mode) => + mode === "normal" + ? layout$.pipe( + map( + (l) => + l.type === "spotlight-landscape" || + l.type === "spotlight-expanded", + ), + ) + : of(false), + ), + distinctUntilChanged(), + map((enabled) => + enabled ? (): void => spotlightExpandedToggle$.next() : null, + ), + ), + ); + + const screenTap$ = new Subject(); + const controlsTap$ = new Subject(); + const screenHover$ = new Subject(); + const screenUnhover$ = new Subject(); + + /*PUBLIC*/ const showHeader$ = scope.behavior( + windowMode$.pipe(map((mode) => mode !== "pip" && mode !== "flat")), + ); + + /*PUBLIC*/ const showFooter$ = scope.behavior( + windowMode$.pipe( + switchMap((mode) => { + switch (mode) { + case "pip": + return of(false); + case "normal": + case "narrow": + return of(true); + case "flat": + // Sadly Firefox has some layering glitches that prevent the footer + // from appearing properly. They happen less often if we never hide + // the footer. + if (isFirefox()) return of(true); + // Show/hide the footer in response to interactions + return merge( + screenTap$.pipe(map(() => "tap screen" as const)), + controlsTap$.pipe(map(() => "tap controls" as const)), + screenHover$.pipe(map(() => "hover" as const)), + ).pipe( + switchScan((state, interaction) => { + switch (interaction) { + case "tap screen": + return state + ? // Toggle visibility on tap + of(false) + : // Hide after a timeout + timer(showFooterMs).pipe( + map(() => false), + startWith(true), + ); + case "tap controls": + // The user is interacting with things, so reset the timeout + return timer(showFooterMs).pipe( + map(() => false), + startWith(true), + ); + case "hover": + // Show on hover and hide after a timeout + return race( + timer(showFooterMs), + screenUnhover$.pipe(take(1)), + ).pipe( + map(() => false), + startWith(true), + ); + } + }, false), + startWith(false), + ); + } + }), + ), + ); + + /** + * Whether audio is currently being output through the earpiece. + */ + /*PUBLIC*/ const earpieceMode$ = scope.behavior( + combineLatest( + [mediaDevices.audioOutput.available$, mediaDevices.audioOutput.selected$], + (available, selected) => + selected !== undefined && + available.get(selected.id)?.type === "earpiece", + ), + ); + + /** + * Callback to toggle between the earpiece and the loudspeaker. + * + * This will be `null` in case the target does not exist in the list + * of available audio outputs. + */ + /*PUBLIC*/ const audioOutputSwitcher$ = scope.behavior<{ + targetOutput: "earpiece" | "speaker"; + switch: () => void; + } | null>( + combineLatest( + [mediaDevices.audioOutput.available$, mediaDevices.audioOutput.selected$], + (available, selected) => { + const selectionType = selected && available.get(selected.id)?.type; + + // If we are in any output mode other than speaker switch to speaker. + const newSelectionType: "earpiece" | "speaker" = + selectionType === "speaker" ? "earpiece" : "speaker"; + const newSelection = [...available].find( + ([, d]) => d.type === newSelectionType, + ); + if (newSelection === undefined) return null; + + const [id] = newSelection; + return { + targetOutput: newSelectionType, + switch: (): void => mediaDevices.audioOutput.select(id), + }; + }, + ), + ); + + /** + * Emits an array of reactions that should be visible on the screen. + */ + // DISCUSSION move this into a reaction file + // const {visibleReactions$, audibleReactions$} = reactionsObservables$(showReactionSetting$, ) + const visibleReactions$ = scope.behavior( + showReactions.value$.pipe( + switchMap((show) => (show ? reactions$ : of({}))), + scan< + Record, + { sender: string; emoji: string; startX: number }[] + >((acc, latest) => { + const newSet: { sender: string; emoji: string; startX: number }[] = []; + for (const [sender, reaction] of Object.entries(latest)) { + const startX = + acc.find((v) => v.sender === sender && v.emoji)?.startX ?? + Math.ceil(Math.random() * 80) + 10; + newSet.push({ sender, emoji: reaction.emoji, startX }); + } + return newSet; + }, []), + ), + ); + + /** + * Emits an array of reactions that should be played. + */ + const audibleReactions$ = playReactionsSound.value$.pipe( + switchMap((show) => + show ? reactions$ : of>({}), + ), + map((reactions) => Object.values(reactions).map((v) => v.name)), + scan( + (acc, latest) => { + return { + playing: latest.filter( + (v) => acc.playing.includes(v) || acc.newSounds.includes(v), + ), + newSounds: latest.filter( + (v) => !acc.playing.includes(v) && !acc.newSounds.includes(v), + ), + }; + }, + { playing: [], newSounds: [] }, + ), + map((v) => v.newSounds), + ); + + const newHandRaised$ = handsRaised$.pipe( + map((v) => Object.keys(v).length), + scan( + (acc, newValue) => ({ + value: newValue, + playSounds: newValue > acc.value, + }), + { value: 0, playSounds: false }, + ), + filter((v) => v.playSounds), + ); + + const newScreenShare$ = screenShares$.pipe( + map((v) => v.length), + scan( + (acc, newValue) => ({ + value: newValue, + playSounds: newValue > acc.value, + }), + { value: 0, playSounds: false }, + ), + filter((v) => v.playSounds), + ); + + /** + * Whether we are sharing our screen. + */ + // reassigned here to make it publicly accessible + const sharingScreen$ = localMembership.sharingScreen$; + + /** + * Callback to toggle screen sharing. If null, screen sharing is not possible. + */ + // reassigned here to make it publicly accessible + const toggleScreenSharing = localMembership.toggleScreenSharing; + + const join = localMembership.requestConnect; + join(); // TODO-MULTI-SFU: Use this view model for the lobby as well, and only call this once 'join' is clicked? + + return { + autoLeave$: callLifecycle.autoLeave$, + callPickupState$, + leave$, + hangup: () => userHangup$.next(), + join, + toggleScreenSharing, + sharingScreen$, + + tapScreen: () => screenTap$.next(), + tapControls: () => controlsTap$.next(), + hoverScreen: () => screenHover$.next(), + unhoverScreen: () => screenUnhover$.next(), + + configError$: localMembership.configError$, + participantCount$, + audioParticipants$, + isOneOnOneWith$, + localUserIsAlone$, + + handsRaised$, + reactions$, + joinSoundEffect$, + leaveSoundEffect$, + newHandRaised$, + newScreenShare$, + audibleReactions$, + visibleReactions$, + + windowMode$, + spotlightExpanded$, + toggleSpotlightExpanded$, + gridMode$, + setGridMode, + grid$, + spotlight$, + pip$, + layout$, + tileStoreGeneration$, + showSpotlightIndicators$, + showSpeakingIndicators$, + showHeader$, + showFooter$, + earpieceMode$, + audioOutputSwitcher$, + reconnecting$, + }; +} +// TODO-MULTI-SFU // Setup and update the keyProvider which was create by `createRoom` was a thing before. Now we never update if the E2EEsystem changes +// do we need this? + +function getE2eeKeyProvider( + e2eeSystem: EncryptionSystem, + rtcSession: MatrixRTCSession, +): BaseKeyProvider | undefined { + if (e2eeSystem.kind === E2eeType.NONE) return undefined; + + if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) { + const keyProvider = new MatrixKeyProvider(); + keyProvider.setRTCSession(rtcSession); + return keyProvider; + } else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) { + const keyProvider = new ExternalE2EEKeyProvider(); + keyProvider + .setKey(e2eeSystem.secret) + .catch((e) => logger.error("Failed to set shared key for E2EE", e)); + return keyProvider; + } +} diff --git a/src/state/CallViewModel/CallViewModelOld.ts b/src/state/CallViewModel/CallViewModelOld.ts new file mode 100644 index 00000000..ff4a6269 --- /dev/null +++ b/src/state/CallViewModel/CallViewModelOld.ts @@ -0,0 +1,1335 @@ +/* +Copyright 2023, 2024, 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + type BaseKeyProvider, + type ConnectionState, + ExternalE2EEKeyProvider, + type Room as LivekitRoom, + type RoomOptions, +} from "livekit-client"; +import { type Room as MatrixRoom } from "matrix-js-sdk"; +import { + combineLatest, + distinctUntilChanged, + EMPTY, + filter, + fromEvent, + map, + merge, + NEVER, + type Observable, + of, + pairwise, + race, + scan, + skip, + skipWhile, + startWith, + Subject, + switchAll, + switchMap, + switchScan, + take, + tap, + throttleTime, + timer, +} from "rxjs"; +import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; +import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; +import { type IWidgetApiRequest } from "matrix-widget-api"; + +import { + LocalUserMediaViewModel, + type MediaViewModel, + type RemoteUserMediaViewModel, + ScreenShareViewModel, + type UserMediaViewModel, +} from "../MediaViewModel"; +import { accumulate, generateItems, pauseWhen } from "../../utils/observable"; +import { + duplicateTiles, + MatrixRTCMode, + matrixRTCMode, + playReactionsSound, + showReactions, +} from "../../settings/settings"; +import { isFirefox } from "../../Platform"; +import { setPipEnabled$ } from "../../controls"; +import { TileStore } from "../TileStore"; +import { gridLikeLayout } from "../GridLikeLayout"; +import { spotlightExpandedLayout } from "../SpotlightExpandedLayout"; +import { oneOnOneLayout } from "../OneOnOneLayout"; +import { pipLayout } from "../PipLayout"; +import { type EncryptionSystem } from "../../e2ee/sharedKeyManagement"; +import { + type RaisedHandInfo, + type ReactionInfo, + type ReactionOption, +} from "../../reactions"; +import { shallowEquals } from "../../utils/array"; +import { type MediaDevices } from "../MediaDevices"; +import { type Behavior } from "../Behavior"; +import { E2eeType } from "../../e2ee/e2eeType"; +import { MatrixKeyProvider } from "../../e2ee/matrixKeyProvider"; +import { type MuteStates } from "../MuteStates"; +import { getUrlParams } from "../../UrlParams"; +import { type ProcessorState } from "../../livekit/TrackProcessorContext"; +import { ElementWidgetActions, widget } from "../../widget"; +import { UserMedia } from "../UserMedia.ts"; +import { ScreenShare } from "../ScreenShare.ts"; +import { + type GridLayoutMedia, + type Layout, + type LayoutMedia, + type OneOnOneLayoutMedia, + type SpotlightExpandedLayoutMedia, + type SpotlightLandscapeLayoutMedia, + type SpotlightPortraitLayoutMedia, +} from "../layout-types.ts"; +import { type ElementCallError } from "../../utils/errors.ts"; +import { type ObservableScope } from "../ObservableScope.ts"; +import { createLocalMembership$ } from "./localMember/LocalMembership.ts"; +import { createLocalTransport$ } from "./localMember/LocalTransport.ts"; +import { + createMemberships$, + membershipsAndTransports$, +} from "../SessionBehaviors.ts"; +import { ECConnectionFactory } from "./remoteMembers/ConnectionFactory.ts"; +import { createConnectionManager$ } from "./remoteMembers/ConnectionManager.ts"; +import { + createMatrixLivekitMembers$, + type MatrixLivekitMember, +} from "./remoteMembers/MatrixLivekitMembers.ts"; +import { + createCallNotificationLifecycle$, + createReceivedDecline$, + createSentCallNotification$, +} from "./CallNotificationLifecycle.ts"; +import { + createMatrixMemberMetadata$, + createRoomMembers$, +} from "./remoteMembers/MatrixMemberMetadata.ts"; + +const logger = rootLogger.getChild("[CallViewModel]"); +//TODO +// Larger rename +// member,membership -> rtcMember +// participant -> livekitParticipant +// matrixLivekitItem -> callMember +// js-sdk +// callMembership -> rtcMembership +export interface CallViewModelOptions { + encryptionSystem: EncryptionSystem; + autoLeaveWhenOthersLeft?: boolean; + /** + * If the call is started in a way where we want it to behave like a telephone usecase + * If we sent a notification event, we want the ui to show a ringing state + */ + waitForCallPickup?: boolean; + /** Optional factory to create LiveKit rooms, mainly for testing purposes. */ + livekitRoomFactory?: (options?: RoomOptions) => LivekitRoom; + /** Optional behavior overriding the local connection state, mainly for testing purposes. */ + connectionState$?: Behavior; +} + +// Do not play any sounds if the participant count has exceeded this +// number. +export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8; +export const THROTTLE_SOUND_EFFECT_MS = 500; + +// This is the number of participants that we think constitutes a "small" call +// on mobile. No spotlight tile should be shown below this threshold. +const smallMobileCallThreshold = 3; + +// How long the footer should be shown for when hovering over or interacting +// with the interface +const showFooterMs = 4000; + +export type GridMode = "grid" | "spotlight"; + +export type WindowMode = "normal" | "narrow" | "flat" | "pip"; + +interface LayoutScanState { + layout: Layout | null; + tiles: TileStore; +} + +type MediaItem = UserMedia | ScreenShare; +type AudioLivekitItem = { + livekitRoom: LivekitRoom; + participants: string[]; + url: string; +}; +/** + * A view model providing all the application logic needed to show the in-call + * UI (may eventually be expanded to cover the lobby and feedback screens in the + * future). + */ +// Throughout this class and related code we must distinguish between MatrixRTC +// state and LiveKit state. We use the common terminology of room "members", RTC +// "memberships", and LiveKit "participants". +export class CallViewModel { + private readonly userId = this.matrixRoom.client.getUserId()!; + private readonly deviceId = this.matrixRoom.client.getDeviceId()!; + + private readonly livekitKeyProvider = getE2eeKeyProvider( + this.options.encryptionSystem, + this.matrixRTCSession, + ); + + // Each hbar seperates a block of input variables required for the CallViewModel to function. + // The outputs of this block is written under the hbar. + // + // For mocking purposes it is recommended to only mock the functions creating those outputs. + // All other fields are just temp computations for the mentioned output. + // The class does not need anything except the values underneath the bar. + // The creation of the values under the bar are all tested independently and testing the callViewModel Should + // not test their cretation. Call view model only needs: + // - memberships$ via createMemberships$ + // - localMembership via createLocalMembership$ + // - callLifecycle via createCallNotificationLifecycle$ + // - matrixMemberMetadataStore via createMatrixMemberMetadata$ + + // ------------------------------------------------------------------------ + // memberships$ + private memberships$ = createMemberships$(this.scope, this.matrixRTCSession); + + // ------------------------------------------------------------------------ + // matrixLivekitMembers$ AND localMembership + + private membershipsAndTransports = membershipsAndTransports$( + this.scope, + this.memberships$, + ); + + private localTransport$ = createLocalTransport$({ + scope: this.scope, + memberships$: this.memberships$, + client: this.matrixRoom.client, + roomId: this.matrixRoom.roomId, + useOldestMember$: this.scope.behavior( + matrixRTCMode.value$.pipe(map((v) => v === MatrixRTCMode.Legacy)), + ), + }); + + private connectionFactory = new ECConnectionFactory( + this.matrixRoom.client, + this.mediaDevices, + this.trackProcessorState$, + this.livekitKeyProvider, + getUrlParams().controlledAudioDevices, + this.options.livekitRoomFactory, + ); + + private connectionManager = createConnectionManager$({ + scope: this.scope, + connectionFactory: this.connectionFactory, + inputTransports$: this.scope.behavior( + combineLatest( + [this.localTransport$, this.membershipsAndTransports.transports$], + (localTransport, transports) => { + const localTransportAsArray = localTransport ? [localTransport] : []; + return transports.mapInner((transports) => [ + ...localTransportAsArray, + ...transports, + ]); + }, + ), + ), + logger: logger, + }); + + private matrixLivekitMembers$ = createMatrixLivekitMembers$({ + scope: this.scope, + membershipsWithTransport$: + this.membershipsAndTransports.membershipsWithTransport$, + connectionManager: this.connectionManager, + }); + + private connectOptions$ = this.scope.behavior( + matrixRTCMode.value$.pipe( + map((mode) => ({ + encryptMedia: this.livekitKeyProvider !== undefined, + // TODO. This might need to get called again on each cahnge of matrixRTCMode... + matrixRTCMode: mode, + })), + ), + ); + + private localMembership = createLocalMembership$({ + scope: this.scope, + muteStates: this.muteStates, + mediaDevices: this.mediaDevices, + connectionManager: this.connectionManager, + matrixRTCSession: this.matrixRTCSession, + matrixRoom: this.matrixRoom, + localTransport$: this.localTransport$, + trackProcessorState$: this.trackProcessorState$, + widget, + options: this.connectOptions$, + logger: logger.getChild(`[${Date.now()}]`), + }); + + private localRtcMembership$ = this.scope.behavior( + this.memberships$.pipe( + map( + (memberships) => + memberships.value.find( + (membership) => + membership.userId === this.userId && + membership.deviceId === this.deviceId, + ) ?? null, + ), + ), + ); + + private localMatrixLivekitMemberUninitialized = { + membership$: this.localRtcMembership$, + participant$: this.localMembership.participant$, + connection$: this.localMembership.connection$, + userId: this.userId, + }; + + private localMatrixLivekitMember$: Behavior = + this.scope.behavior( + this.localRtcMembership$.pipe( + switchMap((membership) => { + if (!membership) return of(null); + return of( + // casting is save here since we know that localRtcMembership$ is !== null since we reached this case. + this.localMatrixLivekitMemberUninitialized as MatrixLivekitMember, + ); + }), + ), + ); + + // ------------------------------------------------------------------------ + // callLifecycle + + private callLifecycle = createCallNotificationLifecycle$({ + scope: this.scope, + memberships$: this.memberships$, + sentCallNotification$: createSentCallNotification$( + this.scope, + this.matrixRTCSession, + ), + receivedDecline$: createReceivedDecline$(this.matrixRoom), + options: this.options, + localUser: { userId: this.userId, deviceId: this.deviceId }, + }); + public autoLeave$ = this.callLifecycle.autoLeave$; + + // ------------------------------------------------------------------------ + // matrixMemberMetadataStore + + private matrixMemberMetadataStore = createMatrixMemberMetadata$( + this.scope, + this.scope.behavior(this.memberships$.pipe(map((mems) => mems.value))), + createRoomMembers$(this.scope, this.matrixRoom), + ); + + /** + * If there is a configuration error with the call (e.g. misconfigured E2EE). + * This is a fatal error that prevents the call from being created/joined. + * Should render a blocking error screen. + */ + public get configError$(): Behavior { + return this.localMembership.configError$; + } + + public join = this.localMembership.requestConnect; + + // CODESMELL? + // This is functionally the same Observable as leave$, except here it's + // hoisted to the top of the class. This enables the cyclic dependency between + // leave$ -> autoLeave$ -> callPickupState$ -> livekitConnectionState$ -> + // localConnection$ -> transports$ -> joined$ -> leave$. + private readonly leaveHoisted$ = new Subject< + "user" | "timeout" | "decline" | "allOthersLeft" + >(); + + /** + * Whether various media/event sources should pretend to be disconnected from + * all network input, even if their connection still technically works. + */ + // We do this when the app is in the 'reconnecting' state, because it might be + // that the LiveKit connection is still functional while the homeserver is + // down, for example, and we want to avoid making people worry that the app is + // in a split-brained state. + // DISCUSSION own membership manager ALSO this probably can be simplifis + public reconnecting$ = this.localMembership.reconnecting$; + private readonly pretendToBeDisconnected$ = this.reconnecting$; + + public readonly audioParticipants$ = this.scope.behavior( + this.matrixLivekitMembers$.pipe( + switchMap((membersWithEpoch) => { + const members = membersWithEpoch.value; + const a$ = combineLatest( + members.map((member) => + combineLatest([member.connection$, member.participant$]).pipe( + map(([connection, participant]) => { + // do not render audio for local participant + if (!connection || !participant || participant.isLocal) + return null; + const livekitRoom = connection.livekitRoom; + const url = connection.transport.livekit_service_url; + + return { url, livekitRoom, participant: participant.identity }; + }), + ), + ), + ); + return a$; + }), + map((members) => + members.reduce((acc, curr) => { + if (!curr) return acc; + + const existing = acc.find((item) => item.url === curr.url); + if (existing) { + existing.participants.push(curr.participant); + } else { + acc.push({ + livekitRoom: curr.livekitRoom, + participants: [curr.participant], + url: curr.url, + }); + } + return acc; + }, []), + ), + ), + [], + ); + + public readonly handsRaised$ = this.scope.behavior( + this.handsRaisedSubject$.pipe(pauseWhen(this.pretendToBeDisconnected$)), + ); + + public readonly reactions$ = this.scope.behavior( + this.reactionsSubject$.pipe( + map((v) => + Object.fromEntries( + Object.entries(v).map(([a, { reactionOption }]) => [ + a, + reactionOption, + ]), + ), + ), + pauseWhen(this.pretendToBeDisconnected$), + ), + ); + + /** + * List of user media (camera feeds) that we want tiles for. + */ + // TODO this also needs the local participant to be added. + private readonly userMedia$ = this.scope.behavior( + combineLatest([ + this.localMatrixLivekitMember$, + this.matrixLivekitMembers$, + duplicateTiles.value$, + ]).pipe( + // Generate a collection of MediaItems from the list of expected (whether + // present or missing) LiveKit participants. + generateItems( + function* ([ + localMatrixLivekitMember, + { value: matrixLivekitMembers }, + duplicateTiles, + ]) { + let localParticipantId = undefined; + // add local member if available + if (localMatrixLivekitMember) { + const { userId, participant$, connection$, membership$ } = + localMatrixLivekitMember; + localParticipantId = `${userId}:${membership$.value.deviceId}`; // should be membership$.value.membershipID which is not optional + // const participantId = membership$.value.membershipID; + if (localParticipantId) { + for (let dup = 0; dup < 1 + duplicateTiles; dup++) { + yield { + keys: [ + dup, + localParticipantId, + userId, + participant$, + connection$, + ], + data: undefined, + }; + } + } + } + // add remote members that are available + for (const { + userId, + participant$, + connection$, + membership$, + } of matrixLivekitMembers) { + const participantId = `${userId}:${membership$.value.deviceId}`; + if (participantId === localParticipantId) continue; + // const participantId = membership$.value?.identity; + for (let dup = 0; dup < 1 + duplicateTiles; dup++) { + yield { + keys: [dup, participantId, userId, participant$, connection$], + data: undefined, + }; + } + } + }, + ( + scope, + _data$, + dup, + participantId, + userId, + participant$, + connection$, + ) => { + const livekitRoom$ = scope.behavior( + connection$.pipe(map((c) => c?.livekitRoom)), + ); + const focusUrl$ = scope.behavior( + connection$.pipe(map((c) => c?.transport.livekit_service_url)), + ); + const displayName$ = scope.behavior( + this.matrixMemberMetadataStore + .createDisplayNameBehavior$(userId) + .pipe(map((name) => name ?? userId)), + ); + + return new UserMedia( + scope, + `${participantId}:${dup}`, + userId, + participant$, + this.options.encryptionSystem, + livekitRoom$, + focusUrl$, + this.mediaDevices, + this.pretendToBeDisconnected$, + displayName$, + this.matrixMemberMetadataStore.createAvatarUrlBehavior$(userId), + this.handsRaised$.pipe(map((v) => v[participantId]?.time ?? null)), + this.reactions$.pipe(map((v) => v[participantId] ?? undefined)), + ); + }, + ), + ), + ); + + /** + * List of all media items (user media and screen share media) that we want + * tiles for. + */ + private readonly mediaItems$ = this.scope.behavior( + this.userMedia$.pipe( + switchMap((userMedia) => + userMedia.length === 0 + ? of([]) + : combineLatest( + userMedia.map((m) => m.screenShares$), + (...screenShares) => [...userMedia, ...screenShares.flat(1)], + ), + ), + ), + ); + + /** + * List of MediaItems that we want to display, that are of type ScreenShare + */ + private readonly screenShares$ = this.scope.behavior( + this.mediaItems$.pipe( + map((mediaItems) => + mediaItems.filter((m): m is ScreenShare => m instanceof ScreenShare), + ), + ), + ); + + public readonly joinSoundEffect$ = this.userMedia$.pipe( + pairwise(), + filter( + ([prev, current]) => + current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && + current.length > prev.length, + ), + map(() => {}), + throttleTime(THROTTLE_SOUND_EFFECT_MS), + ); + + /** + * The number of participants currently in the call. + * + * - Each participant has a corresponding MatrixRTC membership state event + * - There can be multiple participants for one Matrix user if they join from + * multiple devices. + */ + public readonly participantCount$ = this.scope.behavior( + this.matrixLivekitMembers$.pipe(map((ms) => ms.value.length)), + ); + + // only public to expose to the view. + // TODO if we are in "unknown" state we need a loading rendering (or empty screen) + // Otherwise it looks like we already connected and only than the ringing starts which is weird. + public readonly callPickupState$ = this.callLifecycle.callPickupState$; + + public readonly leaveSoundEffect$ = combineLatest([ + this.callLifecycle.callPickupState$, + this.userMedia$, + ]).pipe( + // Until the call is successful, do not play a leave sound. + // If callPickupState$ is null, then we always play the sound as it will not conflict with a decline sound. + skipWhile(([c]) => c !== null && c !== "success"), + map(([, userMedia]) => userMedia), + pairwise(), + filter( + ([prev, current]) => + current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && + current.length < prev.length, + ), + map(() => {}), + throttleTime(THROTTLE_SOUND_EFFECT_MS), + ); + + private readonly userHangup$ = new Subject(); + public hangup(): void { + this.userHangup$.next(); + } + + private readonly widgetHangup$ = + widget === null + ? NEVER + : ( + fromEvent( + widget.lazyActions, + ElementWidgetActions.HangupCall, + ) as Observable> + ).pipe( + tap((ev) => { + widget!.api.transport.reply(ev.detail, {}); + }), + ); + + public readonly leave$: Observable< + "user" | "timeout" | "decline" | "allOthersLeft" + > = merge( + this.callLifecycle.autoLeave$, + merge(this.userHangup$, this.widgetHangup$).pipe( + map(() => "user" as const), + ), + ).pipe( + this.scope.share, + tap((reason) => this.leaveHoisted$.next(reason)), + ); + + private readonly spotlightSpeaker$ = + this.scope.behavior( + this.userMedia$.pipe( + switchMap((mediaItems) => + mediaItems.length === 0 + ? of([]) + : combineLatest( + mediaItems.map((m) => + m.vm.speaking$.pipe(map((s) => [m, s] as const)), + ), + ), + ), + scan<(readonly [UserMedia, boolean])[], UserMedia | undefined, null>( + (prev, mediaItems) => { + // Only remote users that are still in the call should be sticky + const [stickyMedia, stickySpeaking] = + (!prev?.vm.local && mediaItems.find(([m]) => m === prev)) || []; + // Decide who to spotlight: + // If the previous speaker is still speaking, stick with them rather + // than switching eagerly to someone else + return stickySpeaking + ? stickyMedia! + : // Otherwise, select any remote user who is speaking + (mediaItems.find(([m, s]) => !m.vm.local && s)?.[0] ?? + // Otherwise, stick with the person who was last speaking + stickyMedia ?? + // Otherwise, spotlight an arbitrary remote user + mediaItems.find(([m]) => !m.vm.local)?.[0] ?? + // Otherwise, spotlight the local user + mediaItems.find(([m]) => m.vm.local)?.[0]); + }, + null, + ), + map((speaker) => speaker?.vm ?? null), + ), + ); + + private readonly grid$ = this.scope.behavior( + this.userMedia$.pipe( + switchMap((mediaItems) => { + const bins = mediaItems.map((m) => + m.bin$.pipe(map((bin) => [m, bin] as const)), + ); + // Sort the media by bin order and generate a tile for each one + return bins.length === 0 + ? of([]) + : combineLatest(bins, (...bins) => + bins.sort(([, bin1], [, bin2]) => bin1 - bin2).map(([m]) => m.vm), + ); + }), + distinctUntilChanged(shallowEquals), + ), + ); + + private readonly spotlight$ = this.scope.behavior( + this.screenShares$.pipe( + switchMap((screenShares) => { + if (screenShares.length > 0) { + return of(screenShares.map((m) => m.vm)); + } + + return this.spotlightSpeaker$.pipe( + map((speaker) => (speaker ? [speaker] : [])), + ); + }), + distinctUntilChanged(shallowEquals), + ), + ); + + private readonly pip$ = this.scope.behavior( + combineLatest([ + // TODO This also needs epoch logic to dedupe the screenshares and mediaItems emits + this.screenShares$, + this.spotlightSpeaker$, + this.mediaItems$, + ]).pipe( + switchMap(([screenShares, spotlight, mediaItems]) => { + if (screenShares.length > 0) { + return this.spotlightSpeaker$; + } + if (!spotlight || spotlight.local) { + return of(null); + } + + const localUserMedia = mediaItems.find( + (m) => m.vm instanceof LocalUserMediaViewModel, + ) as UserMedia | undefined; + + const localUserMediaViewModel = localUserMedia?.vm as + | LocalUserMediaViewModel + | undefined; + + if (!localUserMediaViewModel) { + return of(null); + } + return localUserMediaViewModel.alwaysShow$.pipe( + map((alwaysShow) => { + if (alwaysShow) { + return localUserMediaViewModel; + } + + return null; + }), + ); + }), + ), + ); + + private readonly hasRemoteScreenShares$: Observable = + this.spotlight$.pipe( + map((spotlight) => + spotlight.some((vm) => !vm.local && vm instanceof ScreenShareViewModel), + ), + distinctUntilChanged(), + ); + + private readonly pipEnabled$ = this.scope.behavior(setPipEnabled$, false); + + private readonly naturalWindowMode$ = this.scope.behavior( + fromEvent(window, "resize").pipe( + map(() => { + const height = window.innerHeight; + const width = window.innerWidth; + if (height <= 400 && width <= 340) return "pip"; + // Our layouts for flat windows are better at adapting to a small width + // than our layouts for narrow windows are at adapting to a small height, + // so we give "flat" precedence here + if (height <= 600) return "flat"; + if (width <= 600) return "narrow"; + return "normal"; + }), + ), + "normal", + ); + + /** + * The general shape of the window. + */ + public readonly windowMode$ = this.scope.behavior( + this.pipEnabled$.pipe( + switchMap((pip) => + pip ? of("pip") : this.naturalWindowMode$, + ), + ), + ); + + private readonly spotlightExpandedToggle$ = new Subject(); + public readonly spotlightExpanded$ = this.scope.behavior( + this.spotlightExpandedToggle$.pipe( + accumulate(false, (expanded) => !expanded), + ), + ); + + private readonly gridModeUserSelection$ = new Subject(); + /** + * The layout mode of the media tile grid. + */ + public readonly gridMode$ = + // If the user hasn't selected spotlight and somebody starts screen sharing, + // automatically switch to spotlight mode and reset when screen sharing ends + this.scope.behavior( + this.gridModeUserSelection$.pipe( + switchMap((userSelection) => + (userSelection === "spotlight" + ? EMPTY + : combineLatest([ + this.hasRemoteScreenShares$, + this.windowMode$, + ]).pipe( + skip(userSelection === null ? 0 : 1), + map( + ([hasScreenShares, windowMode]): GridMode => + hasScreenShares || windowMode === "flat" + ? "spotlight" + : "grid", + ), + ) + ).pipe(startWith(userSelection ?? "grid")), + ), + ), + "grid", + ); + + public setGridMode(value: GridMode): void { + this.gridModeUserSelection$.next(value); + } + + private readonly gridLayoutMedia$: Observable = + combineLatest([this.grid$, this.spotlight$], (grid, spotlight) => ({ + type: "grid", + spotlight: spotlight.some((vm) => vm instanceof ScreenShareViewModel) + ? spotlight + : undefined, + grid, + })); + + private readonly spotlightLandscapeLayoutMedia$: Observable = + combineLatest([this.grid$, this.spotlight$], (grid, spotlight) => ({ + type: "spotlight-landscape", + spotlight, + grid, + })); + + private readonly spotlightPortraitLayoutMedia$: Observable = + combineLatest([this.grid$, this.spotlight$], (grid, spotlight) => ({ + type: "spotlight-portrait", + spotlight, + grid, + })); + + private readonly spotlightExpandedLayoutMedia$: Observable = + combineLatest([this.spotlight$, this.pip$], (spotlight, pip) => ({ + type: "spotlight-expanded", + spotlight, + pip: pip ?? undefined, + })); + + private readonly oneOnOneLayoutMedia$: Observable = + this.mediaItems$.pipe( + map((mediaItems) => { + if (mediaItems.length !== 2) return null; + const local = mediaItems.find((vm) => vm.vm.local)?.vm as + | LocalUserMediaViewModel + | undefined; + const remote = mediaItems.find((vm) => !vm.vm.local)?.vm as + | RemoteUserMediaViewModel + | undefined; + // There might not be a remote tile if there are screen shares, or if + // only the local user is in the call and they're using the duplicate + // tiles option + if (!remote || !local) return null; + + return { type: "one-on-one", local, remote }; + }), + ); + + private readonly pipLayoutMedia$: Observable = + this.spotlight$.pipe(map((spotlight) => ({ type: "pip", spotlight }))); + + /** + * The media to be used to produce a layout. + */ + private readonly layoutMedia$ = this.scope.behavior( + this.windowMode$.pipe( + switchMap((windowMode) => { + switch (windowMode) { + case "normal": + return this.gridMode$.pipe( + switchMap((gridMode) => { + switch (gridMode) { + case "grid": + return this.oneOnOneLayoutMedia$.pipe( + switchMap((oneOnOne) => + oneOnOne === null + ? this.gridLayoutMedia$ + : of(oneOnOne), + ), + ); + case "spotlight": + return this.spotlightExpanded$.pipe( + switchMap((expanded) => + expanded + ? this.spotlightExpandedLayoutMedia$ + : this.spotlightLandscapeLayoutMedia$, + ), + ); + } + }), + ); + case "narrow": + return this.oneOnOneLayoutMedia$.pipe( + switchMap((oneOnOne) => + oneOnOne === null + ? combineLatest( + [this.grid$, this.spotlight$], + (grid, spotlight) => + grid.length > smallMobileCallThreshold || + spotlight.some( + (vm) => vm instanceof ScreenShareViewModel, + ) + ? this.spotlightPortraitLayoutMedia$ + : this.gridLayoutMedia$, + ).pipe(switchAll()) + : // The expanded spotlight layout makes for a better one-on-one + // experience in narrow windows + this.spotlightExpandedLayoutMedia$, + ), + ); + case "flat": + return this.gridMode$.pipe( + switchMap((gridMode) => { + switch (gridMode) { + case "grid": + // Yes, grid mode actually gets you a "spotlight" layout in + // this window mode. + return this.spotlightLandscapeLayoutMedia$; + case "spotlight": + return this.spotlightExpandedLayoutMedia$; + } + }), + ); + case "pip": + return this.pipLayoutMedia$; + } + }), + ), + ); + + // There is a cyclical dependency here: the layout algorithms want to know + // which tiles are on screen, but to know which tiles are on screen we have to + // first render a layout. To deal with this we assume initially that no tiles + // are visible, and loop the data back into the layouts with a Subject. + private readonly visibleTiles$ = new Subject(); + private readonly setVisibleTiles = (value: number): void => + this.visibleTiles$.next(value); + + private readonly layoutInternals$ = this.scope.behavior< + LayoutScanState & { layout: Layout } + >( + combineLatest([ + this.layoutMedia$, + this.visibleTiles$.pipe(startWith(0), distinctUntilChanged()), + ]).pipe( + scan< + [LayoutMedia, number], + LayoutScanState & { layout: Layout }, + LayoutScanState + >( + ({ tiles: prevTiles }, [media, visibleTiles]) => { + let layout: Layout; + let newTiles: TileStore; + switch (media.type) { + case "grid": + case "spotlight-landscape": + case "spotlight-portrait": + [layout, newTiles] = gridLikeLayout( + media, + visibleTiles, + this.setVisibleTiles, + prevTiles, + ); + break; + case "spotlight-expanded": + [layout, newTiles] = spotlightExpandedLayout(media, prevTiles); + break; + case "one-on-one": + [layout, newTiles] = oneOnOneLayout(media, prevTiles); + break; + case "pip": + [layout, newTiles] = pipLayout(media, prevTiles); + break; + } + + return { layout, tiles: newTiles }; + }, + { layout: null, tiles: TileStore.empty() }, + ), + ), + ); + + /** + * The layout of tiles in the call interface. + */ + public readonly layout$ = this.scope.behavior( + this.layoutInternals$.pipe(map(({ layout }) => layout)), + ); + + /** + * The current generation of the tile store, exposed for debugging purposes. + */ + public readonly tileStoreGeneration$ = this.scope.behavior( + this.layoutInternals$.pipe(map(({ tiles }) => tiles.generation)), + ); + + public showSpotlightIndicators$ = this.scope.behavior( + this.layout$.pipe(map((l) => l.type !== "grid")), + ); + + public showSpeakingIndicators$ = this.scope.behavior( + this.layout$.pipe( + switchMap((l) => { + switch (l.type) { + case "spotlight-landscape": + case "spotlight-portrait": + // If the spotlight is showing the active speaker, we can do without + // speaking indicators as they're a redundant visual cue. But if + // screen sharing feeds are in the spotlight we still need them. + return l.spotlight.media$.pipe( + map((models: MediaViewModel[]) => + models.some((m) => m instanceof ScreenShareViewModel), + ), + ); + // In expanded spotlight layout, the active speaker is always shown in + // the picture-in-picture tile so there is no need for speaking + // indicators. And in one-on-one layout there's no question as to who is + // speaking. + case "spotlight-expanded": + case "one-on-one": + return of(false); + default: + return of(true); + } + }), + ), + ); + + public readonly toggleSpotlightExpanded$ = this.scope.behavior< + (() => void) | null + >( + this.windowMode$.pipe( + switchMap((mode) => + mode === "normal" + ? this.layout$.pipe( + map( + (l) => + l.type === "spotlight-landscape" || + l.type === "spotlight-expanded", + ), + ) + : of(false), + ), + distinctUntilChanged(), + map((enabled) => + enabled ? (): void => this.spotlightExpandedToggle$.next() : null, + ), + ), + ); + + private readonly screenTap$ = new Subject(); + private readonly controlsTap$ = new Subject(); + private readonly screenHover$ = new Subject(); + private readonly screenUnhover$ = new Subject(); + + /** + * Callback for when the user taps the call view. + */ + public tapScreen(): void { + this.screenTap$.next(); + } + + /** + * Callback for when the user taps the call's controls. + */ + public tapControls(): void { + this.controlsTap$.next(); + } + + /** + * Callback for when the user hovers over the call view. + */ + public hoverScreen(): void { + this.screenHover$.next(); + } + + /** + * Callback for when the user stops hovering over the call view. + */ + public unhoverScreen(): void { + this.screenUnhover$.next(); + } + + public readonly showHeader$ = this.scope.behavior( + this.windowMode$.pipe(map((mode) => mode !== "pip" && mode !== "flat")), + ); + + public readonly showFooter$ = this.scope.behavior( + this.windowMode$.pipe( + switchMap((mode) => { + switch (mode) { + case "pip": + return of(false); + case "normal": + case "narrow": + return of(true); + case "flat": + // Sadly Firefox has some layering glitches that prevent the footer + // from appearing properly. They happen less often if we never hide + // the footer. + if (isFirefox()) return of(true); + // Show/hide the footer in response to interactions + return merge( + this.screenTap$.pipe(map(() => "tap screen" as const)), + this.controlsTap$.pipe(map(() => "tap controls" as const)), + this.screenHover$.pipe(map(() => "hover" as const)), + ).pipe( + switchScan((state, interaction) => { + switch (interaction) { + case "tap screen": + return state + ? // Toggle visibility on tap + of(false) + : // Hide after a timeout + timer(showFooterMs).pipe( + map(() => false), + startWith(true), + ); + case "tap controls": + // The user is interacting with things, so reset the timeout + return timer(showFooterMs).pipe( + map(() => false), + startWith(true), + ); + case "hover": + // Show on hover and hide after a timeout + return race( + timer(showFooterMs), + this.screenUnhover$.pipe(take(1)), + ).pipe( + map(() => false), + startWith(true), + ); + } + }, false), + startWith(false), + ); + } + }), + ), + ); + + /** + * Whether audio is currently being output through the earpiece. + */ + public readonly earpieceMode$ = this.scope.behavior( + combineLatest( + [ + this.mediaDevices.audioOutput.available$, + this.mediaDevices.audioOutput.selected$, + ], + (available, selected) => + selected !== undefined && + available.get(selected.id)?.type === "earpiece", + ), + ); + + /** + * Callback to toggle between the earpiece and the loudspeaker. + * + * This will be `null` in case the target does not exist in the list + * of available audio outputs. + */ + public readonly audioOutputSwitcher$ = this.scope.behavior<{ + targetOutput: "earpiece" | "speaker"; + switch: () => void; + } | null>( + combineLatest( + [ + this.mediaDevices.audioOutput.available$, + this.mediaDevices.audioOutput.selected$, + ], + (available, selected) => { + const selectionType = selected && available.get(selected.id)?.type; + + // If we are in any output mode other than speaker switch to speaker. + const newSelectionType: "earpiece" | "speaker" = + selectionType === "speaker" ? "earpiece" : "speaker"; + const newSelection = [...available].find( + ([, d]) => d.type === newSelectionType, + ); + if (newSelection === undefined) return null; + + const [id] = newSelection; + return { + targetOutput: newSelectionType, + switch: (): void => this.mediaDevices.audioOutput.select(id), + }; + }, + ), + ); + + /** + * Emits an array of reactions that should be visible on the screen. + */ + // DISCUSSION move this into a reaction file + // const {visibleReactions$, audibleReactions$} = reactionsObservables$(showReactionSetting$, ) + public readonly visibleReactions$ = this.scope.behavior( + showReactions.value$.pipe( + switchMap((show) => (show ? this.reactions$ : of({}))), + scan< + Record, + { sender: string; emoji: string; startX: number }[] + >((acc, latest) => { + const newSet: { sender: string; emoji: string; startX: number }[] = []; + for (const [sender, reaction] of Object.entries(latest)) { + const startX = + acc.find((v) => v.sender === sender && v.emoji)?.startX ?? + Math.ceil(Math.random() * 80) + 10; + newSet.push({ sender, emoji: reaction.emoji, startX }); + } + return newSet; + }, []), + ), + ); + + /** + * Emits an array of reactions that should be played. + */ + public readonly audibleReactions$ = playReactionsSound.value$.pipe( + switchMap((show) => + show ? this.reactions$ : of>({}), + ), + map((reactions) => Object.values(reactions).map((v) => v.name)), + scan( + (acc, latest) => { + return { + playing: latest.filter( + (v) => acc.playing.includes(v) || acc.newSounds.includes(v), + ), + newSounds: latest.filter( + (v) => !acc.playing.includes(v) && !acc.newSounds.includes(v), + ), + }; + }, + { playing: [], newSounds: [] }, + ), + map((v) => v.newSounds), + ); + + /** + * Emits an event every time a new hand is raised in + * the call. + */ + public readonly newHandRaised$ = this.handsRaised$.pipe( + map((v) => Object.keys(v).length), + scan( + (acc, newValue) => ({ + value: newValue, + playSounds: newValue > acc.value, + }), + { value: 0, playSounds: false }, + ), + filter((v) => v.playSounds), + ); + + /** + * Emits an event every time a new screenshare is started in + * the call. + */ + public readonly newScreenShare$ = this.screenShares$.pipe( + map((v) => v.length), + scan( + (acc, newValue) => ({ + value: newValue, + playSounds: newValue > acc.value, + }), + { value: 0, playSounds: false }, + ), + filter((v) => v.playSounds), + ); + + /** + * Whether we are sharing our screen. + */ + // reassigned here to make it publicly accessible + public readonly sharingScreen$ = this.localMembership.sharingScreen$; + + /** + * Callback to toggle screen sharing. If null, screen sharing is not possible. + */ + // reassigned here to make it publicly accessible + public readonly toggleScreenSharing = + this.localMembership.toggleScreenSharing; + + public constructor( + private readonly scope: ObservableScope, + // A call is permanently tied to a single Matrix room + private readonly matrixRTCSession: MatrixRTCSession, + private readonly matrixRoom: MatrixRoom, + private readonly mediaDevices: MediaDevices, + private readonly muteStates: MuteStates, + private readonly options: CallViewModelOptions, + private readonly handsRaisedSubject$: Observable< + Record + >, + private readonly reactionsSubject$: Observable< + Record + >, + private readonly trackProcessorState$: Behavior, + ) { + // Join automatically + this.join(); // TODO-MULTI-SFU: Use this view model for the lobby as well, and only call this once 'join' is clicked? + } +} + +// TODO-MULTI-SFU // Setup and update the keyProvider which was create by `createRoom` was a thing before. Now we never update if the E2EEsystem changes +// do we need this? + +function getE2eeKeyProvider( + e2eeSystem: EncryptionSystem, + rtcSession: MatrixRTCSession, +): BaseKeyProvider | undefined { + if (e2eeSystem.kind === E2eeType.NONE) return undefined; + + if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) { + const keyProvider = new MatrixKeyProvider(); + keyProvider.setRTCSession(rtcSession); + return keyProvider; + } else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) { + const keyProvider = new ExternalE2EEKeyProvider(); + keyProvider + .setKey(e2eeSystem.secret) + .catch((e) => logger.error("Failed to set shared key for E2EE", e)); + return keyProvider; + } +} From 7ab90d3c1f152a890d7ce73fc22005fbc2eeec81 Mon Sep 17 00:00:00 2001 From: Timo K Date: Mon, 17 Nov 2025 12:00:28 +0100 Subject: [PATCH 2/5] Delete unused files --- .../CallViewModel/CallViewModelFunction.ts | 1400 ----------------- src/state/CallViewModel/CallViewModelOld.ts | 1335 ---------------- 2 files changed, 2735 deletions(-) delete mode 100644 src/state/CallViewModel/CallViewModelFunction.ts delete mode 100644 src/state/CallViewModel/CallViewModelOld.ts diff --git a/src/state/CallViewModel/CallViewModelFunction.ts b/src/state/CallViewModel/CallViewModelFunction.ts deleted file mode 100644 index 20e37395..00000000 --- a/src/state/CallViewModel/CallViewModelFunction.ts +++ /dev/null @@ -1,1400 +0,0 @@ -/* -Copyright 2023, 2024, 2025 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - type BaseKeyProvider, - type ConnectionState, - ExternalE2EEKeyProvider, - type Room as LivekitRoom, - type RoomOptions, -} from "livekit-client"; -import { type RoomMember, type Room as MatrixRoom } from "matrix-js-sdk"; -import { - combineLatest, - distinctUntilChanged, - EMPTY, - filter, - fromEvent, - map, - merge, - NEVER, - type Observable, - of, - pairwise, - race, - scan, - skip, - skipWhile, - startWith, - Subject, - switchAll, - switchMap, - switchScan, - take, - tap, - throttleTime, - timer, -} from "rxjs"; -import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; -import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; -import { type IWidgetApiRequest } from "matrix-widget-api"; - -import { - LocalUserMediaViewModel, - type MediaViewModel, - type RemoteUserMediaViewModel, - ScreenShareViewModel, - type UserMediaViewModel, -} from "../MediaViewModel"; -import { accumulate, generateItems, pauseWhen } from "../../utils/observable"; -import { - duplicateTiles, - MatrixRTCMode, - matrixRTCMode, - playReactionsSound, - showReactions, -} from "../../settings/settings"; -import { isFirefox } from "../../Platform"; -import { setPipEnabled$ } from "../../controls"; -import { TileStore } from "../TileStore"; -import { gridLikeLayout } from "../GridLikeLayout"; -import { spotlightExpandedLayout } from "../SpotlightExpandedLayout"; -import { oneOnOneLayout } from "../OneOnOneLayout"; -import { pipLayout } from "../PipLayout"; -import { type EncryptionSystem } from "../../e2ee/sharedKeyManagement"; -import { - type RaisedHandInfo, - type ReactionInfo, - type ReactionOption, -} from "../../reactions"; -import { shallowEquals } from "../../utils/array"; -import { type MediaDevices } from "../MediaDevices"; -import { type Behavior } from "../Behavior"; -import { E2eeType } from "../../e2ee/e2eeType"; -import { MatrixKeyProvider } from "../../e2ee/matrixKeyProvider"; -import { type MuteStates } from "../MuteStates"; -import { getUrlParams } from "../../UrlParams"; -import { type ProcessorState } from "../../livekit/TrackProcessorContext"; -import { ElementWidgetActions, widget } from "../../widget"; -import { UserMedia } from "../UserMedia.ts"; -import { ScreenShare } from "../ScreenShare.ts"; -import { - type GridLayoutMedia, - type Layout, - type LayoutMedia, - type OneOnOneLayoutMedia, - type SpotlightExpandedLayoutMedia, - type SpotlightLandscapeLayoutMedia, - type SpotlightPortraitLayoutMedia, -} from "../layout-types.ts"; -import { type ElementCallError } from "../../utils/errors.ts"; -import { type ObservableScope } from "../ObservableScope.ts"; -import { createLocalMembership$ } from "./localMember/LocalMembership.ts"; -import { createLocalTransport$ } from "./localMember/LocalTransport.ts"; -import { - createMemberships$, - membershipsAndTransports$, -} from "../SessionBehaviors.ts"; -import { ECConnectionFactory } from "./remoteMembers/ConnectionFactory.ts"; -import { createConnectionManager$ } from "./remoteMembers/ConnectionManager.ts"; -import { - createMatrixLivekitMembers$, - type MatrixLivekitMember, -} from "./remoteMembers/MatrixLivekitMembers.ts"; -import { - type AutoLeaveReason, - createCallNotificationLifecycle$, - createReceivedDecline$, - createSentCallNotification$, -} from "./CallNotificationLifecycle.ts"; -import { - createMatrixMemberMetadata$, - createRoomMembers$, -} from "./remoteMembers/MatrixMemberMetadata.ts"; - -const logger = rootLogger.getChild("[CallViewModel]"); -//TODO -// Larger rename -// member,membership -> rtcMember -// participant -> livekitParticipant -// matrixLivekitItem -> callMember -// js-sdk -// callMembership -> rtcMembership -export interface CallViewModelOptions { - encryptionSystem: EncryptionSystem; - autoLeaveWhenOthersLeft?: boolean; - /** - * If the call is started in a way where we want it to behave like a telephone usecase - * If we sent a notification event, we want the ui to show a ringing state - */ - waitForCallPickup?: boolean; - /** Optional factory to create LiveKit rooms, mainly for testing purposes. */ - livekitRoomFactory?: (options?: RoomOptions) => LivekitRoom; - /** Optional behavior overriding the local connection state, mainly for testing purposes. */ - connectionState$?: Behavior; -} - -// Do not play any sounds if the participant count has exceeded this -// number. -export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8; -export const THROTTLE_SOUND_EFFECT_MS = 500; - -// This is the number of participants that we think constitutes a "small" call -// on mobile. No spotlight tile should be shown below this threshold. -const smallMobileCallThreshold = 3; - -// How long the footer should be shown for when hovering over or interacting -// with the interface -const showFooterMs = 4000; - -export type GridMode = "grid" | "spotlight"; - -export type WindowMode = "normal" | "narrow" | "flat" | "pip"; - -interface LayoutScanState { - layout: Layout | null; - tiles: TileStore; -} - -type MediaItem = UserMedia | ScreenShare; -type AudioLivekitItem = { - livekitRoom: LivekitRoom; - participants: string[]; - url: string; -}; - -type JoinReturn = ReturnType< - ReturnType["requestConnect"] ->; - -/** - * A view model providing all the application logic needed to show the in-call - * UI (may eventually be expanded to cover the lobby and feedback screens in the - * future). - */ -// Throughout this class and related code we must distinguish between MatrixRTC -// state and LiveKit state. We use the common terminology of room "members", RTC -// "memberships", and LiveKit "participants". -export function createCallViewModel$( - scope: ObservableScope, - // A call is permanently tied to a single Matrix room - matrixRTCSession: MatrixRTCSession, - matrixRoom: MatrixRoom, - mediaDevices: MediaDevices, - muteStates: MuteStates, - options: CallViewModelOptions, - handsRaisedSubject$: Observable>, - reactionsSubject$: Observable>, - trackProcessorState$: Behavior, -): { - // lifecycle - autoLeave$: Observable; - callPickupState$: Behavior< - "unknown" | "ringing" | "timeout" | "decline" | "success" | null - >; - leave$: Observable<"user" | AutoLeaveReason>; - hangup: () => void; - - // joining - join: () => JoinReturn; - - // screen sharing - toggleScreenSharing: (() => void) | null; - sharingScreen$: Behavior; - - // UI interactions - tapScreen: () => void; - tapControls: () => void; - hoverScreen: () => void; - unhoverScreen: () => void; - - // errors - configError$: Behavior; - - // participants and counts - participantCount$: Behavior; - audioParticipants$: Behavior; - handsRaised$: Behavior>; - reactions$: Behavior>; - isOneOnOneWith$: Behavior | null>; - localUserIsAlone$: Behavior; - // sounds and events - joinSoundEffect$: Observable; - leaveSoundEffect$: Observable; - newHandRaised$: Observable<{ value: number; playSounds: boolean }>; - newScreenShare$: Observable<{ value: number; playSounds: boolean }>; - audibleReactions$: Observable; - visibleReactions$: Behavior< - { sender: string; emoji: string; startX: number }[] - >; - - // window/layout - windowMode$: Behavior; - spotlightExpanded$: Behavior; - toggleSpotlightExpanded$: Behavior<(() => void) | null>; - gridMode$: Behavior; - setGridMode: (value: GridMode) => void; - - // media view models and layout - grid$: Behavior; - spotlight$: Behavior; - pip$: Behavior; - layout$: Behavior; - tileStoreGeneration$: Behavior; - showSpotlightIndicators$: Behavior; - showSpeakingIndicators$: Behavior; - - // header/footer visibility - showHeader$: Behavior; - showFooter$: Behavior; - - // audio routing - earpieceMode$: Behavior; - audioOutputSwitcher$: Behavior<{ - targetOutput: "earpiece" | "speaker"; - switch: () => void; - } | null>; - - // connection state - reconnecting$: Behavior; -} { - const userId = matrixRoom.client.getUserId()!; - const deviceId = matrixRoom.client.getDeviceId()!; - - const livekitKeyProvider = getE2eeKeyProvider( - options.encryptionSystem, - matrixRTCSession, - ); - - // Each hbar seperates a block of input variables required for the CallViewModel to function. - // The outputs of this block is written under the hbar. - // - // For mocking purposes it is recommended to only mock the functions creating those outputs. - // All other fields are just temp computations for the mentioned output. - // The class does not need anything except the values underneath the bar. - // The creation of the values under the bar are all tested independently and testing the callViewModel Should - // not test their cretation. Call view model only needs: - // - memberships$ via createMemberships$ - // - localMembership via createLocalMembership$ - // - callLifecycle via createCallNotificationLifecycle$ - // - matrixMemberMetadataStore via createMatrixMemberMetadata$ - - // ------------------------------------------------------------------------ - // memberships$ - const memberships$ = createMemberships$(scope, matrixRTCSession); - - // ------------------------------------------------------------------------ - // matrixLivekitMembers$ AND localMembership - - const membershipsAndTransports = membershipsAndTransports$( - scope, - memberships$, - ); - - const localTransport$ = createLocalTransport$({ - scope: scope, - memberships$: memberships$, - client: matrixRoom.client, - roomId: matrixRoom.roomId, - useOldestMember$: scope.behavior( - matrixRTCMode.value$.pipe(map((v) => v === MatrixRTCMode.Legacy)), - ), - }); - - const connectionFactory = new ECConnectionFactory( - matrixRoom.client, - mediaDevices, - trackProcessorState$, - livekitKeyProvider, - getUrlParams().controlledAudioDevices, - options.livekitRoomFactory, - ); - - const connectionManager = createConnectionManager$({ - scope: scope, - connectionFactory: connectionFactory, - inputTransports$: scope.behavior( - combineLatest( - [localTransport$, membershipsAndTransports.transports$], - (localTransport, transports) => { - const localTransportAsArray = localTransport ? [localTransport] : []; - return transports.mapInner((transports) => [ - ...localTransportAsArray, - ...transports, - ]); - }, - ), - ), - logger: logger, - }); - - const matrixLivekitMembers$ = createMatrixLivekitMembers$({ - scope: scope, - membershipsWithTransport$: - membershipsAndTransports.membershipsWithTransport$, - connectionManager: connectionManager, - }); - - const connectOptions$ = scope.behavior( - matrixRTCMode.value$.pipe( - map((mode) => ({ - encryptMedia: livekitKeyProvider !== undefined, - // TODO. This might need to get called again on each cahnge of matrixRTCMode... - matrixRTCMode: mode, - })), - ), - ); - - const localMembership = createLocalMembership$({ - scope: scope, - muteStates: muteStates, - mediaDevices: mediaDevices, - connectionManager: connectionManager, - matrixRTCSession: matrixRTCSession, - matrixRoom: matrixRoom, - localTransport$: localTransport$, - trackProcessorState$: trackProcessorState$, - widget, - options: connectOptions$, - logger: logger.getChild(`[${Date.now()}]`), - }); - - const localRtcMembership$ = scope.behavior( - memberships$.pipe( - map( - (memberships) => - memberships.value.find( - (membership) => - membership.userId === userId && membership.deviceId === deviceId, - ) ?? null, - ), - ), - ); - - const localMatrixLivekitMemberUninitialized = { - membership$: localRtcMembership$, - participant$: localMembership.participant$, - connection$: localMembership.connection$, - userId: userId, - }; - - const localMatrixLivekitMember$: Behavior = - scope.behavior( - localRtcMembership$.pipe( - switchMap((membership) => { - if (!membership) return of(null); - return of( - // casting is save here since we know that localRtcMembership$ is !== null since we reached this case. - localMatrixLivekitMemberUninitialized as MatrixLivekitMember, - ); - }), - ), - ); - - // ------------------------------------------------------------------------ - // callLifecycle - - const callLifecycle = createCallNotificationLifecycle$({ - scope: scope, - memberships$: memberships$, - sentCallNotification$: createSentCallNotification$(scope, matrixRTCSession), - receivedDecline$: createReceivedDecline$(matrixRoom), - options: options, - localUser: { userId: userId, deviceId: deviceId }, - }); - - // ------------------------------------------------------------------------ - // matrixMemberMetadataStore - - const matrixRoomMembers$ = createRoomMembers$(scope, matrixRoom); - const matrixMemberMetadataStore = createMatrixMemberMetadata$( - scope, - scope.behavior(memberships$.pipe(map((mems) => mems.value))), - matrixRoomMembers$, - ); - - /** - * Returns the Member {userId, getMxcAvatarUrl, rawDisplayName} of the other user in the call, if it's a one-on-one call. - */ - const isOneOnOneWith$ = scope.behavior( - matrixRoomMembers$.pipe( - map((roomMembersMap) => { - const otherMembers = Array.from(roomMembersMap.values()).filter( - (member) => member.userId !== userId, - ); - return otherMembers.length === 1 ? otherMembers[0] : null; - }), - ), - ); - - const localUserIsAlone$ = scope.behavior( - matrixRoomMembers$.pipe( - map( - (roomMembersMap) => - roomMembersMap.size === 1 && roomMembersMap.get(userId) !== undefined, - ), - ), - ); - - // CODESMELL? - // This is functionally the same Observable as leave$, except here it's - // hoisted to the top of the class. This enables the cyclic dependency between - // leave$ -> autoLeave$ -> callPickupState$ -> livekitConnectionState$ -> - // localConnection$ -> transports$ -> joined$ -> leave$. - const leaveHoisted$ = new Subject< - "user" | "timeout" | "decline" | "allOthersLeft" - >(); - - /** - * Whether various media/event sources should pretend to be disconnected from - * all network input, even if their connection still technically works. - */ - // We do this when the app is in the 'reconnecting' state, because it might be - // that the LiveKit connection is still functional while the homeserver is - // down, for example, and we want to avoid making people worry that the app is - // in a split-brained state. - // DISCUSSION own membership manager ALSO this probably can be simplifis - /*PUBLIC*/ const reconnecting$ = localMembership.reconnecting$; - const pretendToBeDisconnected$ = reconnecting$; - - /*PUBLIC*/ const audioParticipants$ = scope.behavior( - matrixLivekitMembers$.pipe( - switchMap((membersWithEpoch) => { - const members = membersWithEpoch.value; - const a$ = combineLatest( - members.map((member) => - combineLatest([member.connection$, member.participant$]).pipe( - map(([connection, participant]) => { - // do not render audio for local participant - if (!connection || !participant || participant.isLocal) - return null; - const livekitRoom = connection.livekitRoom; - const url = connection.transport.livekit_service_url; - - return { url, livekitRoom, participant: participant.identity }; - }), - ), - ), - ); - return a$; - }), - map((members) => - members.reduce((acc, curr) => { - if (!curr) return acc; - - const existing = acc.find((item) => item.url === curr.url); - if (existing) { - existing.participants.push(curr.participant); - } else { - acc.push({ - livekitRoom: curr.livekitRoom, - participants: [curr.participant], - url: curr.url, - }); - } - return acc; - }, []), - ), - ), - [], - ); - - /*PUBLIC*/ const handsRaised$ = scope.behavior( - handsRaisedSubject$.pipe(pauseWhen(pretendToBeDisconnected$)), - ); - - /*PUBLIC*/ const reactions$ = scope.behavior( - reactionsSubject$.pipe( - map((v) => - Object.fromEntries( - Object.entries(v).map(([a, { reactionOption }]) => [ - a, - reactionOption, - ]), - ), - ), - pauseWhen(pretendToBeDisconnected$), - ), - ); - - /** - * List of user media (camera feeds) that we want tiles for. - */ - // TODO this also needs the local participant to be added. - const userMedia$ = scope.behavior( - combineLatest([ - localMatrixLivekitMember$, - matrixLivekitMembers$, - duplicateTiles.value$, - ]).pipe( - // Generate a collection of MediaItems from the list of expected (whether - // present or missing) LiveKit participants. - generateItems( - function* ([ - localMatrixLivekitMember, - { value: matrixLivekitMembers }, - duplicateTiles, - ]) { - let localParticipantId = undefined; - // add local member if available - if (localMatrixLivekitMember) { - const { userId, participant$, connection$, membership$ } = - localMatrixLivekitMember; - localParticipantId = `${userId}:${membership$.value.deviceId}`; // should be membership$.value.membershipID which is not optional - // const participantId = membership$.value.membershipID; - if (localParticipantId) { - for (let dup = 0; dup < 1 + duplicateTiles; dup++) { - yield { - keys: [ - dup, - localParticipantId, - userId, - participant$, - connection$, - ], - data: undefined, - }; - } - } - } - // add remote members that are available - for (const { - userId, - participant$, - connection$, - membership$, - } of matrixLivekitMembers) { - const participantId = `${userId}:${membership$.value.deviceId}`; - if (participantId === localParticipantId) continue; - // const participantId = membership$.value?.identity; - for (let dup = 0; dup < 1 + duplicateTiles; dup++) { - yield { - keys: [dup, participantId, userId, participant$, connection$], - data: undefined, - }; - } - } - }, - ( - scope, - _data$, - dup, - participantId, - userId, - participant$, - connection$, - ) => { - const livekitRoom$ = scope.behavior( - connection$.pipe(map((c) => c?.livekitRoom)), - ); - const focusUrl$ = scope.behavior( - connection$.pipe(map((c) => c?.transport.livekit_service_url)), - ); - const displayName$ = scope.behavior( - matrixMemberMetadataStore - .createDisplayNameBehavior$(userId) - .pipe(map((name) => name ?? userId)), - ); - - return new UserMedia( - scope, - `${participantId}:${dup}`, - userId, - participant$, - options.encryptionSystem, - livekitRoom$, - focusUrl$, - mediaDevices, - pretendToBeDisconnected$, - displayName$, - matrixMemberMetadataStore.createAvatarUrlBehavior$(userId), - handsRaised$.pipe(map((v) => v[participantId]?.time ?? null)), - reactions$.pipe(map((v) => v[participantId] ?? undefined)), - ); - }, - ), - ), - ); - - /** - * List of all media items (user media and screen share media) that we want - * tiles for. - */ - const mediaItems$ = scope.behavior( - userMedia$.pipe( - switchMap((userMedia) => - userMedia.length === 0 - ? of([]) - : combineLatest( - userMedia.map((m) => m.screenShares$), - (...screenShares) => [...userMedia, ...screenShares.flat(1)], - ), - ), - ), - ); - - /** - * List of MediaItems that we want to display, that are of type ScreenShare - */ - const screenShares$ = scope.behavior( - mediaItems$.pipe( - map((mediaItems) => - mediaItems.filter((m): m is ScreenShare => m instanceof ScreenShare), - ), - ), - ); - - /*PUBLIC*/ const joinSoundEffect$ = userMedia$.pipe( - pairwise(), - filter( - ([prev, current]) => - current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && - current.length > prev.length, - ), - map(() => {}), - throttleTime(THROTTLE_SOUND_EFFECT_MS), - ); - - /** - * The number of participants currently in the call. - * - * - Each participant has a corresponding MatrixRTC membership state event - * - There can be multiple participants for one Matrix user if they join from - * multiple devices. - */ - /*PUBLIC*/ const participantCount$ = scope.behavior( - matrixLivekitMembers$.pipe(map((ms) => ms.value.length)), - ); - - // only public to expose to the view. - // TODO if we are in "unknown" state we need a loading rendering (or empty screen) - // Otherwise it looks like we already connected and only than the ringing starts which is weird. - /*PUBLIC*/ const callPickupState$ = callLifecycle.callPickupState$; - - /*PUBLIC*/ const leaveSoundEffect$ = combineLatest([ - callLifecycle.callPickupState$, - userMedia$, - ]).pipe( - // Until the call is successful, do not play a leave sound. - // If callPickupState$ is null, then we always play the sound as it will not conflict with a decline sound. - skipWhile(([c]) => c !== null && c !== "success"), - map(([, userMedia]) => userMedia), - pairwise(), - filter( - ([prev, current]) => - current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && - current.length < prev.length, - ), - map(() => {}), - throttleTime(THROTTLE_SOUND_EFFECT_MS), - ); - - const userHangup$ = new Subject(); - - const widgetHangup$ = - widget === null - ? NEVER - : ( - fromEvent( - widget.lazyActions, - ElementWidgetActions.HangupCall, - ) as Observable> - ).pipe( - tap((ev) => { - widget!.api.transport.reply(ev.detail, {}); - }), - ); - - const leave$: Observable<"user" | "timeout" | "decline" | "allOthersLeft"> = - merge( - callLifecycle.autoLeave$, - merge(userHangup$, widgetHangup$).pipe(map(() => "user" as const)), - ).pipe( - scope.share, - tap((reason) => leaveHoisted$.next(reason)), - ); - - const spotlightSpeaker$ = scope.behavior( - userMedia$.pipe( - switchMap((mediaItems) => - mediaItems.length === 0 - ? of([]) - : combineLatest( - mediaItems.map((m) => - m.vm.speaking$.pipe(map((s) => [m, s] as const)), - ), - ), - ), - scan<(readonly [UserMedia, boolean])[], UserMedia | undefined, null>( - (prev, mediaItems) => { - // Only remote users that are still in the call should be sticky - const [stickyMedia, stickySpeaking] = - (!prev?.vm.local && mediaItems.find(([m]) => m === prev)) || []; - // Decide who to spotlight: - // If the previous speaker is still speaking, stick with them rather - // than switching eagerly to someone else - return stickySpeaking - ? stickyMedia! - : // Otherwise, select any remote user who is speaking - (mediaItems.find(([m, s]) => !m.vm.local && s)?.[0] ?? - // Otherwise, stick with the person who was last speaking - stickyMedia ?? - // Otherwise, spotlight an arbitrary remote user - mediaItems.find(([m]) => !m.vm.local)?.[0] ?? - // Otherwise, spotlight the local user - mediaItems.find(([m]) => m.vm.local)?.[0]); - }, - null, - ), - map((speaker) => speaker?.vm ?? null), - ), - ); - - const grid$ = scope.behavior( - userMedia$.pipe( - switchMap((mediaItems) => { - const bins = mediaItems.map((m) => - m.bin$.pipe(map((bin) => [m, bin] as const)), - ); - // Sort the media by bin order and generate a tile for each one - return bins.length === 0 - ? of([]) - : combineLatest(bins, (...bins) => - bins.sort(([, bin1], [, bin2]) => bin1 - bin2).map(([m]) => m.vm), - ); - }), - distinctUntilChanged(shallowEquals), - ), - ); - - const spotlight$ = scope.behavior( - screenShares$.pipe( - switchMap((screenShares) => { - if (screenShares.length > 0) { - return of(screenShares.map((m) => m.vm)); - } - - return spotlightSpeaker$.pipe( - map((speaker) => (speaker ? [speaker] : [])), - ); - }), - distinctUntilChanged(shallowEquals), - ), - ); - - const pip$ = scope.behavior( - combineLatest([ - // TODO This also needs epoch logic to dedupe the screenshares and mediaItems emits - screenShares$, - spotlightSpeaker$, - mediaItems$, - ]).pipe( - switchMap(([screenShares, spotlight, mediaItems]) => { - if (screenShares.length > 0) { - return spotlightSpeaker$; - } - if (!spotlight || spotlight.local) { - return of(null); - } - - const localUserMedia = mediaItems.find( - (m) => m.vm instanceof LocalUserMediaViewModel, - ) as UserMedia | undefined; - - const localUserMediaViewModel = localUserMedia?.vm as - | LocalUserMediaViewModel - | undefined; - - if (!localUserMediaViewModel) { - return of(null); - } - return localUserMediaViewModel.alwaysShow$.pipe( - map((alwaysShow) => { - if (alwaysShow) { - return localUserMediaViewModel; - } - - return null; - }), - ); - }), - ), - ); - - const hasRemoteScreenShares$: Observable = spotlight$.pipe( - map((spotlight) => - spotlight.some((vm) => !vm.local && vm instanceof ScreenShareViewModel), - ), - distinctUntilChanged(), - ); - - const pipEnabled$ = scope.behavior(setPipEnabled$, false); - - const naturalWindowMode$ = scope.behavior( - fromEvent(window, "resize").pipe( - map(() => { - const height = window.innerHeight; - const width = window.innerWidth; - if (height <= 400 && width <= 340) return "pip"; - // Our layouts for flat windows are better at adapting to a small width - // than our layouts for narrow windows are at adapting to a small height, - // so we give "flat" precedence here - if (height <= 600) return "flat"; - if (width <= 600) return "narrow"; - return "normal"; - }), - ), - "normal", - ); - - /** - * The general shape of the window. - */ - /*PUBLIC*/ const windowMode$ = scope.behavior( - pipEnabled$.pipe( - switchMap((pip) => (pip ? of("pip") : naturalWindowMode$)), - ), - ); - - const spotlightExpandedToggle$ = new Subject(); - /*PUBLIC*/ const spotlightExpanded$ = scope.behavior( - spotlightExpandedToggle$.pipe(accumulate(false, (expanded) => !expanded)), - ); - - const gridModeUserSelection$ = new Subject(); - /** - * The layout mode of the media tile grid. - */ - /*PUBLIC*/ const gridMode$ = - // If the user hasn't selected spotlight and somebody starts screen sharing, - // automatically switch to spotlight mode and reset when screen sharing ends - scope.behavior( - gridModeUserSelection$.pipe( - switchMap((userSelection) => - (userSelection === "spotlight" - ? EMPTY - : combineLatest([hasRemoteScreenShares$, windowMode$]).pipe( - skip(userSelection === null ? 0 : 1), - map( - ([hasScreenShares, windowMode]): GridMode => - hasScreenShares || windowMode === "flat" - ? "spotlight" - : "grid", - ), - ) - ).pipe(startWith(userSelection ?? "grid")), - ), - ), - "grid", - ); - - /*PUBLIC*/ const setGridMode = (value: GridMode): void => { - gridModeUserSelection$.next(value); - }; - - const gridLayoutMedia$: Observable = combineLatest( - [grid$, spotlight$], - (grid, spotlight) => ({ - type: "grid", - spotlight: spotlight.some((vm) => vm instanceof ScreenShareViewModel) - ? spotlight - : undefined, - grid, - }), - ); - - const spotlightLandscapeLayoutMedia$: Observable = - combineLatest([grid$, spotlight$], (grid, spotlight) => ({ - type: "spotlight-landscape", - spotlight, - grid, - })); - - const spotlightPortraitLayoutMedia$: Observable = - combineLatest([grid$, spotlight$], (grid, spotlight) => ({ - type: "spotlight-portrait", - spotlight, - grid, - })); - - const spotlightExpandedLayoutMedia$: Observable = - combineLatest([spotlight$, pip$], (spotlight, pip) => ({ - type: "spotlight-expanded", - spotlight, - pip: pip ?? undefined, - })); - - const oneOnOneLayoutMedia$: Observable = - mediaItems$.pipe( - map((mediaItems) => { - if (mediaItems.length !== 2) return null; - const local = mediaItems.find((vm) => vm.vm.local)?.vm as - | LocalUserMediaViewModel - | undefined; - const remote = mediaItems.find((vm) => !vm.vm.local)?.vm as - | RemoteUserMediaViewModel - | undefined; - // There might not be a remote tile if there are screen shares, or if - // only the local user is in the call and they're using the duplicate - // tiles option - if (!remote || !local) return null; - - return { type: "one-on-one", local, remote }; - }), - ); - - const pipLayoutMedia$: Observable = spotlight$.pipe( - map((spotlight) => ({ type: "pip", spotlight })), - ); - - /** - * The media to be used to produce a layout. - */ - const layoutMedia$ = scope.behavior( - windowMode$.pipe( - switchMap((windowMode) => { - switch (windowMode) { - case "normal": - return gridMode$.pipe( - switchMap((gridMode) => { - switch (gridMode) { - case "grid": - return oneOnOneLayoutMedia$.pipe( - switchMap((oneOnOne) => - oneOnOne === null ? gridLayoutMedia$ : of(oneOnOne), - ), - ); - case "spotlight": - return spotlightExpanded$.pipe( - switchMap((expanded) => - expanded - ? spotlightExpandedLayoutMedia$ - : spotlightLandscapeLayoutMedia$, - ), - ); - } - }), - ); - case "narrow": - return oneOnOneLayoutMedia$.pipe( - switchMap((oneOnOne) => - oneOnOne === null - ? combineLatest([grid$, spotlight$], (grid, spotlight) => - grid.length > smallMobileCallThreshold || - spotlight.some((vm) => vm instanceof ScreenShareViewModel) - ? spotlightPortraitLayoutMedia$ - : gridLayoutMedia$, - ).pipe(switchAll()) - : // The expanded spotlight layout makes for a better one-on-one - // experience in narrow windows - spotlightExpandedLayoutMedia$, - ), - ); - case "flat": - return gridMode$.pipe( - switchMap((gridMode) => { - switch (gridMode) { - case "grid": - // Yes, grid mode actually gets you a "spotlight" layout in - // this window mode. - return spotlightLandscapeLayoutMedia$; - case "spotlight": - return spotlightExpandedLayoutMedia$; - } - }), - ); - case "pip": - return pipLayoutMedia$; - } - }), - ), - ); - - // There is a cyclical dependency here: the layout algorithms want to know - // which tiles are on screen, but to know which tiles are on screen we have to - // first render a layout. To deal with this we assume initially that no tiles - // are visible, and loop the data back into the layouts with a Subject. - const visibleTiles$ = new Subject(); - const setVisibleTiles = (value: number): void => visibleTiles$.next(value); - - const layoutInternals$ = scope.behavior( - combineLatest([ - layoutMedia$, - visibleTiles$.pipe(startWith(0), distinctUntilChanged()), - ]).pipe( - scan< - [LayoutMedia, number], - LayoutScanState & { layout: Layout }, - LayoutScanState - >( - ({ tiles: prevTiles }, [media, visibleTiles]) => { - let layout: Layout; - let newTiles: TileStore; - switch (media.type) { - case "grid": - case "spotlight-landscape": - case "spotlight-portrait": - [layout, newTiles] = gridLikeLayout( - media, - visibleTiles, - setVisibleTiles, - prevTiles, - ); - break; - case "spotlight-expanded": - [layout, newTiles] = spotlightExpandedLayout(media, prevTiles); - break; - case "one-on-one": - [layout, newTiles] = oneOnOneLayout(media, prevTiles); - break; - case "pip": - [layout, newTiles] = pipLayout(media, prevTiles); - break; - } - - return { layout, tiles: newTiles }; - }, - { layout: null, tiles: TileStore.empty() }, - ), - ), - ); - - /** - * The layout of tiles in the call interface. - */ - /*PUBLIC*/ const layout$ = scope.behavior( - layoutInternals$.pipe(map(({ layout }) => layout)), - ); - - /** - * The current generation of the tile store, exposed for debugging purposes. - */ - /*PUBLIC*/ const tileStoreGeneration$ = scope.behavior( - layoutInternals$.pipe(map(({ tiles }) => tiles.generation)), - ); - - /*PUBLIC*/ const showSpotlightIndicators$ = scope.behavior( - layout$.pipe(map((l) => l.type !== "grid")), - ); - - /*PUBLIC*/ const showSpeakingIndicators$ = scope.behavior( - layout$.pipe( - switchMap((l) => { - switch (l.type) { - case "spotlight-landscape": - case "spotlight-portrait": - // If the spotlight is showing the active speaker, we can do without - // speaking indicators as they're a redundant visual cue. But if - // screen sharing feeds are in the spotlight we still need them. - return l.spotlight.media$.pipe( - map((models: MediaViewModel[]) => - models.some((m) => m instanceof ScreenShareViewModel), - ), - ); - // In expanded spotlight layout, the active speaker is always shown in - // the picture-in-picture tile so there is no need for speaking - // indicators. And in one-on-one layout there's no question as to who is - // speaking. - case "spotlight-expanded": - case "one-on-one": - return of(false); - default: - return of(true); - } - }), - ), - ); - - /*PUBLIC*/ const toggleSpotlightExpanded$ = scope.behavior< - (() => void) | null - >( - windowMode$.pipe( - switchMap((mode) => - mode === "normal" - ? layout$.pipe( - map( - (l) => - l.type === "spotlight-landscape" || - l.type === "spotlight-expanded", - ), - ) - : of(false), - ), - distinctUntilChanged(), - map((enabled) => - enabled ? (): void => spotlightExpandedToggle$.next() : null, - ), - ), - ); - - const screenTap$ = new Subject(); - const controlsTap$ = new Subject(); - const screenHover$ = new Subject(); - const screenUnhover$ = new Subject(); - - /*PUBLIC*/ const showHeader$ = scope.behavior( - windowMode$.pipe(map((mode) => mode !== "pip" && mode !== "flat")), - ); - - /*PUBLIC*/ const showFooter$ = scope.behavior( - windowMode$.pipe( - switchMap((mode) => { - switch (mode) { - case "pip": - return of(false); - case "normal": - case "narrow": - return of(true); - case "flat": - // Sadly Firefox has some layering glitches that prevent the footer - // from appearing properly. They happen less often if we never hide - // the footer. - if (isFirefox()) return of(true); - // Show/hide the footer in response to interactions - return merge( - screenTap$.pipe(map(() => "tap screen" as const)), - controlsTap$.pipe(map(() => "tap controls" as const)), - screenHover$.pipe(map(() => "hover" as const)), - ).pipe( - switchScan((state, interaction) => { - switch (interaction) { - case "tap screen": - return state - ? // Toggle visibility on tap - of(false) - : // Hide after a timeout - timer(showFooterMs).pipe( - map(() => false), - startWith(true), - ); - case "tap controls": - // The user is interacting with things, so reset the timeout - return timer(showFooterMs).pipe( - map(() => false), - startWith(true), - ); - case "hover": - // Show on hover and hide after a timeout - return race( - timer(showFooterMs), - screenUnhover$.pipe(take(1)), - ).pipe( - map(() => false), - startWith(true), - ); - } - }, false), - startWith(false), - ); - } - }), - ), - ); - - /** - * Whether audio is currently being output through the earpiece. - */ - /*PUBLIC*/ const earpieceMode$ = scope.behavior( - combineLatest( - [mediaDevices.audioOutput.available$, mediaDevices.audioOutput.selected$], - (available, selected) => - selected !== undefined && - available.get(selected.id)?.type === "earpiece", - ), - ); - - /** - * Callback to toggle between the earpiece and the loudspeaker. - * - * This will be `null` in case the target does not exist in the list - * of available audio outputs. - */ - /*PUBLIC*/ const audioOutputSwitcher$ = scope.behavior<{ - targetOutput: "earpiece" | "speaker"; - switch: () => void; - } | null>( - combineLatest( - [mediaDevices.audioOutput.available$, mediaDevices.audioOutput.selected$], - (available, selected) => { - const selectionType = selected && available.get(selected.id)?.type; - - // If we are in any output mode other than speaker switch to speaker. - const newSelectionType: "earpiece" | "speaker" = - selectionType === "speaker" ? "earpiece" : "speaker"; - const newSelection = [...available].find( - ([, d]) => d.type === newSelectionType, - ); - if (newSelection === undefined) return null; - - const [id] = newSelection; - return { - targetOutput: newSelectionType, - switch: (): void => mediaDevices.audioOutput.select(id), - }; - }, - ), - ); - - /** - * Emits an array of reactions that should be visible on the screen. - */ - // DISCUSSION move this into a reaction file - // const {visibleReactions$, audibleReactions$} = reactionsObservables$(showReactionSetting$, ) - const visibleReactions$ = scope.behavior( - showReactions.value$.pipe( - switchMap((show) => (show ? reactions$ : of({}))), - scan< - Record, - { sender: string; emoji: string; startX: number }[] - >((acc, latest) => { - const newSet: { sender: string; emoji: string; startX: number }[] = []; - for (const [sender, reaction] of Object.entries(latest)) { - const startX = - acc.find((v) => v.sender === sender && v.emoji)?.startX ?? - Math.ceil(Math.random() * 80) + 10; - newSet.push({ sender, emoji: reaction.emoji, startX }); - } - return newSet; - }, []), - ), - ); - - /** - * Emits an array of reactions that should be played. - */ - const audibleReactions$ = playReactionsSound.value$.pipe( - switchMap((show) => - show ? reactions$ : of>({}), - ), - map((reactions) => Object.values(reactions).map((v) => v.name)), - scan( - (acc, latest) => { - return { - playing: latest.filter( - (v) => acc.playing.includes(v) || acc.newSounds.includes(v), - ), - newSounds: latest.filter( - (v) => !acc.playing.includes(v) && !acc.newSounds.includes(v), - ), - }; - }, - { playing: [], newSounds: [] }, - ), - map((v) => v.newSounds), - ); - - const newHandRaised$ = handsRaised$.pipe( - map((v) => Object.keys(v).length), - scan( - (acc, newValue) => ({ - value: newValue, - playSounds: newValue > acc.value, - }), - { value: 0, playSounds: false }, - ), - filter((v) => v.playSounds), - ); - - const newScreenShare$ = screenShares$.pipe( - map((v) => v.length), - scan( - (acc, newValue) => ({ - value: newValue, - playSounds: newValue > acc.value, - }), - { value: 0, playSounds: false }, - ), - filter((v) => v.playSounds), - ); - - /** - * Whether we are sharing our screen. - */ - // reassigned here to make it publicly accessible - const sharingScreen$ = localMembership.sharingScreen$; - - /** - * Callback to toggle screen sharing. If null, screen sharing is not possible. - */ - // reassigned here to make it publicly accessible - const toggleScreenSharing = localMembership.toggleScreenSharing; - - const join = localMembership.requestConnect; - join(); // TODO-MULTI-SFU: Use this view model for the lobby as well, and only call this once 'join' is clicked? - - return { - autoLeave$: callLifecycle.autoLeave$, - callPickupState$, - leave$, - hangup: () => userHangup$.next(), - join, - toggleScreenSharing, - sharingScreen$, - - tapScreen: () => screenTap$.next(), - tapControls: () => controlsTap$.next(), - hoverScreen: () => screenHover$.next(), - unhoverScreen: () => screenUnhover$.next(), - - configError$: localMembership.configError$, - participantCount$, - audioParticipants$, - isOneOnOneWith$, - localUserIsAlone$, - - handsRaised$, - reactions$, - joinSoundEffect$, - leaveSoundEffect$, - newHandRaised$, - newScreenShare$, - audibleReactions$, - visibleReactions$, - - windowMode$, - spotlightExpanded$, - toggleSpotlightExpanded$, - gridMode$, - setGridMode, - grid$, - spotlight$, - pip$, - layout$, - tileStoreGeneration$, - showSpotlightIndicators$, - showSpeakingIndicators$, - showHeader$, - showFooter$, - earpieceMode$, - audioOutputSwitcher$, - reconnecting$, - }; -} -// TODO-MULTI-SFU // Setup and update the keyProvider which was create by `createRoom` was a thing before. Now we never update if the E2EEsystem changes -// do we need this? - -function getE2eeKeyProvider( - e2eeSystem: EncryptionSystem, - rtcSession: MatrixRTCSession, -): BaseKeyProvider | undefined { - if (e2eeSystem.kind === E2eeType.NONE) return undefined; - - if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) { - const keyProvider = new MatrixKeyProvider(); - keyProvider.setRTCSession(rtcSession); - return keyProvider; - } else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) { - const keyProvider = new ExternalE2EEKeyProvider(); - keyProvider - .setKey(e2eeSystem.secret) - .catch((e) => logger.error("Failed to set shared key for E2EE", e)); - return keyProvider; - } -} diff --git a/src/state/CallViewModel/CallViewModelOld.ts b/src/state/CallViewModel/CallViewModelOld.ts deleted file mode 100644 index ff4a6269..00000000 --- a/src/state/CallViewModel/CallViewModelOld.ts +++ /dev/null @@ -1,1335 +0,0 @@ -/* -Copyright 2023, 2024, 2025 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - type BaseKeyProvider, - type ConnectionState, - ExternalE2EEKeyProvider, - type Room as LivekitRoom, - type RoomOptions, -} from "livekit-client"; -import { type Room as MatrixRoom } from "matrix-js-sdk"; -import { - combineLatest, - distinctUntilChanged, - EMPTY, - filter, - fromEvent, - map, - merge, - NEVER, - type Observable, - of, - pairwise, - race, - scan, - skip, - skipWhile, - startWith, - Subject, - switchAll, - switchMap, - switchScan, - take, - tap, - throttleTime, - timer, -} from "rxjs"; -import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; -import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; -import { type IWidgetApiRequest } from "matrix-widget-api"; - -import { - LocalUserMediaViewModel, - type MediaViewModel, - type RemoteUserMediaViewModel, - ScreenShareViewModel, - type UserMediaViewModel, -} from "../MediaViewModel"; -import { accumulate, generateItems, pauseWhen } from "../../utils/observable"; -import { - duplicateTiles, - MatrixRTCMode, - matrixRTCMode, - playReactionsSound, - showReactions, -} from "../../settings/settings"; -import { isFirefox } from "../../Platform"; -import { setPipEnabled$ } from "../../controls"; -import { TileStore } from "../TileStore"; -import { gridLikeLayout } from "../GridLikeLayout"; -import { spotlightExpandedLayout } from "../SpotlightExpandedLayout"; -import { oneOnOneLayout } from "../OneOnOneLayout"; -import { pipLayout } from "../PipLayout"; -import { type EncryptionSystem } from "../../e2ee/sharedKeyManagement"; -import { - type RaisedHandInfo, - type ReactionInfo, - type ReactionOption, -} from "../../reactions"; -import { shallowEquals } from "../../utils/array"; -import { type MediaDevices } from "../MediaDevices"; -import { type Behavior } from "../Behavior"; -import { E2eeType } from "../../e2ee/e2eeType"; -import { MatrixKeyProvider } from "../../e2ee/matrixKeyProvider"; -import { type MuteStates } from "../MuteStates"; -import { getUrlParams } from "../../UrlParams"; -import { type ProcessorState } from "../../livekit/TrackProcessorContext"; -import { ElementWidgetActions, widget } from "../../widget"; -import { UserMedia } from "../UserMedia.ts"; -import { ScreenShare } from "../ScreenShare.ts"; -import { - type GridLayoutMedia, - type Layout, - type LayoutMedia, - type OneOnOneLayoutMedia, - type SpotlightExpandedLayoutMedia, - type SpotlightLandscapeLayoutMedia, - type SpotlightPortraitLayoutMedia, -} from "../layout-types.ts"; -import { type ElementCallError } from "../../utils/errors.ts"; -import { type ObservableScope } from "../ObservableScope.ts"; -import { createLocalMembership$ } from "./localMember/LocalMembership.ts"; -import { createLocalTransport$ } from "./localMember/LocalTransport.ts"; -import { - createMemberships$, - membershipsAndTransports$, -} from "../SessionBehaviors.ts"; -import { ECConnectionFactory } from "./remoteMembers/ConnectionFactory.ts"; -import { createConnectionManager$ } from "./remoteMembers/ConnectionManager.ts"; -import { - createMatrixLivekitMembers$, - type MatrixLivekitMember, -} from "./remoteMembers/MatrixLivekitMembers.ts"; -import { - createCallNotificationLifecycle$, - createReceivedDecline$, - createSentCallNotification$, -} from "./CallNotificationLifecycle.ts"; -import { - createMatrixMemberMetadata$, - createRoomMembers$, -} from "./remoteMembers/MatrixMemberMetadata.ts"; - -const logger = rootLogger.getChild("[CallViewModel]"); -//TODO -// Larger rename -// member,membership -> rtcMember -// participant -> livekitParticipant -// matrixLivekitItem -> callMember -// js-sdk -// callMembership -> rtcMembership -export interface CallViewModelOptions { - encryptionSystem: EncryptionSystem; - autoLeaveWhenOthersLeft?: boolean; - /** - * If the call is started in a way where we want it to behave like a telephone usecase - * If we sent a notification event, we want the ui to show a ringing state - */ - waitForCallPickup?: boolean; - /** Optional factory to create LiveKit rooms, mainly for testing purposes. */ - livekitRoomFactory?: (options?: RoomOptions) => LivekitRoom; - /** Optional behavior overriding the local connection state, mainly for testing purposes. */ - connectionState$?: Behavior; -} - -// Do not play any sounds if the participant count has exceeded this -// number. -export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8; -export const THROTTLE_SOUND_EFFECT_MS = 500; - -// This is the number of participants that we think constitutes a "small" call -// on mobile. No spotlight tile should be shown below this threshold. -const smallMobileCallThreshold = 3; - -// How long the footer should be shown for when hovering over or interacting -// with the interface -const showFooterMs = 4000; - -export type GridMode = "grid" | "spotlight"; - -export type WindowMode = "normal" | "narrow" | "flat" | "pip"; - -interface LayoutScanState { - layout: Layout | null; - tiles: TileStore; -} - -type MediaItem = UserMedia | ScreenShare; -type AudioLivekitItem = { - livekitRoom: LivekitRoom; - participants: string[]; - url: string; -}; -/** - * A view model providing all the application logic needed to show the in-call - * UI (may eventually be expanded to cover the lobby and feedback screens in the - * future). - */ -// Throughout this class and related code we must distinguish between MatrixRTC -// state and LiveKit state. We use the common terminology of room "members", RTC -// "memberships", and LiveKit "participants". -export class CallViewModel { - private readonly userId = this.matrixRoom.client.getUserId()!; - private readonly deviceId = this.matrixRoom.client.getDeviceId()!; - - private readonly livekitKeyProvider = getE2eeKeyProvider( - this.options.encryptionSystem, - this.matrixRTCSession, - ); - - // Each hbar seperates a block of input variables required for the CallViewModel to function. - // The outputs of this block is written under the hbar. - // - // For mocking purposes it is recommended to only mock the functions creating those outputs. - // All other fields are just temp computations for the mentioned output. - // The class does not need anything except the values underneath the bar. - // The creation of the values under the bar are all tested independently and testing the callViewModel Should - // not test their cretation. Call view model only needs: - // - memberships$ via createMemberships$ - // - localMembership via createLocalMembership$ - // - callLifecycle via createCallNotificationLifecycle$ - // - matrixMemberMetadataStore via createMatrixMemberMetadata$ - - // ------------------------------------------------------------------------ - // memberships$ - private memberships$ = createMemberships$(this.scope, this.matrixRTCSession); - - // ------------------------------------------------------------------------ - // matrixLivekitMembers$ AND localMembership - - private membershipsAndTransports = membershipsAndTransports$( - this.scope, - this.memberships$, - ); - - private localTransport$ = createLocalTransport$({ - scope: this.scope, - memberships$: this.memberships$, - client: this.matrixRoom.client, - roomId: this.matrixRoom.roomId, - useOldestMember$: this.scope.behavior( - matrixRTCMode.value$.pipe(map((v) => v === MatrixRTCMode.Legacy)), - ), - }); - - private connectionFactory = new ECConnectionFactory( - this.matrixRoom.client, - this.mediaDevices, - this.trackProcessorState$, - this.livekitKeyProvider, - getUrlParams().controlledAudioDevices, - this.options.livekitRoomFactory, - ); - - private connectionManager = createConnectionManager$({ - scope: this.scope, - connectionFactory: this.connectionFactory, - inputTransports$: this.scope.behavior( - combineLatest( - [this.localTransport$, this.membershipsAndTransports.transports$], - (localTransport, transports) => { - const localTransportAsArray = localTransport ? [localTransport] : []; - return transports.mapInner((transports) => [ - ...localTransportAsArray, - ...transports, - ]); - }, - ), - ), - logger: logger, - }); - - private matrixLivekitMembers$ = createMatrixLivekitMembers$({ - scope: this.scope, - membershipsWithTransport$: - this.membershipsAndTransports.membershipsWithTransport$, - connectionManager: this.connectionManager, - }); - - private connectOptions$ = this.scope.behavior( - matrixRTCMode.value$.pipe( - map((mode) => ({ - encryptMedia: this.livekitKeyProvider !== undefined, - // TODO. This might need to get called again on each cahnge of matrixRTCMode... - matrixRTCMode: mode, - })), - ), - ); - - private localMembership = createLocalMembership$({ - scope: this.scope, - muteStates: this.muteStates, - mediaDevices: this.mediaDevices, - connectionManager: this.connectionManager, - matrixRTCSession: this.matrixRTCSession, - matrixRoom: this.matrixRoom, - localTransport$: this.localTransport$, - trackProcessorState$: this.trackProcessorState$, - widget, - options: this.connectOptions$, - logger: logger.getChild(`[${Date.now()}]`), - }); - - private localRtcMembership$ = this.scope.behavior( - this.memberships$.pipe( - map( - (memberships) => - memberships.value.find( - (membership) => - membership.userId === this.userId && - membership.deviceId === this.deviceId, - ) ?? null, - ), - ), - ); - - private localMatrixLivekitMemberUninitialized = { - membership$: this.localRtcMembership$, - participant$: this.localMembership.participant$, - connection$: this.localMembership.connection$, - userId: this.userId, - }; - - private localMatrixLivekitMember$: Behavior = - this.scope.behavior( - this.localRtcMembership$.pipe( - switchMap((membership) => { - if (!membership) return of(null); - return of( - // casting is save here since we know that localRtcMembership$ is !== null since we reached this case. - this.localMatrixLivekitMemberUninitialized as MatrixLivekitMember, - ); - }), - ), - ); - - // ------------------------------------------------------------------------ - // callLifecycle - - private callLifecycle = createCallNotificationLifecycle$({ - scope: this.scope, - memberships$: this.memberships$, - sentCallNotification$: createSentCallNotification$( - this.scope, - this.matrixRTCSession, - ), - receivedDecline$: createReceivedDecline$(this.matrixRoom), - options: this.options, - localUser: { userId: this.userId, deviceId: this.deviceId }, - }); - public autoLeave$ = this.callLifecycle.autoLeave$; - - // ------------------------------------------------------------------------ - // matrixMemberMetadataStore - - private matrixMemberMetadataStore = createMatrixMemberMetadata$( - this.scope, - this.scope.behavior(this.memberships$.pipe(map((mems) => mems.value))), - createRoomMembers$(this.scope, this.matrixRoom), - ); - - /** - * If there is a configuration error with the call (e.g. misconfigured E2EE). - * This is a fatal error that prevents the call from being created/joined. - * Should render a blocking error screen. - */ - public get configError$(): Behavior { - return this.localMembership.configError$; - } - - public join = this.localMembership.requestConnect; - - // CODESMELL? - // This is functionally the same Observable as leave$, except here it's - // hoisted to the top of the class. This enables the cyclic dependency between - // leave$ -> autoLeave$ -> callPickupState$ -> livekitConnectionState$ -> - // localConnection$ -> transports$ -> joined$ -> leave$. - private readonly leaveHoisted$ = new Subject< - "user" | "timeout" | "decline" | "allOthersLeft" - >(); - - /** - * Whether various media/event sources should pretend to be disconnected from - * all network input, even if their connection still technically works. - */ - // We do this when the app is in the 'reconnecting' state, because it might be - // that the LiveKit connection is still functional while the homeserver is - // down, for example, and we want to avoid making people worry that the app is - // in a split-brained state. - // DISCUSSION own membership manager ALSO this probably can be simplifis - public reconnecting$ = this.localMembership.reconnecting$; - private readonly pretendToBeDisconnected$ = this.reconnecting$; - - public readonly audioParticipants$ = this.scope.behavior( - this.matrixLivekitMembers$.pipe( - switchMap((membersWithEpoch) => { - const members = membersWithEpoch.value; - const a$ = combineLatest( - members.map((member) => - combineLatest([member.connection$, member.participant$]).pipe( - map(([connection, participant]) => { - // do not render audio for local participant - if (!connection || !participant || participant.isLocal) - return null; - const livekitRoom = connection.livekitRoom; - const url = connection.transport.livekit_service_url; - - return { url, livekitRoom, participant: participant.identity }; - }), - ), - ), - ); - return a$; - }), - map((members) => - members.reduce((acc, curr) => { - if (!curr) return acc; - - const existing = acc.find((item) => item.url === curr.url); - if (existing) { - existing.participants.push(curr.participant); - } else { - acc.push({ - livekitRoom: curr.livekitRoom, - participants: [curr.participant], - url: curr.url, - }); - } - return acc; - }, []), - ), - ), - [], - ); - - public readonly handsRaised$ = this.scope.behavior( - this.handsRaisedSubject$.pipe(pauseWhen(this.pretendToBeDisconnected$)), - ); - - public readonly reactions$ = this.scope.behavior( - this.reactionsSubject$.pipe( - map((v) => - Object.fromEntries( - Object.entries(v).map(([a, { reactionOption }]) => [ - a, - reactionOption, - ]), - ), - ), - pauseWhen(this.pretendToBeDisconnected$), - ), - ); - - /** - * List of user media (camera feeds) that we want tiles for. - */ - // TODO this also needs the local participant to be added. - private readonly userMedia$ = this.scope.behavior( - combineLatest([ - this.localMatrixLivekitMember$, - this.matrixLivekitMembers$, - duplicateTiles.value$, - ]).pipe( - // Generate a collection of MediaItems from the list of expected (whether - // present or missing) LiveKit participants. - generateItems( - function* ([ - localMatrixLivekitMember, - { value: matrixLivekitMembers }, - duplicateTiles, - ]) { - let localParticipantId = undefined; - // add local member if available - if (localMatrixLivekitMember) { - const { userId, participant$, connection$, membership$ } = - localMatrixLivekitMember; - localParticipantId = `${userId}:${membership$.value.deviceId}`; // should be membership$.value.membershipID which is not optional - // const participantId = membership$.value.membershipID; - if (localParticipantId) { - for (let dup = 0; dup < 1 + duplicateTiles; dup++) { - yield { - keys: [ - dup, - localParticipantId, - userId, - participant$, - connection$, - ], - data: undefined, - }; - } - } - } - // add remote members that are available - for (const { - userId, - participant$, - connection$, - membership$, - } of matrixLivekitMembers) { - const participantId = `${userId}:${membership$.value.deviceId}`; - if (participantId === localParticipantId) continue; - // const participantId = membership$.value?.identity; - for (let dup = 0; dup < 1 + duplicateTiles; dup++) { - yield { - keys: [dup, participantId, userId, participant$, connection$], - data: undefined, - }; - } - } - }, - ( - scope, - _data$, - dup, - participantId, - userId, - participant$, - connection$, - ) => { - const livekitRoom$ = scope.behavior( - connection$.pipe(map((c) => c?.livekitRoom)), - ); - const focusUrl$ = scope.behavior( - connection$.pipe(map((c) => c?.transport.livekit_service_url)), - ); - const displayName$ = scope.behavior( - this.matrixMemberMetadataStore - .createDisplayNameBehavior$(userId) - .pipe(map((name) => name ?? userId)), - ); - - return new UserMedia( - scope, - `${participantId}:${dup}`, - userId, - participant$, - this.options.encryptionSystem, - livekitRoom$, - focusUrl$, - this.mediaDevices, - this.pretendToBeDisconnected$, - displayName$, - this.matrixMemberMetadataStore.createAvatarUrlBehavior$(userId), - this.handsRaised$.pipe(map((v) => v[participantId]?.time ?? null)), - this.reactions$.pipe(map((v) => v[participantId] ?? undefined)), - ); - }, - ), - ), - ); - - /** - * List of all media items (user media and screen share media) that we want - * tiles for. - */ - private readonly mediaItems$ = this.scope.behavior( - this.userMedia$.pipe( - switchMap((userMedia) => - userMedia.length === 0 - ? of([]) - : combineLatest( - userMedia.map((m) => m.screenShares$), - (...screenShares) => [...userMedia, ...screenShares.flat(1)], - ), - ), - ), - ); - - /** - * List of MediaItems that we want to display, that are of type ScreenShare - */ - private readonly screenShares$ = this.scope.behavior( - this.mediaItems$.pipe( - map((mediaItems) => - mediaItems.filter((m): m is ScreenShare => m instanceof ScreenShare), - ), - ), - ); - - public readonly joinSoundEffect$ = this.userMedia$.pipe( - pairwise(), - filter( - ([prev, current]) => - current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && - current.length > prev.length, - ), - map(() => {}), - throttleTime(THROTTLE_SOUND_EFFECT_MS), - ); - - /** - * The number of participants currently in the call. - * - * - Each participant has a corresponding MatrixRTC membership state event - * - There can be multiple participants for one Matrix user if they join from - * multiple devices. - */ - public readonly participantCount$ = this.scope.behavior( - this.matrixLivekitMembers$.pipe(map((ms) => ms.value.length)), - ); - - // only public to expose to the view. - // TODO if we are in "unknown" state we need a loading rendering (or empty screen) - // Otherwise it looks like we already connected and only than the ringing starts which is weird. - public readonly callPickupState$ = this.callLifecycle.callPickupState$; - - public readonly leaveSoundEffect$ = combineLatest([ - this.callLifecycle.callPickupState$, - this.userMedia$, - ]).pipe( - // Until the call is successful, do not play a leave sound. - // If callPickupState$ is null, then we always play the sound as it will not conflict with a decline sound. - skipWhile(([c]) => c !== null && c !== "success"), - map(([, userMedia]) => userMedia), - pairwise(), - filter( - ([prev, current]) => - current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && - current.length < prev.length, - ), - map(() => {}), - throttleTime(THROTTLE_SOUND_EFFECT_MS), - ); - - private readonly userHangup$ = new Subject(); - public hangup(): void { - this.userHangup$.next(); - } - - private readonly widgetHangup$ = - widget === null - ? NEVER - : ( - fromEvent( - widget.lazyActions, - ElementWidgetActions.HangupCall, - ) as Observable> - ).pipe( - tap((ev) => { - widget!.api.transport.reply(ev.detail, {}); - }), - ); - - public readonly leave$: Observable< - "user" | "timeout" | "decline" | "allOthersLeft" - > = merge( - this.callLifecycle.autoLeave$, - merge(this.userHangup$, this.widgetHangup$).pipe( - map(() => "user" as const), - ), - ).pipe( - this.scope.share, - tap((reason) => this.leaveHoisted$.next(reason)), - ); - - private readonly spotlightSpeaker$ = - this.scope.behavior( - this.userMedia$.pipe( - switchMap((mediaItems) => - mediaItems.length === 0 - ? of([]) - : combineLatest( - mediaItems.map((m) => - m.vm.speaking$.pipe(map((s) => [m, s] as const)), - ), - ), - ), - scan<(readonly [UserMedia, boolean])[], UserMedia | undefined, null>( - (prev, mediaItems) => { - // Only remote users that are still in the call should be sticky - const [stickyMedia, stickySpeaking] = - (!prev?.vm.local && mediaItems.find(([m]) => m === prev)) || []; - // Decide who to spotlight: - // If the previous speaker is still speaking, stick with them rather - // than switching eagerly to someone else - return stickySpeaking - ? stickyMedia! - : // Otherwise, select any remote user who is speaking - (mediaItems.find(([m, s]) => !m.vm.local && s)?.[0] ?? - // Otherwise, stick with the person who was last speaking - stickyMedia ?? - // Otherwise, spotlight an arbitrary remote user - mediaItems.find(([m]) => !m.vm.local)?.[0] ?? - // Otherwise, spotlight the local user - mediaItems.find(([m]) => m.vm.local)?.[0]); - }, - null, - ), - map((speaker) => speaker?.vm ?? null), - ), - ); - - private readonly grid$ = this.scope.behavior( - this.userMedia$.pipe( - switchMap((mediaItems) => { - const bins = mediaItems.map((m) => - m.bin$.pipe(map((bin) => [m, bin] as const)), - ); - // Sort the media by bin order and generate a tile for each one - return bins.length === 0 - ? of([]) - : combineLatest(bins, (...bins) => - bins.sort(([, bin1], [, bin2]) => bin1 - bin2).map(([m]) => m.vm), - ); - }), - distinctUntilChanged(shallowEquals), - ), - ); - - private readonly spotlight$ = this.scope.behavior( - this.screenShares$.pipe( - switchMap((screenShares) => { - if (screenShares.length > 0) { - return of(screenShares.map((m) => m.vm)); - } - - return this.spotlightSpeaker$.pipe( - map((speaker) => (speaker ? [speaker] : [])), - ); - }), - distinctUntilChanged(shallowEquals), - ), - ); - - private readonly pip$ = this.scope.behavior( - combineLatest([ - // TODO This also needs epoch logic to dedupe the screenshares and mediaItems emits - this.screenShares$, - this.spotlightSpeaker$, - this.mediaItems$, - ]).pipe( - switchMap(([screenShares, spotlight, mediaItems]) => { - if (screenShares.length > 0) { - return this.spotlightSpeaker$; - } - if (!spotlight || spotlight.local) { - return of(null); - } - - const localUserMedia = mediaItems.find( - (m) => m.vm instanceof LocalUserMediaViewModel, - ) as UserMedia | undefined; - - const localUserMediaViewModel = localUserMedia?.vm as - | LocalUserMediaViewModel - | undefined; - - if (!localUserMediaViewModel) { - return of(null); - } - return localUserMediaViewModel.alwaysShow$.pipe( - map((alwaysShow) => { - if (alwaysShow) { - return localUserMediaViewModel; - } - - return null; - }), - ); - }), - ), - ); - - private readonly hasRemoteScreenShares$: Observable = - this.spotlight$.pipe( - map((spotlight) => - spotlight.some((vm) => !vm.local && vm instanceof ScreenShareViewModel), - ), - distinctUntilChanged(), - ); - - private readonly pipEnabled$ = this.scope.behavior(setPipEnabled$, false); - - private readonly naturalWindowMode$ = this.scope.behavior( - fromEvent(window, "resize").pipe( - map(() => { - const height = window.innerHeight; - const width = window.innerWidth; - if (height <= 400 && width <= 340) return "pip"; - // Our layouts for flat windows are better at adapting to a small width - // than our layouts for narrow windows are at adapting to a small height, - // so we give "flat" precedence here - if (height <= 600) return "flat"; - if (width <= 600) return "narrow"; - return "normal"; - }), - ), - "normal", - ); - - /** - * The general shape of the window. - */ - public readonly windowMode$ = this.scope.behavior( - this.pipEnabled$.pipe( - switchMap((pip) => - pip ? of("pip") : this.naturalWindowMode$, - ), - ), - ); - - private readonly spotlightExpandedToggle$ = new Subject(); - public readonly spotlightExpanded$ = this.scope.behavior( - this.spotlightExpandedToggle$.pipe( - accumulate(false, (expanded) => !expanded), - ), - ); - - private readonly gridModeUserSelection$ = new Subject(); - /** - * The layout mode of the media tile grid. - */ - public readonly gridMode$ = - // If the user hasn't selected spotlight and somebody starts screen sharing, - // automatically switch to spotlight mode and reset when screen sharing ends - this.scope.behavior( - this.gridModeUserSelection$.pipe( - switchMap((userSelection) => - (userSelection === "spotlight" - ? EMPTY - : combineLatest([ - this.hasRemoteScreenShares$, - this.windowMode$, - ]).pipe( - skip(userSelection === null ? 0 : 1), - map( - ([hasScreenShares, windowMode]): GridMode => - hasScreenShares || windowMode === "flat" - ? "spotlight" - : "grid", - ), - ) - ).pipe(startWith(userSelection ?? "grid")), - ), - ), - "grid", - ); - - public setGridMode(value: GridMode): void { - this.gridModeUserSelection$.next(value); - } - - private readonly gridLayoutMedia$: Observable = - combineLatest([this.grid$, this.spotlight$], (grid, spotlight) => ({ - type: "grid", - spotlight: spotlight.some((vm) => vm instanceof ScreenShareViewModel) - ? spotlight - : undefined, - grid, - })); - - private readonly spotlightLandscapeLayoutMedia$: Observable = - combineLatest([this.grid$, this.spotlight$], (grid, spotlight) => ({ - type: "spotlight-landscape", - spotlight, - grid, - })); - - private readonly spotlightPortraitLayoutMedia$: Observable = - combineLatest([this.grid$, this.spotlight$], (grid, spotlight) => ({ - type: "spotlight-portrait", - spotlight, - grid, - })); - - private readonly spotlightExpandedLayoutMedia$: Observable = - combineLatest([this.spotlight$, this.pip$], (spotlight, pip) => ({ - type: "spotlight-expanded", - spotlight, - pip: pip ?? undefined, - })); - - private readonly oneOnOneLayoutMedia$: Observable = - this.mediaItems$.pipe( - map((mediaItems) => { - if (mediaItems.length !== 2) return null; - const local = mediaItems.find((vm) => vm.vm.local)?.vm as - | LocalUserMediaViewModel - | undefined; - const remote = mediaItems.find((vm) => !vm.vm.local)?.vm as - | RemoteUserMediaViewModel - | undefined; - // There might not be a remote tile if there are screen shares, or if - // only the local user is in the call and they're using the duplicate - // tiles option - if (!remote || !local) return null; - - return { type: "one-on-one", local, remote }; - }), - ); - - private readonly pipLayoutMedia$: Observable = - this.spotlight$.pipe(map((spotlight) => ({ type: "pip", spotlight }))); - - /** - * The media to be used to produce a layout. - */ - private readonly layoutMedia$ = this.scope.behavior( - this.windowMode$.pipe( - switchMap((windowMode) => { - switch (windowMode) { - case "normal": - return this.gridMode$.pipe( - switchMap((gridMode) => { - switch (gridMode) { - case "grid": - return this.oneOnOneLayoutMedia$.pipe( - switchMap((oneOnOne) => - oneOnOne === null - ? this.gridLayoutMedia$ - : of(oneOnOne), - ), - ); - case "spotlight": - return this.spotlightExpanded$.pipe( - switchMap((expanded) => - expanded - ? this.spotlightExpandedLayoutMedia$ - : this.spotlightLandscapeLayoutMedia$, - ), - ); - } - }), - ); - case "narrow": - return this.oneOnOneLayoutMedia$.pipe( - switchMap((oneOnOne) => - oneOnOne === null - ? combineLatest( - [this.grid$, this.spotlight$], - (grid, spotlight) => - grid.length > smallMobileCallThreshold || - spotlight.some( - (vm) => vm instanceof ScreenShareViewModel, - ) - ? this.spotlightPortraitLayoutMedia$ - : this.gridLayoutMedia$, - ).pipe(switchAll()) - : // The expanded spotlight layout makes for a better one-on-one - // experience in narrow windows - this.spotlightExpandedLayoutMedia$, - ), - ); - case "flat": - return this.gridMode$.pipe( - switchMap((gridMode) => { - switch (gridMode) { - case "grid": - // Yes, grid mode actually gets you a "spotlight" layout in - // this window mode. - return this.spotlightLandscapeLayoutMedia$; - case "spotlight": - return this.spotlightExpandedLayoutMedia$; - } - }), - ); - case "pip": - return this.pipLayoutMedia$; - } - }), - ), - ); - - // There is a cyclical dependency here: the layout algorithms want to know - // which tiles are on screen, but to know which tiles are on screen we have to - // first render a layout. To deal with this we assume initially that no tiles - // are visible, and loop the data back into the layouts with a Subject. - private readonly visibleTiles$ = new Subject(); - private readonly setVisibleTiles = (value: number): void => - this.visibleTiles$.next(value); - - private readonly layoutInternals$ = this.scope.behavior< - LayoutScanState & { layout: Layout } - >( - combineLatest([ - this.layoutMedia$, - this.visibleTiles$.pipe(startWith(0), distinctUntilChanged()), - ]).pipe( - scan< - [LayoutMedia, number], - LayoutScanState & { layout: Layout }, - LayoutScanState - >( - ({ tiles: prevTiles }, [media, visibleTiles]) => { - let layout: Layout; - let newTiles: TileStore; - switch (media.type) { - case "grid": - case "spotlight-landscape": - case "spotlight-portrait": - [layout, newTiles] = gridLikeLayout( - media, - visibleTiles, - this.setVisibleTiles, - prevTiles, - ); - break; - case "spotlight-expanded": - [layout, newTiles] = spotlightExpandedLayout(media, prevTiles); - break; - case "one-on-one": - [layout, newTiles] = oneOnOneLayout(media, prevTiles); - break; - case "pip": - [layout, newTiles] = pipLayout(media, prevTiles); - break; - } - - return { layout, tiles: newTiles }; - }, - { layout: null, tiles: TileStore.empty() }, - ), - ), - ); - - /** - * The layout of tiles in the call interface. - */ - public readonly layout$ = this.scope.behavior( - this.layoutInternals$.pipe(map(({ layout }) => layout)), - ); - - /** - * The current generation of the tile store, exposed for debugging purposes. - */ - public readonly tileStoreGeneration$ = this.scope.behavior( - this.layoutInternals$.pipe(map(({ tiles }) => tiles.generation)), - ); - - public showSpotlightIndicators$ = this.scope.behavior( - this.layout$.pipe(map((l) => l.type !== "grid")), - ); - - public showSpeakingIndicators$ = this.scope.behavior( - this.layout$.pipe( - switchMap((l) => { - switch (l.type) { - case "spotlight-landscape": - case "spotlight-portrait": - // If the spotlight is showing the active speaker, we can do without - // speaking indicators as they're a redundant visual cue. But if - // screen sharing feeds are in the spotlight we still need them. - return l.spotlight.media$.pipe( - map((models: MediaViewModel[]) => - models.some((m) => m instanceof ScreenShareViewModel), - ), - ); - // In expanded spotlight layout, the active speaker is always shown in - // the picture-in-picture tile so there is no need for speaking - // indicators. And in one-on-one layout there's no question as to who is - // speaking. - case "spotlight-expanded": - case "one-on-one": - return of(false); - default: - return of(true); - } - }), - ), - ); - - public readonly toggleSpotlightExpanded$ = this.scope.behavior< - (() => void) | null - >( - this.windowMode$.pipe( - switchMap((mode) => - mode === "normal" - ? this.layout$.pipe( - map( - (l) => - l.type === "spotlight-landscape" || - l.type === "spotlight-expanded", - ), - ) - : of(false), - ), - distinctUntilChanged(), - map((enabled) => - enabled ? (): void => this.spotlightExpandedToggle$.next() : null, - ), - ), - ); - - private readonly screenTap$ = new Subject(); - private readonly controlsTap$ = new Subject(); - private readonly screenHover$ = new Subject(); - private readonly screenUnhover$ = new Subject(); - - /** - * Callback for when the user taps the call view. - */ - public tapScreen(): void { - this.screenTap$.next(); - } - - /** - * Callback for when the user taps the call's controls. - */ - public tapControls(): void { - this.controlsTap$.next(); - } - - /** - * Callback for when the user hovers over the call view. - */ - public hoverScreen(): void { - this.screenHover$.next(); - } - - /** - * Callback for when the user stops hovering over the call view. - */ - public unhoverScreen(): void { - this.screenUnhover$.next(); - } - - public readonly showHeader$ = this.scope.behavior( - this.windowMode$.pipe(map((mode) => mode !== "pip" && mode !== "flat")), - ); - - public readonly showFooter$ = this.scope.behavior( - this.windowMode$.pipe( - switchMap((mode) => { - switch (mode) { - case "pip": - return of(false); - case "normal": - case "narrow": - return of(true); - case "flat": - // Sadly Firefox has some layering glitches that prevent the footer - // from appearing properly. They happen less often if we never hide - // the footer. - if (isFirefox()) return of(true); - // Show/hide the footer in response to interactions - return merge( - this.screenTap$.pipe(map(() => "tap screen" as const)), - this.controlsTap$.pipe(map(() => "tap controls" as const)), - this.screenHover$.pipe(map(() => "hover" as const)), - ).pipe( - switchScan((state, interaction) => { - switch (interaction) { - case "tap screen": - return state - ? // Toggle visibility on tap - of(false) - : // Hide after a timeout - timer(showFooterMs).pipe( - map(() => false), - startWith(true), - ); - case "tap controls": - // The user is interacting with things, so reset the timeout - return timer(showFooterMs).pipe( - map(() => false), - startWith(true), - ); - case "hover": - // Show on hover and hide after a timeout - return race( - timer(showFooterMs), - this.screenUnhover$.pipe(take(1)), - ).pipe( - map(() => false), - startWith(true), - ); - } - }, false), - startWith(false), - ); - } - }), - ), - ); - - /** - * Whether audio is currently being output through the earpiece. - */ - public readonly earpieceMode$ = this.scope.behavior( - combineLatest( - [ - this.mediaDevices.audioOutput.available$, - this.mediaDevices.audioOutput.selected$, - ], - (available, selected) => - selected !== undefined && - available.get(selected.id)?.type === "earpiece", - ), - ); - - /** - * Callback to toggle between the earpiece and the loudspeaker. - * - * This will be `null` in case the target does not exist in the list - * of available audio outputs. - */ - public readonly audioOutputSwitcher$ = this.scope.behavior<{ - targetOutput: "earpiece" | "speaker"; - switch: () => void; - } | null>( - combineLatest( - [ - this.mediaDevices.audioOutput.available$, - this.mediaDevices.audioOutput.selected$, - ], - (available, selected) => { - const selectionType = selected && available.get(selected.id)?.type; - - // If we are in any output mode other than speaker switch to speaker. - const newSelectionType: "earpiece" | "speaker" = - selectionType === "speaker" ? "earpiece" : "speaker"; - const newSelection = [...available].find( - ([, d]) => d.type === newSelectionType, - ); - if (newSelection === undefined) return null; - - const [id] = newSelection; - return { - targetOutput: newSelectionType, - switch: (): void => this.mediaDevices.audioOutput.select(id), - }; - }, - ), - ); - - /** - * Emits an array of reactions that should be visible on the screen. - */ - // DISCUSSION move this into a reaction file - // const {visibleReactions$, audibleReactions$} = reactionsObservables$(showReactionSetting$, ) - public readonly visibleReactions$ = this.scope.behavior( - showReactions.value$.pipe( - switchMap((show) => (show ? this.reactions$ : of({}))), - scan< - Record, - { sender: string; emoji: string; startX: number }[] - >((acc, latest) => { - const newSet: { sender: string; emoji: string; startX: number }[] = []; - for (const [sender, reaction] of Object.entries(latest)) { - const startX = - acc.find((v) => v.sender === sender && v.emoji)?.startX ?? - Math.ceil(Math.random() * 80) + 10; - newSet.push({ sender, emoji: reaction.emoji, startX }); - } - return newSet; - }, []), - ), - ); - - /** - * Emits an array of reactions that should be played. - */ - public readonly audibleReactions$ = playReactionsSound.value$.pipe( - switchMap((show) => - show ? this.reactions$ : of>({}), - ), - map((reactions) => Object.values(reactions).map((v) => v.name)), - scan( - (acc, latest) => { - return { - playing: latest.filter( - (v) => acc.playing.includes(v) || acc.newSounds.includes(v), - ), - newSounds: latest.filter( - (v) => !acc.playing.includes(v) && !acc.newSounds.includes(v), - ), - }; - }, - { playing: [], newSounds: [] }, - ), - map((v) => v.newSounds), - ); - - /** - * Emits an event every time a new hand is raised in - * the call. - */ - public readonly newHandRaised$ = this.handsRaised$.pipe( - map((v) => Object.keys(v).length), - scan( - (acc, newValue) => ({ - value: newValue, - playSounds: newValue > acc.value, - }), - { value: 0, playSounds: false }, - ), - filter((v) => v.playSounds), - ); - - /** - * Emits an event every time a new screenshare is started in - * the call. - */ - public readonly newScreenShare$ = this.screenShares$.pipe( - map((v) => v.length), - scan( - (acc, newValue) => ({ - value: newValue, - playSounds: newValue > acc.value, - }), - { value: 0, playSounds: false }, - ), - filter((v) => v.playSounds), - ); - - /** - * Whether we are sharing our screen. - */ - // reassigned here to make it publicly accessible - public readonly sharingScreen$ = this.localMembership.sharingScreen$; - - /** - * Callback to toggle screen sharing. If null, screen sharing is not possible. - */ - // reassigned here to make it publicly accessible - public readonly toggleScreenSharing = - this.localMembership.toggleScreenSharing; - - public constructor( - private readonly scope: ObservableScope, - // A call is permanently tied to a single Matrix room - private readonly matrixRTCSession: MatrixRTCSession, - private readonly matrixRoom: MatrixRoom, - private readonly mediaDevices: MediaDevices, - private readonly muteStates: MuteStates, - private readonly options: CallViewModelOptions, - private readonly handsRaisedSubject$: Observable< - Record - >, - private readonly reactionsSubject$: Observable< - Record - >, - private readonly trackProcessorState$: Behavior, - ) { - // Join automatically - this.join(); // TODO-MULTI-SFU: Use this view model for the lobby as well, and only call this once 'join' is clicked? - } -} - -// TODO-MULTI-SFU // Setup and update the keyProvider which was create by `createRoom` was a thing before. Now we never update if the E2EEsystem changes -// do we need this? - -function getE2eeKeyProvider( - e2eeSystem: EncryptionSystem, - rtcSession: MatrixRTCSession, -): BaseKeyProvider | undefined { - if (e2eeSystem.kind === E2eeType.NONE) return undefined; - - if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) { - const keyProvider = new MatrixKeyProvider(); - keyProvider.setRTCSession(rtcSession); - return keyProvider; - } else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) { - const keyProvider = new ExternalE2EEKeyProvider(); - keyProvider - .setKey(e2eeSystem.secret) - .catch((e) => logger.error("Failed to set shared key for E2EE", e)); - return keyProvider; - } -} From e9f400e9f15c568080331ee7399a248499775383 Mon Sep 17 00:00:00 2001 From: Timo K Date: Mon, 17 Nov 2025 14:30:16 +0100 Subject: [PATCH 3/5] remove ReturnType --- src/state/CallViewModel/CallViewModel.ts | 11 +++++------ .../CallViewModel/localMember/LocalMembership.ts | 1 + 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index ff2ed0cb..0fd71d3d 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -93,7 +93,10 @@ import { } from "../layout-types.ts"; import { type ElementCallError } from "../../utils/errors.ts"; import { type ObservableScope } from "../ObservableScope.ts"; -import { createLocalMembership$ } from "./localMember/LocalMembership.ts"; +import { + createLocalMembership$, + type LocalMemberConnectionState, +} from "./localMember/LocalMembership.ts"; import { createLocalTransport$ } from "./localMember/LocalTransport.ts"; import { createMemberships$, @@ -167,10 +170,6 @@ type AudioLivekitItem = { url: string; }; -type JoinReturn = ReturnType< - ReturnType["requestConnect"] ->; - /** * A view model providing all the application logic needed to show the in-call * UI (may eventually be expanded to cover the lobby and feedback screens in the @@ -189,7 +188,7 @@ export class CallViewModel { public hangup: () => void; // joining - public join: () => JoinReturn; + public join: () => LocalMemberConnectionState; // screen sharing public toggleScreenSharing: (() => void) | null; diff --git a/src/state/CallViewModel/localMember/LocalMembership.ts b/src/state/CallViewModel/localMember/LocalMembership.ts index ae6c1c7c..d9a3887b 100644 --- a/src/state/CallViewModel/localMember/LocalMembership.ts +++ b/src/state/CallViewModel/localMember/LocalMembership.ts @@ -335,6 +335,7 @@ export const createLocalMembership$ = ({ }); combineLatest([localTransport$, connectRequested$]).subscribe( + // TODO reconnect on options change. ([transport, connectRequested]) => { if ( transport === null || From 4414fe3c8243021f733c5265c1fde1551e32a75b Mon Sep 17 00:00:00 2001 From: Timo K Date: Mon, 17 Nov 2025 14:39:24 +0100 Subject: [PATCH 4/5] remove all `/*PUBLIC*/` --- src/state/CallViewModel/CallViewModel.ts | 44 +++++++++++------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 0fd71d3d..3e3d8190 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -469,10 +469,10 @@ export class CallViewModel { // down, for example, and we want to avoid making people worry that the app is // in a split-brained state. // DISCUSSION own membership manager ALSO this probably can be simplifis - /*PUBLIC*/ const reconnecting$ = localMembership.reconnecting$; + const reconnecting$ = localMembership.reconnecting$; const pretendToBeDisconnected$ = reconnecting$; - /*PUBLIC*/ const audioParticipants$ = scope.behavior( + const audioParticipants$ = scope.behavior( matrixLivekitMembers$.pipe( switchMap((membersWithEpoch) => { const members = membersWithEpoch.value; @@ -518,11 +518,11 @@ export class CallViewModel { [], ); - /*PUBLIC*/ const handsRaised$ = scope.behavior( + const handsRaised$ = scope.behavior( handsRaisedSubject$.pipe(pauseWhen(pretendToBeDisconnected$)), ); - /*PUBLIC*/ const reactions$ = scope.behavior( + const reactions$ = scope.behavior( reactionsSubject$.pipe( map((v) => Object.fromEntries( @@ -663,7 +663,7 @@ export class CallViewModel { ), ); - /*PUBLIC*/ const joinSoundEffect$ = userMedia$.pipe( + const joinSoundEffect$ = userMedia$.pipe( pairwise(), filter( ([prev, current]) => @@ -681,16 +681,16 @@ export class CallViewModel { * - There can be multiple participants for one Matrix user if they join from * multiple devices. */ - /*PUBLIC*/ const participantCount$ = scope.behavior( + const participantCount$ = scope.behavior( matrixLivekitMembers$.pipe(map((ms) => ms.value.length)), ); // only public to expose to the view. // TODO if we are in "unknown" state we need a loading rendering (or empty screen) // Otherwise it looks like we already connected and only than the ringing starts which is weird. - /*PUBLIC*/ const callPickupState$ = callLifecycle.callPickupState$; + const callPickupState$ = callLifecycle.callPickupState$; - /*PUBLIC*/ const leaveSoundEffect$ = combineLatest([ + const leaveSoundEffect$ = combineLatest([ callLifecycle.callPickupState$, userMedia$, ]).pipe( @@ -871,14 +871,14 @@ export class CallViewModel { /** * The general shape of the window. */ - /*PUBLIC*/ const windowMode$ = scope.behavior( + const windowMode$ = scope.behavior( pipEnabled$.pipe( switchMap((pip) => (pip ? of("pip") : naturalWindowMode$)), ), ); const spotlightExpandedToggle$ = new Subject(); - /*PUBLIC*/ const spotlightExpanded$ = scope.behavior( + const spotlightExpanded$ = scope.behavior( spotlightExpandedToggle$.pipe(accumulate(false, (expanded) => !expanded)), ); @@ -886,7 +886,7 @@ export class CallViewModel { /** * The layout mode of the media tile grid. */ - /*PUBLIC*/ const gridMode$ = + const gridMode$ = // If the user hasn't selected spotlight and somebody starts screen sharing, // automatically switch to spotlight mode and reset when screen sharing ends scope.behavior( @@ -909,7 +909,7 @@ export class CallViewModel { "grid", ); - /*PUBLIC*/ const setGridMode = (value: GridMode): void => { + const setGridMode = (value: GridMode): void => { gridModeUserSelection$.next(value); }; @@ -1087,22 +1087,22 @@ export class CallViewModel { /** * The layout of tiles in the call interface. */ - /*PUBLIC*/ const layout$ = scope.behavior( + const layout$ = scope.behavior( layoutInternals$.pipe(map(({ layout }) => layout)), ); /** * The current generation of the tile store, exposed for debugging purposes. */ - /*PUBLIC*/ const tileStoreGeneration$ = scope.behavior( + const tileStoreGeneration$ = scope.behavior( layoutInternals$.pipe(map(({ tiles }) => tiles.generation)), ); - /*PUBLIC*/ const showSpotlightIndicators$ = scope.behavior( + const showSpotlightIndicators$ = scope.behavior( layout$.pipe(map((l) => l.type !== "grid")), ); - /*PUBLIC*/ const showSpeakingIndicators$ = scope.behavior( + const showSpeakingIndicators$ = scope.behavior( layout$.pipe( switchMap((l) => { switch (l.type) { @@ -1130,9 +1130,7 @@ export class CallViewModel { ), ); - /*PUBLIC*/ const toggleSpotlightExpanded$ = scope.behavior< - (() => void) | null - >( + const toggleSpotlightExpanded$ = scope.behavior<(() => void) | null>( windowMode$.pipe( switchMap((mode) => mode === "normal" @@ -1157,11 +1155,11 @@ export class CallViewModel { const screenHover$ = new Subject(); const screenUnhover$ = new Subject(); - /*PUBLIC*/ const showHeader$ = scope.behavior( + const showHeader$ = scope.behavior( windowMode$.pipe(map((mode) => mode !== "pip" && mode !== "flat")), ); - /*PUBLIC*/ const showFooter$ = scope.behavior( + const showFooter$ = scope.behavior( windowMode$.pipe( switchMap((mode) => { switch (mode) { @@ -1219,7 +1217,7 @@ export class CallViewModel { /** * Whether audio is currently being output through the earpiece. */ - /*PUBLIC*/ const earpieceMode$ = scope.behavior( + const earpieceMode$ = scope.behavior( combineLatest( [ mediaDevices.audioOutput.available$, @@ -1237,7 +1235,7 @@ export class CallViewModel { * This will be `null` in case the target does not exist in the list * of available audio outputs. */ - /*PUBLIC*/ const audioOutputSwitcher$ = scope.behavior<{ + const audioOutputSwitcher$ = scope.behavior<{ targetOutput: "earpiece" | "speaker"; switch: () => void; } | null>( From f518111887048e382cdf92fdaf95d8cd6434b692 Mon Sep 17 00:00:00 2001 From: Timo K Date: Mon, 17 Nov 2025 14:55:00 +0100 Subject: [PATCH 5/5] add back docstrings --- src/reactions/ReactionsReader.ts | 1 + src/state/CallViewModel/CallViewModel.ts | 78 ++++++++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/src/reactions/ReactionsReader.ts b/src/reactions/ReactionsReader.ts index c1f78b51..74b47c77 100644 --- a/src/reactions/ReactionsReader.ts +++ b/src/reactions/ReactionsReader.ts @@ -266,6 +266,7 @@ export class ReactionsReader { ); return; } + // TODO refactor to use memer id `membershipEvent.membershipID` (needs to happen in combination with other memberId refactors) const identifier = `${membershipEvent.userId}:${membershipEvent.deviceId}`; if (!content.emoji) { diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 3e3d8190..063a953e 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -181,32 +181,68 @@ type AudioLivekitItem = { export class CallViewModel { // lifecycle public autoLeave$: Observable; + // TODO if we are in "unknown" state we need a loading rendering (or empty screen) + // Otherwise it looks like we already connected and only than the ringing starts which is weird. public callPickupState$: Behavior< "unknown" | "ringing" | "timeout" | "decline" | "success" | null >; public leave$: Observable<"user" | AutoLeaveReason>; + /** Call to initiate hangup. Use in conbination with connectino state track the async hangup process. */ public hangup: () => void; // joining public join: () => LocalMemberConnectionState; // screen sharing + /** + * Callback to toggle screen sharing. If null, screen sharing is not possible. + */ public toggleScreenSharing: (() => void) | null; + /** + * Whether we are sharing our screen. + */ public sharingScreen$: Behavior; // UI interactions + /** + * Callback for when the user taps the call view. + */ public tapScreen: () => void; + /** + * Callback for when the user taps the call's controls. + */ public tapControls: () => void; + /** + * Callback for when the user hovers over the call view. + */ public hoverScreen: () => void; + /** + * Callback for when the user stops hovering over the call view. + */ public unhoverScreen: () => void; // errors + /** + * If there is a configuration error with the call (e.g. misconfigured E2EE). + * This is a fatal error that prevents the call from being created/joined. + * Should render a blocking error screen. + */ public configError$: Behavior; // participants and counts + /** + * The number of participants currently in the call. + * + * - Each participant has a corresponding MatrixRTC membership state event + * - There can be multiple participants for one Matrix user if they join from + * multiple devices. + */ public participantCount$: Behavior; + /** Participants sorted by livekit room so they can be used in the audio rendering */ public audioParticipants$: Behavior; + /** List of participants raising their hand */ public handsRaised$: Behavior>; + /** List of reactions. Keys are: membership.membershipId (currently predefined as: `${membershipEvent.userId}:${membershipEvent.deviceId}`)*/ public reactions$: Behavior>; public isOneOnOneWith$: Behavior; public leaveSoundEffect$: Observable; + /** + * Emits an event every time a new hand is raised in + * the call. + */ public newHandRaised$: Observable<{ value: number; playSounds: boolean }>; + /** + * Emits an event every time a new screenshare is started in + * the call. + */ public newScreenShare$: Observable<{ value: number; playSounds: boolean }>; + /** + * Emits an array of reactions that should be played. + */ public audibleReactions$: Observable; + /** + * Emits an array of reactions that should be visible on the screen. + */ + // DISCUSSION move this into a reaction file public visibleReactions$: Behavior< { sender: string; emoji: string; startX: number }[] >; // window/layout + /** + * The general shape of the window. + */ public windowMode$: Behavior; public spotlightExpanded$: Behavior; public toggleSpotlightExpanded$: Behavior<(() => void) | null>; @@ -234,7 +288,13 @@ export class CallViewModel { public grid$: Behavior; public spotlight$: Behavior; public pip$: Behavior; + /** + * The layout of tiles in the call interface. + */ public layout$: Behavior; + /** + * The current generation of the tile store, exposed for debugging purposes. + */ public tileStoreGeneration$: Behavior; public showSpotlightIndicators$: Behavior; public showSpeakingIndicators$: Behavior; @@ -244,13 +304,31 @@ export class CallViewModel { public showFooter$: Behavior; // audio routing + /** + * Whether audio is currently being output through the earpiece. + */ public earpieceMode$: Behavior; + /** + * Callback to toggle between the earpiece and the loudspeaker. + * + * This will be `null` in case the target does not exist in the list + * of available audio outputs. + */ public audioOutputSwitcher$: Behavior<{ targetOutput: "earpiece" | "speaker"; switch: () => void; } | null>; // connection state + /** + * Whether various media/event sources should pretend to be disconnected from + * all network input, even if their connection still technically works. + */ + // We do this when the app is in the 'reconnecting' state, because it might be + // that the LiveKit connection is still functional while the homeserver is + // down, for example, and we want to avoid making people worry that the app is + // in a split-brained state. + // DISCUSSION own membership manager ALSO this probably can be simplifis public reconnecting$: Behavior; // THIS has to be the last public field declaration