diff --git a/src/reactions/ReactionsReader.ts b/src/reactions/ReactionsReader.ts index c1f78b51..74b47c77 100644 --- a/src/reactions/ReactionsReader.ts +++ b/src/reactions/ReactionsReader.ts @@ -266,6 +266,7 @@ export class ReactionsReader { ); return; } + // TODO refactor to use memer id `membershipEvent.membershipID` (needs to happen in combination with other memberId refactors) const identifier = `${membershipEvent.userId}:${membershipEvent.deviceId}`; if (!content.emoji) { diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index ff4a6269..063a953e 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -12,7 +12,7 @@ import { type Room as LivekitRoom, type RoomOptions, } from "livekit-client"; -import { type Room as MatrixRoom } from "matrix-js-sdk"; +import { type RoomMember, type Room as MatrixRoom } from "matrix-js-sdk"; import { combineLatest, distinctUntilChanged, @@ -93,7 +93,10 @@ import { } from "../layout-types.ts"; import { type ElementCallError } from "../../utils/errors.ts"; import { type ObservableScope } from "../ObservableScope.ts"; -import { createLocalMembership$ } from "./localMember/LocalMembership.ts"; +import { + createLocalMembership$, + type LocalMemberConnectionState, +} from "./localMember/LocalMembership.ts"; import { createLocalTransport$ } from "./localMember/LocalTransport.ts"; import { createMemberships$, @@ -106,6 +109,7 @@ import { type MatrixLivekitMember, } from "./remoteMembers/MatrixLivekitMembers.ts"; import { + type AutoLeaveReason, createCallNotificationLifecycle$, createReceivedDecline$, createSentCallNotification$, @@ -165,6 +169,7 @@ type AudioLivekitItem = { participants: string[]; url: string; }; + /** * A view model providing all the application logic needed to show the in-call * UI (may eventually be expanded to cover the lobby and feedback screens in the @@ -174,185 +179,147 @@ type AudioLivekitItem = { // state and LiveKit state. We use the common terminology of room "members", RTC // "memberships", and LiveKit "participants". export class CallViewModel { - private readonly userId = this.matrixRoom.client.getUserId()!; - private readonly deviceId = this.matrixRoom.client.getDeviceId()!; + // lifecycle + public autoLeave$: Observable; + // TODO if we are in "unknown" state we need a loading rendering (or empty screen) + // Otherwise it looks like we already connected and only than the ringing starts which is weird. + public callPickupState$: Behavior< + "unknown" | "ringing" | "timeout" | "decline" | "success" | null + >; + public leave$: Observable<"user" | AutoLeaveReason>; + /** Call to initiate hangup. Use in conbination with connectino state track the async hangup process. */ + public hangup: () => void; - private readonly livekitKeyProvider = getE2eeKeyProvider( - this.options.encryptionSystem, - this.matrixRTCSession, - ); + // joining + public join: () => LocalMemberConnectionState; - // Each hbar seperates a block of input variables required for the CallViewModel to function. - // The outputs of this block is written under the hbar. - // - // For mocking purposes it is recommended to only mock the functions creating those outputs. - // All other fields are just temp computations for the mentioned output. - // The class does not need anything except the values underneath the bar. - // The creation of the values under the bar are all tested independently and testing the callViewModel Should - // not test their cretation. Call view model only needs: - // - memberships$ via createMemberships$ - // - localMembership via createLocalMembership$ - // - callLifecycle via createCallNotificationLifecycle$ - // - matrixMemberMetadataStore via createMatrixMemberMetadata$ + // screen sharing + /** + * Callback to toggle screen sharing. If null, screen sharing is not possible. + */ + public toggleScreenSharing: (() => void) | null; + /** + * Whether we are sharing our screen. + */ + public sharingScreen$: Behavior; - // ------------------------------------------------------------------------ - // memberships$ - private memberships$ = createMemberships$(this.scope, this.matrixRTCSession); - - // ------------------------------------------------------------------------ - // matrixLivekitMembers$ AND localMembership - - private membershipsAndTransports = membershipsAndTransports$( - this.scope, - this.memberships$, - ); - - private localTransport$ = createLocalTransport$({ - scope: this.scope, - memberships$: this.memberships$, - client: this.matrixRoom.client, - roomId: this.matrixRoom.roomId, - useOldestMember$: this.scope.behavior( - matrixRTCMode.value$.pipe(map((v) => v === MatrixRTCMode.Legacy)), - ), - }); - - private connectionFactory = new ECConnectionFactory( - this.matrixRoom.client, - this.mediaDevices, - this.trackProcessorState$, - this.livekitKeyProvider, - getUrlParams().controlledAudioDevices, - this.options.livekitRoomFactory, - ); - - private connectionManager = createConnectionManager$({ - scope: this.scope, - connectionFactory: this.connectionFactory, - inputTransports$: this.scope.behavior( - combineLatest( - [this.localTransport$, this.membershipsAndTransports.transports$], - (localTransport, transports) => { - const localTransportAsArray = localTransport ? [localTransport] : []; - return transports.mapInner((transports) => [ - ...localTransportAsArray, - ...transports, - ]); - }, - ), - ), - logger: logger, - }); - - private matrixLivekitMembers$ = createMatrixLivekitMembers$({ - scope: this.scope, - membershipsWithTransport$: - this.membershipsAndTransports.membershipsWithTransport$, - connectionManager: this.connectionManager, - }); - - private connectOptions$ = this.scope.behavior( - matrixRTCMode.value$.pipe( - map((mode) => ({ - encryptMedia: this.livekitKeyProvider !== undefined, - // TODO. This might need to get called again on each cahnge of matrixRTCMode... - matrixRTCMode: mode, - })), - ), - ); - - private localMembership = createLocalMembership$({ - scope: this.scope, - muteStates: this.muteStates, - mediaDevices: this.mediaDevices, - connectionManager: this.connectionManager, - matrixRTCSession: this.matrixRTCSession, - matrixRoom: this.matrixRoom, - localTransport$: this.localTransport$, - trackProcessorState$: this.trackProcessorState$, - widget, - options: this.connectOptions$, - logger: logger.getChild(`[${Date.now()}]`), - }); - - private localRtcMembership$ = this.scope.behavior( - this.memberships$.pipe( - map( - (memberships) => - memberships.value.find( - (membership) => - membership.userId === this.userId && - membership.deviceId === this.deviceId, - ) ?? null, - ), - ), - ); - - private localMatrixLivekitMemberUninitialized = { - membership$: this.localRtcMembership$, - participant$: this.localMembership.participant$, - connection$: this.localMembership.connection$, - userId: this.userId, - }; - - private localMatrixLivekitMember$: Behavior = - this.scope.behavior( - this.localRtcMembership$.pipe( - switchMap((membership) => { - if (!membership) return of(null); - return of( - // casting is save here since we know that localRtcMembership$ is !== null since we reached this case. - this.localMatrixLivekitMemberUninitialized as MatrixLivekitMember, - ); - }), - ), - ); - - // ------------------------------------------------------------------------ - // callLifecycle - - private callLifecycle = createCallNotificationLifecycle$({ - scope: this.scope, - memberships$: this.memberships$, - sentCallNotification$: createSentCallNotification$( - this.scope, - this.matrixRTCSession, - ), - receivedDecline$: createReceivedDecline$(this.matrixRoom), - options: this.options, - localUser: { userId: this.userId, deviceId: this.deviceId }, - }); - public autoLeave$ = this.callLifecycle.autoLeave$; - - // ------------------------------------------------------------------------ - // matrixMemberMetadataStore - - private matrixMemberMetadataStore = createMatrixMemberMetadata$( - this.scope, - this.scope.behavior(this.memberships$.pipe(map((mems) => mems.value))), - createRoomMembers$(this.scope, this.matrixRoom), - ); + // UI interactions + /** + * Callback for when the user taps the call view. + */ + public tapScreen: () => void; + /** + * Callback for when the user taps the call's controls. + */ + public tapControls: () => void; + /** + * Callback for when the user hovers over the call view. + */ + public hoverScreen: () => void; + /** + * Callback for when the user stops hovering over the call view. + */ + public unhoverScreen: () => void; + // errors /** * If there is a configuration error with the call (e.g. misconfigured E2EE). * This is a fatal error that prevents the call from being created/joined. * Should render a blocking error screen. */ - public get configError$(): Behavior { - return this.localMembership.configError$; - } + public configError$: Behavior; - public join = this.localMembership.requestConnect; + // participants and counts + /** + * The number of participants currently in the call. + * + * - Each participant has a corresponding MatrixRTC membership state event + * - There can be multiple participants for one Matrix user if they join from + * multiple devices. + */ + public participantCount$: Behavior; + /** Participants sorted by livekit room so they can be used in the audio rendering */ + public audioParticipants$: Behavior; + /** List of participants raising their hand */ + public handsRaised$: Behavior>; + /** List of reactions. Keys are: membership.membershipId (currently predefined as: `${membershipEvent.userId}:${membershipEvent.deviceId}`)*/ + public reactions$: Behavior>; + public isOneOnOneWith$: Behavior | null>; + public localUserIsAlone$: Behavior; + // sounds and events + public joinSoundEffect$: Observable; + public leaveSoundEffect$: Observable; + /** + * Emits an event every time a new hand is raised in + * the call. + */ + public newHandRaised$: Observable<{ value: number; playSounds: boolean }>; + /** + * Emits an event every time a new screenshare is started in + * the call. + */ + public newScreenShare$: Observable<{ value: number; playSounds: boolean }>; + /** + * Emits an array of reactions that should be played. + */ + public audibleReactions$: Observable; + /** + * Emits an array of reactions that should be visible on the screen. + */ + // DISCUSSION move this into a reaction file + public visibleReactions$: Behavior< + { sender: string; emoji: string; startX: number }[] + >; - // CODESMELL? - // This is functionally the same Observable as leave$, except here it's - // hoisted to the top of the class. This enables the cyclic dependency between - // leave$ -> autoLeave$ -> callPickupState$ -> livekitConnectionState$ -> - // localConnection$ -> transports$ -> joined$ -> leave$. - private readonly leaveHoisted$ = new Subject< - "user" | "timeout" | "decline" | "allOthersLeft" - >(); + // window/layout + /** + * The general shape of the window. + */ + public windowMode$: Behavior; + public spotlightExpanded$: Behavior; + public toggleSpotlightExpanded$: Behavior<(() => void) | null>; + public gridMode$: Behavior; + public setGridMode: (value: GridMode) => void; + // media view models and layout + public grid$: Behavior; + public spotlight$: Behavior; + public pip$: Behavior; + /** + * The layout of tiles in the call interface. + */ + public layout$: Behavior; + /** + * The current generation of the tile store, exposed for debugging purposes. + */ + public tileStoreGeneration$: Behavior; + public showSpotlightIndicators$: Behavior; + public showSpeakingIndicators$: Behavior; + + // header/footer visibility + public showHeader$: Behavior; + public showFooter$: Behavior; + + // audio routing + /** + * Whether audio is currently being output through the earpiece. + */ + public earpieceMode$: Behavior; + /** + * Callback to toggle between the earpiece and the loudspeaker. + * + * This will be `null` in case the target does not exist in the list + * of available audio outputs. + */ + public audioOutputSwitcher$: Behavior<{ + targetOutput: "earpiece" | "speaker"; + switch: () => void; + } | null>; + + // connection state /** * Whether various media/event sources should pretend to be disconnected from * all network input, even if their connection still technically works. @@ -362,275 +329,490 @@ export class CallViewModel { // down, for example, and we want to avoid making people worry that the app is // in a split-brained state. // DISCUSSION own membership manager ALSO this probably can be simplifis - public reconnecting$ = this.localMembership.reconnecting$; - private readonly pretendToBeDisconnected$ = this.reconnecting$; + public reconnecting$: Behavior; - public readonly audioParticipants$ = this.scope.behavior( - this.matrixLivekitMembers$.pipe( - switchMap((membersWithEpoch) => { - const members = membersWithEpoch.value; - const a$ = combineLatest( - members.map((member) => - combineLatest([member.connection$, member.participant$]).pipe( - map(([connection, participant]) => { - // do not render audio for local participant - if (!connection || !participant || participant.isLocal) - return null; - const livekitRoom = connection.livekitRoom; - const url = connection.transport.livekit_service_url; + // THIS has to be the last public field declaration + public constructor( + scope: ObservableScope, + // A call is permanently tied to a single Matrix room + matrixRTCSession: MatrixRTCSession, + matrixRoom: MatrixRoom, + mediaDevices: MediaDevices, + muteStates: MuteStates, + options: CallViewModelOptions, + handsRaisedSubject$: Observable>, + reactionsSubject$: Observable>, + trackProcessorState$: Behavior, + ) { + const userId = matrixRoom.client.getUserId()!; + const deviceId = matrixRoom.client.getDeviceId()!; - return { url, livekitRoom, participant: participant.identity }; - }), - ), - ), - ); - return a$; - }), - map((members) => - members.reduce((acc, curr) => { - if (!curr) return acc; + const livekitKeyProvider = getE2eeKeyProvider( + options.encryptionSystem, + matrixRTCSession, + ); - const existing = acc.find((item) => item.url === curr.url); - if (existing) { - existing.participants.push(curr.participant); - } else { - acc.push({ - livekitRoom: curr.livekitRoom, - participants: [curr.participant], - url: curr.url, - }); - } - return acc; - }, []), + // Each hbar seperates a block of input variables required for the CallViewModel to function. + // The outputs of this block is written under the hbar. + // + // For mocking purposes it is recommended to only mock the functions creating those outputs. + // All other fields are just temp computations for the mentioned output. + // The class does not need anything except the values underneath the bar. + // The creation of the values under the bar are all tested independently and testing the callViewModel Should + // not test their cretation. Call view model only needs: + // - memberships$ via createMemberships$ + // - localMembership via createLocalMembership$ + // - callLifecycle via createCallNotificationLifecycle$ + // - matrixMemberMetadataStore via createMatrixMemberMetadata$ + + // ------------------------------------------------------------------------ + // memberships$ + const memberships$ = createMemberships$(scope, matrixRTCSession); + + // ------------------------------------------------------------------------ + // matrixLivekitMembers$ AND localMembership + + const membershipsAndTransports = membershipsAndTransports$( + scope, + memberships$, + ); + + const localTransport$ = createLocalTransport$({ + scope: scope, + memberships$: memberships$, + client: matrixRoom.client, + roomId: matrixRoom.roomId, + useOldestMember$: scope.behavior( + matrixRTCMode.value$.pipe(map((v) => v === MatrixRTCMode.Legacy)), ), - ), - [], - ); + }); - public readonly handsRaised$ = this.scope.behavior( - this.handsRaisedSubject$.pipe(pauseWhen(this.pretendToBeDisconnected$)), - ); + const connectionFactory = new ECConnectionFactory( + matrixRoom.client, + mediaDevices, + trackProcessorState$, + livekitKeyProvider, + getUrlParams().controlledAudioDevices, + options.livekitRoomFactory, + ); - public readonly reactions$ = this.scope.behavior( - this.reactionsSubject$.pipe( - map((v) => - Object.fromEntries( - Object.entries(v).map(([a, { reactionOption }]) => [ - a, - reactionOption, - ]), + const connectionManager = createConnectionManager$({ + scope: scope, + connectionFactory: connectionFactory, + inputTransports$: scope.behavior( + combineLatest( + [localTransport$, membershipsAndTransports.transports$], + (localTransport, transports) => { + const localTransportAsArray = localTransport + ? [localTransport] + : []; + return transports.mapInner((transports) => [ + ...localTransportAsArray, + ...transports, + ]); + }, ), ), - pauseWhen(this.pretendToBeDisconnected$), - ), - ); + logger: logger, + }); - /** - * List of user media (camera feeds) that we want tiles for. - */ - // TODO this also needs the local participant to be added. - private readonly userMedia$ = this.scope.behavior( - combineLatest([ - this.localMatrixLivekitMember$, - this.matrixLivekitMembers$, - duplicateTiles.value$, - ]).pipe( - // Generate a collection of MediaItems from the list of expected (whether - // present or missing) LiveKit participants. - generateItems( - function* ([ - localMatrixLivekitMember, - { value: matrixLivekitMembers }, - duplicateTiles, - ]) { - let localParticipantId = undefined; - // add local member if available - if (localMatrixLivekitMember) { - const { userId, participant$, connection$, membership$ } = - localMatrixLivekitMember; - localParticipantId = `${userId}:${membership$.value.deviceId}`; // should be membership$.value.membershipID which is not optional - // const participantId = membership$.value.membershipID; - if (localParticipantId) { + const matrixLivekitMembers$ = createMatrixLivekitMembers$({ + scope: scope, + membershipsWithTransport$: + membershipsAndTransports.membershipsWithTransport$, + connectionManager: connectionManager, + }); + + const connectOptions$ = scope.behavior( + matrixRTCMode.value$.pipe( + map((mode) => ({ + encryptMedia: livekitKeyProvider !== undefined, + // TODO. This might need to get called again on each cahnge of matrixRTCMode... + matrixRTCMode: mode, + })), + ), + ); + + const localMembership = createLocalMembership$({ + scope: scope, + muteStates: muteStates, + mediaDevices: mediaDevices, + connectionManager: connectionManager, + matrixRTCSession: matrixRTCSession, + matrixRoom: matrixRoom, + localTransport$: localTransport$, + trackProcessorState$: trackProcessorState$, + widget, + options: connectOptions$, + logger: logger.getChild(`[${Date.now()}]`), + }); + + const localRtcMembership$ = scope.behavior( + memberships$.pipe( + map( + (memberships) => + memberships.value.find( + (membership) => + membership.userId === userId && + membership.deviceId === deviceId, + ) ?? null, + ), + ), + ); + + const localMatrixLivekitMemberUninitialized = { + membership$: localRtcMembership$, + participant$: localMembership.participant$, + connection$: localMembership.connection$, + userId: userId, + }; + + const localMatrixLivekitMember$: Behavior = + scope.behavior( + localRtcMembership$.pipe( + switchMap((membership) => { + if (!membership) return of(null); + return of( + // casting is save here since we know that localRtcMembership$ is !== null since we reached this case. + localMatrixLivekitMemberUninitialized as MatrixLivekitMember, + ); + }), + ), + ); + + // ------------------------------------------------------------------------ + // callLifecycle + + const callLifecycle = createCallNotificationLifecycle$({ + scope: scope, + memberships$: memberships$, + sentCallNotification$: createSentCallNotification$( + scope, + matrixRTCSession, + ), + receivedDecline$: createReceivedDecline$(matrixRoom), + options: options, + localUser: { userId: userId, deviceId: deviceId }, + }); + + // ------------------------------------------------------------------------ + // matrixMemberMetadataStore + + const matrixRoomMembers$ = createRoomMembers$(scope, matrixRoom); + const matrixMemberMetadataStore = createMatrixMemberMetadata$( + scope, + scope.behavior(memberships$.pipe(map((mems) => mems.value))), + matrixRoomMembers$, + ); + + /** + * Returns the Member {userId, getMxcAvatarUrl, rawDisplayName} of the other user in the call, if it's a one-on-one call. + */ + const isOneOnOneWith$ = scope.behavior( + matrixRoomMembers$.pipe( + map((roomMembersMap) => { + const otherMembers = Array.from(roomMembersMap.values()).filter( + (member) => member.userId !== userId, + ); + return otherMembers.length === 1 ? otherMembers[0] : null; + }), + ), + ); + + const localUserIsAlone$ = scope.behavior( + matrixRoomMembers$.pipe( + map( + (roomMembersMap) => + roomMembersMap.size === 1 && + roomMembersMap.get(userId) !== undefined, + ), + ), + ); + + // CODESMELL? + // This is functionally the same Observable as leave$, except here it's + // hoisted to the top of the class. This enables the cyclic dependency between + // leave$ -> autoLeave$ -> callPickupState$ -> livekitConnectionState$ -> + // localConnection$ -> transports$ -> joined$ -> leave$. + const leaveHoisted$ = new Subject< + "user" | "timeout" | "decline" | "allOthersLeft" + >(); + + /** + * Whether various media/event sources should pretend to be disconnected from + * all network input, even if their connection still technically works. + */ + // We do this when the app is in the 'reconnecting' state, because it might be + // that the LiveKit connection is still functional while the homeserver is + // down, for example, and we want to avoid making people worry that the app is + // in a split-brained state. + // DISCUSSION own membership manager ALSO this probably can be simplifis + const reconnecting$ = localMembership.reconnecting$; + const pretendToBeDisconnected$ = reconnecting$; + + const audioParticipants$ = scope.behavior( + matrixLivekitMembers$.pipe( + switchMap((membersWithEpoch) => { + const members = membersWithEpoch.value; + const a$ = combineLatest( + members.map((member) => + combineLatest([member.connection$, member.participant$]).pipe( + map(([connection, participant]) => { + // do not render audio for local participant + if (!connection || !participant || participant.isLocal) + return null; + const livekitRoom = connection.livekitRoom; + const url = connection.transport.livekit_service_url; + + return { + url, + livekitRoom, + participant: participant.identity, + }; + }), + ), + ), + ); + return a$; + }), + map((members) => + members.reduce((acc, curr) => { + if (!curr) return acc; + + const existing = acc.find((item) => item.url === curr.url); + if (existing) { + existing.participants.push(curr.participant); + } else { + acc.push({ + livekitRoom: curr.livekitRoom, + participants: [curr.participant], + url: curr.url, + }); + } + return acc; + }, []), + ), + ), + [], + ); + + const handsRaised$ = scope.behavior( + handsRaisedSubject$.pipe(pauseWhen(pretendToBeDisconnected$)), + ); + + const reactions$ = scope.behavior( + reactionsSubject$.pipe( + map((v) => + Object.fromEntries( + Object.entries(v).map(([a, { reactionOption }]) => [ + a, + reactionOption, + ]), + ), + ), + pauseWhen(pretendToBeDisconnected$), + ), + ); + + /** + * List of user media (camera feeds) that we want tiles for. + */ + // TODO this also needs the local participant to be added. + const userMedia$ = scope.behavior( + combineLatest([ + localMatrixLivekitMember$, + matrixLivekitMembers$, + duplicateTiles.value$, + ]).pipe( + // Generate a collection of MediaItems from the list of expected (whether + // present or missing) LiveKit participants. + generateItems( + function* ([ + localMatrixLivekitMember, + { value: matrixLivekitMembers }, + duplicateTiles, + ]) { + let localParticipantId = undefined; + // add local member if available + if (localMatrixLivekitMember) { + const { userId, participant$, connection$, membership$ } = + localMatrixLivekitMember; + localParticipantId = `${userId}:${membership$.value.deviceId}`; // should be membership$.value.membershipID which is not optional + // const participantId = membership$.value.membershipID; + if (localParticipantId) { + for (let dup = 0; dup < 1 + duplicateTiles; dup++) { + yield { + keys: [ + dup, + localParticipantId, + userId, + participant$, + connection$, + ], + data: undefined, + }; + } + } + } + // add remote members that are available + for (const { + userId, + participant$, + connection$, + membership$, + } of matrixLivekitMembers) { + const participantId = `${userId}:${membership$.value.deviceId}`; + if (participantId === localParticipantId) continue; + // const participantId = membership$.value?.identity; for (let dup = 0; dup < 1 + duplicateTiles; dup++) { yield { - keys: [ - dup, - localParticipantId, - userId, - participant$, - connection$, - ], + keys: [dup, participantId, userId, participant$, connection$], data: undefined, }; } } - } - // add remote members that are available - for (const { + }, + ( + scope, + _data$, + dup, + participantId, userId, participant$, connection$, - membership$, - } of matrixLivekitMembers) { - const participantId = `${userId}:${membership$.value.deviceId}`; - if (participantId === localParticipantId) continue; - // const participantId = membership$.value?.identity; - for (let dup = 0; dup < 1 + duplicateTiles; dup++) { - yield { - keys: [dup, participantId, userId, participant$, connection$], - data: undefined, - }; - } - } - }, - ( - scope, - _data$, - dup, - participantId, - userId, - participant$, - connection$, - ) => { - const livekitRoom$ = scope.behavior( - connection$.pipe(map((c) => c?.livekitRoom)), - ); - const focusUrl$ = scope.behavior( - connection$.pipe(map((c) => c?.transport.livekit_service_url)), - ); - const displayName$ = scope.behavior( - this.matrixMemberMetadataStore - .createDisplayNameBehavior$(userId) - .pipe(map((name) => name ?? userId)), - ); + ) => { + const livekitRoom$ = scope.behavior( + connection$.pipe(map((c) => c?.livekitRoom)), + ); + const focusUrl$ = scope.behavior( + connection$.pipe(map((c) => c?.transport.livekit_service_url)), + ); + const displayName$ = scope.behavior( + matrixMemberMetadataStore + .createDisplayNameBehavior$(userId) + .pipe(map((name) => name ?? userId)), + ); - return new UserMedia( - scope, - `${participantId}:${dup}`, - userId, - participant$, - this.options.encryptionSystem, - livekitRoom$, - focusUrl$, - this.mediaDevices, - this.pretendToBeDisconnected$, - displayName$, - this.matrixMemberMetadataStore.createAvatarUrlBehavior$(userId), - this.handsRaised$.pipe(map((v) => v[participantId]?.time ?? null)), - this.reactions$.pipe(map((v) => v[participantId] ?? undefined)), - ); - }, + return new UserMedia( + scope, + `${participantId}:${dup}`, + userId, + participant$, + options.encryptionSystem, + livekitRoom$, + focusUrl$, + mediaDevices, + pretendToBeDisconnected$, + displayName$, + matrixMemberMetadataStore.createAvatarUrlBehavior$(userId), + handsRaised$.pipe(map((v) => v[participantId]?.time ?? null)), + reactions$.pipe(map((v) => v[participantId] ?? undefined)), + ); + }, + ), ), - ), - ); + ); - /** - * List of all media items (user media and screen share media) that we want - * tiles for. - */ - private readonly mediaItems$ = this.scope.behavior( - this.userMedia$.pipe( - switchMap((userMedia) => - userMedia.length === 0 - ? of([]) - : combineLatest( - userMedia.map((m) => m.screenShares$), - (...screenShares) => [...userMedia, ...screenShares.flat(1)], - ), + /** + * List of all media items (user media and screen share media) that we want + * tiles for. + */ + const mediaItems$ = scope.behavior( + userMedia$.pipe( + switchMap((userMedia) => + userMedia.length === 0 + ? of([]) + : combineLatest( + userMedia.map((m) => m.screenShares$), + (...screenShares) => [...userMedia, ...screenShares.flat(1)], + ), + ), ), - ), - ); + ); - /** - * List of MediaItems that we want to display, that are of type ScreenShare - */ - private readonly screenShares$ = this.scope.behavior( - this.mediaItems$.pipe( - map((mediaItems) => - mediaItems.filter((m): m is ScreenShare => m instanceof ScreenShare), + /** + * List of MediaItems that we want to display, that are of type ScreenShare + */ + const screenShares$ = scope.behavior( + mediaItems$.pipe( + map((mediaItems) => + mediaItems.filter((m): m is ScreenShare => m instanceof ScreenShare), + ), ), - ), - ); + ); - public readonly joinSoundEffect$ = this.userMedia$.pipe( - pairwise(), - filter( - ([prev, current]) => - current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && - current.length > prev.length, - ), - map(() => {}), - throttleTime(THROTTLE_SOUND_EFFECT_MS), - ); + const joinSoundEffect$ = userMedia$.pipe( + pairwise(), + filter( + ([prev, current]) => + current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && + current.length > prev.length, + ), + map(() => {}), + throttleTime(THROTTLE_SOUND_EFFECT_MS), + ); - /** - * The number of participants currently in the call. - * - * - Each participant has a corresponding MatrixRTC membership state event - * - There can be multiple participants for one Matrix user if they join from - * multiple devices. - */ - public readonly participantCount$ = this.scope.behavior( - this.matrixLivekitMembers$.pipe(map((ms) => ms.value.length)), - ); + /** + * The number of participants currently in the call. + * + * - Each participant has a corresponding MatrixRTC membership state event + * - There can be multiple participants for one Matrix user if they join from + * multiple devices. + */ + const participantCount$ = scope.behavior( + matrixLivekitMembers$.pipe(map((ms) => ms.value.length)), + ); - // only public to expose to the view. - // TODO if we are in "unknown" state we need a loading rendering (or empty screen) - // Otherwise it looks like we already connected and only than the ringing starts which is weird. - public readonly callPickupState$ = this.callLifecycle.callPickupState$; + // only public to expose to the view. + // TODO if we are in "unknown" state we need a loading rendering (or empty screen) + // Otherwise it looks like we already connected and only than the ringing starts which is weird. + const callPickupState$ = callLifecycle.callPickupState$; - public readonly leaveSoundEffect$ = combineLatest([ - this.callLifecycle.callPickupState$, - this.userMedia$, - ]).pipe( - // Until the call is successful, do not play a leave sound. - // If callPickupState$ is null, then we always play the sound as it will not conflict with a decline sound. - skipWhile(([c]) => c !== null && c !== "success"), - map(([, userMedia]) => userMedia), - pairwise(), - filter( - ([prev, current]) => - current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && - current.length < prev.length, - ), - map(() => {}), - throttleTime(THROTTLE_SOUND_EFFECT_MS), - ); + const leaveSoundEffect$ = combineLatest([ + callLifecycle.callPickupState$, + userMedia$, + ]).pipe( + // Until the call is successful, do not play a leave sound. + // If callPickupState$ is null, then we always play the sound as it will not conflict with a decline sound. + skipWhile(([c]) => c !== null && c !== "success"), + map(([, userMedia]) => userMedia), + pairwise(), + filter( + ([prev, current]) => + current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && + current.length < prev.length, + ), + map(() => {}), + throttleTime(THROTTLE_SOUND_EFFECT_MS), + ); - private readonly userHangup$ = new Subject(); - public hangup(): void { - this.userHangup$.next(); - } + const userHangup$ = new Subject(); - private readonly widgetHangup$ = - widget === null - ? NEVER - : ( - fromEvent( - widget.lazyActions, - ElementWidgetActions.HangupCall, - ) as Observable> - ).pipe( - tap((ev) => { - widget!.api.transport.reply(ev.detail, {}); - }), - ); + const widgetHangup$ = + widget === null + ? NEVER + : ( + fromEvent( + widget.lazyActions, + ElementWidgetActions.HangupCall, + ) as Observable> + ).pipe( + tap((ev) => { + widget!.api.transport.reply(ev.detail, {}); + }), + ); - public readonly leave$: Observable< - "user" | "timeout" | "decline" | "allOthersLeft" - > = merge( - this.callLifecycle.autoLeave$, - merge(this.userHangup$, this.widgetHangup$).pipe( - map(() => "user" as const), - ), - ).pipe( - this.scope.share, - tap((reason) => this.leaveHoisted$.next(reason)), - ); + const leave$: Observable<"user" | "timeout" | "decline" | "allOthersLeft"> = + merge( + callLifecycle.autoLeave$, + merge(userHangup$, widgetHangup$).pipe(map(() => "user" as const)), + ).pipe( + scope.share, + tap((reason) => leaveHoisted$.next(reason)), + ); - private readonly spotlightSpeaker$ = - this.scope.behavior( - this.userMedia$.pipe( + const spotlightSpeaker$ = scope.behavior( + userMedia$.pipe( switchMap((mediaItems) => mediaItems.length === 0 ? of([]) @@ -665,653 +847,636 @@ export class CallViewModel { ), ); - private readonly grid$ = this.scope.behavior( - this.userMedia$.pipe( - switchMap((mediaItems) => { - const bins = mediaItems.map((m) => - m.bin$.pipe(map((bin) => [m, bin] as const)), - ); - // Sort the media by bin order and generate a tile for each one - return bins.length === 0 - ? of([]) - : combineLatest(bins, (...bins) => - bins.sort(([, bin1], [, bin2]) => bin1 - bin2).map(([m]) => m.vm), - ); - }), - distinctUntilChanged(shallowEquals), - ), - ); + const grid$ = scope.behavior( + userMedia$.pipe( + switchMap((mediaItems) => { + const bins = mediaItems.map((m) => + m.bin$.pipe(map((bin) => [m, bin] as const)), + ); + // Sort the media by bin order and generate a tile for each one + return bins.length === 0 + ? of([]) + : combineLatest(bins, (...bins) => + bins + .sort(([, bin1], [, bin2]) => bin1 - bin2) + .map(([m]) => m.vm), + ); + }), + distinctUntilChanged(shallowEquals), + ), + ); - private readonly spotlight$ = this.scope.behavior( - this.screenShares$.pipe( - switchMap((screenShares) => { - if (screenShares.length > 0) { - return of(screenShares.map((m) => m.vm)); - } + const spotlight$ = scope.behavior( + screenShares$.pipe( + switchMap((screenShares) => { + if (screenShares.length > 0) { + return of(screenShares.map((m) => m.vm)); + } - return this.spotlightSpeaker$.pipe( - map((speaker) => (speaker ? [speaker] : [])), - ); - }), - distinctUntilChanged(shallowEquals), - ), - ); + return spotlightSpeaker$.pipe( + map((speaker) => (speaker ? [speaker] : [])), + ); + }), + distinctUntilChanged(shallowEquals), + ), + ); - private readonly pip$ = this.scope.behavior( - combineLatest([ - // TODO This also needs epoch logic to dedupe the screenshares and mediaItems emits - this.screenShares$, - this.spotlightSpeaker$, - this.mediaItems$, - ]).pipe( - switchMap(([screenShares, spotlight, mediaItems]) => { - if (screenShares.length > 0) { - return this.spotlightSpeaker$; - } - if (!spotlight || spotlight.local) { - return of(null); - } + const pip$ = scope.behavior( + combineLatest([ + // TODO This also needs epoch logic to dedupe the screenshares and mediaItems emits + screenShares$, + spotlightSpeaker$, + mediaItems$, + ]).pipe( + switchMap(([screenShares, spotlight, mediaItems]) => { + if (screenShares.length > 0) { + return spotlightSpeaker$; + } + if (!spotlight || spotlight.local) { + return of(null); + } - const localUserMedia = mediaItems.find( - (m) => m.vm instanceof LocalUserMediaViewModel, - ) as UserMedia | undefined; + const localUserMedia = mediaItems.find( + (m) => m.vm instanceof LocalUserMediaViewModel, + ) as UserMedia | undefined; - const localUserMediaViewModel = localUserMedia?.vm as - | LocalUserMediaViewModel - | undefined; + const localUserMediaViewModel = localUserMedia?.vm as + | LocalUserMediaViewModel + | undefined; - if (!localUserMediaViewModel) { - return of(null); - } - return localUserMediaViewModel.alwaysShow$.pipe( - map((alwaysShow) => { - if (alwaysShow) { - return localUserMediaViewModel; - } + if (!localUserMediaViewModel) { + return of(null); + } + return localUserMediaViewModel.alwaysShow$.pipe( + map((alwaysShow) => { + if (alwaysShow) { + return localUserMediaViewModel; + } - return null; - }), - ); - }), - ), - ); + return null; + }), + ); + }), + ), + ); - private readonly hasRemoteScreenShares$: Observable = - this.spotlight$.pipe( + const hasRemoteScreenShares$: Observable = spotlight$.pipe( map((spotlight) => spotlight.some((vm) => !vm.local && vm instanceof ScreenShareViewModel), ), distinctUntilChanged(), ); - private readonly pipEnabled$ = this.scope.behavior(setPipEnabled$, false); + const pipEnabled$ = scope.behavior(setPipEnabled$, false); - private readonly naturalWindowMode$ = this.scope.behavior( - fromEvent(window, "resize").pipe( - map(() => { - const height = window.innerHeight; - const width = window.innerWidth; - if (height <= 400 && width <= 340) return "pip"; - // Our layouts for flat windows are better at adapting to a small width - // than our layouts for narrow windows are at adapting to a small height, - // so we give "flat" precedence here - if (height <= 600) return "flat"; - if (width <= 600) return "narrow"; - return "normal"; - }), - ), - "normal", - ); - - /** - * The general shape of the window. - */ - public readonly windowMode$ = this.scope.behavior( - this.pipEnabled$.pipe( - switchMap((pip) => - pip ? of("pip") : this.naturalWindowMode$, + const naturalWindowMode$ = scope.behavior( + fromEvent(window, "resize").pipe( + map(() => { + const height = window.innerHeight; + const width = window.innerWidth; + if (height <= 400 && width <= 340) return "pip"; + // Our layouts for flat windows are better at adapting to a small width + // than our layouts for narrow windows are at adapting to a small height, + // so we give "flat" precedence here + if (height <= 600) return "flat"; + if (width <= 600) return "narrow"; + return "normal"; + }), ), - ), - ); + "normal", + ); - private readonly spotlightExpandedToggle$ = new Subject(); - public readonly spotlightExpanded$ = this.scope.behavior( - this.spotlightExpandedToggle$.pipe( - accumulate(false, (expanded) => !expanded), - ), - ); + /** + * The general shape of the window. + */ + const windowMode$ = scope.behavior( + pipEnabled$.pipe( + switchMap((pip) => (pip ? of("pip") : naturalWindowMode$)), + ), + ); - private readonly gridModeUserSelection$ = new Subject(); - /** - * The layout mode of the media tile grid. - */ - public readonly gridMode$ = - // If the user hasn't selected spotlight and somebody starts screen sharing, - // automatically switch to spotlight mode and reset when screen sharing ends - this.scope.behavior( - this.gridModeUserSelection$.pipe( - switchMap((userSelection) => - (userSelection === "spotlight" - ? EMPTY - : combineLatest([ - this.hasRemoteScreenShares$, - this.windowMode$, - ]).pipe( - skip(userSelection === null ? 0 : 1), - map( - ([hasScreenShares, windowMode]): GridMode => - hasScreenShares || windowMode === "flat" - ? "spotlight" - : "grid", - ), - ) - ).pipe(startWith(userSelection ?? "grid")), + const spotlightExpandedToggle$ = new Subject(); + const spotlightExpanded$ = scope.behavior( + spotlightExpandedToggle$.pipe(accumulate(false, (expanded) => !expanded)), + ); + + const gridModeUserSelection$ = new Subject(); + /** + * The layout mode of the media tile grid. + */ + const gridMode$ = + // If the user hasn't selected spotlight and somebody starts screen sharing, + // automatically switch to spotlight mode and reset when screen sharing ends + scope.behavior( + gridModeUserSelection$.pipe( + switchMap((userSelection) => + (userSelection === "spotlight" + ? EMPTY + : combineLatest([hasRemoteScreenShares$, windowMode$]).pipe( + skip(userSelection === null ? 0 : 1), + map( + ([hasScreenShares, windowMode]): GridMode => + hasScreenShares || windowMode === "flat" + ? "spotlight" + : "grid", + ), + ) + ).pipe(startWith(userSelection ?? "grid")), + ), ), - ), - "grid", - ); + "grid", + ); - public setGridMode(value: GridMode): void { - this.gridModeUserSelection$.next(value); - } + const setGridMode = (value: GridMode): void => { + gridModeUserSelection$.next(value); + }; - private readonly gridLayoutMedia$: Observable = - combineLatest([this.grid$, this.spotlight$], (grid, spotlight) => ({ - type: "grid", - spotlight: spotlight.some((vm) => vm instanceof ScreenShareViewModel) - ? spotlight - : undefined, - grid, - })); - - private readonly spotlightLandscapeLayoutMedia$: Observable = - combineLatest([this.grid$, this.spotlight$], (grid, spotlight) => ({ - type: "spotlight-landscape", - spotlight, - grid, - })); - - private readonly spotlightPortraitLayoutMedia$: Observable = - combineLatest([this.grid$, this.spotlight$], (grid, spotlight) => ({ - type: "spotlight-portrait", - spotlight, - grid, - })); - - private readonly spotlightExpandedLayoutMedia$: Observable = - combineLatest([this.spotlight$, this.pip$], (spotlight, pip) => ({ - type: "spotlight-expanded", - spotlight, - pip: pip ?? undefined, - })); - - private readonly oneOnOneLayoutMedia$: Observable = - this.mediaItems$.pipe( - map((mediaItems) => { - if (mediaItems.length !== 2) return null; - const local = mediaItems.find((vm) => vm.vm.local)?.vm as - | LocalUserMediaViewModel - | undefined; - const remote = mediaItems.find((vm) => !vm.vm.local)?.vm as - | RemoteUserMediaViewModel - | undefined; - // There might not be a remote tile if there are screen shares, or if - // only the local user is in the call and they're using the duplicate - // tiles option - if (!remote || !local) return null; - - return { type: "one-on-one", local, remote }; + const gridLayoutMedia$: Observable = combineLatest( + [grid$, spotlight$], + (grid, spotlight) => ({ + type: "grid", + spotlight: spotlight.some((vm) => vm instanceof ScreenShareViewModel) + ? spotlight + : undefined, + grid, }), ); - private readonly pipLayoutMedia$: Observable = - this.spotlight$.pipe(map((spotlight) => ({ type: "pip", spotlight }))); + const spotlightLandscapeLayoutMedia$: Observable = + combineLatest([grid$, spotlight$], (grid, spotlight) => ({ + type: "spotlight-landscape", + spotlight, + grid, + })); - /** - * The media to be used to produce a layout. - */ - private readonly layoutMedia$ = this.scope.behavior( - this.windowMode$.pipe( - switchMap((windowMode) => { - switch (windowMode) { - case "normal": - return this.gridMode$.pipe( - switchMap((gridMode) => { - switch (gridMode) { - case "grid": - return this.oneOnOneLayoutMedia$.pipe( - switchMap((oneOnOne) => - oneOnOne === null - ? this.gridLayoutMedia$ - : of(oneOnOne), - ), - ); - case "spotlight": - return this.spotlightExpanded$.pipe( - switchMap((expanded) => - expanded - ? this.spotlightExpandedLayoutMedia$ - : this.spotlightLandscapeLayoutMedia$, - ), - ); - } - }), - ); - case "narrow": - return this.oneOnOneLayoutMedia$.pipe( - switchMap((oneOnOne) => - oneOnOne === null - ? combineLatest( - [this.grid$, this.spotlight$], - (grid, spotlight) => + const spotlightPortraitLayoutMedia$: Observable = + combineLatest([grid$, spotlight$], (grid, spotlight) => ({ + type: "spotlight-portrait", + spotlight, + grid, + })); + + const spotlightExpandedLayoutMedia$: Observable = + combineLatest([spotlight$, pip$], (spotlight, pip) => ({ + type: "spotlight-expanded", + spotlight, + pip: pip ?? undefined, + })); + + const oneOnOneLayoutMedia$: Observable = + mediaItems$.pipe( + map((mediaItems) => { + if (mediaItems.length !== 2) return null; + const local = mediaItems.find((vm) => vm.vm.local)?.vm as + | LocalUserMediaViewModel + | undefined; + const remote = mediaItems.find((vm) => !vm.vm.local)?.vm as + | RemoteUserMediaViewModel + | undefined; + // There might not be a remote tile if there are screen shares, or if + // only the local user is in the call and they're using the duplicate + // tiles option + if (!remote || !local) return null; + + return { type: "one-on-one", local, remote }; + }), + ); + + const pipLayoutMedia$: Observable = spotlight$.pipe( + map((spotlight) => ({ type: "pip", spotlight })), + ); + + /** + * The media to be used to produce a layout. + */ + const layoutMedia$ = scope.behavior( + windowMode$.pipe( + switchMap((windowMode) => { + switch (windowMode) { + case "normal": + return gridMode$.pipe( + switchMap((gridMode) => { + switch (gridMode) { + case "grid": + return oneOnOneLayoutMedia$.pipe( + switchMap((oneOnOne) => + oneOnOne === null ? gridLayoutMedia$ : of(oneOnOne), + ), + ); + case "spotlight": + return spotlightExpanded$.pipe( + switchMap((expanded) => + expanded + ? spotlightExpandedLayoutMedia$ + : spotlightLandscapeLayoutMedia$, + ), + ); + } + }), + ); + case "narrow": + return oneOnOneLayoutMedia$.pipe( + switchMap((oneOnOne) => + oneOnOne === null + ? combineLatest([grid$, spotlight$], (grid, spotlight) => grid.length > smallMobileCallThreshold || spotlight.some( (vm) => vm instanceof ScreenShareViewModel, ) - ? this.spotlightPortraitLayoutMedia$ - : this.gridLayoutMedia$, - ).pipe(switchAll()) - : // The expanded spotlight layout makes for a better one-on-one - // experience in narrow windows - this.spotlightExpandedLayoutMedia$, - ), - ); - case "flat": - return this.gridMode$.pipe( - switchMap((gridMode) => { - switch (gridMode) { - case "grid": - // Yes, grid mode actually gets you a "spotlight" layout in - // this window mode. - return this.spotlightLandscapeLayoutMedia$; - case "spotlight": - return this.spotlightExpandedLayoutMedia$; - } - }), - ); - case "pip": - return this.pipLayoutMedia$; - } - }), - ), - ); + ? spotlightPortraitLayoutMedia$ + : gridLayoutMedia$, + ).pipe(switchAll()) + : // The expanded spotlight layout makes for a better one-on-one + // experience in narrow windows + spotlightExpandedLayoutMedia$, + ), + ); + case "flat": + return gridMode$.pipe( + switchMap((gridMode) => { + switch (gridMode) { + case "grid": + // Yes, grid mode actually gets you a "spotlight" layout in + // this window mode. + return spotlightLandscapeLayoutMedia$; + case "spotlight": + return spotlightExpandedLayoutMedia$; + } + }), + ); + case "pip": + return pipLayoutMedia$; + } + }), + ), + ); - // There is a cyclical dependency here: the layout algorithms want to know - // which tiles are on screen, but to know which tiles are on screen we have to - // first render a layout. To deal with this we assume initially that no tiles - // are visible, and loop the data back into the layouts with a Subject. - private readonly visibleTiles$ = new Subject(); - private readonly setVisibleTiles = (value: number): void => - this.visibleTiles$.next(value); + // There is a cyclical dependency here: the layout algorithms want to know + // which tiles are on screen, but to know which tiles are on screen we have to + // first render a layout. To deal with this we assume initially that no tiles + // are visible, and loop the data back into the layouts with a Subject. + const visibleTiles$ = new Subject(); + const setVisibleTiles = (value: number): void => visibleTiles$.next(value); - private readonly layoutInternals$ = this.scope.behavior< - LayoutScanState & { layout: Layout } - >( - combineLatest([ - this.layoutMedia$, - this.visibleTiles$.pipe(startWith(0), distinctUntilChanged()), - ]).pipe( - scan< - [LayoutMedia, number], - LayoutScanState & { layout: Layout }, - LayoutScanState - >( - ({ tiles: prevTiles }, [media, visibleTiles]) => { - let layout: Layout; - let newTiles: TileStore; - switch (media.type) { - case "grid": + const layoutInternals$ = scope.behavior< + LayoutScanState & { layout: Layout } + >( + combineLatest([ + layoutMedia$, + visibleTiles$.pipe(startWith(0), distinctUntilChanged()), + ]).pipe( + scan< + [LayoutMedia, number], + LayoutScanState & { layout: Layout }, + LayoutScanState + >( + ({ tiles: prevTiles }, [media, visibleTiles]) => { + let layout: Layout; + let newTiles: TileStore; + switch (media.type) { + case "grid": + case "spotlight-landscape": + case "spotlight-portrait": + [layout, newTiles] = gridLikeLayout( + media, + visibleTiles, + setVisibleTiles, + prevTiles, + ); + break; + case "spotlight-expanded": + [layout, newTiles] = spotlightExpandedLayout(media, prevTiles); + break; + case "one-on-one": + [layout, newTiles] = oneOnOneLayout(media, prevTiles); + break; + case "pip": + [layout, newTiles] = pipLayout(media, prevTiles); + break; + } + + return { layout, tiles: newTiles }; + }, + { layout: null, tiles: TileStore.empty() }, + ), + ), + ); + + /** + * The layout of tiles in the call interface. + */ + const layout$ = scope.behavior( + layoutInternals$.pipe(map(({ layout }) => layout)), + ); + + /** + * The current generation of the tile store, exposed for debugging purposes. + */ + const tileStoreGeneration$ = scope.behavior( + layoutInternals$.pipe(map(({ tiles }) => tiles.generation)), + ); + + const showSpotlightIndicators$ = scope.behavior( + layout$.pipe(map((l) => l.type !== "grid")), + ); + + const showSpeakingIndicators$ = scope.behavior( + layout$.pipe( + switchMap((l) => { + switch (l.type) { case "spotlight-landscape": case "spotlight-portrait": - [layout, newTiles] = gridLikeLayout( - media, - visibleTiles, - this.setVisibleTiles, - prevTiles, + // If the spotlight is showing the active speaker, we can do without + // speaking indicators as they're a redundant visual cue. But if + // screen sharing feeds are in the spotlight we still need them. + return l.spotlight.media$.pipe( + map((models: MediaViewModel[]) => + models.some((m) => m instanceof ScreenShareViewModel), + ), ); - break; + // In expanded spotlight layout, the active speaker is always shown in + // the picture-in-picture tile so there is no need for speaking + // indicators. And in one-on-one layout there's no question as to who is + // speaking. case "spotlight-expanded": - [layout, newTiles] = spotlightExpandedLayout(media, prevTiles); - break; case "one-on-one": - [layout, newTiles] = oneOnOneLayout(media, prevTiles); - break; - case "pip": - [layout, newTiles] = pipLayout(media, prevTiles); - break; + return of(false); + default: + return of(true); } + }), + ), + ); - return { layout, tiles: newTiles }; + const toggleSpotlightExpanded$ = scope.behavior<(() => void) | null>( + windowMode$.pipe( + switchMap((mode) => + mode === "normal" + ? layout$.pipe( + map( + (l) => + l.type === "spotlight-landscape" || + l.type === "spotlight-expanded", + ), + ) + : of(false), + ), + distinctUntilChanged(), + map((enabled) => + enabled ? (): void => spotlightExpandedToggle$.next() : null, + ), + ), + ); + + const screenTap$ = new Subject(); + const controlsTap$ = new Subject(); + const screenHover$ = new Subject(); + const screenUnhover$ = new Subject(); + + const showHeader$ = scope.behavior( + windowMode$.pipe(map((mode) => mode !== "pip" && mode !== "flat")), + ); + + const showFooter$ = scope.behavior( + windowMode$.pipe( + switchMap((mode) => { + switch (mode) { + case "pip": + return of(false); + case "normal": + case "narrow": + return of(true); + case "flat": + // Sadly Firefox has some layering glitches that prevent the footer + // from appearing properly. They happen less often if we never hide + // the footer. + if (isFirefox()) return of(true); + // Show/hide the footer in response to interactions + return merge( + screenTap$.pipe(map(() => "tap screen" as const)), + controlsTap$.pipe(map(() => "tap controls" as const)), + screenHover$.pipe(map(() => "hover" as const)), + ).pipe( + switchScan((state, interaction) => { + switch (interaction) { + case "tap screen": + return state + ? // Toggle visibility on tap + of(false) + : // Hide after a timeout + timer(showFooterMs).pipe( + map(() => false), + startWith(true), + ); + case "tap controls": + // The user is interacting with things, so reset the timeout + return timer(showFooterMs).pipe( + map(() => false), + startWith(true), + ); + case "hover": + // Show on hover and hide after a timeout + return race( + timer(showFooterMs), + screenUnhover$.pipe(take(1)), + ).pipe( + map(() => false), + startWith(true), + ); + } + }, false), + startWith(false), + ); + } + }), + ), + ); + + /** + * Whether audio is currently being output through the earpiece. + */ + const earpieceMode$ = scope.behavior( + combineLatest( + [ + mediaDevices.audioOutput.available$, + mediaDevices.audioOutput.selected$, + ], + (available, selected) => + selected !== undefined && + available.get(selected.id)?.type === "earpiece", + ), + ); + + /** + * Callback to toggle between the earpiece and the loudspeaker. + * + * This will be `null` in case the target does not exist in the list + * of available audio outputs. + */ + const audioOutputSwitcher$ = scope.behavior<{ + targetOutput: "earpiece" | "speaker"; + switch: () => void; + } | null>( + combineLatest( + [ + mediaDevices.audioOutput.available$, + mediaDevices.audioOutput.selected$, + ], + (available, selected) => { + const selectionType = selected && available.get(selected.id)?.type; + + // If we are in any output mode other than speaker switch to speaker. + const newSelectionType: "earpiece" | "speaker" = + selectionType === "speaker" ? "earpiece" : "speaker"; + const newSelection = [...available].find( + ([, d]) => d.type === newSelectionType, + ); + if (newSelection === undefined) return null; + + const [id] = newSelection; + return { + targetOutput: newSelectionType, + switch: (): void => mediaDevices.audioOutput.select(id), + }; }, - { layout: null, tiles: TileStore.empty() }, ), - ), - ); + ); - /** - * The layout of tiles in the call interface. - */ - public readonly layout$ = this.scope.behavior( - this.layoutInternals$.pipe(map(({ layout }) => layout)), - ); - - /** - * The current generation of the tile store, exposed for debugging purposes. - */ - public readonly tileStoreGeneration$ = this.scope.behavior( - this.layoutInternals$.pipe(map(({ tiles }) => tiles.generation)), - ); - - public showSpotlightIndicators$ = this.scope.behavior( - this.layout$.pipe(map((l) => l.type !== "grid")), - ); - - public showSpeakingIndicators$ = this.scope.behavior( - this.layout$.pipe( - switchMap((l) => { - switch (l.type) { - case "spotlight-landscape": - case "spotlight-portrait": - // If the spotlight is showing the active speaker, we can do without - // speaking indicators as they're a redundant visual cue. But if - // screen sharing feeds are in the spotlight we still need them. - return l.spotlight.media$.pipe( - map((models: MediaViewModel[]) => - models.some((m) => m instanceof ScreenShareViewModel), - ), - ); - // In expanded spotlight layout, the active speaker is always shown in - // the picture-in-picture tile so there is no need for speaking - // indicators. And in one-on-one layout there's no question as to who is - // speaking. - case "spotlight-expanded": - case "one-on-one": - return of(false); - default: - return of(true); - } - }), - ), - ); - - public readonly toggleSpotlightExpanded$ = this.scope.behavior< - (() => void) | null - >( - this.windowMode$.pipe( - switchMap((mode) => - mode === "normal" - ? this.layout$.pipe( - map( - (l) => - l.type === "spotlight-landscape" || - l.type === "spotlight-expanded", - ), - ) - : of(false), + /** + * Emits an array of reactions that should be visible on the screen. + */ + // DISCUSSION move this into a reaction file + // const {visibleReactions$, audibleReactions$} = reactionsObservables$(showReactionSetting$, ) + const visibleReactions$ = scope.behavior( + showReactions.value$.pipe( + switchMap((show) => (show ? reactions$ : of({}))), + scan< + Record, + { sender: string; emoji: string; startX: number }[] + >((acc, latest) => { + const newSet: { sender: string; emoji: string; startX: number }[] = + []; + for (const [sender, reaction] of Object.entries(latest)) { + const startX = + acc.find((v) => v.sender === sender && v.emoji)?.startX ?? + Math.ceil(Math.random() * 80) + 10; + newSet.push({ sender, emoji: reaction.emoji, startX }); + } + return newSet; + }, []), ), - distinctUntilChanged(), - map((enabled) => - enabled ? (): void => this.spotlightExpandedToggle$.next() : null, + ); + + /** + * Emits an array of reactions that should be played. + */ + const audibleReactions$ = playReactionsSound.value$.pipe( + switchMap((show) => + show ? reactions$ : of>({}), ), - ), - ); + map((reactions) => Object.values(reactions).map((v) => v.name)), + scan( + (acc, latest) => { + return { + playing: latest.filter( + (v) => acc.playing.includes(v) || acc.newSounds.includes(v), + ), + newSounds: latest.filter( + (v) => !acc.playing.includes(v) && !acc.newSounds.includes(v), + ), + }; + }, + { playing: [], newSounds: [] }, + ), + map((v) => v.newSounds), + ); - private readonly screenTap$ = new Subject(); - private readonly controlsTap$ = new Subject(); - private readonly screenHover$ = new Subject(); - private readonly screenUnhover$ = new Subject(); + const newHandRaised$ = handsRaised$.pipe( + map((v) => Object.keys(v).length), + scan( + (acc, newValue) => ({ + value: newValue, + playSounds: newValue > acc.value, + }), + { value: 0, playSounds: false }, + ), + filter((v) => v.playSounds), + ); - /** - * Callback for when the user taps the call view. - */ - public tapScreen(): void { - this.screenTap$.next(); - } + const newScreenShare$ = screenShares$.pipe( + map((v) => v.length), + scan( + (acc, newValue) => ({ + value: newValue, + playSounds: newValue > acc.value, + }), + { value: 0, playSounds: false }, + ), + filter((v) => v.playSounds), + ); - /** - * Callback for when the user taps the call's controls. - */ - public tapControls(): void { - this.controlsTap$.next(); - } + /** + * Whether we are sharing our screen. + */ + // reassigned here to make it publicly accessible + const sharingScreen$ = localMembership.sharingScreen$; - /** - * Callback for when the user hovers over the call view. - */ - public hoverScreen(): void { - this.screenHover$.next(); - } + /** + * Callback to toggle screen sharing. If null, screen sharing is not possible. + */ + // reassigned here to make it publicly accessible + const toggleScreenSharing = localMembership.toggleScreenSharing; - /** - * Callback for when the user stops hovering over the call view. - */ - public unhoverScreen(): void { - this.screenUnhover$.next(); - } + const join = localMembership.requestConnect; + join(); // TODO-MULTI-SFU: Use this view model for the lobby as well, and only call this once 'join' is clicked? - public readonly showHeader$ = this.scope.behavior( - this.windowMode$.pipe(map((mode) => mode !== "pip" && mode !== "flat")), - ); + this.autoLeave$ = callLifecycle.autoLeave$; + this.callPickupState$ = callPickupState$; + this.leave$ = leave$; + this.hangup = (): void => userHangup$.next(); + this.join = join; + this.toggleScreenSharing = toggleScreenSharing; + this.sharingScreen$ = sharingScreen$; - public readonly showFooter$ = this.scope.behavior( - this.windowMode$.pipe( - switchMap((mode) => { - switch (mode) { - case "pip": - return of(false); - case "normal": - case "narrow": - return of(true); - case "flat": - // Sadly Firefox has some layering glitches that prevent the footer - // from appearing properly. They happen less often if we never hide - // the footer. - if (isFirefox()) return of(true); - // Show/hide the footer in response to interactions - return merge( - this.screenTap$.pipe(map(() => "tap screen" as const)), - this.controlsTap$.pipe(map(() => "tap controls" as const)), - this.screenHover$.pipe(map(() => "hover" as const)), - ).pipe( - switchScan((state, interaction) => { - switch (interaction) { - case "tap screen": - return state - ? // Toggle visibility on tap - of(false) - : // Hide after a timeout - timer(showFooterMs).pipe( - map(() => false), - startWith(true), - ); - case "tap controls": - // The user is interacting with things, so reset the timeout - return timer(showFooterMs).pipe( - map(() => false), - startWith(true), - ); - case "hover": - // Show on hover and hide after a timeout - return race( - timer(showFooterMs), - this.screenUnhover$.pipe(take(1)), - ).pipe( - map(() => false), - startWith(true), - ); - } - }, false), - startWith(false), - ); - } - }), - ), - ); + this.tapScreen = (): void => screenTap$.next(); + this.tapControls = (): void => controlsTap$.next(); + this.hoverScreen = (): void => screenHover$.next(); + this.unhoverScreen = (): void => screenUnhover$.next(); - /** - * Whether audio is currently being output through the earpiece. - */ - public readonly earpieceMode$ = this.scope.behavior( - combineLatest( - [ - this.mediaDevices.audioOutput.available$, - this.mediaDevices.audioOutput.selected$, - ], - (available, selected) => - selected !== undefined && - available.get(selected.id)?.type === "earpiece", - ), - ); + this.configError$ = localMembership.configError$; + this.participantCount$ = participantCount$; + this.audioParticipants$ = audioParticipants$; + this.isOneOnOneWith$ = isOneOnOneWith$; + this.localUserIsAlone$ = localUserIsAlone$; - /** - * Callback to toggle between the earpiece and the loudspeaker. - * - * This will be `null` in case the target does not exist in the list - * of available audio outputs. - */ - public readonly audioOutputSwitcher$ = this.scope.behavior<{ - targetOutput: "earpiece" | "speaker"; - switch: () => void; - } | null>( - combineLatest( - [ - this.mediaDevices.audioOutput.available$, - this.mediaDevices.audioOutput.selected$, - ], - (available, selected) => { - const selectionType = selected && available.get(selected.id)?.type; + this.handsRaised$ = handsRaised$; + this.reactions$ = reactions$; + this.joinSoundEffect$ = joinSoundEffect$; + this.leaveSoundEffect$ = leaveSoundEffect$; + this.newHandRaised$ = newHandRaised$; + this.newScreenShare$ = newScreenShare$; + this.audibleReactions$ = audibleReactions$; + this.visibleReactions$ = visibleReactions$; - // If we are in any output mode other than speaker switch to speaker. - const newSelectionType: "earpiece" | "speaker" = - selectionType === "speaker" ? "earpiece" : "speaker"; - const newSelection = [...available].find( - ([, d]) => d.type === newSelectionType, - ); - if (newSelection === undefined) return null; - - const [id] = newSelection; - return { - targetOutput: newSelectionType, - switch: (): void => this.mediaDevices.audioOutput.select(id), - }; - }, - ), - ); - - /** - * Emits an array of reactions that should be visible on the screen. - */ - // DISCUSSION move this into a reaction file - // const {visibleReactions$, audibleReactions$} = reactionsObservables$(showReactionSetting$, ) - public readonly visibleReactions$ = this.scope.behavior( - showReactions.value$.pipe( - switchMap((show) => (show ? this.reactions$ : of({}))), - scan< - Record, - { sender: string; emoji: string; startX: number }[] - >((acc, latest) => { - const newSet: { sender: string; emoji: string; startX: number }[] = []; - for (const [sender, reaction] of Object.entries(latest)) { - const startX = - acc.find((v) => v.sender === sender && v.emoji)?.startX ?? - Math.ceil(Math.random() * 80) + 10; - newSet.push({ sender, emoji: reaction.emoji, startX }); - } - return newSet; - }, []), - ), - ); - - /** - * Emits an array of reactions that should be played. - */ - public readonly audibleReactions$ = playReactionsSound.value$.pipe( - switchMap((show) => - show ? this.reactions$ : of>({}), - ), - map((reactions) => Object.values(reactions).map((v) => v.name)), - scan( - (acc, latest) => { - return { - playing: latest.filter( - (v) => acc.playing.includes(v) || acc.newSounds.includes(v), - ), - newSounds: latest.filter( - (v) => !acc.playing.includes(v) && !acc.newSounds.includes(v), - ), - }; - }, - { playing: [], newSounds: [] }, - ), - map((v) => v.newSounds), - ); - - /** - * Emits an event every time a new hand is raised in - * the call. - */ - public readonly newHandRaised$ = this.handsRaised$.pipe( - map((v) => Object.keys(v).length), - scan( - (acc, newValue) => ({ - value: newValue, - playSounds: newValue > acc.value, - }), - { value: 0, playSounds: false }, - ), - filter((v) => v.playSounds), - ); - - /** - * Emits an event every time a new screenshare is started in - * the call. - */ - public readonly newScreenShare$ = this.screenShares$.pipe( - map((v) => v.length), - scan( - (acc, newValue) => ({ - value: newValue, - playSounds: newValue > acc.value, - }), - { value: 0, playSounds: false }, - ), - filter((v) => v.playSounds), - ); - - /** - * Whether we are sharing our screen. - */ - // reassigned here to make it publicly accessible - public readonly sharingScreen$ = this.localMembership.sharingScreen$; - - /** - * Callback to toggle screen sharing. If null, screen sharing is not possible. - */ - // reassigned here to make it publicly accessible - public readonly toggleScreenSharing = - this.localMembership.toggleScreenSharing; - - public constructor( - private readonly scope: ObservableScope, - // A call is permanently tied to a single Matrix room - private readonly matrixRTCSession: MatrixRTCSession, - private readonly matrixRoom: MatrixRoom, - private readonly mediaDevices: MediaDevices, - private readonly muteStates: MuteStates, - private readonly options: CallViewModelOptions, - private readonly handsRaisedSubject$: Observable< - Record - >, - private readonly reactionsSubject$: Observable< - Record - >, - private readonly trackProcessorState$: Behavior, - ) { - // Join automatically - this.join(); // TODO-MULTI-SFU: Use this view model for the lobby as well, and only call this once 'join' is clicked? + this.windowMode$ = windowMode$; + this.spotlightExpanded$ = spotlightExpanded$; + this.toggleSpotlightExpanded$ = toggleSpotlightExpanded$; + this.gridMode$ = gridMode$; + this.setGridMode = setGridMode; + this.grid$ = grid$; + this.spotlight$ = spotlight$; + this.pip$ = pip$; + this.layout$ = layout$; + this.tileStoreGeneration$ = tileStoreGeneration$; + this.showSpotlightIndicators$ = showSpotlightIndicators$; + this.showSpeakingIndicators$ = showSpeakingIndicators$; + this.showHeader$ = showHeader$; + this.showFooter$ = showFooter$; + this.earpieceMode$ = earpieceMode$; + this.audioOutputSwitcher$ = audioOutputSwitcher$; + this.reconnecting$ = reconnecting$; } } - // TODO-MULTI-SFU // Setup and update the keyProvider which was create by `createRoom` was a thing before. Now we never update if the E2EEsystem changes // do we need this? diff --git a/src/state/CallViewModel/localMember/LocalMembership.ts b/src/state/CallViewModel/localMember/LocalMembership.ts index ae6c1c7c..d9a3887b 100644 --- a/src/state/CallViewModel/localMember/LocalMembership.ts +++ b/src/state/CallViewModel/localMember/LocalMembership.ts @@ -335,6 +335,7 @@ export const createLocalMembership$ = ({ }); combineLatest([localTransport$, connectRequested$]).subscribe( + // TODO reconnect on options change. ([transport, connectRequested]) => { if ( transport === null ||