diff --git a/src/state/CallViewModel/CallViewModelFunction.ts b/src/state/CallViewModel/CallViewModelFunction.ts deleted file mode 100644 index 20e37395..00000000 --- a/src/state/CallViewModel/CallViewModelFunction.ts +++ /dev/null @@ -1,1400 +0,0 @@ -/* -Copyright 2023, 2024, 2025 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - type BaseKeyProvider, - type ConnectionState, - ExternalE2EEKeyProvider, - type Room as LivekitRoom, - type RoomOptions, -} from "livekit-client"; -import { type RoomMember, type Room as MatrixRoom } from "matrix-js-sdk"; -import { - combineLatest, - distinctUntilChanged, - EMPTY, - filter, - fromEvent, - map, - merge, - NEVER, - type Observable, - of, - pairwise, - race, - scan, - skip, - skipWhile, - startWith, - Subject, - switchAll, - switchMap, - switchScan, - take, - tap, - throttleTime, - timer, -} from "rxjs"; -import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; -import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; -import { type IWidgetApiRequest } from "matrix-widget-api"; - -import { - LocalUserMediaViewModel, - type MediaViewModel, - type RemoteUserMediaViewModel, - ScreenShareViewModel, - type UserMediaViewModel, -} from "../MediaViewModel"; -import { accumulate, generateItems, pauseWhen } from "../../utils/observable"; -import { - duplicateTiles, - MatrixRTCMode, - matrixRTCMode, - playReactionsSound, - showReactions, -} from "../../settings/settings"; -import { isFirefox } from "../../Platform"; -import { setPipEnabled$ } from "../../controls"; -import { TileStore } from "../TileStore"; -import { gridLikeLayout } from "../GridLikeLayout"; -import { spotlightExpandedLayout } from "../SpotlightExpandedLayout"; -import { oneOnOneLayout } from "../OneOnOneLayout"; -import { pipLayout } from "../PipLayout"; -import { type EncryptionSystem } from "../../e2ee/sharedKeyManagement"; -import { - type RaisedHandInfo, - type ReactionInfo, - type ReactionOption, -} from "../../reactions"; -import { shallowEquals } from "../../utils/array"; -import { type MediaDevices } from "../MediaDevices"; -import { type Behavior } from "../Behavior"; -import { E2eeType } from "../../e2ee/e2eeType"; -import { MatrixKeyProvider } from "../../e2ee/matrixKeyProvider"; -import { type MuteStates } from "../MuteStates"; -import { getUrlParams } from "../../UrlParams"; -import { type ProcessorState } from "../../livekit/TrackProcessorContext"; -import { ElementWidgetActions, widget } from "../../widget"; -import { UserMedia } from "../UserMedia.ts"; -import { ScreenShare } from "../ScreenShare.ts"; -import { - type GridLayoutMedia, - type Layout, - type LayoutMedia, - type OneOnOneLayoutMedia, - type SpotlightExpandedLayoutMedia, - type SpotlightLandscapeLayoutMedia, - type SpotlightPortraitLayoutMedia, -} from "../layout-types.ts"; -import { type ElementCallError } from "../../utils/errors.ts"; -import { type ObservableScope } from "../ObservableScope.ts"; -import { createLocalMembership$ } from "./localMember/LocalMembership.ts"; -import { createLocalTransport$ } from "./localMember/LocalTransport.ts"; -import { - createMemberships$, - membershipsAndTransports$, -} from "../SessionBehaviors.ts"; -import { ECConnectionFactory } from "./remoteMembers/ConnectionFactory.ts"; -import { createConnectionManager$ } from "./remoteMembers/ConnectionManager.ts"; -import { - createMatrixLivekitMembers$, - type MatrixLivekitMember, -} from "./remoteMembers/MatrixLivekitMembers.ts"; -import { - type AutoLeaveReason, - createCallNotificationLifecycle$, - createReceivedDecline$, - createSentCallNotification$, -} from "./CallNotificationLifecycle.ts"; -import { - createMatrixMemberMetadata$, - createRoomMembers$, -} from "./remoteMembers/MatrixMemberMetadata.ts"; - -const logger = rootLogger.getChild("[CallViewModel]"); -//TODO -// Larger rename -// member,membership -> rtcMember -// participant -> livekitParticipant -// matrixLivekitItem -> callMember -// js-sdk -// callMembership -> rtcMembership -export interface CallViewModelOptions { - encryptionSystem: EncryptionSystem; - autoLeaveWhenOthersLeft?: boolean; - /** - * If the call is started in a way where we want it to behave like a telephone usecase - * If we sent a notification event, we want the ui to show a ringing state - */ - waitForCallPickup?: boolean; - /** Optional factory to create LiveKit rooms, mainly for testing purposes. */ - livekitRoomFactory?: (options?: RoomOptions) => LivekitRoom; - /** Optional behavior overriding the local connection state, mainly for testing purposes. */ - connectionState$?: Behavior; -} - -// Do not play any sounds if the participant count has exceeded this -// number. -export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8; -export const THROTTLE_SOUND_EFFECT_MS = 500; - -// This is the number of participants that we think constitutes a "small" call -// on mobile. No spotlight tile should be shown below this threshold. -const smallMobileCallThreshold = 3; - -// How long the footer should be shown for when hovering over or interacting -// with the interface -const showFooterMs = 4000; - -export type GridMode = "grid" | "spotlight"; - -export type WindowMode = "normal" | "narrow" | "flat" | "pip"; - -interface LayoutScanState { - layout: Layout | null; - tiles: TileStore; -} - -type MediaItem = UserMedia | ScreenShare; -type AudioLivekitItem = { - livekitRoom: LivekitRoom; - participants: string[]; - url: string; -}; - -type JoinReturn = ReturnType< - ReturnType["requestConnect"] ->; - -/** - * A view model providing all the application logic needed to show the in-call - * UI (may eventually be expanded to cover the lobby and feedback screens in the - * future). - */ -// Throughout this class and related code we must distinguish between MatrixRTC -// state and LiveKit state. We use the common terminology of room "members", RTC -// "memberships", and LiveKit "participants". -export function createCallViewModel$( - scope: ObservableScope, - // A call is permanently tied to a single Matrix room - matrixRTCSession: MatrixRTCSession, - matrixRoom: MatrixRoom, - mediaDevices: MediaDevices, - muteStates: MuteStates, - options: CallViewModelOptions, - handsRaisedSubject$: Observable>, - reactionsSubject$: Observable>, - trackProcessorState$: Behavior, -): { - // lifecycle - autoLeave$: Observable; - callPickupState$: Behavior< - "unknown" | "ringing" | "timeout" | "decline" | "success" | null - >; - leave$: Observable<"user" | AutoLeaveReason>; - hangup: () => void; - - // joining - join: () => JoinReturn; - - // screen sharing - toggleScreenSharing: (() => void) | null; - sharingScreen$: Behavior; - - // UI interactions - tapScreen: () => void; - tapControls: () => void; - hoverScreen: () => void; - unhoverScreen: () => void; - - // errors - configError$: Behavior; - - // participants and counts - participantCount$: Behavior; - audioParticipants$: Behavior; - handsRaised$: Behavior>; - reactions$: Behavior>; - isOneOnOneWith$: Behavior | null>; - localUserIsAlone$: Behavior; - // sounds and events - joinSoundEffect$: Observable; - leaveSoundEffect$: Observable; - newHandRaised$: Observable<{ value: number; playSounds: boolean }>; - newScreenShare$: Observable<{ value: number; playSounds: boolean }>; - audibleReactions$: Observable; - visibleReactions$: Behavior< - { sender: string; emoji: string; startX: number }[] - >; - - // window/layout - windowMode$: Behavior; - spotlightExpanded$: Behavior; - toggleSpotlightExpanded$: Behavior<(() => void) | null>; - gridMode$: Behavior; - setGridMode: (value: GridMode) => void; - - // media view models and layout - grid$: Behavior; - spotlight$: Behavior; - pip$: Behavior; - layout$: Behavior; - tileStoreGeneration$: Behavior; - showSpotlightIndicators$: Behavior; - showSpeakingIndicators$: Behavior; - - // header/footer visibility - showHeader$: Behavior; - showFooter$: Behavior; - - // audio routing - earpieceMode$: Behavior; - audioOutputSwitcher$: Behavior<{ - targetOutput: "earpiece" | "speaker"; - switch: () => void; - } | null>; - - // connection state - reconnecting$: Behavior; -} { - const userId = matrixRoom.client.getUserId()!; - const deviceId = matrixRoom.client.getDeviceId()!; - - const livekitKeyProvider = getE2eeKeyProvider( - options.encryptionSystem, - matrixRTCSession, - ); - - // Each hbar seperates a block of input variables required for the CallViewModel to function. - // The outputs of this block is written under the hbar. - // - // For mocking purposes it is recommended to only mock the functions creating those outputs. - // All other fields are just temp computations for the mentioned output. - // The class does not need anything except the values underneath the bar. - // The creation of the values under the bar are all tested independently and testing the callViewModel Should - // not test their cretation. Call view model only needs: - // - memberships$ via createMemberships$ - // - localMembership via createLocalMembership$ - // - callLifecycle via createCallNotificationLifecycle$ - // - matrixMemberMetadataStore via createMatrixMemberMetadata$ - - // ------------------------------------------------------------------------ - // memberships$ - const memberships$ = createMemberships$(scope, matrixRTCSession); - - // ------------------------------------------------------------------------ - // matrixLivekitMembers$ AND localMembership - - const membershipsAndTransports = membershipsAndTransports$( - scope, - memberships$, - ); - - const localTransport$ = createLocalTransport$({ - scope: scope, - memberships$: memberships$, - client: matrixRoom.client, - roomId: matrixRoom.roomId, - useOldestMember$: scope.behavior( - matrixRTCMode.value$.pipe(map((v) => v === MatrixRTCMode.Legacy)), - ), - }); - - const connectionFactory = new ECConnectionFactory( - matrixRoom.client, - mediaDevices, - trackProcessorState$, - livekitKeyProvider, - getUrlParams().controlledAudioDevices, - options.livekitRoomFactory, - ); - - const connectionManager = createConnectionManager$({ - scope: scope, - connectionFactory: connectionFactory, - inputTransports$: scope.behavior( - combineLatest( - [localTransport$, membershipsAndTransports.transports$], - (localTransport, transports) => { - const localTransportAsArray = localTransport ? [localTransport] : []; - return transports.mapInner((transports) => [ - ...localTransportAsArray, - ...transports, - ]); - }, - ), - ), - logger: logger, - }); - - const matrixLivekitMembers$ = createMatrixLivekitMembers$({ - scope: scope, - membershipsWithTransport$: - membershipsAndTransports.membershipsWithTransport$, - connectionManager: connectionManager, - }); - - const connectOptions$ = scope.behavior( - matrixRTCMode.value$.pipe( - map((mode) => ({ - encryptMedia: livekitKeyProvider !== undefined, - // TODO. This might need to get called again on each cahnge of matrixRTCMode... - matrixRTCMode: mode, - })), - ), - ); - - const localMembership = createLocalMembership$({ - scope: scope, - muteStates: muteStates, - mediaDevices: mediaDevices, - connectionManager: connectionManager, - matrixRTCSession: matrixRTCSession, - matrixRoom: matrixRoom, - localTransport$: localTransport$, - trackProcessorState$: trackProcessorState$, - widget, - options: connectOptions$, - logger: logger.getChild(`[${Date.now()}]`), - }); - - const localRtcMembership$ = scope.behavior( - memberships$.pipe( - map( - (memberships) => - memberships.value.find( - (membership) => - membership.userId === userId && membership.deviceId === deviceId, - ) ?? null, - ), - ), - ); - - const localMatrixLivekitMemberUninitialized = { - membership$: localRtcMembership$, - participant$: localMembership.participant$, - connection$: localMembership.connection$, - userId: userId, - }; - - const localMatrixLivekitMember$: Behavior = - scope.behavior( - localRtcMembership$.pipe( - switchMap((membership) => { - if (!membership) return of(null); - return of( - // casting is save here since we know that localRtcMembership$ is !== null since we reached this case. - localMatrixLivekitMemberUninitialized as MatrixLivekitMember, - ); - }), - ), - ); - - // ------------------------------------------------------------------------ - // callLifecycle - - const callLifecycle = createCallNotificationLifecycle$({ - scope: scope, - memberships$: memberships$, - sentCallNotification$: createSentCallNotification$(scope, matrixRTCSession), - receivedDecline$: createReceivedDecline$(matrixRoom), - options: options, - localUser: { userId: userId, deviceId: deviceId }, - }); - - // ------------------------------------------------------------------------ - // matrixMemberMetadataStore - - const matrixRoomMembers$ = createRoomMembers$(scope, matrixRoom); - const matrixMemberMetadataStore = createMatrixMemberMetadata$( - scope, - scope.behavior(memberships$.pipe(map((mems) => mems.value))), - matrixRoomMembers$, - ); - - /** - * Returns the Member {userId, getMxcAvatarUrl, rawDisplayName} of the other user in the call, if it's a one-on-one call. - */ - const isOneOnOneWith$ = scope.behavior( - matrixRoomMembers$.pipe( - map((roomMembersMap) => { - const otherMembers = Array.from(roomMembersMap.values()).filter( - (member) => member.userId !== userId, - ); - return otherMembers.length === 1 ? otherMembers[0] : null; - }), - ), - ); - - const localUserIsAlone$ = scope.behavior( - matrixRoomMembers$.pipe( - map( - (roomMembersMap) => - roomMembersMap.size === 1 && roomMembersMap.get(userId) !== undefined, - ), - ), - ); - - // CODESMELL? - // This is functionally the same Observable as leave$, except here it's - // hoisted to the top of the class. This enables the cyclic dependency between - // leave$ -> autoLeave$ -> callPickupState$ -> livekitConnectionState$ -> - // localConnection$ -> transports$ -> joined$ -> leave$. - const leaveHoisted$ = new Subject< - "user" | "timeout" | "decline" | "allOthersLeft" - >(); - - /** - * Whether various media/event sources should pretend to be disconnected from - * all network input, even if their connection still technically works. - */ - // We do this when the app is in the 'reconnecting' state, because it might be - // that the LiveKit connection is still functional while the homeserver is - // down, for example, and we want to avoid making people worry that the app is - // in a split-brained state. - // DISCUSSION own membership manager ALSO this probably can be simplifis - /*PUBLIC*/ const reconnecting$ = localMembership.reconnecting$; - const pretendToBeDisconnected$ = reconnecting$; - - /*PUBLIC*/ const audioParticipants$ = scope.behavior( - matrixLivekitMembers$.pipe( - switchMap((membersWithEpoch) => { - const members = membersWithEpoch.value; - const a$ = combineLatest( - members.map((member) => - combineLatest([member.connection$, member.participant$]).pipe( - map(([connection, participant]) => { - // do not render audio for local participant - if (!connection || !participant || participant.isLocal) - return null; - const livekitRoom = connection.livekitRoom; - const url = connection.transport.livekit_service_url; - - return { url, livekitRoom, participant: participant.identity }; - }), - ), - ), - ); - return a$; - }), - map((members) => - members.reduce((acc, curr) => { - if (!curr) return acc; - - const existing = acc.find((item) => item.url === curr.url); - if (existing) { - existing.participants.push(curr.participant); - } else { - acc.push({ - livekitRoom: curr.livekitRoom, - participants: [curr.participant], - url: curr.url, - }); - } - return acc; - }, []), - ), - ), - [], - ); - - /*PUBLIC*/ const handsRaised$ = scope.behavior( - handsRaisedSubject$.pipe(pauseWhen(pretendToBeDisconnected$)), - ); - - /*PUBLIC*/ const reactions$ = scope.behavior( - reactionsSubject$.pipe( - map((v) => - Object.fromEntries( - Object.entries(v).map(([a, { reactionOption }]) => [ - a, - reactionOption, - ]), - ), - ), - pauseWhen(pretendToBeDisconnected$), - ), - ); - - /** - * List of user media (camera feeds) that we want tiles for. - */ - // TODO this also needs the local participant to be added. - const userMedia$ = scope.behavior( - combineLatest([ - localMatrixLivekitMember$, - matrixLivekitMembers$, - duplicateTiles.value$, - ]).pipe( - // Generate a collection of MediaItems from the list of expected (whether - // present or missing) LiveKit participants. - generateItems( - function* ([ - localMatrixLivekitMember, - { value: matrixLivekitMembers }, - duplicateTiles, - ]) { - let localParticipantId = undefined; - // add local member if available - if (localMatrixLivekitMember) { - const { userId, participant$, connection$, membership$ } = - localMatrixLivekitMember; - localParticipantId = `${userId}:${membership$.value.deviceId}`; // should be membership$.value.membershipID which is not optional - // const participantId = membership$.value.membershipID; - if (localParticipantId) { - for (let dup = 0; dup < 1 + duplicateTiles; dup++) { - yield { - keys: [ - dup, - localParticipantId, - userId, - participant$, - connection$, - ], - data: undefined, - }; - } - } - } - // add remote members that are available - for (const { - userId, - participant$, - connection$, - membership$, - } of matrixLivekitMembers) { - const participantId = `${userId}:${membership$.value.deviceId}`; - if (participantId === localParticipantId) continue; - // const participantId = membership$.value?.identity; - for (let dup = 0; dup < 1 + duplicateTiles; dup++) { - yield { - keys: [dup, participantId, userId, participant$, connection$], - data: undefined, - }; - } - } - }, - ( - scope, - _data$, - dup, - participantId, - userId, - participant$, - connection$, - ) => { - const livekitRoom$ = scope.behavior( - connection$.pipe(map((c) => c?.livekitRoom)), - ); - const focusUrl$ = scope.behavior( - connection$.pipe(map((c) => c?.transport.livekit_service_url)), - ); - const displayName$ = scope.behavior( - matrixMemberMetadataStore - .createDisplayNameBehavior$(userId) - .pipe(map((name) => name ?? userId)), - ); - - return new UserMedia( - scope, - `${participantId}:${dup}`, - userId, - participant$, - options.encryptionSystem, - livekitRoom$, - focusUrl$, - mediaDevices, - pretendToBeDisconnected$, - displayName$, - matrixMemberMetadataStore.createAvatarUrlBehavior$(userId), - handsRaised$.pipe(map((v) => v[participantId]?.time ?? null)), - reactions$.pipe(map((v) => v[participantId] ?? undefined)), - ); - }, - ), - ), - ); - - /** - * List of all media items (user media and screen share media) that we want - * tiles for. - */ - const mediaItems$ = scope.behavior( - userMedia$.pipe( - switchMap((userMedia) => - userMedia.length === 0 - ? of([]) - : combineLatest( - userMedia.map((m) => m.screenShares$), - (...screenShares) => [...userMedia, ...screenShares.flat(1)], - ), - ), - ), - ); - - /** - * List of MediaItems that we want to display, that are of type ScreenShare - */ - const screenShares$ = scope.behavior( - mediaItems$.pipe( - map((mediaItems) => - mediaItems.filter((m): m is ScreenShare => m instanceof ScreenShare), - ), - ), - ); - - /*PUBLIC*/ const joinSoundEffect$ = userMedia$.pipe( - pairwise(), - filter( - ([prev, current]) => - current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && - current.length > prev.length, - ), - map(() => {}), - throttleTime(THROTTLE_SOUND_EFFECT_MS), - ); - - /** - * The number of participants currently in the call. - * - * - Each participant has a corresponding MatrixRTC membership state event - * - There can be multiple participants for one Matrix user if they join from - * multiple devices. - */ - /*PUBLIC*/ const participantCount$ = scope.behavior( - matrixLivekitMembers$.pipe(map((ms) => ms.value.length)), - ); - - // only public to expose to the view. - // TODO if we are in "unknown" state we need a loading rendering (or empty screen) - // Otherwise it looks like we already connected and only than the ringing starts which is weird. - /*PUBLIC*/ const callPickupState$ = callLifecycle.callPickupState$; - - /*PUBLIC*/ const leaveSoundEffect$ = combineLatest([ - callLifecycle.callPickupState$, - userMedia$, - ]).pipe( - // Until the call is successful, do not play a leave sound. - // If callPickupState$ is null, then we always play the sound as it will not conflict with a decline sound. - skipWhile(([c]) => c !== null && c !== "success"), - map(([, userMedia]) => userMedia), - pairwise(), - filter( - ([prev, current]) => - current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && - current.length < prev.length, - ), - map(() => {}), - throttleTime(THROTTLE_SOUND_EFFECT_MS), - ); - - const userHangup$ = new Subject(); - - const widgetHangup$ = - widget === null - ? NEVER - : ( - fromEvent( - widget.lazyActions, - ElementWidgetActions.HangupCall, - ) as Observable> - ).pipe( - tap((ev) => { - widget!.api.transport.reply(ev.detail, {}); - }), - ); - - const leave$: Observable<"user" | "timeout" | "decline" | "allOthersLeft"> = - merge( - callLifecycle.autoLeave$, - merge(userHangup$, widgetHangup$).pipe(map(() => "user" as const)), - ).pipe( - scope.share, - tap((reason) => leaveHoisted$.next(reason)), - ); - - const spotlightSpeaker$ = scope.behavior( - userMedia$.pipe( - switchMap((mediaItems) => - mediaItems.length === 0 - ? of([]) - : combineLatest( - mediaItems.map((m) => - m.vm.speaking$.pipe(map((s) => [m, s] as const)), - ), - ), - ), - scan<(readonly [UserMedia, boolean])[], UserMedia | undefined, null>( - (prev, mediaItems) => { - // Only remote users that are still in the call should be sticky - const [stickyMedia, stickySpeaking] = - (!prev?.vm.local && mediaItems.find(([m]) => m === prev)) || []; - // Decide who to spotlight: - // If the previous speaker is still speaking, stick with them rather - // than switching eagerly to someone else - return stickySpeaking - ? stickyMedia! - : // Otherwise, select any remote user who is speaking - (mediaItems.find(([m, s]) => !m.vm.local && s)?.[0] ?? - // Otherwise, stick with the person who was last speaking - stickyMedia ?? - // Otherwise, spotlight an arbitrary remote user - mediaItems.find(([m]) => !m.vm.local)?.[0] ?? - // Otherwise, spotlight the local user - mediaItems.find(([m]) => m.vm.local)?.[0]); - }, - null, - ), - map((speaker) => speaker?.vm ?? null), - ), - ); - - const grid$ = scope.behavior( - userMedia$.pipe( - switchMap((mediaItems) => { - const bins = mediaItems.map((m) => - m.bin$.pipe(map((bin) => [m, bin] as const)), - ); - // Sort the media by bin order and generate a tile for each one - return bins.length === 0 - ? of([]) - : combineLatest(bins, (...bins) => - bins.sort(([, bin1], [, bin2]) => bin1 - bin2).map(([m]) => m.vm), - ); - }), - distinctUntilChanged(shallowEquals), - ), - ); - - const spotlight$ = scope.behavior( - screenShares$.pipe( - switchMap((screenShares) => { - if (screenShares.length > 0) { - return of(screenShares.map((m) => m.vm)); - } - - return spotlightSpeaker$.pipe( - map((speaker) => (speaker ? [speaker] : [])), - ); - }), - distinctUntilChanged(shallowEquals), - ), - ); - - const pip$ = scope.behavior( - combineLatest([ - // TODO This also needs epoch logic to dedupe the screenshares and mediaItems emits - screenShares$, - spotlightSpeaker$, - mediaItems$, - ]).pipe( - switchMap(([screenShares, spotlight, mediaItems]) => { - if (screenShares.length > 0) { - return spotlightSpeaker$; - } - if (!spotlight || spotlight.local) { - return of(null); - } - - const localUserMedia = mediaItems.find( - (m) => m.vm instanceof LocalUserMediaViewModel, - ) as UserMedia | undefined; - - const localUserMediaViewModel = localUserMedia?.vm as - | LocalUserMediaViewModel - | undefined; - - if (!localUserMediaViewModel) { - return of(null); - } - return localUserMediaViewModel.alwaysShow$.pipe( - map((alwaysShow) => { - if (alwaysShow) { - return localUserMediaViewModel; - } - - return null; - }), - ); - }), - ), - ); - - const hasRemoteScreenShares$: Observable = spotlight$.pipe( - map((spotlight) => - spotlight.some((vm) => !vm.local && vm instanceof ScreenShareViewModel), - ), - distinctUntilChanged(), - ); - - const pipEnabled$ = scope.behavior(setPipEnabled$, false); - - const naturalWindowMode$ = scope.behavior( - fromEvent(window, "resize").pipe( - map(() => { - const height = window.innerHeight; - const width = window.innerWidth; - if (height <= 400 && width <= 340) return "pip"; - // Our layouts for flat windows are better at adapting to a small width - // than our layouts for narrow windows are at adapting to a small height, - // so we give "flat" precedence here - if (height <= 600) return "flat"; - if (width <= 600) return "narrow"; - return "normal"; - }), - ), - "normal", - ); - - /** - * The general shape of the window. - */ - /*PUBLIC*/ const windowMode$ = scope.behavior( - pipEnabled$.pipe( - switchMap((pip) => (pip ? of("pip") : naturalWindowMode$)), - ), - ); - - const spotlightExpandedToggle$ = new Subject(); - /*PUBLIC*/ const spotlightExpanded$ = scope.behavior( - spotlightExpandedToggle$.pipe(accumulate(false, (expanded) => !expanded)), - ); - - const gridModeUserSelection$ = new Subject(); - /** - * The layout mode of the media tile grid. - */ - /*PUBLIC*/ const gridMode$ = - // If the user hasn't selected spotlight and somebody starts screen sharing, - // automatically switch to spotlight mode and reset when screen sharing ends - scope.behavior( - gridModeUserSelection$.pipe( - switchMap((userSelection) => - (userSelection === "spotlight" - ? EMPTY - : combineLatest([hasRemoteScreenShares$, windowMode$]).pipe( - skip(userSelection === null ? 0 : 1), - map( - ([hasScreenShares, windowMode]): GridMode => - hasScreenShares || windowMode === "flat" - ? "spotlight" - : "grid", - ), - ) - ).pipe(startWith(userSelection ?? "grid")), - ), - ), - "grid", - ); - - /*PUBLIC*/ const setGridMode = (value: GridMode): void => { - gridModeUserSelection$.next(value); - }; - - const gridLayoutMedia$: Observable = combineLatest( - [grid$, spotlight$], - (grid, spotlight) => ({ - type: "grid", - spotlight: spotlight.some((vm) => vm instanceof ScreenShareViewModel) - ? spotlight - : undefined, - grid, - }), - ); - - const spotlightLandscapeLayoutMedia$: Observable = - combineLatest([grid$, spotlight$], (grid, spotlight) => ({ - type: "spotlight-landscape", - spotlight, - grid, - })); - - const spotlightPortraitLayoutMedia$: Observable = - combineLatest([grid$, spotlight$], (grid, spotlight) => ({ - type: "spotlight-portrait", - spotlight, - grid, - })); - - const spotlightExpandedLayoutMedia$: Observable = - combineLatest([spotlight$, pip$], (spotlight, pip) => ({ - type: "spotlight-expanded", - spotlight, - pip: pip ?? undefined, - })); - - const oneOnOneLayoutMedia$: Observable = - mediaItems$.pipe( - map((mediaItems) => { - if (mediaItems.length !== 2) return null; - const local = mediaItems.find((vm) => vm.vm.local)?.vm as - | LocalUserMediaViewModel - | undefined; - const remote = mediaItems.find((vm) => !vm.vm.local)?.vm as - | RemoteUserMediaViewModel - | undefined; - // There might not be a remote tile if there are screen shares, or if - // only the local user is in the call and they're using the duplicate - // tiles option - if (!remote || !local) return null; - - return { type: "one-on-one", local, remote }; - }), - ); - - const pipLayoutMedia$: Observable = spotlight$.pipe( - map((spotlight) => ({ type: "pip", spotlight })), - ); - - /** - * The media to be used to produce a layout. - */ - const layoutMedia$ = scope.behavior( - windowMode$.pipe( - switchMap((windowMode) => { - switch (windowMode) { - case "normal": - return gridMode$.pipe( - switchMap((gridMode) => { - switch (gridMode) { - case "grid": - return oneOnOneLayoutMedia$.pipe( - switchMap((oneOnOne) => - oneOnOne === null ? gridLayoutMedia$ : of(oneOnOne), - ), - ); - case "spotlight": - return spotlightExpanded$.pipe( - switchMap((expanded) => - expanded - ? spotlightExpandedLayoutMedia$ - : spotlightLandscapeLayoutMedia$, - ), - ); - } - }), - ); - case "narrow": - return oneOnOneLayoutMedia$.pipe( - switchMap((oneOnOne) => - oneOnOne === null - ? combineLatest([grid$, spotlight$], (grid, spotlight) => - grid.length > smallMobileCallThreshold || - spotlight.some((vm) => vm instanceof ScreenShareViewModel) - ? spotlightPortraitLayoutMedia$ - : gridLayoutMedia$, - ).pipe(switchAll()) - : // The expanded spotlight layout makes for a better one-on-one - // experience in narrow windows - spotlightExpandedLayoutMedia$, - ), - ); - case "flat": - return gridMode$.pipe( - switchMap((gridMode) => { - switch (gridMode) { - case "grid": - // Yes, grid mode actually gets you a "spotlight" layout in - // this window mode. - return spotlightLandscapeLayoutMedia$; - case "spotlight": - return spotlightExpandedLayoutMedia$; - } - }), - ); - case "pip": - return pipLayoutMedia$; - } - }), - ), - ); - - // There is a cyclical dependency here: the layout algorithms want to know - // which tiles are on screen, but to know which tiles are on screen we have to - // first render a layout. To deal with this we assume initially that no tiles - // are visible, and loop the data back into the layouts with a Subject. - const visibleTiles$ = new Subject(); - const setVisibleTiles = (value: number): void => visibleTiles$.next(value); - - const layoutInternals$ = scope.behavior( - combineLatest([ - layoutMedia$, - visibleTiles$.pipe(startWith(0), distinctUntilChanged()), - ]).pipe( - scan< - [LayoutMedia, number], - LayoutScanState & { layout: Layout }, - LayoutScanState - >( - ({ tiles: prevTiles }, [media, visibleTiles]) => { - let layout: Layout; - let newTiles: TileStore; - switch (media.type) { - case "grid": - case "spotlight-landscape": - case "spotlight-portrait": - [layout, newTiles] = gridLikeLayout( - media, - visibleTiles, - setVisibleTiles, - prevTiles, - ); - break; - case "spotlight-expanded": - [layout, newTiles] = spotlightExpandedLayout(media, prevTiles); - break; - case "one-on-one": - [layout, newTiles] = oneOnOneLayout(media, prevTiles); - break; - case "pip": - [layout, newTiles] = pipLayout(media, prevTiles); - break; - } - - return { layout, tiles: newTiles }; - }, - { layout: null, tiles: TileStore.empty() }, - ), - ), - ); - - /** - * The layout of tiles in the call interface. - */ - /*PUBLIC*/ const layout$ = scope.behavior( - layoutInternals$.pipe(map(({ layout }) => layout)), - ); - - /** - * The current generation of the tile store, exposed for debugging purposes. - */ - /*PUBLIC*/ const tileStoreGeneration$ = scope.behavior( - layoutInternals$.pipe(map(({ tiles }) => tiles.generation)), - ); - - /*PUBLIC*/ const showSpotlightIndicators$ = scope.behavior( - layout$.pipe(map((l) => l.type !== "grid")), - ); - - /*PUBLIC*/ const showSpeakingIndicators$ = scope.behavior( - layout$.pipe( - switchMap((l) => { - switch (l.type) { - case "spotlight-landscape": - case "spotlight-portrait": - // If the spotlight is showing the active speaker, we can do without - // speaking indicators as they're a redundant visual cue. But if - // screen sharing feeds are in the spotlight we still need them. - return l.spotlight.media$.pipe( - map((models: MediaViewModel[]) => - models.some((m) => m instanceof ScreenShareViewModel), - ), - ); - // In expanded spotlight layout, the active speaker is always shown in - // the picture-in-picture tile so there is no need for speaking - // indicators. And in one-on-one layout there's no question as to who is - // speaking. - case "spotlight-expanded": - case "one-on-one": - return of(false); - default: - return of(true); - } - }), - ), - ); - - /*PUBLIC*/ const toggleSpotlightExpanded$ = scope.behavior< - (() => void) | null - >( - windowMode$.pipe( - switchMap((mode) => - mode === "normal" - ? layout$.pipe( - map( - (l) => - l.type === "spotlight-landscape" || - l.type === "spotlight-expanded", - ), - ) - : of(false), - ), - distinctUntilChanged(), - map((enabled) => - enabled ? (): void => spotlightExpandedToggle$.next() : null, - ), - ), - ); - - const screenTap$ = new Subject(); - const controlsTap$ = new Subject(); - const screenHover$ = new Subject(); - const screenUnhover$ = new Subject(); - - /*PUBLIC*/ const showHeader$ = scope.behavior( - windowMode$.pipe(map((mode) => mode !== "pip" && mode !== "flat")), - ); - - /*PUBLIC*/ const showFooter$ = scope.behavior( - windowMode$.pipe( - switchMap((mode) => { - switch (mode) { - case "pip": - return of(false); - case "normal": - case "narrow": - return of(true); - case "flat": - // Sadly Firefox has some layering glitches that prevent the footer - // from appearing properly. They happen less often if we never hide - // the footer. - if (isFirefox()) return of(true); - // Show/hide the footer in response to interactions - return merge( - screenTap$.pipe(map(() => "tap screen" as const)), - controlsTap$.pipe(map(() => "tap controls" as const)), - screenHover$.pipe(map(() => "hover" as const)), - ).pipe( - switchScan((state, interaction) => { - switch (interaction) { - case "tap screen": - return state - ? // Toggle visibility on tap - of(false) - : // Hide after a timeout - timer(showFooterMs).pipe( - map(() => false), - startWith(true), - ); - case "tap controls": - // The user is interacting with things, so reset the timeout - return timer(showFooterMs).pipe( - map(() => false), - startWith(true), - ); - case "hover": - // Show on hover and hide after a timeout - return race( - timer(showFooterMs), - screenUnhover$.pipe(take(1)), - ).pipe( - map(() => false), - startWith(true), - ); - } - }, false), - startWith(false), - ); - } - }), - ), - ); - - /** - * Whether audio is currently being output through the earpiece. - */ - /*PUBLIC*/ const earpieceMode$ = scope.behavior( - combineLatest( - [mediaDevices.audioOutput.available$, mediaDevices.audioOutput.selected$], - (available, selected) => - selected !== undefined && - available.get(selected.id)?.type === "earpiece", - ), - ); - - /** - * Callback to toggle between the earpiece and the loudspeaker. - * - * This will be `null` in case the target does not exist in the list - * of available audio outputs. - */ - /*PUBLIC*/ const audioOutputSwitcher$ = scope.behavior<{ - targetOutput: "earpiece" | "speaker"; - switch: () => void; - } | null>( - combineLatest( - [mediaDevices.audioOutput.available$, mediaDevices.audioOutput.selected$], - (available, selected) => { - const selectionType = selected && available.get(selected.id)?.type; - - // If we are in any output mode other than speaker switch to speaker. - const newSelectionType: "earpiece" | "speaker" = - selectionType === "speaker" ? "earpiece" : "speaker"; - const newSelection = [...available].find( - ([, d]) => d.type === newSelectionType, - ); - if (newSelection === undefined) return null; - - const [id] = newSelection; - return { - targetOutput: newSelectionType, - switch: (): void => mediaDevices.audioOutput.select(id), - }; - }, - ), - ); - - /** - * Emits an array of reactions that should be visible on the screen. - */ - // DISCUSSION move this into a reaction file - // const {visibleReactions$, audibleReactions$} = reactionsObservables$(showReactionSetting$, ) - const visibleReactions$ = scope.behavior( - showReactions.value$.pipe( - switchMap((show) => (show ? reactions$ : of({}))), - scan< - Record, - { sender: string; emoji: string; startX: number }[] - >((acc, latest) => { - const newSet: { sender: string; emoji: string; startX: number }[] = []; - for (const [sender, reaction] of Object.entries(latest)) { - const startX = - acc.find((v) => v.sender === sender && v.emoji)?.startX ?? - Math.ceil(Math.random() * 80) + 10; - newSet.push({ sender, emoji: reaction.emoji, startX }); - } - return newSet; - }, []), - ), - ); - - /** - * Emits an array of reactions that should be played. - */ - const audibleReactions$ = playReactionsSound.value$.pipe( - switchMap((show) => - show ? reactions$ : of>({}), - ), - map((reactions) => Object.values(reactions).map((v) => v.name)), - scan( - (acc, latest) => { - return { - playing: latest.filter( - (v) => acc.playing.includes(v) || acc.newSounds.includes(v), - ), - newSounds: latest.filter( - (v) => !acc.playing.includes(v) && !acc.newSounds.includes(v), - ), - }; - }, - { playing: [], newSounds: [] }, - ), - map((v) => v.newSounds), - ); - - const newHandRaised$ = handsRaised$.pipe( - map((v) => Object.keys(v).length), - scan( - (acc, newValue) => ({ - value: newValue, - playSounds: newValue > acc.value, - }), - { value: 0, playSounds: false }, - ), - filter((v) => v.playSounds), - ); - - const newScreenShare$ = screenShares$.pipe( - map((v) => v.length), - scan( - (acc, newValue) => ({ - value: newValue, - playSounds: newValue > acc.value, - }), - { value: 0, playSounds: false }, - ), - filter((v) => v.playSounds), - ); - - /** - * Whether we are sharing our screen. - */ - // reassigned here to make it publicly accessible - const sharingScreen$ = localMembership.sharingScreen$; - - /** - * Callback to toggle screen sharing. If null, screen sharing is not possible. - */ - // reassigned here to make it publicly accessible - const toggleScreenSharing = localMembership.toggleScreenSharing; - - const join = localMembership.requestConnect; - join(); // TODO-MULTI-SFU: Use this view model for the lobby as well, and only call this once 'join' is clicked? - - return { - autoLeave$: callLifecycle.autoLeave$, - callPickupState$, - leave$, - hangup: () => userHangup$.next(), - join, - toggleScreenSharing, - sharingScreen$, - - tapScreen: () => screenTap$.next(), - tapControls: () => controlsTap$.next(), - hoverScreen: () => screenHover$.next(), - unhoverScreen: () => screenUnhover$.next(), - - configError$: localMembership.configError$, - participantCount$, - audioParticipants$, - isOneOnOneWith$, - localUserIsAlone$, - - handsRaised$, - reactions$, - joinSoundEffect$, - leaveSoundEffect$, - newHandRaised$, - newScreenShare$, - audibleReactions$, - visibleReactions$, - - windowMode$, - spotlightExpanded$, - toggleSpotlightExpanded$, - gridMode$, - setGridMode, - grid$, - spotlight$, - pip$, - layout$, - tileStoreGeneration$, - showSpotlightIndicators$, - showSpeakingIndicators$, - showHeader$, - showFooter$, - earpieceMode$, - audioOutputSwitcher$, - reconnecting$, - }; -} -// TODO-MULTI-SFU // Setup and update the keyProvider which was create by `createRoom` was a thing before. Now we never update if the E2EEsystem changes -// do we need this? - -function getE2eeKeyProvider( - e2eeSystem: EncryptionSystem, - rtcSession: MatrixRTCSession, -): BaseKeyProvider | undefined { - if (e2eeSystem.kind === E2eeType.NONE) return undefined; - - if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) { - const keyProvider = new MatrixKeyProvider(); - keyProvider.setRTCSession(rtcSession); - return keyProvider; - } else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) { - const keyProvider = new ExternalE2EEKeyProvider(); - keyProvider - .setKey(e2eeSystem.secret) - .catch((e) => logger.error("Failed to set shared key for E2EE", e)); - return keyProvider; - } -} diff --git a/src/state/CallViewModel/CallViewModelOld.ts b/src/state/CallViewModel/CallViewModelOld.ts deleted file mode 100644 index ff4a6269..00000000 --- a/src/state/CallViewModel/CallViewModelOld.ts +++ /dev/null @@ -1,1335 +0,0 @@ -/* -Copyright 2023, 2024, 2025 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - type BaseKeyProvider, - type ConnectionState, - ExternalE2EEKeyProvider, - type Room as LivekitRoom, - type RoomOptions, -} from "livekit-client"; -import { type Room as MatrixRoom } from "matrix-js-sdk"; -import { - combineLatest, - distinctUntilChanged, - EMPTY, - filter, - fromEvent, - map, - merge, - NEVER, - type Observable, - of, - pairwise, - race, - scan, - skip, - skipWhile, - startWith, - Subject, - switchAll, - switchMap, - switchScan, - take, - tap, - throttleTime, - timer, -} from "rxjs"; -import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; -import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; -import { type IWidgetApiRequest } from "matrix-widget-api"; - -import { - LocalUserMediaViewModel, - type MediaViewModel, - type RemoteUserMediaViewModel, - ScreenShareViewModel, - type UserMediaViewModel, -} from "../MediaViewModel"; -import { accumulate, generateItems, pauseWhen } from "../../utils/observable"; -import { - duplicateTiles, - MatrixRTCMode, - matrixRTCMode, - playReactionsSound, - showReactions, -} from "../../settings/settings"; -import { isFirefox } from "../../Platform"; -import { setPipEnabled$ } from "../../controls"; -import { TileStore } from "../TileStore"; -import { gridLikeLayout } from "../GridLikeLayout"; -import { spotlightExpandedLayout } from "../SpotlightExpandedLayout"; -import { oneOnOneLayout } from "../OneOnOneLayout"; -import { pipLayout } from "../PipLayout"; -import { type EncryptionSystem } from "../../e2ee/sharedKeyManagement"; -import { - type RaisedHandInfo, - type ReactionInfo, - type ReactionOption, -} from "../../reactions"; -import { shallowEquals } from "../../utils/array"; -import { type MediaDevices } from "../MediaDevices"; -import { type Behavior } from "../Behavior"; -import { E2eeType } from "../../e2ee/e2eeType"; -import { MatrixKeyProvider } from "../../e2ee/matrixKeyProvider"; -import { type MuteStates } from "../MuteStates"; -import { getUrlParams } from "../../UrlParams"; -import { type ProcessorState } from "../../livekit/TrackProcessorContext"; -import { ElementWidgetActions, widget } from "../../widget"; -import { UserMedia } from "../UserMedia.ts"; -import { ScreenShare } from "../ScreenShare.ts"; -import { - type GridLayoutMedia, - type Layout, - type LayoutMedia, - type OneOnOneLayoutMedia, - type SpotlightExpandedLayoutMedia, - type SpotlightLandscapeLayoutMedia, - type SpotlightPortraitLayoutMedia, -} from "../layout-types.ts"; -import { type ElementCallError } from "../../utils/errors.ts"; -import { type ObservableScope } from "../ObservableScope.ts"; -import { createLocalMembership$ } from "./localMember/LocalMembership.ts"; -import { createLocalTransport$ } from "./localMember/LocalTransport.ts"; -import { - createMemberships$, - membershipsAndTransports$, -} from "../SessionBehaviors.ts"; -import { ECConnectionFactory } from "./remoteMembers/ConnectionFactory.ts"; -import { createConnectionManager$ } from "./remoteMembers/ConnectionManager.ts"; -import { - createMatrixLivekitMembers$, - type MatrixLivekitMember, -} from "./remoteMembers/MatrixLivekitMembers.ts"; -import { - createCallNotificationLifecycle$, - createReceivedDecline$, - createSentCallNotification$, -} from "./CallNotificationLifecycle.ts"; -import { - createMatrixMemberMetadata$, - createRoomMembers$, -} from "./remoteMembers/MatrixMemberMetadata.ts"; - -const logger = rootLogger.getChild("[CallViewModel]"); -//TODO -// Larger rename -// member,membership -> rtcMember -// participant -> livekitParticipant -// matrixLivekitItem -> callMember -// js-sdk -// callMembership -> rtcMembership -export interface CallViewModelOptions { - encryptionSystem: EncryptionSystem; - autoLeaveWhenOthersLeft?: boolean; - /** - * If the call is started in a way where we want it to behave like a telephone usecase - * If we sent a notification event, we want the ui to show a ringing state - */ - waitForCallPickup?: boolean; - /** Optional factory to create LiveKit rooms, mainly for testing purposes. */ - livekitRoomFactory?: (options?: RoomOptions) => LivekitRoom; - /** Optional behavior overriding the local connection state, mainly for testing purposes. */ - connectionState$?: Behavior; -} - -// Do not play any sounds if the participant count has exceeded this -// number. -export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8; -export const THROTTLE_SOUND_EFFECT_MS = 500; - -// This is the number of participants that we think constitutes a "small" call -// on mobile. No spotlight tile should be shown below this threshold. -const smallMobileCallThreshold = 3; - -// How long the footer should be shown for when hovering over or interacting -// with the interface -const showFooterMs = 4000; - -export type GridMode = "grid" | "spotlight"; - -export type WindowMode = "normal" | "narrow" | "flat" | "pip"; - -interface LayoutScanState { - layout: Layout | null; - tiles: TileStore; -} - -type MediaItem = UserMedia | ScreenShare; -type AudioLivekitItem = { - livekitRoom: LivekitRoom; - participants: string[]; - url: string; -}; -/** - * A view model providing all the application logic needed to show the in-call - * UI (may eventually be expanded to cover the lobby and feedback screens in the - * future). - */ -// Throughout this class and related code we must distinguish between MatrixRTC -// state and LiveKit state. We use the common terminology of room "members", RTC -// "memberships", and LiveKit "participants". -export class CallViewModel { - private readonly userId = this.matrixRoom.client.getUserId()!; - private readonly deviceId = this.matrixRoom.client.getDeviceId()!; - - private readonly livekitKeyProvider = getE2eeKeyProvider( - this.options.encryptionSystem, - this.matrixRTCSession, - ); - - // Each hbar seperates a block of input variables required for the CallViewModel to function. - // The outputs of this block is written under the hbar. - // - // For mocking purposes it is recommended to only mock the functions creating those outputs. - // All other fields are just temp computations for the mentioned output. - // The class does not need anything except the values underneath the bar. - // The creation of the values under the bar are all tested independently and testing the callViewModel Should - // not test their cretation. Call view model only needs: - // - memberships$ via createMemberships$ - // - localMembership via createLocalMembership$ - // - callLifecycle via createCallNotificationLifecycle$ - // - matrixMemberMetadataStore via createMatrixMemberMetadata$ - - // ------------------------------------------------------------------------ - // memberships$ - private memberships$ = createMemberships$(this.scope, this.matrixRTCSession); - - // ------------------------------------------------------------------------ - // matrixLivekitMembers$ AND localMembership - - private membershipsAndTransports = membershipsAndTransports$( - this.scope, - this.memberships$, - ); - - private localTransport$ = createLocalTransport$({ - scope: this.scope, - memberships$: this.memberships$, - client: this.matrixRoom.client, - roomId: this.matrixRoom.roomId, - useOldestMember$: this.scope.behavior( - matrixRTCMode.value$.pipe(map((v) => v === MatrixRTCMode.Legacy)), - ), - }); - - private connectionFactory = new ECConnectionFactory( - this.matrixRoom.client, - this.mediaDevices, - this.trackProcessorState$, - this.livekitKeyProvider, - getUrlParams().controlledAudioDevices, - this.options.livekitRoomFactory, - ); - - private connectionManager = createConnectionManager$({ - scope: this.scope, - connectionFactory: this.connectionFactory, - inputTransports$: this.scope.behavior( - combineLatest( - [this.localTransport$, this.membershipsAndTransports.transports$], - (localTransport, transports) => { - const localTransportAsArray = localTransport ? [localTransport] : []; - return transports.mapInner((transports) => [ - ...localTransportAsArray, - ...transports, - ]); - }, - ), - ), - logger: logger, - }); - - private matrixLivekitMembers$ = createMatrixLivekitMembers$({ - scope: this.scope, - membershipsWithTransport$: - this.membershipsAndTransports.membershipsWithTransport$, - connectionManager: this.connectionManager, - }); - - private connectOptions$ = this.scope.behavior( - matrixRTCMode.value$.pipe( - map((mode) => ({ - encryptMedia: this.livekitKeyProvider !== undefined, - // TODO. This might need to get called again on each cahnge of matrixRTCMode... - matrixRTCMode: mode, - })), - ), - ); - - private localMembership = createLocalMembership$({ - scope: this.scope, - muteStates: this.muteStates, - mediaDevices: this.mediaDevices, - connectionManager: this.connectionManager, - matrixRTCSession: this.matrixRTCSession, - matrixRoom: this.matrixRoom, - localTransport$: this.localTransport$, - trackProcessorState$: this.trackProcessorState$, - widget, - options: this.connectOptions$, - logger: logger.getChild(`[${Date.now()}]`), - }); - - private localRtcMembership$ = this.scope.behavior( - this.memberships$.pipe( - map( - (memberships) => - memberships.value.find( - (membership) => - membership.userId === this.userId && - membership.deviceId === this.deviceId, - ) ?? null, - ), - ), - ); - - private localMatrixLivekitMemberUninitialized = { - membership$: this.localRtcMembership$, - participant$: this.localMembership.participant$, - connection$: this.localMembership.connection$, - userId: this.userId, - }; - - private localMatrixLivekitMember$: Behavior = - this.scope.behavior( - this.localRtcMembership$.pipe( - switchMap((membership) => { - if (!membership) return of(null); - return of( - // casting is save here since we know that localRtcMembership$ is !== null since we reached this case. - this.localMatrixLivekitMemberUninitialized as MatrixLivekitMember, - ); - }), - ), - ); - - // ------------------------------------------------------------------------ - // callLifecycle - - private callLifecycle = createCallNotificationLifecycle$({ - scope: this.scope, - memberships$: this.memberships$, - sentCallNotification$: createSentCallNotification$( - this.scope, - this.matrixRTCSession, - ), - receivedDecline$: createReceivedDecline$(this.matrixRoom), - options: this.options, - localUser: { userId: this.userId, deviceId: this.deviceId }, - }); - public autoLeave$ = this.callLifecycle.autoLeave$; - - // ------------------------------------------------------------------------ - // matrixMemberMetadataStore - - private matrixMemberMetadataStore = createMatrixMemberMetadata$( - this.scope, - this.scope.behavior(this.memberships$.pipe(map((mems) => mems.value))), - createRoomMembers$(this.scope, this.matrixRoom), - ); - - /** - * If there is a configuration error with the call (e.g. misconfigured E2EE). - * This is a fatal error that prevents the call from being created/joined. - * Should render a blocking error screen. - */ - public get configError$(): Behavior { - return this.localMembership.configError$; - } - - public join = this.localMembership.requestConnect; - - // CODESMELL? - // This is functionally the same Observable as leave$, except here it's - // hoisted to the top of the class. This enables the cyclic dependency between - // leave$ -> autoLeave$ -> callPickupState$ -> livekitConnectionState$ -> - // localConnection$ -> transports$ -> joined$ -> leave$. - private readonly leaveHoisted$ = new Subject< - "user" | "timeout" | "decline" | "allOthersLeft" - >(); - - /** - * Whether various media/event sources should pretend to be disconnected from - * all network input, even if their connection still technically works. - */ - // We do this when the app is in the 'reconnecting' state, because it might be - // that the LiveKit connection is still functional while the homeserver is - // down, for example, and we want to avoid making people worry that the app is - // in a split-brained state. - // DISCUSSION own membership manager ALSO this probably can be simplifis - public reconnecting$ = this.localMembership.reconnecting$; - private readonly pretendToBeDisconnected$ = this.reconnecting$; - - public readonly audioParticipants$ = this.scope.behavior( - this.matrixLivekitMembers$.pipe( - switchMap((membersWithEpoch) => { - const members = membersWithEpoch.value; - const a$ = combineLatest( - members.map((member) => - combineLatest([member.connection$, member.participant$]).pipe( - map(([connection, participant]) => { - // do not render audio for local participant - if (!connection || !participant || participant.isLocal) - return null; - const livekitRoom = connection.livekitRoom; - const url = connection.transport.livekit_service_url; - - return { url, livekitRoom, participant: participant.identity }; - }), - ), - ), - ); - return a$; - }), - map((members) => - members.reduce((acc, curr) => { - if (!curr) return acc; - - const existing = acc.find((item) => item.url === curr.url); - if (existing) { - existing.participants.push(curr.participant); - } else { - acc.push({ - livekitRoom: curr.livekitRoom, - participants: [curr.participant], - url: curr.url, - }); - } - return acc; - }, []), - ), - ), - [], - ); - - public readonly handsRaised$ = this.scope.behavior( - this.handsRaisedSubject$.pipe(pauseWhen(this.pretendToBeDisconnected$)), - ); - - public readonly reactions$ = this.scope.behavior( - this.reactionsSubject$.pipe( - map((v) => - Object.fromEntries( - Object.entries(v).map(([a, { reactionOption }]) => [ - a, - reactionOption, - ]), - ), - ), - pauseWhen(this.pretendToBeDisconnected$), - ), - ); - - /** - * List of user media (camera feeds) that we want tiles for. - */ - // TODO this also needs the local participant to be added. - private readonly userMedia$ = this.scope.behavior( - combineLatest([ - this.localMatrixLivekitMember$, - this.matrixLivekitMembers$, - duplicateTiles.value$, - ]).pipe( - // Generate a collection of MediaItems from the list of expected (whether - // present or missing) LiveKit participants. - generateItems( - function* ([ - localMatrixLivekitMember, - { value: matrixLivekitMembers }, - duplicateTiles, - ]) { - let localParticipantId = undefined; - // add local member if available - if (localMatrixLivekitMember) { - const { userId, participant$, connection$, membership$ } = - localMatrixLivekitMember; - localParticipantId = `${userId}:${membership$.value.deviceId}`; // should be membership$.value.membershipID which is not optional - // const participantId = membership$.value.membershipID; - if (localParticipantId) { - for (let dup = 0; dup < 1 + duplicateTiles; dup++) { - yield { - keys: [ - dup, - localParticipantId, - userId, - participant$, - connection$, - ], - data: undefined, - }; - } - } - } - // add remote members that are available - for (const { - userId, - participant$, - connection$, - membership$, - } of matrixLivekitMembers) { - const participantId = `${userId}:${membership$.value.deviceId}`; - if (participantId === localParticipantId) continue; - // const participantId = membership$.value?.identity; - for (let dup = 0; dup < 1 + duplicateTiles; dup++) { - yield { - keys: [dup, participantId, userId, participant$, connection$], - data: undefined, - }; - } - } - }, - ( - scope, - _data$, - dup, - participantId, - userId, - participant$, - connection$, - ) => { - const livekitRoom$ = scope.behavior( - connection$.pipe(map((c) => c?.livekitRoom)), - ); - const focusUrl$ = scope.behavior( - connection$.pipe(map((c) => c?.transport.livekit_service_url)), - ); - const displayName$ = scope.behavior( - this.matrixMemberMetadataStore - .createDisplayNameBehavior$(userId) - .pipe(map((name) => name ?? userId)), - ); - - return new UserMedia( - scope, - `${participantId}:${dup}`, - userId, - participant$, - this.options.encryptionSystem, - livekitRoom$, - focusUrl$, - this.mediaDevices, - this.pretendToBeDisconnected$, - displayName$, - this.matrixMemberMetadataStore.createAvatarUrlBehavior$(userId), - this.handsRaised$.pipe(map((v) => v[participantId]?.time ?? null)), - this.reactions$.pipe(map((v) => v[participantId] ?? undefined)), - ); - }, - ), - ), - ); - - /** - * List of all media items (user media and screen share media) that we want - * tiles for. - */ - private readonly mediaItems$ = this.scope.behavior( - this.userMedia$.pipe( - switchMap((userMedia) => - userMedia.length === 0 - ? of([]) - : combineLatest( - userMedia.map((m) => m.screenShares$), - (...screenShares) => [...userMedia, ...screenShares.flat(1)], - ), - ), - ), - ); - - /** - * List of MediaItems that we want to display, that are of type ScreenShare - */ - private readonly screenShares$ = this.scope.behavior( - this.mediaItems$.pipe( - map((mediaItems) => - mediaItems.filter((m): m is ScreenShare => m instanceof ScreenShare), - ), - ), - ); - - public readonly joinSoundEffect$ = this.userMedia$.pipe( - pairwise(), - filter( - ([prev, current]) => - current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && - current.length > prev.length, - ), - map(() => {}), - throttleTime(THROTTLE_SOUND_EFFECT_MS), - ); - - /** - * The number of participants currently in the call. - * - * - Each participant has a corresponding MatrixRTC membership state event - * - There can be multiple participants for one Matrix user if they join from - * multiple devices. - */ - public readonly participantCount$ = this.scope.behavior( - this.matrixLivekitMembers$.pipe(map((ms) => ms.value.length)), - ); - - // only public to expose to the view. - // TODO if we are in "unknown" state we need a loading rendering (or empty screen) - // Otherwise it looks like we already connected and only than the ringing starts which is weird. - public readonly callPickupState$ = this.callLifecycle.callPickupState$; - - public readonly leaveSoundEffect$ = combineLatest([ - this.callLifecycle.callPickupState$, - this.userMedia$, - ]).pipe( - // Until the call is successful, do not play a leave sound. - // If callPickupState$ is null, then we always play the sound as it will not conflict with a decline sound. - skipWhile(([c]) => c !== null && c !== "success"), - map(([, userMedia]) => userMedia), - pairwise(), - filter( - ([prev, current]) => - current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && - current.length < prev.length, - ), - map(() => {}), - throttleTime(THROTTLE_SOUND_EFFECT_MS), - ); - - private readonly userHangup$ = new Subject(); - public hangup(): void { - this.userHangup$.next(); - } - - private readonly widgetHangup$ = - widget === null - ? NEVER - : ( - fromEvent( - widget.lazyActions, - ElementWidgetActions.HangupCall, - ) as Observable> - ).pipe( - tap((ev) => { - widget!.api.transport.reply(ev.detail, {}); - }), - ); - - public readonly leave$: Observable< - "user" | "timeout" | "decline" | "allOthersLeft" - > = merge( - this.callLifecycle.autoLeave$, - merge(this.userHangup$, this.widgetHangup$).pipe( - map(() => "user" as const), - ), - ).pipe( - this.scope.share, - tap((reason) => this.leaveHoisted$.next(reason)), - ); - - private readonly spotlightSpeaker$ = - this.scope.behavior( - this.userMedia$.pipe( - switchMap((mediaItems) => - mediaItems.length === 0 - ? of([]) - : combineLatest( - mediaItems.map((m) => - m.vm.speaking$.pipe(map((s) => [m, s] as const)), - ), - ), - ), - scan<(readonly [UserMedia, boolean])[], UserMedia | undefined, null>( - (prev, mediaItems) => { - // Only remote users that are still in the call should be sticky - const [stickyMedia, stickySpeaking] = - (!prev?.vm.local && mediaItems.find(([m]) => m === prev)) || []; - // Decide who to spotlight: - // If the previous speaker is still speaking, stick with them rather - // than switching eagerly to someone else - return stickySpeaking - ? stickyMedia! - : // Otherwise, select any remote user who is speaking - (mediaItems.find(([m, s]) => !m.vm.local && s)?.[0] ?? - // Otherwise, stick with the person who was last speaking - stickyMedia ?? - // Otherwise, spotlight an arbitrary remote user - mediaItems.find(([m]) => !m.vm.local)?.[0] ?? - // Otherwise, spotlight the local user - mediaItems.find(([m]) => m.vm.local)?.[0]); - }, - null, - ), - map((speaker) => speaker?.vm ?? null), - ), - ); - - private readonly grid$ = this.scope.behavior( - this.userMedia$.pipe( - switchMap((mediaItems) => { - const bins = mediaItems.map((m) => - m.bin$.pipe(map((bin) => [m, bin] as const)), - ); - // Sort the media by bin order and generate a tile for each one - return bins.length === 0 - ? of([]) - : combineLatest(bins, (...bins) => - bins.sort(([, bin1], [, bin2]) => bin1 - bin2).map(([m]) => m.vm), - ); - }), - distinctUntilChanged(shallowEquals), - ), - ); - - private readonly spotlight$ = this.scope.behavior( - this.screenShares$.pipe( - switchMap((screenShares) => { - if (screenShares.length > 0) { - return of(screenShares.map((m) => m.vm)); - } - - return this.spotlightSpeaker$.pipe( - map((speaker) => (speaker ? [speaker] : [])), - ); - }), - distinctUntilChanged(shallowEquals), - ), - ); - - private readonly pip$ = this.scope.behavior( - combineLatest([ - // TODO This also needs epoch logic to dedupe the screenshares and mediaItems emits - this.screenShares$, - this.spotlightSpeaker$, - this.mediaItems$, - ]).pipe( - switchMap(([screenShares, spotlight, mediaItems]) => { - if (screenShares.length > 0) { - return this.spotlightSpeaker$; - } - if (!spotlight || spotlight.local) { - return of(null); - } - - const localUserMedia = mediaItems.find( - (m) => m.vm instanceof LocalUserMediaViewModel, - ) as UserMedia | undefined; - - const localUserMediaViewModel = localUserMedia?.vm as - | LocalUserMediaViewModel - | undefined; - - if (!localUserMediaViewModel) { - return of(null); - } - return localUserMediaViewModel.alwaysShow$.pipe( - map((alwaysShow) => { - if (alwaysShow) { - return localUserMediaViewModel; - } - - return null; - }), - ); - }), - ), - ); - - private readonly hasRemoteScreenShares$: Observable = - this.spotlight$.pipe( - map((spotlight) => - spotlight.some((vm) => !vm.local && vm instanceof ScreenShareViewModel), - ), - distinctUntilChanged(), - ); - - private readonly pipEnabled$ = this.scope.behavior(setPipEnabled$, false); - - private readonly naturalWindowMode$ = this.scope.behavior( - fromEvent(window, "resize").pipe( - map(() => { - const height = window.innerHeight; - const width = window.innerWidth; - if (height <= 400 && width <= 340) return "pip"; - // Our layouts for flat windows are better at adapting to a small width - // than our layouts for narrow windows are at adapting to a small height, - // so we give "flat" precedence here - if (height <= 600) return "flat"; - if (width <= 600) return "narrow"; - return "normal"; - }), - ), - "normal", - ); - - /** - * The general shape of the window. - */ - public readonly windowMode$ = this.scope.behavior( - this.pipEnabled$.pipe( - switchMap((pip) => - pip ? of("pip") : this.naturalWindowMode$, - ), - ), - ); - - private readonly spotlightExpandedToggle$ = new Subject(); - public readonly spotlightExpanded$ = this.scope.behavior( - this.spotlightExpandedToggle$.pipe( - accumulate(false, (expanded) => !expanded), - ), - ); - - private readonly gridModeUserSelection$ = new Subject(); - /** - * The layout mode of the media tile grid. - */ - public readonly gridMode$ = - // If the user hasn't selected spotlight and somebody starts screen sharing, - // automatically switch to spotlight mode and reset when screen sharing ends - this.scope.behavior( - this.gridModeUserSelection$.pipe( - switchMap((userSelection) => - (userSelection === "spotlight" - ? EMPTY - : combineLatest([ - this.hasRemoteScreenShares$, - this.windowMode$, - ]).pipe( - skip(userSelection === null ? 0 : 1), - map( - ([hasScreenShares, windowMode]): GridMode => - hasScreenShares || windowMode === "flat" - ? "spotlight" - : "grid", - ), - ) - ).pipe(startWith(userSelection ?? "grid")), - ), - ), - "grid", - ); - - public setGridMode(value: GridMode): void { - this.gridModeUserSelection$.next(value); - } - - private readonly gridLayoutMedia$: Observable = - combineLatest([this.grid$, this.spotlight$], (grid, spotlight) => ({ - type: "grid", - spotlight: spotlight.some((vm) => vm instanceof ScreenShareViewModel) - ? spotlight - : undefined, - grid, - })); - - private readonly spotlightLandscapeLayoutMedia$: Observable = - combineLatest([this.grid$, this.spotlight$], (grid, spotlight) => ({ - type: "spotlight-landscape", - spotlight, - grid, - })); - - private readonly spotlightPortraitLayoutMedia$: Observable = - combineLatest([this.grid$, this.spotlight$], (grid, spotlight) => ({ - type: "spotlight-portrait", - spotlight, - grid, - })); - - private readonly spotlightExpandedLayoutMedia$: Observable = - combineLatest([this.spotlight$, this.pip$], (spotlight, pip) => ({ - type: "spotlight-expanded", - spotlight, - pip: pip ?? undefined, - })); - - private readonly oneOnOneLayoutMedia$: Observable = - this.mediaItems$.pipe( - map((mediaItems) => { - if (mediaItems.length !== 2) return null; - const local = mediaItems.find((vm) => vm.vm.local)?.vm as - | LocalUserMediaViewModel - | undefined; - const remote = mediaItems.find((vm) => !vm.vm.local)?.vm as - | RemoteUserMediaViewModel - | undefined; - // There might not be a remote tile if there are screen shares, or if - // only the local user is in the call and they're using the duplicate - // tiles option - if (!remote || !local) return null; - - return { type: "one-on-one", local, remote }; - }), - ); - - private readonly pipLayoutMedia$: Observable = - this.spotlight$.pipe(map((spotlight) => ({ type: "pip", spotlight }))); - - /** - * The media to be used to produce a layout. - */ - private readonly layoutMedia$ = this.scope.behavior( - this.windowMode$.pipe( - switchMap((windowMode) => { - switch (windowMode) { - case "normal": - return this.gridMode$.pipe( - switchMap((gridMode) => { - switch (gridMode) { - case "grid": - return this.oneOnOneLayoutMedia$.pipe( - switchMap((oneOnOne) => - oneOnOne === null - ? this.gridLayoutMedia$ - : of(oneOnOne), - ), - ); - case "spotlight": - return this.spotlightExpanded$.pipe( - switchMap((expanded) => - expanded - ? this.spotlightExpandedLayoutMedia$ - : this.spotlightLandscapeLayoutMedia$, - ), - ); - } - }), - ); - case "narrow": - return this.oneOnOneLayoutMedia$.pipe( - switchMap((oneOnOne) => - oneOnOne === null - ? combineLatest( - [this.grid$, this.spotlight$], - (grid, spotlight) => - grid.length > smallMobileCallThreshold || - spotlight.some( - (vm) => vm instanceof ScreenShareViewModel, - ) - ? this.spotlightPortraitLayoutMedia$ - : this.gridLayoutMedia$, - ).pipe(switchAll()) - : // The expanded spotlight layout makes for a better one-on-one - // experience in narrow windows - this.spotlightExpandedLayoutMedia$, - ), - ); - case "flat": - return this.gridMode$.pipe( - switchMap((gridMode) => { - switch (gridMode) { - case "grid": - // Yes, grid mode actually gets you a "spotlight" layout in - // this window mode. - return this.spotlightLandscapeLayoutMedia$; - case "spotlight": - return this.spotlightExpandedLayoutMedia$; - } - }), - ); - case "pip": - return this.pipLayoutMedia$; - } - }), - ), - ); - - // There is a cyclical dependency here: the layout algorithms want to know - // which tiles are on screen, but to know which tiles are on screen we have to - // first render a layout. To deal with this we assume initially that no tiles - // are visible, and loop the data back into the layouts with a Subject. - private readonly visibleTiles$ = new Subject(); - private readonly setVisibleTiles = (value: number): void => - this.visibleTiles$.next(value); - - private readonly layoutInternals$ = this.scope.behavior< - LayoutScanState & { layout: Layout } - >( - combineLatest([ - this.layoutMedia$, - this.visibleTiles$.pipe(startWith(0), distinctUntilChanged()), - ]).pipe( - scan< - [LayoutMedia, number], - LayoutScanState & { layout: Layout }, - LayoutScanState - >( - ({ tiles: prevTiles }, [media, visibleTiles]) => { - let layout: Layout; - let newTiles: TileStore; - switch (media.type) { - case "grid": - case "spotlight-landscape": - case "spotlight-portrait": - [layout, newTiles] = gridLikeLayout( - media, - visibleTiles, - this.setVisibleTiles, - prevTiles, - ); - break; - case "spotlight-expanded": - [layout, newTiles] = spotlightExpandedLayout(media, prevTiles); - break; - case "one-on-one": - [layout, newTiles] = oneOnOneLayout(media, prevTiles); - break; - case "pip": - [layout, newTiles] = pipLayout(media, prevTiles); - break; - } - - return { layout, tiles: newTiles }; - }, - { layout: null, tiles: TileStore.empty() }, - ), - ), - ); - - /** - * The layout of tiles in the call interface. - */ - public readonly layout$ = this.scope.behavior( - this.layoutInternals$.pipe(map(({ layout }) => layout)), - ); - - /** - * The current generation of the tile store, exposed for debugging purposes. - */ - public readonly tileStoreGeneration$ = this.scope.behavior( - this.layoutInternals$.pipe(map(({ tiles }) => tiles.generation)), - ); - - public showSpotlightIndicators$ = this.scope.behavior( - this.layout$.pipe(map((l) => l.type !== "grid")), - ); - - public showSpeakingIndicators$ = this.scope.behavior( - this.layout$.pipe( - switchMap((l) => { - switch (l.type) { - case "spotlight-landscape": - case "spotlight-portrait": - // If the spotlight is showing the active speaker, we can do without - // speaking indicators as they're a redundant visual cue. But if - // screen sharing feeds are in the spotlight we still need them. - return l.spotlight.media$.pipe( - map((models: MediaViewModel[]) => - models.some((m) => m instanceof ScreenShareViewModel), - ), - ); - // In expanded spotlight layout, the active speaker is always shown in - // the picture-in-picture tile so there is no need for speaking - // indicators. And in one-on-one layout there's no question as to who is - // speaking. - case "spotlight-expanded": - case "one-on-one": - return of(false); - default: - return of(true); - } - }), - ), - ); - - public readonly toggleSpotlightExpanded$ = this.scope.behavior< - (() => void) | null - >( - this.windowMode$.pipe( - switchMap((mode) => - mode === "normal" - ? this.layout$.pipe( - map( - (l) => - l.type === "spotlight-landscape" || - l.type === "spotlight-expanded", - ), - ) - : of(false), - ), - distinctUntilChanged(), - map((enabled) => - enabled ? (): void => this.spotlightExpandedToggle$.next() : null, - ), - ), - ); - - private readonly screenTap$ = new Subject(); - private readonly controlsTap$ = new Subject(); - private readonly screenHover$ = new Subject(); - private readonly screenUnhover$ = new Subject(); - - /** - * Callback for when the user taps the call view. - */ - public tapScreen(): void { - this.screenTap$.next(); - } - - /** - * Callback for when the user taps the call's controls. - */ - public tapControls(): void { - this.controlsTap$.next(); - } - - /** - * Callback for when the user hovers over the call view. - */ - public hoverScreen(): void { - this.screenHover$.next(); - } - - /** - * Callback for when the user stops hovering over the call view. - */ - public unhoverScreen(): void { - this.screenUnhover$.next(); - } - - public readonly showHeader$ = this.scope.behavior( - this.windowMode$.pipe(map((mode) => mode !== "pip" && mode !== "flat")), - ); - - public readonly showFooter$ = this.scope.behavior( - this.windowMode$.pipe( - switchMap((mode) => { - switch (mode) { - case "pip": - return of(false); - case "normal": - case "narrow": - return of(true); - case "flat": - // Sadly Firefox has some layering glitches that prevent the footer - // from appearing properly. They happen less often if we never hide - // the footer. - if (isFirefox()) return of(true); - // Show/hide the footer in response to interactions - return merge( - this.screenTap$.pipe(map(() => "tap screen" as const)), - this.controlsTap$.pipe(map(() => "tap controls" as const)), - this.screenHover$.pipe(map(() => "hover" as const)), - ).pipe( - switchScan((state, interaction) => { - switch (interaction) { - case "tap screen": - return state - ? // Toggle visibility on tap - of(false) - : // Hide after a timeout - timer(showFooterMs).pipe( - map(() => false), - startWith(true), - ); - case "tap controls": - // The user is interacting with things, so reset the timeout - return timer(showFooterMs).pipe( - map(() => false), - startWith(true), - ); - case "hover": - // Show on hover and hide after a timeout - return race( - timer(showFooterMs), - this.screenUnhover$.pipe(take(1)), - ).pipe( - map(() => false), - startWith(true), - ); - } - }, false), - startWith(false), - ); - } - }), - ), - ); - - /** - * Whether audio is currently being output through the earpiece. - */ - public readonly earpieceMode$ = this.scope.behavior( - combineLatest( - [ - this.mediaDevices.audioOutput.available$, - this.mediaDevices.audioOutput.selected$, - ], - (available, selected) => - selected !== undefined && - available.get(selected.id)?.type === "earpiece", - ), - ); - - /** - * Callback to toggle between the earpiece and the loudspeaker. - * - * This will be `null` in case the target does not exist in the list - * of available audio outputs. - */ - public readonly audioOutputSwitcher$ = this.scope.behavior<{ - targetOutput: "earpiece" | "speaker"; - switch: () => void; - } | null>( - combineLatest( - [ - this.mediaDevices.audioOutput.available$, - this.mediaDevices.audioOutput.selected$, - ], - (available, selected) => { - const selectionType = selected && available.get(selected.id)?.type; - - // If we are in any output mode other than speaker switch to speaker. - const newSelectionType: "earpiece" | "speaker" = - selectionType === "speaker" ? "earpiece" : "speaker"; - const newSelection = [...available].find( - ([, d]) => d.type === newSelectionType, - ); - if (newSelection === undefined) return null; - - const [id] = newSelection; - return { - targetOutput: newSelectionType, - switch: (): void => this.mediaDevices.audioOutput.select(id), - }; - }, - ), - ); - - /** - * Emits an array of reactions that should be visible on the screen. - */ - // DISCUSSION move this into a reaction file - // const {visibleReactions$, audibleReactions$} = reactionsObservables$(showReactionSetting$, ) - public readonly visibleReactions$ = this.scope.behavior( - showReactions.value$.pipe( - switchMap((show) => (show ? this.reactions$ : of({}))), - scan< - Record, - { sender: string; emoji: string; startX: number }[] - >((acc, latest) => { - const newSet: { sender: string; emoji: string; startX: number }[] = []; - for (const [sender, reaction] of Object.entries(latest)) { - const startX = - acc.find((v) => v.sender === sender && v.emoji)?.startX ?? - Math.ceil(Math.random() * 80) + 10; - newSet.push({ sender, emoji: reaction.emoji, startX }); - } - return newSet; - }, []), - ), - ); - - /** - * Emits an array of reactions that should be played. - */ - public readonly audibleReactions$ = playReactionsSound.value$.pipe( - switchMap((show) => - show ? this.reactions$ : of>({}), - ), - map((reactions) => Object.values(reactions).map((v) => v.name)), - scan( - (acc, latest) => { - return { - playing: latest.filter( - (v) => acc.playing.includes(v) || acc.newSounds.includes(v), - ), - newSounds: latest.filter( - (v) => !acc.playing.includes(v) && !acc.newSounds.includes(v), - ), - }; - }, - { playing: [], newSounds: [] }, - ), - map((v) => v.newSounds), - ); - - /** - * Emits an event every time a new hand is raised in - * the call. - */ - public readonly newHandRaised$ = this.handsRaised$.pipe( - map((v) => Object.keys(v).length), - scan( - (acc, newValue) => ({ - value: newValue, - playSounds: newValue > acc.value, - }), - { value: 0, playSounds: false }, - ), - filter((v) => v.playSounds), - ); - - /** - * Emits an event every time a new screenshare is started in - * the call. - */ - public readonly newScreenShare$ = this.screenShares$.pipe( - map((v) => v.length), - scan( - (acc, newValue) => ({ - value: newValue, - playSounds: newValue > acc.value, - }), - { value: 0, playSounds: false }, - ), - filter((v) => v.playSounds), - ); - - /** - * Whether we are sharing our screen. - */ - // reassigned here to make it publicly accessible - public readonly sharingScreen$ = this.localMembership.sharingScreen$; - - /** - * Callback to toggle screen sharing. If null, screen sharing is not possible. - */ - // reassigned here to make it publicly accessible - public readonly toggleScreenSharing = - this.localMembership.toggleScreenSharing; - - public constructor( - private readonly scope: ObservableScope, - // A call is permanently tied to a single Matrix room - private readonly matrixRTCSession: MatrixRTCSession, - private readonly matrixRoom: MatrixRoom, - private readonly mediaDevices: MediaDevices, - private readonly muteStates: MuteStates, - private readonly options: CallViewModelOptions, - private readonly handsRaisedSubject$: Observable< - Record - >, - private readonly reactionsSubject$: Observable< - Record - >, - private readonly trackProcessorState$: Behavior, - ) { - // Join automatically - this.join(); // TODO-MULTI-SFU: Use this view model for the lobby as well, and only call this once 'join' is clicked? - } -} - -// TODO-MULTI-SFU // Setup and update the keyProvider which was create by `createRoom` was a thing before. Now we never update if the E2EEsystem changes -// do we need this? - -function getE2eeKeyProvider( - e2eeSystem: EncryptionSystem, - rtcSession: MatrixRTCSession, -): BaseKeyProvider | undefined { - if (e2eeSystem.kind === E2eeType.NONE) return undefined; - - if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) { - const keyProvider = new MatrixKeyProvider(); - keyProvider.setRTCSession(rtcSession); - return keyProvider; - } else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) { - const keyProvider = new ExternalE2EEKeyProvider(); - keyProvider - .setKey(e2eeSystem.secret) - .catch((e) => logger.error("Failed to set shared key for E2EE", e)); - return keyProvider; - } -}