/* Copyright 2025 Element Creations Ltd. Copyright 2023, 2024, 2025 New Vector Ltd. SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ import { type BaseKeyProvider, type ConnectionState, ExternalE2EEKeyProvider, type Room as LivekitRoom, type RoomOptions, } from "livekit-client"; import { type Room as MatrixRoom } from "matrix-js-sdk"; import { catchError, combineLatest, distinctUntilChanged, filter, fromEvent, map, merge, NEVER, type Observable, of, pairwise, race, scan, skipWhile, startWith, Subject, switchAll, switchMap, switchScan, take, tap, throttleTime, timer, } from "rxjs"; import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { MembershipManagerEvent, type LivekitTransportConfig, type MatrixRTCSession, } from "matrix-js-sdk/lib/matrixrtc"; import { type IWidgetApiRequest } from "matrix-widget-api"; import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; import { v4 as uuidv4 } from "uuid"; import { type IMembershipManager } from "matrix-js-sdk/lib/matrixrtc/IMembershipManager"; import { createToggle$, filterBehavior, generateItem, generateItems, pauseWhen, } from "../../utils/observable"; import { duplicateTiles, MatrixRTCMode, playReactionsSound, showReactions, } from "../../settings/settings"; import { isFirefox, platform } from "../../Platform"; import { setPipEnabled$ } from "../../controls"; import { TileStore } from "../TileStore"; import { gridLikeLayout } from "../GridLikeLayout"; import { spotlightExpandedLayout } from "../SpotlightExpandedLayout"; import { oneOnOneLayout } from "../OneOnOneLayout"; import { pipLayout } from "../PipLayout"; import { type EncryptionSystem } from "../../e2ee/sharedKeyManagement"; import { type RaisedHandInfo, type ReactionInfo, type ReactionOption, } from "../../reactions"; import { shallowEquals } from "../../utils/array"; import { type MediaDevices } from "../MediaDevices"; import { constant, type Behavior } from "../Behavior"; import { E2eeType } from "../../e2ee/e2eeType"; import { MatrixKeyProvider } from "../../e2ee/matrixKeyProvider"; import { type MuteStates } from "../MuteStates"; import { getUrlParams } from "../../UrlParams"; import { type ProcessorState } from "../../livekit/TrackProcessorContext"; import { ElementWidgetActions, widget } from "../../widget"; import { type GridLayoutMedia, type Layout, type LayoutMedia, type OneOnOneLayoutMedia, type SpotlightExpandedLayoutMedia, type SpotlightLandscapeLayoutMedia, type SpotlightPortraitLayoutMedia, } from "../layout-types.ts"; import { ElementCallError, UnknownCallError } from "../../utils/errors.ts"; import { type Epoch, type ObservableScope } from "../ObservableScope.ts"; import { createHomeserverConnected$ } from "./localMember/HomeserverConnected.ts"; import { createLocalMembership$, enterRTCSession, TransportState, } from "./localMember/LocalMember.ts"; import { createLocalTransport$, JwtEndpointVersion, } from "./localMember/LocalTransport.ts"; import { createMemberships$, membershipsAndTransports$, } from "../SessionBehaviors.ts"; import { ECConnectionFactory } from "./remoteMembers/ConnectionFactory.ts"; import { type ConnectionManagerData, createConnectionManager$, } from "./remoteMembers/ConnectionManager.ts"; import { createMatrixLivekitMembers$, type LocalMatrixLivekitMember, type RemoteMatrixLivekitMember, type MatrixLivekitMember, } from "./remoteMembers/MatrixLivekitMembers.ts"; import { type AutoLeaveReason, createCallNotificationLifecycle$, createReceivedDecline$, createSentCallNotification$, } from "./CallNotificationLifecycle.ts"; import { createMatrixMemberMetadata$, createRoomMembers$, } from "./remoteMembers/MatrixMemberMetadata.ts"; import { Publisher } from "./localMember/Publisher.ts"; import { type Connection } from "./remoteMembers/Connection.ts"; import { createLayoutModeSwitch } from "./LayoutSwitch.ts"; import { createWrappedUserMedia, type WrappedUserMediaViewModel, } from "../media/WrappedUserMediaViewModel.ts"; import { type ScreenShareViewModel } from "../media/ScreenShareViewModel.ts"; import { type UserMediaViewModel } from "../media/UserMediaViewModel.ts"; import { type MediaViewModel } from "../media/MediaViewModel.ts"; import { type LocalUserMediaViewModel } from "../media/LocalUserMediaViewModel.ts"; import { type RemoteUserMediaViewModel } from "../media/RemoteUserMediaViewModel.ts"; import { createRingingMedia, type RingingMediaViewModel, } from "../media/RingingMediaViewModel.ts"; const logger = rootLogger.getChild("[CallViewModel]"); //TODO // Larger rename // member,membership -> rtcMember // participant -> livekitParticipant // matrixLivekitItem -> callMember // js-sdk // callMembership -> rtcMembership export interface CallViewModelOptions { encryptionSystem: EncryptionSystem; autoLeaveWhenOthersLeft?: boolean; /** * If the call is started in a way where we want it to behave like a telephone usecase * If we sent a notification event, we want the ui to show a ringing state */ waitForCallPickup?: boolean; /** Optional factory to create LiveKit rooms, mainly for testing purposes. */ livekitRoomFactory?: (options?: RoomOptions) => LivekitRoom; /** Optional behavior overriding the local connection state, mainly for testing purposes. */ connectionState$?: Behavior; /** Optional behavior overriding the computed window size, mainly for testing purposes. */ windowSize$?: Behavior<{ width: number; height: number }>; /** The version & compatibility mode of MatrixRTC that we should use. */ matrixRTCMode$?: Behavior; } // Do not play any sounds if the participant count has exceeded this // number. export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8; export const THROTTLE_SOUND_EFFECT_MS = 500; // This is the number of participants that we think constitutes a "small" call // on mobile. No spotlight tile should be shown below this threshold. const smallMobileCallThreshold = 3; // How long the footer should be shown for when hovering over or interacting // with the interface const showFooterMs = 4000; export type GridMode = "grid" | "spotlight"; export type WindowMode = "normal" | "narrow" | "flat" | "pip"; interface LayoutScanState { layout: Layout | null; tiles: TileStore; } export type LivekitRoomItem = { livekitRoom: LivekitRoom; participants: string[]; url: string; }; /** * The return of createCallViewModel$ * this interface represents the root source of data for the call view. * They are a list of observables and objects containing observables to allow for a very granular update mechanism. * * This allows to have one huge call view model that represents the entire view without a unnecessary amount of updates. * * (Mocking this interface should allow building a full view in all states.) */ export interface CallViewModel { // lifecycle autoLeave$: Observable; /** * Whether we are ringing a call recipient. */ ringing$: Behavior; /** Observable that emits when the user should leave the call (hangup pressed, widget action, error). * THIS DOES NOT LEAVE THE CALL YET. The only way to leave the call (send the hangup event) is * - by ending the scope * - or calling requestDisconnect * * TODO: it seems more reasonable to add a leave() method (that calls requestDisconnect) that will then update leave$ and remove the hangup pattern */ leave$: Observable<"user" | AutoLeaveReason>; /** Call to initiate hangup. Use in conbination with reconnection state track the async hangup process. */ hangup: () => void; // joining join: () => void; /** * calls requestDisconnect. The async leave state can than be observed via connected$ */ leave: () => void; // screen sharing /** * Callback to toggle screen sharing. If null, screen sharing is not possible. */ toggleScreenSharing: (() => void) | null; /** * Whether we are sharing our screen. */ sharingScreen$: Behavior; // UI interactions /** * Callback for when the user taps the call view. */ tapScreen: () => void; /** * Callback for when the user taps the call's controls. */ tapControls: () => void; /** * Callback for when the user hovers over the call view. */ hoverScreen: () => void; /** * Callback for when the user stops hovering over the call view. */ unhoverScreen: () => void; // errors /** * If there is a configuration error with the call (e.g. misconfigured E2EE). * This is a fatal error that prevents the call from being created/joined. * Should render a blocking error screen. */ fatalError$: Behavior; // participants and counts /** * The number of participants currently in the call. * * - Each participant has a corresponding MatrixRTC membership state event * - There can be multiple participants for one Matrix user if they join from * multiple devices. */ participantCount$: Behavior; allConnections$: Behavior; /** Participants sorted by livekit room so they can be used in the audio rendering */ livekitRoomItems$: Behavior; /** use the layout instead, this is just for the sdk export. */ matrixLivekitMembers$: Behavior; localMatrixLivekitMember$: Behavior; /** List of participants raising their hand */ handsRaised$: Behavior>; /** List of reactions. Keys are: membership.membershipId (currently predefined as: `${membershipEvent.userId}:${membershipEvent.deviceId}`)*/ reactions$: Behavior>; // sounds and events joinSoundEffect$: Observable; leaveSoundEffect$: Observable; /** * Emits an event every time a new hand is raised in * the call. */ newHandRaised$: Observable<{ value: number; playSounds: boolean }>; /** * Emits an event every time a new screenshare is started in * the call. */ newScreenShare$: Observable<{ value: number; playSounds: boolean }>; /** * Emits an array of reactions that should be played. */ audibleReactions$: Observable; /** * Emits an array of reactions that should be visible on the screen. */ // DISCUSSION move this into a reaction file visibleReactions$: Behavior< { sender: string; emoji: string; startX: number }[] >; // window/layout /** * The general shape of the window. */ windowMode$: Behavior; spotlightExpanded$: Behavior; toggleSpotlightExpanded$: Behavior<(() => void) | null>; gridMode$: Behavior; setGridMode: (value: GridMode) => void; /** * The layout of tiles in the call interface. */ layout$: Behavior; /** * The current generation of the tile store, exposed for debugging purposes. */ tileStoreGeneration$: Behavior; showSpotlightIndicators$: Behavior; showSpeakingIndicators$: Behavior; // header/footer visibility showHeader$: Behavior; showFooter$: Behavior; // audio routing /** * Whether audio is currently being output through the earpiece. */ earpieceMode$: Behavior; /** * Callback to toggle between the earpiece and the loudspeaker. * * This will be `null` in case the target does not exist in the list * of available audio outputs. */ audioOutputSwitcher$: Behavior<{ targetOutput: "earpiece" | "speaker"; switch: () => void; } | null>; /** * Whether the app is currently reconnecting to the LiveKit server and/or setting the matrix rtc room state. */ reconnecting$: Behavior; /** * Shortcut for not requireing to parse and combine connectionState.matrix and connectionState.livekit */ connected$: Behavior; } /** * A view model providing all the application logic needed to show the in-call * UI (may eventually be expanded to cover the lobby and feedback screens in the * future). */ // Throughout this class and related code we must distinguish between MatrixRTC // state and LiveKit state. We use the common terminology of room "members", RTC // "memberships", and LiveKit "participants". export function createCallViewModel$( scope: ObservableScope, // A call is permanently tied to a single Matrix room matrixRTCSession: MatrixRTCSession, matrixRoom: MatrixRoom, mediaDevices: MediaDevices, muteStates: MuteStates, options: CallViewModelOptions, handsRaisedSubject$: Observable>, reactionsSubject$: Observable>, trackProcessorState$: Behavior, ): CallViewModel { const client = matrixRoom.client; const userId = client.getUserId(); const deviceId = client.getDeviceId(); if (!(userId && deviceId)) throw new UnknownCallError(new Error("userId and deviceId are required")); const livekitKeyProvider = getE2eeKeyProvider( options.encryptionSystem, matrixRTCSession, ); const matrixRTCMode$ = options.matrixRTCMode$ ?? constant(MatrixRTCMode.Legacy); // Each hbar seperates a block of input variables required for the CallViewModel to function. // The outputs of this block is written under the hbar. // // For mocking purposes it is recommended to only mock the functions creating those outputs. // All other fields are just temp computations for the mentioned output. // The class does not need anything except the values underneath the bar. // The creations of the values under the bar are all tested independently and testing the callViewModel Should // not test their creation. Call view model only needs: // - memberships$ via createMemberships$ // - localMembership via createLocalMembership$ // - callLifecycle via createCallNotificationLifecycle$ // - matrixMemberMetadataStore via createMatrixMemberMetadata$ // ------------------------------------------------------------------------ // memberships$ const memberships$ = createMemberships$(scope, matrixRTCSession); // ------------------------------------------------------------------------ // matrixLivekitMembers$ AND localMembership const membershipsAndTransports = membershipsAndTransports$( scope, memberships$, ); const ownMembershipIdentity: CallMembershipIdentityParts = { userId, deviceId, // This will only be consumed by the sticky membership manager. So it has no impact on legacy calls. memberId: uuidv4(), }; const localTransport$ = scope.behavior( matrixRTCMode$.pipe( generateItem( "CallViewModel localTransport$", // Re-create LocalTransport whenever the mode changes (mode) => ({ keys: [mode], data: undefined }), (scope, _data$, mode) => createLocalTransport$({ scope: scope, memberships$: memberships$, ownMembershipIdentity, client, delayId$: scope.behavior( ( fromEvent( matrixRTCSession, MembershipManagerEvent.DelayIdChanged, // The type of reemitted event includes the original emitted as the second arg. ) as Observable<[string | undefined, IMembershipManager]> ).pipe(map(([delayId]) => delayId ?? null)), matrixRTCSession.delayId ?? null, ), roomId: matrixRoom.roomId, forceJwtEndpoint: mode === MatrixRTCMode.Matrix_2_0 ? JwtEndpointVersion.Matrix_2_0 : JwtEndpointVersion.Legacy, useOldestMember: mode === MatrixRTCMode.Legacy, }), ), ), ); const connectionFactory = new ECConnectionFactory( client, matrixRoom.roomId, mediaDevices, trackProcessorState$, livekitKeyProvider, getUrlParams().controlledAudioDevices, options.livekitRoomFactory, getUrlParams().echoCancellation, getUrlParams().noiseSuppression, ); const connectionManager = createConnectionManager$({ scope: scope, connectionFactory: connectionFactory, localTransport$: scope.behavior( localTransport$.pipe( switchMap((t) => t.active$), catchError((e: unknown) => { logger.info( "could not pass local transport to createConnectionManager$. localTransport$ threw an error", e, ); return of(null); }), ), ), remoteTransports$: membershipsAndTransports.transports$, logger: logger, ownMembershipIdentity, }); const matrixLivekitMembers$: Behavior> = createMatrixLivekitMembers$({ scope: scope, membershipsWithTransport$: membershipsAndTransports.membershipsWithTransport$, connectionManager: connectionManager, }); const connectOptions$ = scope.behavior( matrixRTCMode$.pipe( map((mode) => ({ encryptMedia: livekitKeyProvider !== undefined, // TODO. This might need to get called again on each change of matrixRTCMode... matrixRTCMode: mode, })), ), ); const localMembership = createLocalMembership$({ scope, homeserverConnected: createHomeserverConnected$( scope, client, matrixRTCSession, ), muteStates, joinMatrixRTC: (transport: LivekitTransportConfig) => { return enterRTCSession( matrixRTCSession, ownMembershipIdentity, transport, connectOptions$.value, ); }, createPublisherFactory: (connection: Connection) => { return new Publisher( connection, mediaDevices, muteStates, trackProcessorState$, logger.getChild( "[Publisher " + connection.transport.livekit_service_url + "]", ), ); }, connectionManager, matrixRTCSession, localTransport$: scope.behavior( localTransport$.pipe(switchMap((t) => t.advertised$)), ), logger: logger.getChild(`[${Date.now()}]`), }); const localRtcMembership$ = scope.behavior( memberships$.pipe( map( (memberships) => memberships.value.find( (membership) => membership.userId === userId && membership.deviceId === deviceId, ) ?? null, ), ), ); const localMatrixLivekitMember$: Behavior = scope.behavior( localRtcMembership$.pipe( filterBehavior((membership) => membership !== null), map((membership$) => { if (membership$ === null) return null; return { membership$, participant: { type: "local" as const, value$: localMembership.participant$, }, connection$: localMembership.connection$, userId, }; }), ), ); // ------------------------------------------------------------------------ // callLifecycle // TODO if we are in "unknown" state we need a loading rendering (or empty screen) // Otherwise it looks like we already connected and only than the ringing starts which is weird. const { callPickupState$, autoLeave$ } = createCallNotificationLifecycle$({ scope: scope, memberships$: memberships$, sentCallNotification$: createSentCallNotification$(scope, matrixRTCSession), receivedDecline$: createReceivedDecline$(matrixRoom), options: options, localUser: { userId: userId, deviceId: deviceId }, }); // ------------------------------------------------------------------------ // matrixMemberMetadataStore const matrixRoomMembers$ = createRoomMembers$(scope, matrixRoom); const matrixMemberMetadataStore = createMatrixMemberMetadata$( scope, scope.behavior(memberships$.pipe(map((mems) => mems.value))), matrixRoomMembers$, ); const allConnections$ = scope.behavior( connectionManager.connectionManagerData$.pipe(map((d) => d.value)), ); const livekitRoomItems$ = scope.behavior( matrixLivekitMembers$.pipe( switchMap((members) => { const a$ = combineLatest( members.value.map((member) => combineLatest([member.connection$, member.participant.value$]).pipe( map(([connection, participant]) => { // do not render audio for local participant if (!connection || !participant || participant.isLocal) return null; const livekitRoom = connection.livekitRoom; const url = connection.transport.livekit_service_url; return { url, livekitRoom, participant: participant.identity, }; }), ), ), ); return a$; }), map((members) => members.reduce((acc, curr) => { if (!curr) return acc; const existing = acc.find((item) => item.url === curr.url); if (existing) { existing.participants.push(curr.participant); } else { acc.push({ livekitRoom: curr.livekitRoom, participants: [curr.participant], url: curr.url, }); } return acc; }, []), ), ), [], ); const handsRaised$ = scope.behavior( handsRaisedSubject$.pipe(pauseWhen(localMembership.reconnecting$)), ); const reactions$ = scope.behavior( reactionsSubject$.pipe( map((v) => Object.fromEntries( Object.entries(v).map(([a, { reactionOption }]) => [ a, reactionOption, ]), ), ), pauseWhen(localMembership.reconnecting$), ), ); /** * List of user media (camera feeds) that we want tiles for. */ const userMedia$ = scope.behavior( combineLatest([ localMatrixLivekitMember$, matrixLivekitMembers$, duplicateTiles.value$, ]).pipe( // Generate a collection of user media from the list of expected (whether // present or missing) LiveKit participants. generateItems( "CallViewModel userMedia$", function* ([ localMatrixLivekitMember, matrixLivekitMembers, duplicateTiles, ]) { const computeMediaId = (m: MatrixLivekitMember): string => `${m.userId}:${m.membership$.value.deviceId}`; const localUserMediaId = localMatrixLivekitMember ? computeMediaId(localMatrixLivekitMember) : undefined; const localAsArray = localMatrixLivekitMember ? [localMatrixLivekitMember] : []; const remoteWithoutLocal = matrixLivekitMembers.value.filter( (m) => computeMediaId(m) !== localUserMediaId, ); const allMatrixLivekitMembers = [ ...localAsArray, ...remoteWithoutLocal, ]; for (const matrixLivekitMember of allMatrixLivekitMembers) { const { userId, participant, connection$, membership$ } = matrixLivekitMember; const rtcId = membership$.value.rtcBackendIdentity; // rtcBackendIdentity const mediaId = computeMediaId(matrixLivekitMember); for (let dup = 0; dup < 1 + duplicateTiles; dup++) { yield { keys: [dup, mediaId, userId, participant, connection$, rtcId], data: undefined, }; } } }, (scope, _, dup, mediaId, userId, participant, connection$, rtcId) => createWrappedUserMedia(scope, { id: `${mediaId}:${dup}`, userId, rtcBackendIdentity: rtcId, participant, encryptionSystem: options.encryptionSystem, livekitRoom$: scope.behavior( connection$.pipe(map((c) => c?.livekitRoom)), ), focusUrl$: scope.behavior( connection$.pipe(map((c) => c?.transport.livekit_service_url)), ), mediaDevices, pretendToBeDisconnected$: localMembership.reconnecting$, displayName$: scope.behavior( matrixMemberMetadataStore .createDisplayNameBehavior$(userId) .pipe(map((name) => name ?? userId)), ), mxcAvatarUrl$: matrixMemberMetadataStore.createAvatarUrlBehavior$(userId), handRaised$: scope.behavior( handsRaised$.pipe(map((v) => v[mediaId]?.time ?? null)), ), reaction$: scope.behavior( reactions$.pipe(map((v) => v[mediaId] ?? undefined)), ), }), ), ), ); const ringingMedia$ = scope.behavior( combineLatest([userMedia$, matrixRoomMembers$, callPickupState$]).pipe( generateItems( "CallViewModel ringingMedia$", function* ([userMedia, roomMembers, callPickupState]) { if ( callPickupState === "ringing" || callPickupState === "timeout" || callPickupState === "decline" ) { for (const member of roomMembers.values()) { if (!userMedia.some((vm) => vm.userId === member.userId)) yield { keys: [member.userId], data: callPickupState, }; } } }, (scope, pickupState$, userId) => createRingingMedia({ id: `ringing:${userId}`, userId, displayName$: scope.behavior( matrixRoomMembers$.pipe( map((members) => members.get(userId)?.rawDisplayName || userId), ), ), mxcAvatarUrl$: matrixMemberMetadataStore.createAvatarUrlBehavior$(userId), pickupState$, muteStates, }), ), distinctUntilChanged(shallowEquals), tap((ringingMedia) => { if (ringingMedia.length > 1) // Warn that UI may do something unexpected in this case logger.warn( `Ringing more than one participant is not supported (ringing ${ringingMedia.map((vm) => vm.userId).join(", ")})`, ); }), ), ); /** * All screen share media that we want to display. */ const screenShares$ = scope.behavior( userMedia$.pipe( switchMap((userMedia) => userMedia.length === 0 ? of([]) : combineLatest( userMedia.map((m) => m.screenShares$), (...screenShares) => screenShares.flat(1), ), ), ), ); const joinSoundEffect$ = userMedia$.pipe( pairwise(), filter( ([prev, current]) => current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && current.length > prev.length, ), map(() => {}), throttleTime(THROTTLE_SOUND_EFFECT_MS), ); /** * The number of participants currently in the call. * * - Each participant has a corresponding MatrixRTC membership state event * - There can be multiple participants for one Matrix user if they join from * multiple devices. */ const participantCount$ = scope.behavior( matrixLivekitMembers$.pipe(map((ms) => ms.value.length)), ); const leaveSoundEffect$ = combineLatest([callPickupState$, userMedia$]).pipe( // Until the call is successful, do not play a leave sound. // If callPickupState$ is null, then we always play the sound as it will not conflict with a decline sound. skipWhile(([c]) => c !== null && c !== "success"), map(([, userMedia]) => userMedia), pairwise(), filter( ([prev, current]) => current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && current.length < prev.length, ), map(() => {}), throttleTime(THROTTLE_SOUND_EFFECT_MS), ); const userHangup$ = new Subject(); const widgetHangup$ = widget === null ? NEVER : ( fromEvent( widget.lazyActions, ElementWidgetActions.HangupCall, ) as Observable> ).pipe( tap((ev) => { widget!.api.transport.reply(ev.detail, {}); }), ); const leave$: Observable<"user" | "timeout" | "decline" | "allOthersLeft"> = merge( autoLeave$, merge(userHangup$, widgetHangup$).pipe(map(() => "user" as const)), ).pipe(scope.share); const spotlightSpeaker$ = scope.behavior( userMedia$.pipe( switchMap((mediaItems) => mediaItems.length === 0 ? of([]) : combineLatest( mediaItems.map((m) => m.speaking$.pipe(map((s) => [m, s] as const)), ), ), ), scan< (readonly [UserMediaViewModel, boolean])[], UserMediaViewModel | undefined, undefined >((prev, mediaItems) => { // Only remote users that are still in the call should be sticky const [stickyMedia, stickySpeaking] = (!prev?.local && mediaItems.find(([m]) => m === prev)) || []; // Decide who to spotlight: // If the previous speaker is still speaking, stick with them rather // than switching eagerly to someone else return stickySpeaking ? stickyMedia! : // Otherwise, select any remote user who is speaking (mediaItems.find(([m, s]) => !m.local && s)?.[0] ?? // Otherwise, stick with the person who was last speaking stickyMedia ?? // Otherwise, spotlight an arbitrary remote user mediaItems.find(([m]) => !m.local)?.[0] ?? // Otherwise, spotlight the local user mediaItems.find(([m]) => m.local)?.[0]); }, undefined), ), ); const grid$ = scope.behavior( userMedia$.pipe( switchMap((mediaItems) => { const bins = mediaItems.map((m) => m.bin$.pipe(map((bin) => [m, bin] as const)), ); // Sort the media by bin order and generate a tile for each one return bins.length === 0 ? of([]) : combineLatest(bins, (...bins) => bins.sort(([, bin1], [, bin2]) => bin1 - bin2).map(([m]) => m), ); }), distinctUntilChanged(shallowEquals), ), ); /** * Local user media suitable for displaying in a PiP (undefined if not found * or if user prefers to not see themselves). */ const localUserMediaForPip$ = scope.behavior< LocalUserMediaViewModel | undefined >( userMedia$.pipe( switchMap((userMedia) => { const localUserMedia = userMedia.find( (m): m is WrappedUserMediaViewModel & LocalUserMediaViewModel => m.type === "user" && m.local, ); if (!localUserMedia) return of(undefined); return localUserMedia.alwaysShow$.pipe( map((alwaysShow) => (alwaysShow ? localUserMedia : undefined)), ); }), ), ); const spotlightAndPip$ = scope.behavior<{ spotlight: MediaViewModel[]; pip$: Behavior; }>( ringingMedia$.pipe( switchMap((ringingMedia) => { if (ringingMedia.length > 0) return of({ spotlight: ringingMedia, pip$: localUserMediaForPip$ }); return screenShares$.pipe( switchMap((screenShares) => { if (screenShares.length > 0) return of({ spotlight: screenShares, pip$: spotlightSpeaker$ }); return spotlightSpeaker$.pipe( map((speaker) => ({ spotlight: speaker ? [speaker] : [], pip$: localUserMediaForPip$, })), ); }), ); }), ), ); const spotlight$ = scope.behavior( spotlightAndPip$.pipe( map(({ spotlight }) => spotlight), distinctUntilChanged(shallowEquals), ), ); const hasRemoteScreenShares$ = scope.behavior( spotlight$.pipe( map((spotlight) => spotlight.some((vm) => vm.type === "screen share" && !vm.local), ), ), ); const pipEnabled$ = scope.behavior(setPipEnabled$, false); const windowSize$ = options.windowSize$ ?? scope.behavior<{ width: number; height: number }>( fromEvent(window, "resize").pipe( startWith(null), map(() => ({ width: window.innerWidth, height: window.innerHeight })), ), ); // A guess at what the window's mode should be based on its size and shape. const naturalWindowMode$ = scope.behavior( windowSize$.pipe( map(({ width, height }) => { if (height <= 400 && width <= 340) return "pip"; // Our layouts for flat windows are better at adapting to a small width // than our layouts for narrow windows are at adapting to a small height, // so we give "flat" precedence here if (height <= 600) return "flat"; if (width <= 600) return "narrow"; return "normal"; }), ), ); /** * The general shape of the window. */ const windowMode$ = scope.behavior( pipEnabled$.pipe( switchMap((pip) => (pip ? of("pip") : naturalWindowMode$)), ), ); const spotlightExpandedToggle$ = new Subject(); const spotlightExpanded$ = createToggle$( scope, false, spotlightExpandedToggle$, ); const { setGridMode, gridMode$ } = createLayoutModeSwitch( scope, windowMode$, hasRemoteScreenShares$, ); const gridLayoutMedia$: Observable = combineLatest( [grid$, spotlight$], (grid, spotlight) => ({ type: "grid", spotlight: spotlight.some((vm) => vm.type === "screen share") ? spotlight : undefined, grid, }), ); const spotlightLandscapeLayoutMedia$: Observable = combineLatest([grid$, spotlight$], (grid, spotlight) => ({ type: "spotlight-landscape", spotlight, grid, })); const spotlightPortraitLayoutMedia$: Observable = combineLatest([grid$, spotlight$], (grid, spotlight) => ({ type: "spotlight-portrait", spotlight, grid, })); const spotlightExpandedLayoutMedia$: Observable = spotlightAndPip$.pipe( switchMap(({ spotlight, pip$ }) => pip$.pipe( map((pip) => ({ type: "spotlight-expanded" as const, spotlight, pip: pip ?? undefined, })), ), ), ); const oneOnOneLayoutMedia$: Observable = userMedia$.pipe( switchMap((userMedia) => { if (userMedia.length <= 2) { const local = userMedia.find( (vm): vm is WrappedUserMediaViewModel & LocalUserMediaViewModel => vm.type === "user" && vm.local, ); if (local !== undefined) { const remote = userMedia.find( ( vm, ): vm is WrappedUserMediaViewModel & RemoteUserMediaViewModel => vm.type === "user" && !vm.local, ); if (remote !== undefined) return of({ type: "one-on-one" as const, spotlight: remote, pip: local, }); // If there's no other user media in the call (could still happen in // this branch due to the duplicate tiles option), we could possibly // show ringing media instead if (userMedia.length === 1) return ringingMedia$.pipe( map((ringingMedia) => { return ringingMedia.length === 1 ? { type: "one-on-one" as const, spotlight: local, pip: ringingMedia[0], } : null; }), ); } } return of(null); }), ); const pipLayoutMedia$: Observable = spotlight$.pipe( map((spotlight) => ({ type: "pip", spotlight })), ); /** * The media to be used to produce a layout. */ const layoutMedia$ = scope.behavior( windowMode$.pipe( switchMap((windowMode) => { switch (windowMode) { case "normal": return gridMode$.pipe( switchMap((gridMode) => { switch (gridMode) { case "grid": return oneOnOneLayoutMedia$.pipe( switchMap((oneOnOne) => oneOnOne === null ? gridLayoutMedia$ : of(oneOnOne), ), ); case "spotlight": return spotlightExpanded$.pipe( switchMap((expanded) => expanded ? spotlightExpandedLayoutMedia$ : spotlightLandscapeLayoutMedia$, ), ); } }), ); case "narrow": return oneOnOneLayoutMedia$.pipe( switchMap((oneOnOne) => oneOnOne === null ? combineLatest([grid$, spotlight$], (grid, spotlight) => grid.length > smallMobileCallThreshold || spotlight.some((vm) => vm.type === "screen share") ? spotlightPortraitLayoutMedia$ : gridLayoutMedia$, ).pipe(switchAll()) : // The expanded spotlight layout makes for a better one-on-one // experience in narrow windows spotlightExpandedLayoutMedia$, ), ); case "flat": return gridMode$.pipe( switchMap((gridMode) => { switch (gridMode) { case "grid": // Yes, grid mode actually gets you a "spotlight" layout in // this window mode. return spotlightLandscapeLayoutMedia$; case "spotlight": return spotlightExpandedLayoutMedia$; } }), ); case "pip": return pipLayoutMedia$; } }), ), ); // There is a cyclical dependency here: the layout algorithms want to know // which tiles are on screen, but to know which tiles are on screen we have to // first render a layout. To deal with this we assume initially that no tiles // are visible, and loop the data back into the layouts with a Subject. const visibleTiles$ = new Subject(); const setVisibleTiles = (value: number): void => visibleTiles$.next(value); const layoutInternals$ = scope.behavior( combineLatest([ layoutMedia$, visibleTiles$.pipe(startWith(0), distinctUntilChanged()), ]).pipe( scan< [LayoutMedia, number], LayoutScanState & { layout: Layout }, LayoutScanState >( ({ tiles: prevTiles }, [media, visibleTiles]) => { let layout: Layout; let newTiles: TileStore; switch (media.type) { case "grid": case "spotlight-landscape": case "spotlight-portrait": [layout, newTiles] = gridLikeLayout( media, visibleTiles, setVisibleTiles, prevTiles, ); break; case "spotlight-expanded": [layout, newTiles] = spotlightExpandedLayout(media, prevTiles); break; case "one-on-one": [layout, newTiles] = oneOnOneLayout(media, prevTiles); break; case "pip": [layout, newTiles] = pipLayout(media, prevTiles); break; } return { layout, tiles: newTiles }; }, { layout: null, tiles: TileStore.empty() }, ), ), ); /** * The layout of tiles in the call interface. */ const layout$ = scope.behavior( layoutInternals$.pipe(map(({ layout }) => layout)), ); /** * The current generation of the tile store, exposed for debugging purposes. */ const tileStoreGeneration$ = scope.behavior( layoutInternals$.pipe(map(({ tiles }) => tiles.generation)), ); const showSpotlightIndicators$ = scope.behavior( layout$.pipe(map((l) => l.type !== "grid")), ); const showSpeakingIndicators$ = scope.behavior( layout$.pipe( switchMap((l) => { switch (l.type) { case "spotlight-landscape": case "spotlight-portrait": // If the spotlight is showing the active speaker, we can do without // speaking indicators as they're a redundant visual cue. But if // screen sharing feeds are in the spotlight we still need them. return l.spotlight.media$.pipe( map((models: MediaViewModel[]) => models.some((m) => m.type === "screen share"), ), ); // In expanded spotlight layout, the active speaker is always shown in // the picture-in-picture tile so there is no need for speaking // indicators. And in one-on-one layout there's no question as to who is // speaking. case "spotlight-expanded": case "one-on-one": return of(false); default: return of(true); } }), ), ); const toggleSpotlightExpanded$ = scope.behavior<(() => void) | null>( windowMode$.pipe( switchMap((mode) => mode === "normal" ? layout$.pipe( map( (l) => l.type === "spotlight-landscape" || l.type === "spotlight-expanded", ), ) : of(false), ), distinctUntilChanged(), map((enabled) => enabled ? (): void => spotlightExpandedToggle$.next() : null, ), ), ); const screenTap$ = new Subject(); const controlsTap$ = new Subject(); const screenHover$ = new Subject(); const screenUnhover$ = new Subject(); const showHeader$ = scope.behavior( windowMode$.pipe(map((mode) => mode !== "pip" && mode !== "flat")), ); const showFooter$ = scope.behavior( windowMode$.pipe( switchMap((mode) => { switch (mode) { case "pip": return of(platform === "desktop" ? true : false); case "normal": case "narrow": return of(true); case "flat": // Sadly Firefox has some layering glitches that prevent the footer // from appearing properly. They happen less often if we never hide // the footer. if (isFirefox()) return of(true); // Show/hide the footer in response to interactions return merge( screenTap$.pipe(map(() => "tap screen" as const)), controlsTap$.pipe(map(() => "tap controls" as const)), screenHover$.pipe(map(() => "hover" as const)), ).pipe( switchScan((state, interaction) => { switch (interaction) { case "tap screen": return state ? // Toggle visibility on tap of(false) : // Hide after a timeout timer(showFooterMs).pipe( map(() => false), startWith(true), ); case "tap controls": // The user is interacting with things, so reset the timeout return timer(showFooterMs).pipe( map(() => false), startWith(true), ); case "hover": // Show on hover and hide after a timeout return race( timer(showFooterMs), screenUnhover$.pipe(take(1)), ).pipe( map(() => false), startWith(true), ); } }, false), startWith(false), ); } }), ), ); /** * Whether audio is currently being output through the earpiece. */ const earpieceMode$ = scope.behavior( combineLatest( [mediaDevices.audioOutput.available$, mediaDevices.audioOutput.selected$], (available, selected) => selected !== undefined && available.get(selected.id)?.type === "earpiece", ), ); /** * Callback to toggle between the earpiece and the loudspeaker. * * This will be `null` in case the target does not exist in the list * of available audio outputs. */ const audioOutputSwitcher$ = scope.behavior<{ targetOutput: "earpiece" | "speaker"; switch: () => void; } | null>( combineLatest( [mediaDevices.audioOutput.available$, mediaDevices.audioOutput.selected$], (available, selected) => { const selectionType = selected && available.get(selected.id)?.type; // If we are in any output mode other than speaker switch to speaker. const newSelectionType: "earpiece" | "speaker" = selectionType === "speaker" ? "earpiece" : "speaker"; const newSelection = [...available].find( ([, d]) => d.type === newSelectionType, ); if (newSelection === undefined) return null; const [id] = newSelection; return { targetOutput: newSelectionType, switch: (): void => mediaDevices.audioOutput.select(id), }; }, ), ); /** * Emits an array of reactions that should be visible on the screen. */ // DISCUSSION move this into a reaction file // const {visibleReactions$, audibleReactions$} = reactionsObservables$(showReactionSetting$, ) const visibleReactions$ = scope.behavior( showReactions.value$.pipe( switchMap((show) => (show ? reactions$ : of({}))), scan< Record, { sender: string; emoji: string; startX: number }[] >((acc, latest) => { const newSet: { sender: string; emoji: string; startX: number }[] = []; for (const [sender, reaction] of Object.entries(latest)) { const startX = acc.find((v) => v.sender === sender && v.emoji)?.startX ?? Math.ceil(Math.random() * 80) + 10; newSet.push({ sender, emoji: reaction.emoji, startX }); } return newSet; }, []), ), ); /** * Emits an array of reactions that should be played. */ const audibleReactions$ = playReactionsSound.value$.pipe( switchMap((show) => show ? reactions$ : of>({}), ), map((reactions) => Object.values(reactions).map((v) => v.name)), scan( (acc, latest) => { return { playing: latest.filter( (v) => acc.playing.includes(v) || acc.newSounds.includes(v), ), newSounds: latest.filter( (v) => !acc.playing.includes(v) && !acc.newSounds.includes(v), ), }; }, { playing: [], newSounds: [] }, ), map((v) => v.newSounds), ); const newHandRaised$ = handsRaised$.pipe( map((v) => Object.keys(v).length), scan( (acc, newValue) => ({ value: newValue, playSounds: newValue > acc.value, }), { value: 0, playSounds: false }, ), filter((v) => v.playSounds), ); const newScreenShare$ = screenShares$.pipe( map((v) => v.length), scan( (acc, newValue) => ({ value: newValue, playSounds: newValue > acc.value, }), { value: 0, playSounds: false }, ), filter((v) => v.playSounds), ); /** * Whether we are sharing our screen. */ // reassigned here to make it publicly accessible const sharingScreen$ = localMembership.sharingScreen$; /** * Callback to toggle screen sharing. If null, screen sharing is not possible. */ // reassigned here to make it publicly accessible const toggleScreenSharing = localMembership.toggleScreenSharing; const errors$ = scope.behavior<{ transportError?: ElementCallError; matrixError?: ElementCallError; connectionError?: ElementCallError; publishError?: ElementCallError; } | null>( localMembership.localMemberState$.pipe( map((value) => { const returnObject: { transportError?: ElementCallError; matrixError?: ElementCallError; connectionError?: ElementCallError; publishError?: ElementCallError; } = {}; if (value instanceof ElementCallError) return { transportError: value }; if (value === TransportState.Waiting) return null; if (value.matrix instanceof ElementCallError) returnObject.matrixError = value.matrix; if (value.media instanceof ElementCallError) returnObject.publishError = value.media; else if ( typeof value.media === "object" && value.media.connection instanceof ElementCallError ) returnObject.connectionError = value.media.connection; return returnObject; }), ), null, ); return { autoLeave$: autoLeave$, ringing$: scope.behavior( callPickupState$.pipe(map((state) => state === "ringing")), ), leave$: leave$, hangup: (): void => userHangup$.next(), join: localMembership.requestJoinAndPublish, leave: localMembership.requestDisconnect, toggleScreenSharing: toggleScreenSharing, sharingScreen$: sharingScreen$, tapScreen: (): void => screenTap$.next(), tapControls: (): void => controlsTap$.next(), hoverScreen: (): void => screenHover$.next(), unhoverScreen: (): void => screenUnhover$.next(), fatalError$: scope.behavior( errors$.pipe( map((errors) => { logger.debug("errors$ to compute any fatal errors:", errors); return ( errors?.transportError ?? errors?.matrixError ?? errors?.connectionError ?? null ); }), filter((error) => error !== null), ), null, ), allConnections$, participantCount$: participantCount$, handsRaised$: handsRaised$, reactions$: reactions$, joinSoundEffect$: joinSoundEffect$, leaveSoundEffect$: leaveSoundEffect$, newHandRaised$: newHandRaised$, newScreenShare$: newScreenShare$, audibleReactions$: audibleReactions$, visibleReactions$: visibleReactions$, windowMode$: windowMode$, spotlightExpanded$: spotlightExpanded$, toggleSpotlightExpanded$: toggleSpotlightExpanded$, gridMode$: gridMode$, setGridMode: setGridMode, layout$: layout$, localMatrixLivekitMember$, matrixLivekitMembers$: scope.behavior( matrixLivekitMembers$.pipe( map((members) => members.value), tap((v) => { const listForLogs = v .map( (m) => m.membership$.value.userId + "|" + m.membership$.value.deviceId, ) .join(","); logger.debug( `matrixLivekitMembers$ updated (exported) [${listForLogs}]`, ); }), ), ), tileStoreGeneration$: tileStoreGeneration$, showSpotlightIndicators$: showSpotlightIndicators$, showSpeakingIndicators$: showSpeakingIndicators$, showHeader$: showHeader$, showFooter$: showFooter$, earpieceMode$: earpieceMode$, audioOutputSwitcher$: audioOutputSwitcher$, reconnecting$: localMembership.reconnecting$, livekitRoomItems$, connected$: localMembership.connected$, }; } function getE2eeKeyProvider( e2eeSystem: EncryptionSystem, rtcSession: MatrixRTCSession, ): BaseKeyProvider | undefined { if (e2eeSystem.kind === E2eeType.NONE) return undefined; if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) { const keyProvider = new MatrixKeyProvider(); keyProvider.setRTCSession(rtcSession); return keyProvider; } else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) { const keyProvider = new ExternalE2EEKeyProvider(); keyProvider .setKey(e2eeSystem.secret) .catch((e) => logger.error("Failed to set shared key for E2EE", e)); return keyProvider; } }