diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 61afb7b9..c19c4818 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -51,14 +51,7 @@ import { v4 as uuidv4 } from "uuid"; import { type IMembershipManager } from "matrix-js-sdk/lib/matrixrtc/IMembershipManager"; import { - LocalUserMediaViewModel, - type MediaViewModel, - type RemoteUserMediaViewModel, - ScreenShareViewModel, - type UserMediaViewModel, -} from "../MediaViewModel"; -import { - accumulate, + createToggle$, filterBehavior, generateItem, generateItems, @@ -92,8 +85,6 @@ import { type MuteStates } from "../MuteStates"; import { getUrlParams } from "../../UrlParams"; import { type ProcessorState } from "../../livekit/TrackProcessorContext"; import { ElementWidgetActions, widget } from "../../widget"; -import { UserMedia } from "../UserMedia.ts"; -import { ScreenShare } from "../ScreenShare.ts"; import { type GridLayoutMedia, type Layout, @@ -144,6 +135,14 @@ import { import { Publisher } from "./localMember/Publisher.ts"; import { type Connection } from "./remoteMembers/Connection.ts"; import { createLayoutModeSwitch } from "./LayoutSwitch.ts"; +import { + createWrappedUserMedia, + type MediaItem, + type WrappedUserMediaViewModel, +} from "../media/MediaItem.ts"; +import { type ScreenShareViewModel } from "../media/ScreenShareViewModel.ts"; +import { type UserMediaViewModel } from "../media/UserMediaViewModel.ts"; +import { type MediaViewModel } from "../media/MediaViewModel.ts"; const logger = rootLogger.getChild("[CallViewModel]"); //TODO @@ -193,7 +192,6 @@ interface LayoutScanState { tiles: TileStore; } -type MediaItem = UserMedia | ScreenShare; export type LivekitRoomItem = { livekitRoom: LivekitRoom; participants: string[]; @@ -283,7 +281,6 @@ export interface CallViewModel { allConnections$: Behavior; /** Participants sorted by livekit room so they can be used in the audio rendering */ livekitRoomItems$: Behavior; - userMedia$: Behavior; /** use the layout instead, this is just for the sdk export. */ matrixLivekitMembers$: Behavior; localMatrixLivekitMember$: Behavior; @@ -334,10 +331,6 @@ export interface CallViewModel { gridMode$: Behavior; setGridMode: (value: GridMode) => void; - // media view models and layout - grid$: Behavior; - spotlight$: Behavior; - pip$: Behavior; /** * The layout of tiles in the call interface. */ @@ -721,7 +714,7 @@ export function createCallViewModel$( /** * List of user media (camera feeds) that we want tiles for. */ - const userMedia$ = scope.behavior( + const userMedia$ = scope.behavior( combineLatest([ localMatrixLivekitMember$, matrixLivekitMembers$, @@ -767,36 +760,35 @@ export function createCallViewModel$( } } }, - (scope, _, dup, mediaId, userId, participant, connection$, rtcId) => { - const livekitRoom$ = scope.behavior( - connection$.pipe(map((c) => c?.livekitRoom)), - ); - const focusUrl$ = scope.behavior( - connection$.pipe(map((c) => c?.transport.livekit_service_url)), - ); - const displayName$ = scope.behavior( - matrixMemberMetadataStore - .createDisplayNameBehavior$(userId) - .pipe(map((name) => name ?? userId)), - ); - - return new UserMedia( - scope, - `${mediaId}:${dup}`, + (scope, _, dup, mediaId, userId, participant, connection$, rtcId) => + createWrappedUserMedia(scope, { + id: `${mediaId}:${dup}`, userId, - rtcId, + rtcBackendIdentity: rtcId, participant, - options.encryptionSystem, - livekitRoom$, - focusUrl$, + encryptionSystem: options.encryptionSystem, + livekitRoom$: scope.behavior( + connection$.pipe(map((c) => c?.livekitRoom)), + ), + focusUrl$: scope.behavior( + connection$.pipe(map((c) => c?.transport.livekit_service_url)), + ), mediaDevices, - localMembership.reconnecting$, - displayName$, - matrixMemberMetadataStore.createAvatarUrlBehavior$(userId), - handsRaised$.pipe(map((v) => v[mediaId]?.time ?? null)), - reactions$.pipe(map((v) => v[mediaId] ?? undefined)), - ); - }, + pretendToBeDisconnected$: localMembership.reconnecting$, + displayName$: scope.behavior( + matrixMemberMetadataStore + .createDisplayNameBehavior$(userId) + .pipe(map((name) => name ?? userId)), + ), + mxcAvatarUrl$: + matrixMemberMetadataStore.createAvatarUrlBehavior$(userId), + handRaised$: scope.behavior( + handsRaised$.pipe(map((v) => v[mediaId]?.time ?? null)), + ), + reaction$: scope.behavior( + reactions$.pipe(map((v) => v[mediaId] ?? undefined)), + ), + }), ), ), ); @@ -821,11 +813,9 @@ export function createCallViewModel$( /** * List of MediaItems that we want to display, that are of type ScreenShare */ - const screenShares$ = scope.behavior( + const screenShares$ = scope.behavior( mediaItems$.pipe( - map((mediaItems) => - mediaItems.filter((m): m is ScreenShare => m instanceof ScreenShare), - ), + map((mediaItems) => mediaItems.filter((m) => m.type === "screen share")), ), ); @@ -888,39 +878,39 @@ export function createCallViewModel$( merge(userHangup$, widgetHangup$).pipe(map(() => "user" as const)), ).pipe(scope.share); - const spotlightSpeaker$ = scope.behavior( + const spotlightSpeaker$ = scope.behavior( userMedia$.pipe( switchMap((mediaItems) => mediaItems.length === 0 ? of([]) : combineLatest( mediaItems.map((m) => - m.vm.speaking$.pipe(map((s) => [m, s] as const)), + m.speaking$.pipe(map((s) => [m, s] as const)), ), ), ), - scan<(readonly [UserMedia, boolean])[], UserMedia | undefined, null>( - (prev, mediaItems) => { - // Only remote users that are still in the call should be sticky - const [stickyMedia, stickySpeaking] = - (!prev?.vm.local && mediaItems.find(([m]) => m === prev)) || []; - // Decide who to spotlight: - // If the previous speaker is still speaking, stick with them rather - // than switching eagerly to someone else - return stickySpeaking - ? stickyMedia! - : // Otherwise, select any remote user who is speaking - (mediaItems.find(([m, s]) => !m.vm.local && s)?.[0] ?? - // Otherwise, stick with the person who was last speaking - stickyMedia ?? - // Otherwise, spotlight an arbitrary remote user - mediaItems.find(([m]) => !m.vm.local)?.[0] ?? - // Otherwise, spotlight the local user - mediaItems.find(([m]) => m.vm.local)?.[0]); - }, - null, - ), - map((speaker) => speaker?.vm ?? null), + scan< + (readonly [UserMediaViewModel, boolean])[], + UserMediaViewModel | undefined, + undefined + >((prev, mediaItems) => { + // Only remote users that are still in the call should be sticky + const [stickyMedia, stickySpeaking] = + (!prev?.local && mediaItems.find(([m]) => m === prev)) || []; + // Decide who to spotlight: + // If the previous speaker is still speaking, stick with them rather + // than switching eagerly to someone else + return stickySpeaking + ? stickyMedia! + : // Otherwise, select any remote user who is speaking + (mediaItems.find(([m, s]) => !m.local && s)?.[0] ?? + // Otherwise, stick with the person who was last speaking + stickyMedia ?? + // Otherwise, spotlight an arbitrary remote user + mediaItems.find(([m]) => !m.local)?.[0] ?? + // Otherwise, spotlight the local user + mediaItems.find(([m]) => m.local)?.[0]); + }, undefined), ), ); @@ -934,7 +924,7 @@ export function createCallViewModel$( return bins.length === 0 ? of([]) : combineLatest(bins, (...bins) => - bins.sort(([, bin1], [, bin2]) => bin1 - bin2).map(([m]) => m.vm), + bins.sort(([, bin1], [, bin2]) => bin1 - bin2).map(([m]) => m), ); }), distinctUntilChanged(shallowEquals), @@ -944,9 +934,7 @@ export function createCallViewModel$( const spotlight$ = scope.behavior( screenShares$.pipe( switchMap((screenShares) => { - if (screenShares.length > 0) { - return of(screenShares.map((m) => m.vm)); - } + if (screenShares.length > 0) return of(screenShares); return spotlightSpeaker$.pipe( map((speaker) => (speaker ? [speaker] : [])), @@ -956,7 +944,7 @@ export function createCallViewModel$( ), ); - const pip$ = scope.behavior( + const pip$ = scope.behavior( combineLatest([ // TODO This also needs epoch logic to dedupe the screenshares and mediaItems emits screenShares$, @@ -968,28 +956,17 @@ export function createCallViewModel$( return spotlightSpeaker$; } if (!spotlight || spotlight.local) { - return of(null); + return of(undefined); } const localUserMedia = mediaItems.find( - (m) => m.vm instanceof LocalUserMediaViewModel, - ) as UserMedia | undefined; - - const localUserMediaViewModel = localUserMedia?.vm as - | LocalUserMediaViewModel - | undefined; - - if (!localUserMediaViewModel) { - return of(null); + (m) => m.type === "user" && m.local, + ); + if (!localUserMedia) { + return of(undefined); } - return localUserMediaViewModel.alwaysShow$.pipe( - map((alwaysShow) => { - if (alwaysShow) { - return localUserMediaViewModel; - } - - return null; - }), + return localUserMedia.alwaysShow$.pipe( + map((alwaysShow) => (alwaysShow ? localUserMedia : undefined)), ); }), ), @@ -998,7 +975,7 @@ export function createCallViewModel$( const hasRemoteScreenShares$ = scope.behavior( spotlight$.pipe( map((spotlight) => - spotlight.some((vm) => !vm.local && vm instanceof ScreenShareViewModel), + spotlight.some((vm) => vm.type === "screen share" && !vm.local), ), ), ); @@ -1039,8 +1016,10 @@ export function createCallViewModel$( ); const spotlightExpandedToggle$ = new Subject(); - const spotlightExpanded$ = scope.behavior( - spotlightExpandedToggle$.pipe(accumulate(false, (expanded) => !expanded)), + const spotlightExpanded$ = createToggle$( + scope, + false, + spotlightExpandedToggle$, ); const { setGridMode, gridMode$ } = createLayoutModeSwitch( @@ -1053,7 +1032,7 @@ export function createCallViewModel$( [grid$, spotlight$], (grid, spotlight) => ({ type: "grid", - spotlight: spotlight.some((vm) => vm instanceof ScreenShareViewModel) + spotlight: spotlight.some((vm) => vm.type === "screen share") ? spotlight : undefined, grid, @@ -1085,12 +1064,8 @@ export function createCallViewModel$( mediaItems$.pipe( map((mediaItems) => { if (mediaItems.length !== 2) return null; - const local = mediaItems.find((vm) => vm.vm.local)?.vm as - | LocalUserMediaViewModel - | undefined; - const remote = mediaItems.find((vm) => !vm.vm.local)?.vm as - | RemoteUserMediaViewModel - | undefined; + const local = mediaItems.find((vm) => vm.type === "user" && vm.local); + const remote = mediaItems.find((vm) => vm.type === "user" && !vm.local); // There might not be a remote tile if there are screen shares, or if // only the local user is in the call and they're using the duplicate // tiles option @@ -1138,7 +1113,7 @@ export function createCallViewModel$( oneOnOne === null ? combineLatest([grid$, spotlight$], (grid, spotlight) => grid.length > smallMobileCallThreshold || - spotlight.some((vm) => vm instanceof ScreenShareViewModel) + spotlight.some((vm) => vm.type === "screen share") ? spotlightPortraitLayoutMedia$ : gridLayoutMedia$, ).pipe(switchAll()) @@ -1245,7 +1220,7 @@ export function createCallViewModel$( // screen sharing feeds are in the spotlight we still need them. return l.spotlight.media$.pipe( map((models: MediaViewModel[]) => - models.some((m) => m instanceof ScreenShareViewModel), + models.some((m) => m.type === "screen share"), ), ); // In expanded spotlight layout, the active speaker is always shown in @@ -1552,11 +1527,7 @@ export function createCallViewModel$( toggleSpotlightExpanded$: toggleSpotlightExpanded$, gridMode$: gridMode$, setGridMode: setGridMode, - grid$: grid$, - spotlight$: spotlight$, - pip$: pip$, layout$: layout$, - userMedia$, localMatrixLivekitMember$, matrixLivekitMembers$: scope.behavior( matrixLivekitMembers$.pipe( diff --git a/src/state/CallViewModel/localMember/Publisher.ts b/src/state/CallViewModel/localMember/Publisher.ts index 8df38743..b7841c49 100644 --- a/src/state/CallViewModel/localMember/Publisher.ts +++ b/src/state/CallViewModel/localMember/Publisher.ts @@ -30,7 +30,7 @@ import { trackProcessorSync, } from "../../../livekit/TrackProcessorContext.tsx"; import { getUrlParams } from "../../../UrlParams.ts"; -import { observeTrackReference$ } from "../../MediaViewModel.ts"; +import { observeTrackReference$ } from "../../observeTrackReference"; import { type Connection } from "../remoteMembers/Connection.ts"; import { ObservableScope } from "../../ObservableScope.ts"; diff --git a/src/state/MediaViewModel.ts b/src/state/MediaViewModel.ts deleted file mode 100644 index 7f806697..00000000 --- a/src/state/MediaViewModel.ts +++ /dev/null @@ -1,806 +0,0 @@ -/* -Copyright 2023, 2024 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - type AudioSource, - type VideoSource, - type TrackReference, - observeParticipantEvents, - observeParticipantMedia, - roomEventSelector, -} from "@livekit/components-core"; -import { - type LocalParticipant, - LocalTrack, - LocalVideoTrack, - type Participant, - ParticipantEvent, - type RemoteParticipant, - Track, - TrackEvent, - facingModeFromLocalTrack, - type Room as LivekitRoom, - RoomEvent as LivekitRoomEvent, - RemoteTrack, -} from "livekit-client"; -import { logger } from "matrix-js-sdk/lib/logger"; -import { - BehaviorSubject, - type Observable, - Subject, - combineLatest, - filter, - fromEvent, - interval, - map, - merge, - of, - startWith, - switchMap, - throttleTime, - distinctUntilChanged, -} from "rxjs"; - -import { alwaysShowSelf } from "../settings/settings"; -import { showConnectionStats } from "../settings/settings"; -import { accumulate } from "../utils/observable"; -import { type EncryptionSystem } from "../e2ee/sharedKeyManagement"; -import { E2eeType } from "../e2ee/e2eeType"; -import { type ReactionOption } from "../reactions"; -import { platform } from "../Platform"; -import { type MediaDevices } from "./MediaDevices"; -import { type Behavior } from "./Behavior"; -import { type ObservableScope } from "./ObservableScope"; - -export function observeTrackReference$( - participant: Participant, - source: Track.Source, -): Observable { - return observeParticipantMedia(participant).pipe( - map(() => participant.getTrackPublication(source)), - distinctUntilChanged(), - map((publication) => publication && { participant, publication, source }), - ); -} - -export function observeRtpStreamStats$( - participant: Participant, - source: Track.Source, - type: "inbound-rtp" | "outbound-rtp", -): Observable< - RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined -> { - return combineLatest([ - observeTrackReference$(participant, source), - interval(1000).pipe(startWith(0)), - ]).pipe( - switchMap(async ([trackReference]) => { - const track = trackReference?.publication?.track; - if ( - !track || - !(track instanceof RemoteTrack || track instanceof LocalTrack) - ) { - return undefined; - } - const report = await track.getRTCStatsReport(); - if (!report) { - return undefined; - } - - for (const v of report.values()) { - if (v.type === type) { - return v; - } - } - - return undefined; - }), - startWith(undefined), - ); -} - -export function observeInboundRtpStreamStats$( - participant: Participant, - source: Track.Source, -): Observable { - return observeRtpStreamStats$(participant, source, "inbound-rtp").pipe( - map((x) => x as RTCInboundRtpStreamStats | undefined), - ); -} - -export function observeOutboundRtpStreamStats$( - participant: Participant, - source: Track.Source, -): Observable { - return observeRtpStreamStats$(participant, source, "outbound-rtp").pipe( - map((x) => x as RTCOutboundRtpStreamStats | undefined), - ); -} - -function observeRemoteTrackReceivingOkay$( - participant: Participant, - source: Track.Source, -): Observable { - let lastStats: { - framesDecoded: number | undefined; - framesDropped: number | undefined; - framesReceived: number | undefined; - } = { - framesDecoded: undefined, - framesDropped: undefined, - framesReceived: undefined, - }; - - return observeInboundRtpStreamStats$(participant, source).pipe( - map((stats) => { - if (!stats) return undefined; - const { framesDecoded, framesDropped, framesReceived } = stats; - return { - framesDecoded, - framesDropped, - framesReceived, - }; - }), - filter((newStats) => !!newStats), - map((newStats): boolean | undefined => { - const oldStats = lastStats; - lastStats = newStats; - if ( - typeof newStats.framesReceived === "number" && - typeof oldStats.framesReceived === "number" && - typeof newStats.framesDecoded === "number" && - typeof oldStats.framesDecoded === "number" - ) { - const framesReceivedDelta = - newStats.framesReceived - oldStats.framesReceived; - const framesDecodedDelta = - newStats.framesDecoded - oldStats.framesDecoded; - - // if we received >0 frames and managed to decode >0 frames then we treat that as success - - if (framesReceivedDelta > 0) { - return framesDecodedDelta > 0; - } - } - - // no change - return undefined; - }), - filter((x) => typeof x === "boolean"), - startWith(undefined), - ); -} - -function encryptionErrorObservable$( - room$: Behavior, - participant: Participant, - encryptionSystem: EncryptionSystem, - criteria: string, -): Observable { - return room$.pipe( - switchMap((room) => { - if (room === undefined) return of(false); - return roomEventSelector(room, LivekitRoomEvent.EncryptionError).pipe( - map((e) => { - const [err] = e; - if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) { - return ( - // Ideally we would pull the participant identity from the field on the error. - // However, it gets lost in the serialization process between workers. - // So, instead we do a string match - (err?.message.includes(participant.identity) && - err?.message.includes(criteria)) ?? - false - ); - } else if (encryptionSystem.kind === E2eeType.SHARED_KEY) { - return !!err?.message.includes(criteria); - } - - return false; - }), - ); - }), - distinctUntilChanged(), - throttleTime(1000), // Throttle to avoid spamming the UI - startWith(false), - ); -} - -export enum EncryptionStatus { - Connecting, - Okay, - KeyMissing, - KeyInvalid, - PasswordInvalid, -} - -abstract class BaseMediaViewModel { - /** - * The LiveKit video track for this media. - */ - public readonly video$: Behavior; - /** - * Whether there should be a warning that this media is unencrypted. - */ - public readonly unencryptedWarning$: Behavior; - - public readonly encryptionStatus$: Behavior; - - /** - * Whether this media corresponds to the local participant. - */ - public abstract readonly local: boolean; - - private observeTrackReference$( - source: Track.Source, - ): Behavior { - return this.scope.behavior( - this.participant$.pipe( - switchMap((p) => - !p ? of(undefined) : observeTrackReference$(p, source), - ), - ), - ); - } - - public constructor( - protected readonly scope: ObservableScope, - /** - * An opaque identifier for this media. - */ - public readonly id: string, - /** - * The Matrix user to which this media belongs. - */ - public readonly userId: string, - // We don't necessarily have a participant if a user connects via MatrixRTC but not (yet) through - // livekit. - protected readonly participant$: Behavior< - LocalParticipant | RemoteParticipant | null - >, - - encryptionSystem: EncryptionSystem, - audioSource: AudioSource, - videoSource: VideoSource, - protected readonly livekitRoom$: Behavior, - public readonly focusUrl$: Behavior, - public readonly displayName$: Behavior, - public readonly mxcAvatarUrl$: Behavior, - ) { - const audio$ = this.observeTrackReference$(audioSource); - this.video$ = this.observeTrackReference$(videoSource); - - this.unencryptedWarning$ = this.scope.behavior( - combineLatest( - [audio$, this.video$], - (a, v) => - encryptionSystem.kind !== E2eeType.NONE && - (a?.publication.isEncrypted === false || - v?.publication.isEncrypted === false), - ), - ); - - this.encryptionStatus$ = this.scope.behavior( - this.participant$.pipe( - switchMap((participant): Observable => { - if (!participant) { - return of(EncryptionStatus.Connecting); - } else if ( - participant.isLocal || - encryptionSystem.kind === E2eeType.NONE - ) { - return of(EncryptionStatus.Okay); - } else if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) { - return combineLatest([ - encryptionErrorObservable$( - livekitRoom$, - participant, - encryptionSystem, - "MissingKey", - ), - encryptionErrorObservable$( - livekitRoom$, - participant, - encryptionSystem, - "InvalidKey", - ), - observeRemoteTrackReceivingOkay$(participant, audioSource), - observeRemoteTrackReceivingOkay$(participant, videoSource), - ]).pipe( - map(([keyMissing, keyInvalid, audioOkay, videoOkay]) => { - if (keyMissing) return EncryptionStatus.KeyMissing; - if (keyInvalid) return EncryptionStatus.KeyInvalid; - if (audioOkay || videoOkay) return EncryptionStatus.Okay; - return undefined; // no change - }), - filter((x) => !!x), - startWith(EncryptionStatus.Connecting), - ); - } else { - return combineLatest([ - encryptionErrorObservable$( - livekitRoom$, - participant, - encryptionSystem, - "InvalidKey", - ), - observeRemoteTrackReceivingOkay$(participant, audioSource), - observeRemoteTrackReceivingOkay$(participant, videoSource), - ]).pipe( - map( - ([keyInvalid, audioOkay, videoOkay]): - | EncryptionStatus - | undefined => { - if (keyInvalid) return EncryptionStatus.PasswordInvalid; - if (audioOkay || videoOkay) return EncryptionStatus.Okay; - return undefined; // no change - }, - ), - filter((x) => !!x), - startWith(EncryptionStatus.Connecting), - ); - } - }), - ), - ); - } -} - -/** - * Some participant's media. - */ -export type MediaViewModel = UserMediaViewModel | ScreenShareViewModel; -export type UserMediaViewModel = - | LocalUserMediaViewModel - | RemoteUserMediaViewModel; - -/** - * Some participant's user media. - */ -abstract class BaseUserMediaViewModel extends BaseMediaViewModel { - private readonly _speaking$ = this.scope.behavior( - this.participant$.pipe( - switchMap((p) => - p - ? observeParticipantEvents( - p, - ParticipantEvent.IsSpeakingChanged, - ).pipe(map((p) => p.isSpeaking)) - : of(false), - ), - ), - ); - /** - * Whether the participant is speaking. - */ - // Getter backed by a private field so that subclasses can override it - public get speaking$(): Behavior { - return this._speaking$; - } - - /** - * Whether this participant is sending audio (i.e. is unmuted on their side). - */ - public readonly audioEnabled$: Behavior; - - private readonly _videoEnabled$: Behavior; - /** - * Whether this participant is sending video. - */ - // Getter backed by a private field so that subclasses can override it - public get videoEnabled$(): Behavior { - return this._videoEnabled$; - } - - private readonly _cropVideo$ = new BehaviorSubject(true); - /** - * Whether the tile video should be contained inside the tile or be cropped to fit. - */ - public readonly cropVideo$: Behavior = this._cropVideo$; - - public constructor( - scope: ObservableScope, - id: string, - userId: string, - /** - * The expected identity of the LiveKit participant. Exposed for debugging. - */ - public readonly rtcBackendIdentity: string, - participant$: Behavior, - encryptionSystem: EncryptionSystem, - livekitRoom$: Behavior, - focusUrl$: Behavior, - displayName$: Behavior, - mxcAvatarUrl$: Behavior, - public readonly handRaised$: Behavior, - public readonly reaction$: Behavior, - ) { - super( - scope, - id, - userId, - participant$, - encryptionSystem, - Track.Source.Microphone, - Track.Source.Camera, - livekitRoom$, - focusUrl$, - displayName$, - mxcAvatarUrl$, - ); - - const media$ = this.scope.behavior( - participant$.pipe( - switchMap((p) => (p && observeParticipantMedia(p)) ?? of(undefined)), - ), - ); - this.audioEnabled$ = this.scope.behavior( - media$.pipe(map((m) => m?.microphoneTrack?.isMuted === false)), - ); - this._videoEnabled$ = this.scope.behavior( - media$.pipe(map((m) => m?.cameraTrack?.isMuted === false)), - ); - } - - public toggleFitContain(): void { - this._cropVideo$.next(!this._cropVideo$.value); - } - - public get local(): boolean { - return this instanceof LocalUserMediaViewModel; - } - - public abstract get audioStreamStats$(): Observable< - RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined - >; - public abstract get videoStreamStats$(): Observable< - RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined - >; -} - -/** - * The local participant's user media. - */ -export class LocalUserMediaViewModel extends BaseUserMediaViewModel { - /** - * The local video track as an observable that emits whenever the track - * changes, the camera is switched, or the track is muted. - */ - private readonly videoTrack$: Observable = - this.video$.pipe( - switchMap((v) => { - const track = v?.publication.track; - if (!(track instanceof LocalVideoTrack)) return of(null); - return merge( - // Watch for track restarts because they indicate a camera switch. - // This event is also emitted when unmuting the track object. - fromEvent(track, TrackEvent.Restarted).pipe( - startWith(null), - map(() => track), - ), - // When the track object is muted, reset it to null. - fromEvent(track, TrackEvent.Muted).pipe(map(() => null)), - ); - }), - ); - - /** - * Whether the video should be mirrored. - */ - public readonly mirror$ = this.scope.behavior( - this.videoTrack$.pipe( - // Mirror only front-facing cameras (those that face the user) - map( - (track) => - track !== null && - facingModeFromLocalTrack(track).facingMode === "user", - ), - ), - ); - - /** - * Whether to show this tile in a highly visible location near the start of - * the grid. - */ - public readonly alwaysShow$ = alwaysShowSelf.value$; - public readonly setAlwaysShow = alwaysShowSelf.setValue; - - /** - * Callback for switching between the front and back cameras. - */ - public readonly switchCamera$: Behavior<(() => void) | null> = - this.scope.behavior( - platform === "desktop" - ? of(null) - : this.videoTrack$.pipe( - map((track) => { - if (track === null) return null; - const facingMode = facingModeFromLocalTrack(track).facingMode; - // If the camera isn't front or back-facing, don't provide a switch - // camera shortcut at all - if (facingMode !== "user" && facingMode !== "environment") - return null; - // Restart the track with a camera facing the opposite direction - return (): void => - void track - .restartTrack({ - facingMode: facingMode === "user" ? "environment" : "user", - }) - .then(() => { - // Inform the MediaDevices which camera was chosen - const deviceId = - track.mediaStreamTrack.getSettings().deviceId; - if (deviceId !== undefined) - this.mediaDevices.videoInput.select(deviceId); - }) - .catch((e) => - logger.error("Failed to switch camera", facingMode, e), - ); - }), - ), - ); - - public constructor( - scope: ObservableScope, - id: string, - userId: string, - rtcBackendIdentity: string, - participant$: Behavior, - encryptionSystem: EncryptionSystem, - livekitRoom$: Behavior, - focusUrl$: Behavior, - private readonly mediaDevices: MediaDevices, - displayName$: Behavior, - mxcAvatarUrl$: Behavior, - handRaised$: Behavior, - reaction$: Behavior, - ) { - super( - scope, - id, - userId, - rtcBackendIdentity, - participant$, - encryptionSystem, - livekitRoom$, - focusUrl$, - displayName$, - mxcAvatarUrl$, - handRaised$, - reaction$, - ); - } - - public audioStreamStats$ = combineLatest([ - this.participant$, - showConnectionStats.value$, - ]).pipe( - switchMap(([p, showConnectionStats]) => { - if (!p || !showConnectionStats) return of(undefined); - return observeOutboundRtpStreamStats$(p, Track.Source.Microphone); - }), - ); - - public videoStreamStats$ = combineLatest([ - this.participant$, - showConnectionStats.value$, - ]).pipe( - switchMap(([p, showConnectionStats]) => { - if (!p || !showConnectionStats) return of(undefined); - return observeOutboundRtpStreamStats$(p, Track.Source.Camera); - }), - ); -} - -/** - * A remote participant's user media. - */ -export class RemoteUserMediaViewModel extends BaseUserMediaViewModel { - /** - * Whether we are waiting for this user's LiveKit participant to exist. This - * could be because either we or the remote party are still connecting. - */ - public readonly waitingForMedia$ = this.scope.behavior( - combineLatest( - [this.livekitRoom$, this.participant$], - (livekitRoom, participant) => - // If livekitRoom is undefined, the user is not attempting to publish on - // any transport and so we shouldn't expect a participant. (They might - // be a subscribe-only bot for example.) - livekitRoom !== undefined && participant === null, - ), - ); - - // This private field is used to override the value from the superclass - private __speaking$: Behavior; - public get speaking$(): Behavior { - return this.__speaking$; - } - - private readonly locallyMutedToggle$ = new Subject(); - private readonly localVolumeAdjustment$ = new Subject(); - private readonly localVolumeCommit$ = new Subject(); - - /** - * The volume to which this participant's audio is set, as a scalar - * multiplier. - */ - public readonly localVolume$ = this.scope.behavior( - merge( - this.locallyMutedToggle$.pipe(map(() => "toggle mute" as const)), - this.localVolumeAdjustment$, - this.localVolumeCommit$.pipe(map(() => "commit" as const)), - ).pipe( - accumulate({ volume: 1, committedVolume: 1 }, (state, event) => { - switch (event) { - case "toggle mute": - return { - ...state, - volume: state.volume === 0 ? state.committedVolume : 0, - }; - case "commit": - // Dragging the slider to zero should have the same effect as - // muting: keep the original committed volume, as if it were never - // dragged - return { - ...state, - committedVolume: - state.volume === 0 ? state.committedVolume : state.volume, - }; - default: - // Volume adjustment - return { ...state, volume: event }; - } - }), - map(({ volume }) => volume), - ), - ); - - // This private field is used to override the value from the superclass - private __videoEnabled$: Behavior; - public get videoEnabled$(): Behavior { - return this.__videoEnabled$; - } - - /** - * Whether this participant's audio is disabled. - */ - public readonly locallyMuted$ = this.scope.behavior( - this.localVolume$.pipe(map((volume) => volume === 0)), - ); - - public constructor( - scope: ObservableScope, - id: string, - userId: string, - rtcBackendIdentity: string, - participant$: Behavior, - encryptionSystem: EncryptionSystem, - livekitRoom$: Behavior, - focusUrl$: Behavior, - private readonly pretendToBeDisconnected$: Behavior, - displayName$: Behavior, - mxcAvatarUrl$: Behavior, - handRaised$: Behavior, - reaction$: Behavior, - ) { - super( - scope, - id, - userId, - rtcBackendIdentity, - participant$, - encryptionSystem, - livekitRoom$, - focusUrl$, - displayName$, - mxcAvatarUrl$, - handRaised$, - reaction$, - ); - - this.__speaking$ = this.scope.behavior( - pretendToBeDisconnected$.pipe( - switchMap((disconnected) => - disconnected ? of(false) : super.speaking$, - ), - ), - ); - - this.__videoEnabled$ = this.scope.behavior( - pretendToBeDisconnected$.pipe( - switchMap((disconnected) => - disconnected ? of(false) : super.videoEnabled$, - ), - ), - ); - - // Sync the local volume with LiveKit - combineLatest([ - participant$, - // The local volume, taking into account whether we're supposed to pretend - // that the audio stream is disconnected (since we don't necessarily want - // that to modify the UI state). - this.pretendToBeDisconnected$.pipe( - switchMap((disconnected) => (disconnected ? of(0) : this.localVolume$)), - this.scope.bind(), - ), - ]).subscribe(([p, volume]) => p?.setVolume(volume)); - } - - public toggleLocallyMuted(): void { - this.locallyMutedToggle$.next(); - } - - public setLocalVolume(value: number): void { - this.localVolumeAdjustment$.next(value); - } - - public commitLocalVolume(): void { - this.localVolumeCommit$.next(); - } - - public audioStreamStats$ = combineLatest([ - this.participant$, - showConnectionStats.value$, - ]).pipe( - switchMap(([p, showConnectionStats]) => { - if (!p || !showConnectionStats) return of(undefined); - return observeInboundRtpStreamStats$(p, Track.Source.Microphone); - }), - ); - - public videoStreamStats$ = combineLatest([ - this.participant$, - showConnectionStats.value$, - ]).pipe( - switchMap(([p, showConnectionStats]) => { - if (!p || !showConnectionStats) return of(undefined); - return observeInboundRtpStreamStats$(p, Track.Source.Camera); - }), - ); -} - -/** - * Some participant's screen share media. - */ -export class ScreenShareViewModel extends BaseMediaViewModel { - /** - * Whether this screen share's video should be displayed. - */ - public readonly videoEnabled$ = this.scope.behavior( - this.pretendToBeDisconnected$.pipe(map((disconnected) => !disconnected)), - ); - - public constructor( - scope: ObservableScope, - id: string, - userId: string, - participant$: Behavior, - encryptionSystem: EncryptionSystem, - livekitRoom$: Behavior, - focusUrl$: Behavior, - private readonly pretendToBeDisconnected$: Behavior, - displayName$: Behavior, - mxcAvatarUrl$: Behavior, - public readonly local: boolean, - ) { - super( - scope, - id, - userId, - participant$, - encryptionSystem, - Track.Source.ScreenShareAudio, - Track.Source.ScreenShare, - livekitRoom$, - focusUrl$, - displayName$, - mxcAvatarUrl$, - ); - } -} diff --git a/src/state/ScreenShare.ts b/src/state/ScreenShare.ts deleted file mode 100644 index 6c908b1f..00000000 --- a/src/state/ScreenShare.ts +++ /dev/null @@ -1,53 +0,0 @@ -/* -Copyright 2025 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - type LocalParticipant, - type RemoteParticipant, - type Room as LivekitRoom, -} from "livekit-client"; - -import { type ObservableScope } from "./ObservableScope.ts"; -import { ScreenShareViewModel } from "./MediaViewModel.ts"; -import type { EncryptionSystem } from "../e2ee/sharedKeyManagement.ts"; -import { constant, type Behavior } from "./Behavior.ts"; - -/** - * A screen share media item to be presented in a tile. This is a thin wrapper - * around ScreenShareViewModel which essentially just establishes an - * ObservableScope for behaviors that the view model depends on. - */ -export class ScreenShare { - public readonly vm: ScreenShareViewModel; - - public constructor( - private readonly scope: ObservableScope, - id: string, - userId: string, - participant: LocalParticipant | RemoteParticipant, - encryptionSystem: EncryptionSystem, - livekitRoom$: Behavior, - focusUrl$: Behavior, - pretendToBeDisconnected$: Behavior, - displayName$: Behavior, - mxcAvatarUrl$: Behavior, - ) { - this.vm = new ScreenShareViewModel( - this.scope, - id, - userId, - constant(participant), - encryptionSystem, - livekitRoom$, - focusUrl$, - pretendToBeDisconnected$, - displayName$, - mxcAvatarUrl$, - participant.isLocal, - ); - } -} diff --git a/src/state/TileStore.ts b/src/state/TileStore.ts index 7b95bd8e..a954eb4e 100644 --- a/src/state/TileStore.ts +++ b/src/state/TileStore.ts @@ -8,10 +8,11 @@ Please see LICENSE in the repository root for full details. import { BehaviorSubject } from "rxjs"; import { logger } from "matrix-js-sdk/lib/logger"; -import { type MediaViewModel, type UserMediaViewModel } from "./MediaViewModel"; import { GridTileViewModel, SpotlightTileViewModel } from "./TileViewModel"; import { fillGaps } from "../utils/iter"; import { debugTileLayout } from "../settings/settings"; +import { type MediaViewModel } from "./media/MediaViewModel"; +import { type UserMediaViewModel } from "./media/UserMediaViewModel"; function debugEntries(entries: GridTileData[]): string[] { return entries.map((e) => e.media.displayName$.value); diff --git a/src/state/TileViewModel.ts b/src/state/TileViewModel.ts index a645a0d1..8b13c685 100644 --- a/src/state/TileViewModel.ts +++ b/src/state/TileViewModel.ts @@ -5,8 +5,9 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type MediaViewModel, type UserMediaViewModel } from "./MediaViewModel"; import { type Behavior } from "./Behavior"; +import { type MediaViewModel } from "./media/MediaViewModel"; +import { type UserMediaViewModel } from "./media/UserMediaViewModel"; let nextId = 0; function createId(): string { diff --git a/src/state/UserMedia.ts b/src/state/UserMedia.ts deleted file mode 100644 index 2adc9134..00000000 --- a/src/state/UserMedia.ts +++ /dev/null @@ -1,209 +0,0 @@ -/* -Copyright 2025 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { combineLatest, map, type Observable, of, switchMap } from "rxjs"; -import { - type LocalParticipant, - ParticipantEvent, - type RemoteParticipant, - type Room as LivekitRoom, -} from "livekit-client"; -import { observeParticipantEvents } from "@livekit/components-core"; - -import { type ObservableScope } from "./ObservableScope.ts"; -import { - LocalUserMediaViewModel, - RemoteUserMediaViewModel, - type UserMediaViewModel, -} from "./MediaViewModel.ts"; -import type { Behavior } from "./Behavior.ts"; -import type { EncryptionSystem } from "../e2ee/sharedKeyManagement.ts"; -import type { MediaDevices } from "./MediaDevices.ts"; -import type { ReactionOption } from "../reactions"; -import { observeSpeaker$ } from "./observeSpeaker.ts"; -import { generateItems } from "../utils/observable.ts"; -import { ScreenShare } from "./ScreenShare.ts"; -import { type TaggedParticipant } from "./CallViewModel/remoteMembers/MatrixLivekitMembers.ts"; - -/** - * Sorting bins defining the order in which media tiles appear in the layout. - */ -enum SortingBin { - /** - * Yourself, when the "always show self" option is on. - */ - SelfAlwaysShown, - /** - * Participants that are sharing their screen. - */ - Presenters, - /** - * Participants that have been speaking recently. - */ - Speakers, - /** - * Participants that have their hand raised. - */ - HandRaised, - /** - * Participants with video. - */ - Video, - /** - * Participants not sharing any video. - */ - NoVideo, - /** - * Yourself, when the "always show self" option is off. - */ - SelfNotAlwaysShown, -} - -/** - * A user media item to be presented in a tile. This is a thin wrapper around - * UserMediaViewModel which additionally determines the media item's sorting bin - * for inclusion in the call layout and tracks associated screen shares. - */ -export class UserMedia { - public readonly vm: UserMediaViewModel = - this.participant.type === "local" - ? new LocalUserMediaViewModel( - this.scope, - this.id, - this.userId, - this.rtcBackendIdentity, - this.participant.value$, - this.encryptionSystem, - this.livekitRoom$, - this.focusUrl$, - this.mediaDevices, - this.displayName$, - this.mxcAvatarUrl$, - this.scope.behavior(this.handRaised$), - this.scope.behavior(this.reaction$), - ) - : new RemoteUserMediaViewModel( - this.scope, - this.id, - this.userId, - this.rtcBackendIdentity, - this.participant.value$, - this.encryptionSystem, - this.livekitRoom$, - this.focusUrl$, - this.pretendToBeDisconnected$, - this.displayName$, - this.mxcAvatarUrl$, - this.scope.behavior(this.handRaised$), - this.scope.behavior(this.reaction$), - ); - - private readonly speaker$ = this.scope.behavior( - observeSpeaker$(this.vm.speaking$), - ); - - // TypeScript needs this widening of the type to happen in a separate statement - private readonly participant$: Behavior< - LocalParticipant | RemoteParticipant | null - > = this.participant.value$; - - /** - * All screen share media associated with this user media. - */ - public readonly screenShares$ = this.scope.behavior( - this.participant$.pipe( - switchMap((p) => - p === null - ? of([]) - : observeParticipantEvents( - p, - ParticipantEvent.TrackPublished, - ParticipantEvent.TrackUnpublished, - ParticipantEvent.LocalTrackPublished, - ParticipantEvent.LocalTrackUnpublished, - ).pipe( - // Technically more than one screen share might be possible... our - // MediaViewModels don't support it though since they look for a unique - // track for the given source. So generateItems here is a bit overkill. - generateItems( - `${this.id} screenShares$`, - function* (p) { - if (p.isScreenShareEnabled) - yield { - keys: ["screen-share"], - data: undefined, - }; - }, - (scope, _data$, key) => - new ScreenShare( - scope, - `${this.id}:${key}`, - this.userId, - p, - this.encryptionSystem, - this.livekitRoom$, - this.focusUrl$, - this.pretendToBeDisconnected$, - this.displayName$, - this.mxcAvatarUrl$, - ), - ), - ), - ), - ), - ); - - private readonly presenter$ = this.scope.behavior( - this.screenShares$.pipe(map((screenShares) => screenShares.length > 0)), - ); - - /** - * Which sorting bin the media item should be placed in. - */ - // This is exposed here rather than by UserMediaViewModel because it's only - // relevant to the layout algorithms; the MediaView component should be - // ignorant of this value. - public readonly bin$ = combineLatest( - [ - this.speaker$, - this.presenter$, - this.vm.videoEnabled$, - this.vm.handRaised$, - this.vm instanceof LocalUserMediaViewModel - ? this.vm.alwaysShow$ - : of(false), - ], - (speaker, presenter, video, handRaised, alwaysShow) => { - if (this.vm.local) - return alwaysShow - ? SortingBin.SelfAlwaysShown - : SortingBin.SelfNotAlwaysShown; - else if (presenter) return SortingBin.Presenters; - else if (speaker) return SortingBin.Speakers; - else if (handRaised) return SortingBin.HandRaised; - else if (video) return SortingBin.Video; - else return SortingBin.NoVideo; - }, - ); - - public constructor( - private readonly scope: ObservableScope, - public readonly id: string, - private readonly userId: string, - private readonly rtcBackendIdentity: string, - private readonly participant: TaggedParticipant, - private readonly encryptionSystem: EncryptionSystem, - private readonly livekitRoom$: Behavior, - private readonly focusUrl$: Behavior, - private readonly mediaDevices: MediaDevices, - private readonly pretendToBeDisconnected$: Behavior, - private readonly displayName$: Behavior, - private readonly mxcAvatarUrl$: Behavior, - private readonly handRaised$: Observable, - private readonly reaction$: Observable, - ) {} -} diff --git a/src/state/VolumeControls.ts b/src/state/VolumeControls.ts new file mode 100644 index 00000000..beb7ae00 --- /dev/null +++ b/src/state/VolumeControls.ts @@ -0,0 +1,101 @@ +/* +Copyright 2026 Element Software Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { combineLatest, map, merge, of, Subject, switchMap } from "rxjs"; + +import { type Behavior } from "./Behavior"; +import { type ObservableScope } from "./ObservableScope"; +import { accumulate } from "../utils/observable"; + +/** + * Controls for audio playback volume. + */ +export interface VolumeControls { + /** + * The volume to which the audio is set, as a scalar multiplier. + */ + playbackVolume$: Behavior; + /** + * Whether playback of this audio is disabled. + */ + playbackMuted$: Behavior; + togglePlaybackMuted: () => void; + adjustPlaybackVolume: (value: number) => void; + commitPlaybackVolume: () => void; +} + +interface VolumeControlsInputs { + pretendToBeDisconnected$: Behavior; + /** + * The callback to run to notify the module performing audio playback of the + * requested volume. + */ + sink$: Behavior<(volume: number) => void>; +} + +/** + * Creates a set of controls for audio playback volume and syncs this with the + * audio playback module for the duration of the scope. + */ +export function createVolumeControls( + scope: ObservableScope, + { pretendToBeDisconnected$, sink$ }: VolumeControlsInputs, +): VolumeControls { + const toggleMuted$ = new Subject<"toggle mute">(); + const adjustVolume$ = new Subject(); + const commitVolume$ = new Subject<"commit">(); + + const playbackVolume$ = scope.behavior( + merge(toggleMuted$, adjustVolume$, commitVolume$).pipe( + accumulate({ volume: 1, committedVolume: 1 }, (state, event) => { + switch (event) { + case "toggle mute": + return { + ...state, + volume: state.volume === 0 ? state.committedVolume : 0, + }; + case "commit": + // Dragging the slider to zero should have the same effect as + // muting: keep the original committed volume, as if it were never + // dragged + return { + ...state, + committedVolume: + state.volume === 0 ? state.committedVolume : state.volume, + }; + default: + // Volume adjustment + return { ...state, volume: event }; + } + }), + map(({ volume }) => volume), + ), + ); + + // Sync the requested volume with the audio playback module + combineLatest([ + sink$, + // The playback volume, taking into account whether we're supposed to + // pretend that the audio stream is disconnected (since we don't necessarily + // want that to modify the UI state). + pretendToBeDisconnected$.pipe( + switchMap((disconnected) => (disconnected ? of(0) : playbackVolume$)), + ), + ]) + .pipe(scope.bind()) + .subscribe(([sink, volume]) => sink(volume)); + + return { + playbackVolume$, + playbackMuted$: scope.behavior( + playbackVolume$.pipe(map((volume) => volume === 0)), + ), + togglePlaybackMuted: () => toggleMuted$.next("toggle mute"), + adjustPlaybackVolume: (value: number) => adjustVolume$.next(value), + commitPlaybackVolume: () => commitVolume$.next("commit"), + }; +} diff --git a/src/state/layout-types.ts b/src/state/layout-types.ts index f32869df..33796f66 100644 --- a/src/state/layout-types.ts +++ b/src/state/layout-types.ts @@ -5,16 +5,14 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ +import { type LocalUserMediaViewModel } from "./media/LocalUserMediaViewModel.ts"; +import { type MediaViewModel } from "./media/MediaViewModel.ts"; +import { type RemoteUserMediaViewModel } from "./media/RemoteUserMediaViewModel.ts"; +import { type UserMediaViewModel } from "./media/UserMediaViewModel.ts"; import { type GridTileViewModel, type SpotlightTileViewModel, } from "./TileViewModel.ts"; -import { - type LocalUserMediaViewModel, - type RemoteUserMediaViewModel, - type MediaViewModel, - type UserMediaViewModel, -} from "./MediaViewModel.ts"; export interface GridLayoutMedia { type: "grid"; diff --git a/src/state/media/LocalScreenShareViewModel.ts b/src/state/media/LocalScreenShareViewModel.ts new file mode 100644 index 00000000..b31739d9 --- /dev/null +++ b/src/state/media/LocalScreenShareViewModel.ts @@ -0,0 +1,32 @@ +/* +Copyright 2023, 2024 New Vector Ltd. +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type LocalParticipant } from "livekit-client"; + +import { type Behavior } from "../Behavior"; +import { + type BaseScreenShareInputs, + type BaseScreenShareViewModel, + createBaseScreenShare, +} from "./ScreenShareViewModel"; +import { type ObservableScope } from "../ObservableScope"; + +export interface LocalScreenShareViewModel extends BaseScreenShareViewModel { + local: true; +} + +export interface LocalScreenShareInputs extends BaseScreenShareInputs { + participant$: Behavior; +} + +export function createLocalScreenShare( + scope: ObservableScope, + inputs: LocalScreenShareInputs, +): LocalScreenShareViewModel { + return { ...createBaseScreenShare(scope, inputs), local: true }; +} diff --git a/src/state/media/LocalUserMediaViewModel.ts b/src/state/media/LocalUserMediaViewModel.ts new file mode 100644 index 00000000..fd21428b --- /dev/null +++ b/src/state/media/LocalUserMediaViewModel.ts @@ -0,0 +1,137 @@ +/* +Copyright 2023, 2024 New Vector Ltd. +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + facingModeFromLocalTrack, + type LocalParticipant, + LocalVideoTrack, + TrackEvent, +} from "livekit-client"; +import { + fromEvent, + map, + merge, + type Observable, + of, + startWith, + switchMap, +} from "rxjs"; +import { logger } from "matrix-js-sdk/lib/logger"; + +import { type Behavior } from "../Behavior"; +import { + type BaseUserMediaInputs, + type BaseUserMediaViewModel, + createBaseUserMedia, +} from "./UserMediaViewModel"; +import { type ObservableScope } from "../ObservableScope"; +import { alwaysShowSelf } from "../../settings/settings"; +import { platform } from "../../Platform"; +import { type MediaDevices } from "../MediaDevices"; + +export interface LocalUserMediaViewModel extends BaseUserMediaViewModel { + local: true; + /** + * Whether the video should be mirrored. + */ + mirror$: Behavior; + /** + * Whether to show this tile in a highly visible location near the start of + * the grid. + */ + alwaysShow$: Behavior; + setAlwaysShow: (value: boolean) => void; + switchCamera$: Behavior<(() => void) | null>; +} + +export interface LocalUserMediaInputs extends Omit< + BaseUserMediaInputs, + "statsType" +> { + participant$: Behavior; + mediaDevices: MediaDevices; +} + +export function createLocalUserMedia( + scope: ObservableScope, + { mediaDevices, ...inputs }: LocalUserMediaInputs, +): LocalUserMediaViewModel { + const baseUserMedia = createBaseUserMedia(scope, { + ...inputs, + statsType: "outbound-rtp", + }); + + /** + * The local video track as an observable that emits whenever the track + * changes, the camera is switched, or the track is muted. + */ + const videoTrack$: Observable = + baseUserMedia.video$.pipe( + switchMap((v) => { + const track = v?.publication.track; + if (!(track instanceof LocalVideoTrack)) return of(null); + return merge( + // Watch for track restarts because they indicate a camera switch. + // This event is also emitted when unmuting the track object. + fromEvent(track, TrackEvent.Restarted).pipe( + startWith(null), + map(() => track), + ), + // When the track object is muted, reset it to null. + fromEvent(track, TrackEvent.Muted).pipe(map(() => null)), + ); + }), + ); + + return { + ...baseUserMedia, + local: true, + mirror$: scope.behavior( + videoTrack$.pipe( + // Mirror only front-facing cameras (those that face the user) + map( + (track) => + track !== null && + facingModeFromLocalTrack(track).facingMode === "user", + ), + ), + ), + alwaysShow$: alwaysShowSelf.value$, + setAlwaysShow: alwaysShowSelf.setValue, + switchCamera$: scope.behavior( + platform === "desktop" + ? of(null) + : videoTrack$.pipe( + map((track) => { + if (track === null) return null; + const facingMode = facingModeFromLocalTrack(track).facingMode; + // If the camera isn't front or back-facing, don't provide a switch + // camera shortcut at all + if (facingMode !== "user" && facingMode !== "environment") + return null; + // Restart the track with a camera facing the opposite direction + return (): void => + void track + .restartTrack({ + facingMode: facingMode === "user" ? "environment" : "user", + }) + .then(() => { + // Inform the MediaDevices which camera was chosen + const deviceId = + track.mediaStreamTrack.getSettings().deviceId; + if (deviceId !== undefined) + mediaDevices.videoInput.select(deviceId); + }) + .catch((e) => + logger.error("Failed to switch camera", facingMode, e), + ); + }), + ), + ), + }; +} diff --git a/src/state/media/MediaItem.ts b/src/state/media/MediaItem.ts new file mode 100644 index 00000000..6cd80045 --- /dev/null +++ b/src/state/media/MediaItem.ts @@ -0,0 +1,198 @@ +/* +Copyright 2025-2026 Element Software Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { combineLatest, map, of, switchMap } from "rxjs"; +import { + type LocalParticipant, + ParticipantEvent, + type RemoteParticipant, +} from "livekit-client"; +import { observeParticipantEvents } from "@livekit/components-core"; + +import { type ObservableScope } from "../ObservableScope.ts"; +import type { Behavior } from "../Behavior.ts"; +import type { MediaDevices } from "../MediaDevices.ts"; +import { observeSpeaker$ } from "./observeSpeaker.ts"; +import { generateItems } from "../../utils/observable.ts"; +import { type TaggedParticipant } from "../CallViewModel/remoteMembers/MatrixLivekitMembers.ts"; +import { type UserMediaViewModel } from "./UserMediaViewModel.ts"; +import { type ScreenShareViewModel } from "./ScreenShareViewModel.ts"; +import { + createLocalUserMedia, + type LocalUserMediaInputs, +} from "./LocalUserMediaViewModel.ts"; +import { + createRemoteUserMedia, + type RemoteUserMediaInputs, +} from "./RemoteUserMediaViewModel.ts"; +import { createLocalScreenShare } from "./LocalScreenShareViewModel.ts"; +import { createRemoteScreenShare } from "./RemoteScreenShareViewModel.ts"; + +/** + * Sorting bins defining the order in which media tiles appear in the layout. + */ +enum SortingBin { + /** + * Yourself, when the "always show self" option is on. + */ + SelfAlwaysShown, + /** + * Participants that are sharing their screen. + */ + Presenters, + /** + * Participants that have been speaking recently. + */ + Speakers, + /** + * Participants that have their hand raised. + */ + HandRaised, + /** + * Participants with video. + */ + Video, + /** + * Participants not sharing any video. + */ + NoVideo, + /** + * Yourself, when the "always show self" option is off. + */ + SelfNotAlwaysShown, +} + +/** + * A user media item to be presented in a tile. This is a thin wrapper around + * UserMediaViewModel which additionally carries data relevant to the tile + * layout algorithms (data which the MediaView component should be ignorant of). + */ +export type WrappedUserMediaViewModel = UserMediaViewModel & { + /** + * All screen share media associated with this user media. + */ + screenShares$: Behavior; + /** + * Which sorting bin the media item should be placed in. + */ + bin$: Behavior; +}; + +interface WrappedUserMediaInputs extends Omit< + LocalUserMediaInputs & RemoteUserMediaInputs, + "participant$" +> { + participant: TaggedParticipant; + mediaDevices: MediaDevices; + pretendToBeDisconnected$: Behavior; +} + +export function createWrappedUserMedia( + scope: ObservableScope, + { + participant, + mediaDevices, + pretendToBeDisconnected$, + ...inputs + }: WrappedUserMediaInputs, +): WrappedUserMediaViewModel { + const userMedia = + participant.type === "local" + ? createLocalUserMedia(scope, { + participant$: participant.value$, + mediaDevices, + ...inputs, + }) + : createRemoteUserMedia(scope, { + participant$: participant.value$, + pretendToBeDisconnected$, + ...inputs, + }); + + // TypeScript needs this widening of the type to happen in a separate statement + const participant$: Behavior = + participant.value$; + + const screenShares$ = scope.behavior( + participant$.pipe( + switchMap((p) => + p === null + ? of([]) + : observeParticipantEvents( + p, + ParticipantEvent.TrackPublished, + ParticipantEvent.TrackUnpublished, + ParticipantEvent.LocalTrackPublished, + ParticipantEvent.LocalTrackUnpublished, + ).pipe( + // Technically more than one screen share might be possible... our + // MediaViewModels don't support it though since they look for a unique + // track for the given source. So generateItems here is a bit overkill. + generateItems( + `${inputs.id} screenShares$`, + function* (p) { + if (p.isScreenShareEnabled) + yield { + keys: ["screen-share"], + data: undefined, + }; + }, + (scope, _data$, key) => { + const id = `${inputs.id}:${key}`; + return participant.type === "local" + ? createLocalScreenShare(scope, { + ...inputs, + id, + participant$: participant.value$, + }) + : createRemoteScreenShare(scope, { + ...inputs, + id, + participant$: participant.value$, + pretendToBeDisconnected$, + }); + }, + ), + ), + ), + ), + ); + + const speaker$ = scope.behavior(observeSpeaker$(userMedia.speaking$)); + const presenter$ = scope.behavior( + screenShares$.pipe(map((screenShares) => screenShares.length > 0)), + ); + + return { + ...userMedia, + screenShares$, + bin$: scope.behavior( + combineLatest( + [ + speaker$, + presenter$, + userMedia.videoEnabled$, + userMedia.handRaised$, + userMedia.local ? userMedia.alwaysShow$ : of(null), + ], + (speaker, presenter, video, handRaised, alwaysShow) => { + if (alwaysShow !== null) + return alwaysShow + ? SortingBin.SelfAlwaysShown + : SortingBin.SelfNotAlwaysShown; + else if (presenter) return SortingBin.Presenters; + else if (speaker) return SortingBin.Speakers; + else if (handRaised) return SortingBin.HandRaised; + else if (video) return SortingBin.Video; + else return SortingBin.NoVideo; + }, + ), + ), + }; +} + +export type MediaItem = WrappedUserMediaViewModel | ScreenShareViewModel; diff --git a/src/state/MediaViewModel.test.ts b/src/state/media/MediaViewModel.test.ts similarity index 84% rename from src/state/MediaViewModel.test.ts rename to src/state/media/MediaViewModel.test.ts index 92868216..71475b8c 100644 --- a/src/state/MediaViewModel.test.ts +++ b/src/state/media/MediaViewModel.test.ts @@ -17,13 +17,12 @@ import { mockLocalParticipant, mockMediaDevices, mockRtcMembership, - createLocalMedia, - createRemoteMedia, + mockLocalMedia, + mockRemoteMedia, withTestScheduler, mockRemoteParticipant, -} from "../utils/test"; -import { getValue } from "../utils/observable"; -import { constant } from "./Behavior"; +} from "../../utils/test"; +import { constant } from "../Behavior"; global.MediaStreamTrack = class {} as unknown as { new (): MediaStreamTrack; @@ -35,7 +34,7 @@ global.MediaStream = class {} as unknown as { }; const platformMock = vi.hoisted(() => vi.fn(() => "desktop")); -vi.mock("../Platform", () => ({ +vi.mock("../../Platform", () => ({ get platform(): string { return platformMock(); }, @@ -45,7 +44,7 @@ const rtcMembership = mockRtcMembership("@alice:example.org", "AAAA"); test("control a participant's volume", () => { const setVolumeSpy = vi.fn(); - const vm = createRemoteMedia( + const vm = mockRemoteMedia( rtcMembership, {}, mockRemoteParticipant({ setVolume: setVolumeSpy }), @@ -54,33 +53,33 @@ test("control a participant's volume", () => { schedule("-ab---c---d|", { a() { // Try muting by toggling - vm.toggleLocallyMuted(); + vm.togglePlaybackMuted(); expect(setVolumeSpy).toHaveBeenLastCalledWith(0); }, b() { // Try unmuting by dragging the slider back up - vm.setLocalVolume(0.6); - vm.setLocalVolume(0.8); - vm.commitLocalVolume(); + vm.adjustPlaybackVolume(0.6); + vm.adjustPlaybackVolume(0.8); + vm.commitPlaybackVolume(); expect(setVolumeSpy).toHaveBeenCalledWith(0.6); expect(setVolumeSpy).toHaveBeenLastCalledWith(0.8); }, c() { // Try muting by dragging the slider back down - vm.setLocalVolume(0.2); - vm.setLocalVolume(0); - vm.commitLocalVolume(); + vm.adjustPlaybackVolume(0.2); + vm.adjustPlaybackVolume(0); + vm.commitPlaybackVolume(); expect(setVolumeSpy).toHaveBeenCalledWith(0.2); expect(setVolumeSpy).toHaveBeenLastCalledWith(0); }, d() { // Try unmuting by toggling - vm.toggleLocallyMuted(); + vm.togglePlaybackMuted(); // The volume should return to the last non-zero committed volume expect(setVolumeSpy).toHaveBeenLastCalledWith(0.8); }, }); - expectObservable(vm.localVolume$).toBe("ab(cd)(ef)g", { + expectObservable(vm.playbackVolume$).toBe("ab(cd)(ef)g", { a: 1, b: 0, c: 0.6, @@ -93,11 +92,11 @@ test("control a participant's volume", () => { }); test("toggle fit/contain for a participant's video", () => { - const vm = createRemoteMedia(rtcMembership, {}, mockRemoteParticipant({})); + const vm = mockRemoteMedia(rtcMembership, {}, mockRemoteParticipant({})); withTestScheduler(({ expectObservable, schedule }) => { schedule("-ab|", { - a: () => vm.toggleFitContain(), - b: () => vm.toggleFitContain(), + a: () => vm.toggleCropVideo(), + b: () => vm.toggleCropVideo(), }); expectObservable(vm.cropVideo$).toBe("abc", { a: true, @@ -108,7 +107,7 @@ test("toggle fit/contain for a participant's video", () => { }); test("local media remembers whether it should always be shown", () => { - const vm1 = createLocalMedia( + const vm1 = mockLocalMedia( rtcMembership, {}, mockLocalParticipant({}), @@ -120,7 +119,7 @@ test("local media remembers whether it should always be shown", () => { }); // Next local media should start out *not* always shown - const vm2 = createLocalMedia( + const vm2 = mockLocalMedia( rtcMembership, {}, mockLocalParticipant({}), @@ -166,7 +165,7 @@ test("switch cameras", async () => { const selectVideoInput = vi.fn(); - const vm = createLocalMedia( + const vm = mockLocalMedia( rtcMembership, {}, mockLocalParticipant({ @@ -184,7 +183,7 @@ test("switch cameras", async () => { ); // Switch to back camera - getValue(vm.switchCamera$)!(); + vm.switchCamera$.value!(); expect(restartTrack).toHaveBeenCalledExactlyOnceWith({ facingMode: "environment", }); @@ -195,7 +194,7 @@ test("switch cameras", async () => { expect(deviceId).toBe("back camera"); // Switch to front camera - getValue(vm.switchCamera$)!(); + vm.switchCamera$.value!(); expect(restartTrack).toHaveBeenCalledTimes(2); expect(restartTrack).toHaveBeenLastCalledWith({ facingMode: "user" }); await waitFor(() => { @@ -206,17 +205,17 @@ test("switch cameras", async () => { }); test("remote media is in waiting state when participant has not yet connected", () => { - const vm = createRemoteMedia(rtcMembership, {}, null); // null participant + const vm = mockRemoteMedia(rtcMembership, {}, null); // null participant expect(vm.waitingForMedia$.value).toBe(true); }); test("remote media is not in waiting state when participant is connected", () => { - const vm = createRemoteMedia(rtcMembership, {}, mockRemoteParticipant({})); + const vm = mockRemoteMedia(rtcMembership, {}, mockRemoteParticipant({})); expect(vm.waitingForMedia$.value).toBe(false); }); test("remote media is not in waiting state when participant is connected with no publications", () => { - const vm = createRemoteMedia( + const vm = mockRemoteMedia( rtcMembership, {}, mockRemoteParticipant({ @@ -228,7 +227,7 @@ test("remote media is not in waiting state when participant is connected with no }); test("remote media is not in waiting state when user does not intend to publish anywhere", () => { - const vm = createRemoteMedia( + const vm = mockRemoteMedia( rtcMembership, {}, mockRemoteParticipant({}), diff --git a/src/state/media/MediaViewModel.ts b/src/state/media/MediaViewModel.ts new file mode 100644 index 00000000..bdc4875b --- /dev/null +++ b/src/state/media/MediaViewModel.ts @@ -0,0 +1,44 @@ +/* +Copyright 2023, 2024 New Vector Ltd. +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type Behavior } from "../Behavior"; +import { type ScreenShareViewModel } from "./ScreenShareViewModel"; +import { type UserMediaViewModel } from "./UserMediaViewModel"; + +/** + * A participant's media. + */ +export type MediaViewModel = UserMediaViewModel | ScreenShareViewModel; + +/** + * Properties which are common to all MediaViewModels. + */ +export interface BaseMediaViewModel { + /** + * An opaque identifier for this media. + */ + id: string; + /** + * The Matrix user to which this media belongs. + */ + userId: string; + displayName$: Behavior; + mxcAvatarUrl$: Behavior; +} + +export type BaseMediaInputs = BaseMediaViewModel; + +// All this function does is strip out superfluous data from the input object +export function createBaseMedia({ + id, + userId, + displayName$, + mxcAvatarUrl$, +}: BaseMediaInputs): BaseMediaViewModel { + return { id, userId, displayName$, mxcAvatarUrl$ }; +} diff --git a/src/state/media/MemberMediaViewModel.ts b/src/state/media/MemberMediaViewModel.ts new file mode 100644 index 00000000..e7f57b59 --- /dev/null +++ b/src/state/media/MemberMediaViewModel.ts @@ -0,0 +1,272 @@ +/* +Copyright 2023, 2024 New Vector Ltd. +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + type Room as LivekitRoom, + RoomEvent as LivekitRoomEvent, + type Participant, + type Track, +} from "livekit-client"; +import { + type AudioSource, + roomEventSelector, + type TrackReference, + type VideoSource, +} from "@livekit/components-core"; +import { type LocalParticipant, type RemoteParticipant } from "livekit-client"; +import { + combineLatest, + distinctUntilChanged, + filter, + map, + type Observable, + of, + startWith, + switchMap, + throttleTime, +} from "rxjs"; + +import { type Behavior } from "../Behavior"; +import { type BaseMediaViewModel, createBaseMedia } from "./MediaViewModel"; +import { type EncryptionSystem } from "../../e2ee/sharedKeyManagement"; +import { type ObservableScope } from "../ObservableScope"; +import { observeTrackReference$ } from "../observeTrackReference"; +import { E2eeType } from "../../e2ee/e2eeType"; +import { observeInboundRtpStreamStats$ } from "./observeRtpStreamStats"; + +// TODO: Encryption status is kinda broken and thus unused right now. Remove? +export enum EncryptionStatus { + Connecting, + Okay, + KeyMissing, + KeyInvalid, + PasswordInvalid, +} + +/** + * Media belonging to an active member of the RTC session. + */ +export interface MemberMediaViewModel extends BaseMediaViewModel { + /** + * The LiveKit video track for this media. + */ + video$: Behavior; + /** + * The URL of the LiveKit focus on which this member should be publishing. + * Exposed for debugging. + */ + focusUrl$: Behavior; + /** + * Whether there should be a warning that this media is unencrypted. + */ + unencryptedWarning$: Behavior; + encryptionStatus$: Behavior; +} + +export interface MemberMediaInputs extends BaseMediaViewModel { + participant$: Behavior; + livekitRoom$: Behavior; + audioSource: AudioSource; + videoSource: VideoSource; + focusUrl$: Behavior; + encryptionSystem: EncryptionSystem; +} + +export function createMemberMedia( + scope: ObservableScope, + { + participant$, + livekitRoom$, + audioSource, + videoSource, + focusUrl$, + encryptionSystem, + ...inputs + }: MemberMediaInputs, +): MemberMediaViewModel { + const trackBehavior$ = ( + source: Track.Source, + ): Behavior => + scope.behavior( + participant$.pipe( + switchMap((p) => + !p ? of(undefined) : observeTrackReference$(p, source), + ), + ), + ); + + const audio$ = trackBehavior$(audioSource); + const video$ = trackBehavior$(videoSource); + + return { + ...createBaseMedia(inputs), + video$, + focusUrl$, + unencryptedWarning$: scope.behavior( + combineLatest( + [audio$, video$], + (a, v) => + encryptionSystem.kind !== E2eeType.NONE && + (a?.publication.isEncrypted === false || + v?.publication.isEncrypted === false), + ), + ), + encryptionStatus$: scope.behavior( + participant$.pipe( + switchMap((participant): Observable => { + if (!participant) { + return of(EncryptionStatus.Connecting); + } else if ( + participant.isLocal || + encryptionSystem.kind === E2eeType.NONE + ) { + return of(EncryptionStatus.Okay); + } else if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) { + return combineLatest([ + encryptionErrorObservable$( + livekitRoom$, + participant, + encryptionSystem, + "MissingKey", + ), + encryptionErrorObservable$( + livekitRoom$, + participant, + encryptionSystem, + "InvalidKey", + ), + observeRemoteTrackReceivingOkay$(participant, audioSource), + observeRemoteTrackReceivingOkay$(participant, videoSource), + ]).pipe( + map(([keyMissing, keyInvalid, audioOkay, videoOkay]) => { + if (keyMissing) return EncryptionStatus.KeyMissing; + if (keyInvalid) return EncryptionStatus.KeyInvalid; + if (audioOkay || videoOkay) return EncryptionStatus.Okay; + return undefined; // no change + }), + filter((x) => !!x), + startWith(EncryptionStatus.Connecting), + ); + } else { + return combineLatest([ + encryptionErrorObservable$( + livekitRoom$, + participant, + encryptionSystem, + "InvalidKey", + ), + observeRemoteTrackReceivingOkay$(participant, audioSource), + observeRemoteTrackReceivingOkay$(participant, videoSource), + ]).pipe( + map( + ([keyInvalid, audioOkay, videoOkay]): + | EncryptionStatus + | undefined => { + if (keyInvalid) return EncryptionStatus.PasswordInvalid; + if (audioOkay || videoOkay) return EncryptionStatus.Okay; + return undefined; // no change + }, + ), + filter((x) => !!x), + startWith(EncryptionStatus.Connecting), + ); + } + }), + ), + ), + }; +} + +function encryptionErrorObservable$( + room$: Behavior, + participant: Participant, + encryptionSystem: EncryptionSystem, + criteria: string, +): Observable { + return room$.pipe( + switchMap((room) => { + if (room === undefined) return of(false); + return roomEventSelector(room, LivekitRoomEvent.EncryptionError).pipe( + map((e) => { + const [err] = e; + if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) { + return ( + // Ideally we would pull the participant identity from the field on the error. + // However, it gets lost in the serialization process between workers. + // So, instead we do a string match + (err?.message.includes(participant.identity) && + err?.message.includes(criteria)) ?? + false + ); + } else if (encryptionSystem.kind === E2eeType.SHARED_KEY) { + return !!err?.message.includes(criteria); + } + + return false; + }), + ); + }), + distinctUntilChanged(), + throttleTime(1000), // Throttle to avoid spamming the UI + startWith(false), + ); +} + +function observeRemoteTrackReceivingOkay$( + participant: Participant, + source: Track.Source, +): Observable { + let lastStats: { + framesDecoded: number | undefined; + framesDropped: number | undefined; + framesReceived: number | undefined; + } = { + framesDecoded: undefined, + framesDropped: undefined, + framesReceived: undefined, + }; + + return observeInboundRtpStreamStats$(participant, source).pipe( + map((stats) => { + if (!stats) return undefined; + const { framesDecoded, framesDropped, framesReceived } = stats; + return { + framesDecoded, + framesDropped, + framesReceived, + }; + }), + filter((newStats) => !!newStats), + map((newStats): boolean | undefined => { + const oldStats = lastStats; + lastStats = newStats; + if ( + typeof newStats.framesReceived === "number" && + typeof oldStats.framesReceived === "number" && + typeof newStats.framesDecoded === "number" && + typeof oldStats.framesDecoded === "number" + ) { + const framesReceivedDelta = + newStats.framesReceived - oldStats.framesReceived; + const framesDecodedDelta = + newStats.framesDecoded - oldStats.framesDecoded; + + // if we received >0 frames and managed to decode >0 frames then we treat that as success + + if (framesReceivedDelta > 0) { + return framesDecodedDelta > 0; + } + } + + // no change + return undefined; + }), + filter((x) => typeof x === "boolean"), + startWith(undefined), + ); +} diff --git a/src/state/media/RemoteScreenShareViewModel.ts b/src/state/media/RemoteScreenShareViewModel.ts new file mode 100644 index 00000000..eff6d9c1 --- /dev/null +++ b/src/state/media/RemoteScreenShareViewModel.ts @@ -0,0 +1,44 @@ +/* +Copyright 2023, 2024 New Vector Ltd. +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type RemoteParticipant } from "livekit-client"; +import { map } from "rxjs"; + +import { type Behavior } from "../Behavior"; +import { + type BaseScreenShareInputs, + type BaseScreenShareViewModel, + createBaseScreenShare, +} from "./ScreenShareViewModel"; +import { type ObservableScope } from "../ObservableScope"; + +export interface RemoteScreenShareViewModel extends BaseScreenShareViewModel { + local: false; + /** + * Whether this screen share's video should be displayed. + */ + videoEnabled$: Behavior; +} + +export interface RemoteScreenShareInputs extends BaseScreenShareInputs { + participant$: Behavior; + pretendToBeDisconnected$: Behavior; +} + +export function createRemoteScreenShare( + scope: ObservableScope, + { pretendToBeDisconnected$, ...inputs }: RemoteScreenShareInputs, +): RemoteScreenShareViewModel { + return { + ...createBaseScreenShare(scope, inputs), + local: false, + videoEnabled$: scope.behavior( + pretendToBeDisconnected$.pipe(map((disconnected) => !disconnected)), + ), + }; +} diff --git a/src/state/media/RemoteUserMediaViewModel.ts b/src/state/media/RemoteUserMediaViewModel.ts new file mode 100644 index 00000000..4307dea4 --- /dev/null +++ b/src/state/media/RemoteUserMediaViewModel.ts @@ -0,0 +1,82 @@ +/* +Copyright 2023, 2024 New Vector Ltd. +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type RemoteParticipant } from "livekit-client"; +import { combineLatest, map, of, switchMap } from "rxjs"; + +import { type Behavior } from "../Behavior"; +import { createVolumeControls, type VolumeControls } from "../VolumeControls"; +import { + type BaseUserMediaInputs, + type BaseUserMediaViewModel, + createBaseUserMedia, +} from "./UserMediaViewModel"; +import { type ObservableScope } from "../ObservableScope"; + +export interface RemoteUserMediaViewModel + extends BaseUserMediaViewModel, VolumeControls { + local: false; + /** + * Whether we are waiting for this user's LiveKit participant to exist. This + * could be because either we or the remote party are still connecting. + */ + waitingForMedia$: Behavior; +} + +export interface RemoteUserMediaInputs extends Omit< + BaseUserMediaInputs, + "statsType" +> { + participant$: Behavior; + pretendToBeDisconnected$: Behavior; +} + +export function createRemoteUserMedia( + scope: ObservableScope, + { pretendToBeDisconnected$, ...inputs }: RemoteUserMediaInputs, +): RemoteUserMediaViewModel { + const baseUserMedia = createBaseUserMedia(scope, { + ...inputs, + statsType: "inbound-rtp", + }); + + return { + ...baseUserMedia, + ...createVolumeControls(scope, { + pretendToBeDisconnected$, + sink$: scope.behavior( + inputs.participant$.pipe(map((p) => (volume) => p?.setVolume(volume))), + ), + }), + local: false, + speaking$: scope.behavior( + pretendToBeDisconnected$.pipe( + switchMap((disconnected) => + disconnected ? of(false) : baseUserMedia.speaking$, + ), + ), + ), + videoEnabled$: scope.behavior( + pretendToBeDisconnected$.pipe( + switchMap((disconnected) => + disconnected ? of(false) : baseUserMedia.videoEnabled$, + ), + ), + ), + waitingForMedia$: scope.behavior( + combineLatest( + [inputs.livekitRoom$, inputs.participant$], + (livekitRoom, participant) => + // If livekitRoom is undefined, the user is not attempting to publish on + // any transport and so we shouldn't expect a participant. (They might + // be a subscribe-only bot for example.) + livekitRoom !== undefined && participant === null, + ), + ), + }; +} diff --git a/src/state/media/ScreenShareViewModel.ts b/src/state/media/ScreenShareViewModel.ts new file mode 100644 index 00000000..36cd9440 --- /dev/null +++ b/src/state/media/ScreenShareViewModel.ts @@ -0,0 +1,51 @@ +/* +Copyright 2023, 2024 New Vector Ltd. +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { Track } from "livekit-client"; + +import { type ObservableScope } from "../ObservableScope"; +import { type LocalScreenShareViewModel } from "./LocalScreenShareViewModel"; +import { + createMemberMedia, + type MemberMediaInputs, + type MemberMediaViewModel, +} from "./MemberMediaViewModel"; +import { type RemoteScreenShareViewModel } from "./RemoteScreenShareViewModel"; + +/** + * A participant's screen share media. + */ +export type ScreenShareViewModel = + | LocalScreenShareViewModel + | RemoteScreenShareViewModel; + +/** + * Properties which are common to all ScreenShareViewModels. + */ +export interface BaseScreenShareViewModel extends MemberMediaViewModel { + type: "screen share"; +} + +export type BaseScreenShareInputs = Omit< + MemberMediaInputs, + "audioSource" | "videoSource" +>; + +export function createBaseScreenShare( + scope: ObservableScope, + inputs: BaseScreenShareInputs, +): BaseScreenShareViewModel { + return { + ...createMemberMedia(scope, { + ...inputs, + audioSource: Track.Source.ScreenShareAudio, + videoSource: Track.Source.ScreenShare, + }), + type: "screen share", + }; +} diff --git a/src/state/media/UserMediaViewModel.ts b/src/state/media/UserMediaViewModel.ts new file mode 100644 index 00000000..8da5e63a --- /dev/null +++ b/src/state/media/UserMediaViewModel.ts @@ -0,0 +1,143 @@ +/* +Copyright 2023, 2024 New Vector Ltd. +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + combineLatest, + map, + type Observable, + of, + Subject, + switchMap, +} from "rxjs"; +import { + observeParticipantEvents, + observeParticipantMedia, +} from "@livekit/components-core"; +import { ParticipantEvent, Track } from "livekit-client"; + +import { type ReactionOption } from "../../reactions"; +import { type Behavior } from "../Behavior"; +import { type LocalUserMediaViewModel } from "./LocalUserMediaViewModel"; +import { + createMemberMedia, + type MemberMediaInputs, + type MemberMediaViewModel, +} from "./MemberMediaViewModel"; +import { type RemoteUserMediaViewModel } from "./RemoteUserMediaViewModel"; +import { type ObservableScope } from "../ObservableScope"; +import { createToggle$ } from "../../utils/observable"; +import { showConnectionStats } from "../../settings/settings"; +import { observeRtpStreamStats$ } from "./observeRtpStreamStats"; + +/** + * A participant's user media (i.e. their microphone and camera feed). + */ +export type UserMediaViewModel = + | LocalUserMediaViewModel + | RemoteUserMediaViewModel; + +export interface BaseUserMediaViewModel extends MemberMediaViewModel { + type: "user"; + speaking$: Behavior; + audioEnabled$: Behavior; + videoEnabled$: Behavior; + cropVideo$: Behavior; + toggleCropVideo: () => void; + /** + * The expected identity of the LiveKit participant. Exposed for debugging. + */ + rtcBackendIdentity: string; + handRaised$: Behavior; + reaction$: Behavior; + audioStreamStats$: Observable< + RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined + >; + videoStreamStats$: Observable< + RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined + >; +} + +export interface BaseUserMediaInputs extends Omit< + MemberMediaInputs, + "audioSource" | "videoSource" +> { + rtcBackendIdentity: string; + handRaised$: Behavior; + reaction$: Behavior; + statsType: "inbound-rtp" | "outbound-rtp"; +} + +export function createBaseUserMedia( + scope: ObservableScope, + { + rtcBackendIdentity, + handRaised$, + reaction$, + statsType, + ...inputs + }: BaseUserMediaInputs, +): BaseUserMediaViewModel { + const { participant$ } = inputs; + const media$ = scope.behavior( + participant$.pipe( + switchMap((p) => (p && observeParticipantMedia(p)) ?? of(undefined)), + ), + ); + const toggleCropVideo$ = new Subject(); + + return { + ...createMemberMedia(scope, { + ...inputs, + audioSource: Track.Source.Microphone, + videoSource: Track.Source.Camera, + }), + type: "user", + speaking$: scope.behavior( + participant$.pipe( + switchMap((p) => + p + ? observeParticipantEvents( + p, + ParticipantEvent.IsSpeakingChanged, + ).pipe(map((p) => p.isSpeaking)) + : of(false), + ), + ), + ), + audioEnabled$: scope.behavior( + media$.pipe(map((m) => m?.microphoneTrack?.isMuted === false)), + ), + videoEnabled$: scope.behavior( + media$.pipe(map((m) => m?.cameraTrack?.isMuted === false)), + ), + cropVideo$: createToggle$(scope, true, toggleCropVideo$), + toggleCropVideo: () => toggleCropVideo$.next(), + rtcBackendIdentity, + handRaised$, + reaction$, + audioStreamStats$: combineLatest([ + participant$, + showConnectionStats.value$, + ]).pipe( + switchMap(([p, showConnectionStats]) => { + // + if (!p || !showConnectionStats) return of(undefined); + return observeRtpStreamStats$(p, Track.Source.Microphone, statsType); + }), + ), + videoStreamStats$: combineLatest([ + participant$, + showConnectionStats.value$, + ]).pipe( + switchMap(([p, showConnectionStats]) => { + if (!p || !showConnectionStats) return of(undefined); + return observeRtpStreamStats$(p, Track.Source.Camera, statsType); + }), + ), + }; +} diff --git a/src/state/media/observeRtpStreamStats.ts b/src/state/media/observeRtpStreamStats.ts new file mode 100644 index 00000000..d1640382 --- /dev/null +++ b/src/state/media/observeRtpStreamStats.ts @@ -0,0 +1,69 @@ +/* +Copyright 2023, 2024 New Vector Ltd. +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + LocalTrack, + type Participant, + RemoteTrack, + type Track, +} from "livekit-client"; +import { + combineLatest, + interval, + type Observable, + startWith, + switchMap, + map, +} from "rxjs"; + +import { observeTrackReference$ } from "../observeTrackReference"; + +export function observeRtpStreamStats$( + participant: Participant, + source: Track.Source, + type: "inbound-rtp" | "outbound-rtp", +): Observable< + RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined +> { + return combineLatest([ + observeTrackReference$(participant, source), + interval(1000).pipe(startWith(0)), + ]).pipe( + switchMap(async ([trackReference]) => { + const track = trackReference?.publication?.track; + if ( + !track || + !(track instanceof RemoteTrack || track instanceof LocalTrack) + ) { + return undefined; + } + const report = await track.getRTCStatsReport(); + if (!report) { + return undefined; + } + + for (const v of report.values()) { + if (v.type === type) { + return v; + } + } + + return undefined; + }), + startWith(undefined), + ); +} + +export function observeInboundRtpStreamStats$( + participant: Participant, + source: Track.Source, +): Observable { + return observeRtpStreamStats$(participant, source, "inbound-rtp").pipe( + map((x) => x as RTCInboundRtpStreamStats | undefined), + ); +} diff --git a/src/state/observeSpeaker.test.ts b/src/state/media/observeSpeaker.test.ts similarity index 98% rename from src/state/observeSpeaker.test.ts rename to src/state/media/observeSpeaker.test.ts index 224916d2..18622fb8 100644 --- a/src/state/observeSpeaker.test.ts +++ b/src/state/media/observeSpeaker.test.ts @@ -7,7 +7,7 @@ Please see LICENSE in the repository root for full details. import { describe, test } from "vitest"; -import { withTestScheduler } from "../utils/test"; +import { withTestScheduler } from "../../utils/test"; import { observeSpeaker$ } from "./observeSpeaker"; const yesNo = { diff --git a/src/state/observeSpeaker.ts b/src/state/media/observeSpeaker.ts similarity index 100% rename from src/state/observeSpeaker.ts rename to src/state/media/observeSpeaker.ts diff --git a/src/state/observeTrackReference.ts b/src/state/observeTrackReference.ts new file mode 100644 index 00000000..8e295d05 --- /dev/null +++ b/src/state/observeTrackReference.ts @@ -0,0 +1,28 @@ +/* +Copyright 2023, 2024 New Vector Ltd. +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + observeParticipantMedia, + type TrackReference, +} from "@livekit/components-core"; +import { type Participant, type Track } from "livekit-client"; +import { distinctUntilChanged, map, type Observable } from "rxjs"; + +/** + * Reactively reads a participant's track reference for a given media source. + */ +export function observeTrackReference$( + participant: Participant, + source: Track.Source, +): Observable { + return observeParticipantMedia(participant).pipe( + map(() => participant.getTrackPublication(source)), + distinctUntilChanged(), + map((publication) => publication && { participant, publication, source }), + ); +} diff --git a/src/tile/GridTile.test.tsx b/src/tile/GridTile.test.tsx index 9bc0efb2..02f09a17 100644 --- a/src/tile/GridTile.test.tsx +++ b/src/tile/GridTile.test.tsx @@ -14,7 +14,7 @@ import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; import { GridTile } from "./GridTile"; import { mockRtcMembership, - createRemoteMedia, + mockRemoteMedia, mockRemoteParticipant, } from "../utils/test"; import { GridTileViewModel } from "../state/TileViewModel"; @@ -29,7 +29,7 @@ global.IntersectionObserver = class MockIntersectionObserver { } as unknown as typeof IntersectionObserver; test("GridTile is accessible", async () => { - const vm = createRemoteMedia( + const vm = mockRemoteMedia( mockRtcMembership("@alice:example.org", "AAAA"), { rawDisplayName: "Alice", diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index 92262f05..9c3adea7 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -39,11 +39,6 @@ import { import { useObservableEagerState } from "observable-hooks"; import styles from "./GridTile.module.css"; -import { - type UserMediaViewModel, - LocalUserMediaViewModel, - type RemoteUserMediaViewModel, -} from "../state/MediaViewModel"; import { Slider } from "../Slider"; import { MediaView } from "./MediaView"; import { useLatest } from "../useLatest"; @@ -51,6 +46,9 @@ import { type GridTileViewModel } from "../state/TileViewModel"; import { useMergedRefs } from "../useMergedRefs"; import { useReactionsSender } from "../reactions/useReactionsSender"; import { useBehavior } from "../useBehavior"; +import { type LocalUserMediaViewModel } from "../state/media/LocalUserMediaViewModel"; +import { type RemoteUserMediaViewModel } from "../state/media/RemoteUserMediaViewModel"; +import { type UserMediaViewModel } from "../state/media/UserMediaViewModel"; interface TileProps { ref?: Ref; @@ -68,7 +66,7 @@ interface TileProps { interface UserMediaTileProps extends TileProps { vm: UserMediaViewModel; mirror: boolean; - locallyMuted: boolean; + playbackMuted: boolean; waitingForMedia?: boolean; primaryButton?: ReactNode; menuStart?: ReactNode; @@ -79,7 +77,7 @@ const UserMediaTile: FC = ({ ref, vm, showSpeakingIndicators, - locallyMuted, + playbackMuted, waitingForMedia, primaryButton, menuStart, @@ -109,7 +107,7 @@ const UserMediaTile: FC = ({ const onSelectFitContain = useCallback( (e: Event) => { e.preventDefault(); - vm.toggleFitContain(); + vm.toggleCropVideo(); }, [vm], ); @@ -117,12 +115,12 @@ const UserMediaTile: FC = ({ const handRaised = useBehavior(vm.handRaised$); const reaction = useBehavior(vm.reaction$); - const AudioIcon = locallyMuted + const AudioIcon = playbackMuted ? VolumeOffSolidIcon : audioEnabled ? MicOnSolidIcon : MicOffSolidIcon; - const audioIconLabel = locallyMuted + const audioIconLabel = playbackMuted ? t("video_tile.muted_for_me") : audioEnabled ? t("microphone_on") @@ -166,7 +164,7 @@ const UserMediaTile: FC = ({ width={20} height={20} aria-label={audioIconLabel} - data-muted={locallyMuted || !audioEnabled} + data-muted={playbackMuted || !audioEnabled} className={styles.muteIcon} /> } @@ -245,7 +243,7 @@ const LocalUserMediaTile: FC = ({ = ({ }) => { const { t } = useTranslation(); const waitingForMedia = useBehavior(vm.waitingForMedia$); - const locallyMuted = useBehavior(vm.locallyMuted$); - const localVolume = useBehavior(vm.localVolume$); + const playbackMuted = useBehavior(vm.playbackMuted$); + const playbackVolume = useBehavior(vm.playbackVolume$); const onSelectMute = useCallback( (e: Event) => { e.preventDefault(); - vm.toggleLocallyMuted(); + vm.togglePlaybackMuted(); }, [vm], ); - const onChangeLocalVolume = useCallback( - (v: number) => vm.setLocalVolume(v), - [vm], - ); - const onCommitLocalVolume = useCallback(() => vm.commitLocalVolume(), [vm]); - const VolumeIcon = locallyMuted ? VolumeOffIcon : VolumeOnIcon; + const VolumeIcon = playbackMuted ? VolumeOffIcon : VolumeOnIcon; return ( {/* TODO: Figure out how to make this slider keyboard accessible */} @@ -332,9 +325,9 @@ const RemoteUserMediaTile: FC = ({ = ({ const displayName = useBehavior(media.displayName$); const mxcAvatarUrl = useBehavior(media.mxcAvatarUrl$); - if (media instanceof LocalUserMediaViewModel) { + if (media.local) { return ( { diff --git a/src/tile/MediaView.tsx b/src/tile/MediaView.tsx index fadc9d2b..f912c069 100644 --- a/src/tile/MediaView.tsx +++ b/src/tile/MediaView.tsx @@ -16,10 +16,10 @@ import { ErrorSolidIcon } from "@vector-im/compound-design-tokens/assets/web/ico import styles from "./MediaView.module.css"; import { Avatar } from "../Avatar"; -import { type EncryptionStatus } from "../state/MediaViewModel"; +import { type EncryptionStatus } from "../state/media/MemberMediaViewModel"; import { RaisedHandIndicator } from "../reactions/RaisedHandIndicator"; import { - showConnectionStats, + showConnectionStats as showConnectionStatsSetting, showHandRaisedTimer, useSetting, } from "../settings/settings"; @@ -85,7 +85,7 @@ export const MediaView: FC = ({ }) => { const { t } = useTranslation(); const [handRaiseTimerVisible] = useSetting(showHandRaisedTimer); - const [showConnectioStats] = useSetting(showConnectionStats); + const [showConnectionStats] = useSetting(showConnectionStatsSetting); const avatarSize = Math.round(Math.min(targetWidth, targetHeight) / 2); @@ -139,10 +139,10 @@ export const MediaView: FC = ({ {waitingForMedia && (
{t("video_tile.waiting_for_media")} - {showConnectioStats ? " " + rtcBackendIdentity : ""} + {showConnectionStats ? " " + rtcBackendIdentity : ""}
)} - {(audioStreamStats || videoStreamStats) && ( + {showConnectionStats && ( <> { - const vm1 = createRemoteMedia( + const vm1 = mockRemoteMedia( mockRtcMembership("@alice:example.org", "AAAA"), { rawDisplayName: "Alice", @@ -37,7 +37,7 @@ test("SpotlightTile is accessible", async () => { mockRemoteParticipant({}), ); - const vm2 = createLocalMedia( + const vm2 = mockLocalMedia( mockRtcMembership("@bob:example.org", "BBBB"), { rawDisplayName: "Bob", diff --git a/src/tile/SpotlightTile.tsx b/src/tile/SpotlightTile.tsx index e685327e..75c69479 100644 --- a/src/tile/SpotlightTile.tsx +++ b/src/tile/SpotlightTile.tsx @@ -32,20 +32,19 @@ import FullScreenMaximiseIcon from "../icons/FullScreenMaximise.svg?react"; import FullScreenMinimiseIcon from "../icons/FullScreenMinimise.svg?react"; import { MediaView } from "./MediaView"; import styles from "./SpotlightTile.module.css"; -import { - type EncryptionStatus, - LocalUserMediaViewModel, - type MediaViewModel, - ScreenShareViewModel, - type UserMediaViewModel, - type RemoteUserMediaViewModel, -} from "../state/MediaViewModel"; import { useInitial } from "../useInitial"; import { useMergedRefs } from "../useMergedRefs"; import { useReactiveState } from "../useReactiveState"; import { useLatest } from "../useLatest"; import { type SpotlightTileViewModel } from "../state/TileViewModel"; import { useBehavior } from "../useBehavior"; +import { type EncryptionStatus } from "../state/media/MemberMediaViewModel"; +import { type LocalUserMediaViewModel } from "../state/media/LocalUserMediaViewModel"; +import { type RemoteUserMediaViewModel } from "../state/media/RemoteUserMediaViewModel"; +import { type UserMediaViewModel } from "../state/media/UserMediaViewModel"; +import { type ScreenShareViewModel } from "../state/media/ScreenShareViewModel"; +import { type RemoteScreenShareViewModel } from "../state/media/RemoteScreenShareViewModel"; +import { type MediaViewModel } from "../state/media/MediaViewModel"; interface SpotlightItemBaseProps { ref?: Ref; @@ -54,7 +53,6 @@ interface SpotlightItemBaseProps { targetWidth: number; targetHeight: number; video: TrackReferenceOrPlaceholder | undefined; - videoEnabled: boolean; userId: string; unencryptedWarning: boolean; encryptionStatus: EncryptionStatus; @@ -67,6 +65,7 @@ interface SpotlightItemBaseProps { interface SpotlightUserMediaItemBaseProps extends SpotlightItemBaseProps { videoFit: "contain" | "cover"; + videoEnabled: boolean; } interface SpotlightLocalUserMediaItemProps extends SpotlightUserMediaItemBaseProps { @@ -106,14 +105,16 @@ const SpotlightUserMediaItem: FC = ({ ...props }) => { const cropVideo = useBehavior(vm.cropVideo$); + const videoEnabled = useBehavior(vm.videoEnabled$); const baseProps: SpotlightUserMediaItemBaseProps & RefAttributes = { videoFit: cropVideo ? "cover" : "contain", + videoEnabled, ...props, }; - return vm instanceof LocalUserMediaViewModel ? ( + return vm.local ? ( ) : ( @@ -122,6 +123,31 @@ const SpotlightUserMediaItem: FC = ({ SpotlightUserMediaItem.displayName = "SpotlightUserMediaItem"; +interface SpotlightScreenShareItemProps extends SpotlightItemBaseProps { + vm: ScreenShareViewModel; + videoEnabled: boolean; +} + +const SpotlightScreenShareItem: FC = ({ + vm, + ...props +}) => { + return ; +}; + +interface SpotlightRemoteScreenShareItemProps extends SpotlightItemBaseProps { + vm: RemoteScreenShareViewModel; +} + +const SpotlightRemoteScreenShareItem: FC< + SpotlightRemoteScreenShareItemProps +> = ({ vm, ...props }) => { + const videoEnabled = useBehavior(vm.videoEnabled$); + return ( + + ); +}; + interface SpotlightItemProps { ref?: Ref; vm: MediaViewModel; @@ -152,7 +178,6 @@ const SpotlightItem: FC = ({ const displayName = useBehavior(vm.displayName$); const mxcAvatarUrl = useBehavior(vm.mxcAvatarUrl$); const video = useBehavior(vm.video$); - const videoEnabled = useBehavior(vm.videoEnabled$); const unencryptedWarning = useBehavior(vm.unencryptedWarning$); const encryptionStatus = useBehavior(vm.encryptionStatus$); @@ -178,7 +203,6 @@ const SpotlightItem: FC = ({ targetWidth, targetHeight, video: video ?? undefined, - videoEnabled, userId: vm.userId, unencryptedWarning, focusUrl, @@ -189,10 +213,12 @@ const SpotlightItem: FC = ({ "aria-hidden": ariaHidden, }; - return vm instanceof ScreenShareViewModel ? ( - + if (vm.type === "user") + return ; + return vm.local ? ( + ) : ( - + ); }; diff --git a/src/utils/observable.ts b/src/utils/observable.ts index 2e19748b..353dc877 100644 --- a/src/utils/observable.ts +++ b/src/utils/observable.ts @@ -61,6 +61,20 @@ export function accumulate( events$.pipe(scan(update, initial), startWith(initial)); } +/** + * Given a source of toggle events, creates a Behavior whose value toggles + * between `true` and `false`. + */ +export function createToggle$( + scope: ObservableScope, + initialValue: boolean, + toggle$: Observable, +): Behavior { + return scope.behavior( + toggle$.pipe(accumulate(initialValue, (state) => !state)), + ); +} + const switchSymbol = Symbol("switch"); /** diff --git a/src/utils/test.ts b/src/utils/test.ts index d78bdf42..c1e67927 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -52,10 +52,6 @@ import { } from "matrix-js-sdk/lib/matrixrtc/IKeyTransport"; import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; -import { - LocalUserMediaViewModel, - RemoteUserMediaViewModel, -} from "../state/MediaViewModel"; import { E2eeType } from "../e2ee/e2eeType"; import { DEFAULT_CONFIG, @@ -66,6 +62,14 @@ import { type MediaDevices } from "../state/MediaDevices"; import { type Behavior, constant } from "../state/Behavior"; import { ObservableScope } from "../state/ObservableScope"; import { MuteStates } from "../state/MuteStates"; +import { + createLocalUserMedia, + type LocalUserMediaViewModel, +} from "../state/media/LocalUserMediaViewModel"; +import { + createRemoteUserMedia, + type RemoteUserMediaViewModel, +} from "../state/media/RemoteUserMediaViewModel"; export function withFakeTimers(continuation: () => void): void { vi.useFakeTimers(); @@ -323,30 +327,27 @@ export function mockLocalParticipant( } as Partial as LocalParticipant; } -export function createLocalMedia( +export function mockLocalMedia( rtcMember: CallMembership, roomMember: Partial, localParticipant: LocalParticipant, mediaDevices: MediaDevices, ): LocalUserMediaViewModel { const member = mockMatrixRoomMember(rtcMember, roomMember); - return new LocalUserMediaViewModel( - testScope(), - "local", - member.userId, - rtcMember.rtcBackendIdentity, - constant(localParticipant), - { - kind: E2eeType.PER_PARTICIPANT, - }, - constant(mockLivekitRoom({ localParticipant })), - constant("https://rtc-example.org"), + return createLocalUserMedia(testScope(), { + id: "local", + userId: member.userId, + rtcBackendIdentity: rtcMember.rtcBackendIdentity, + participant$: constant(localParticipant), + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + livekitRoom$: constant(mockLivekitRoom({ localParticipant })), + focusUrl$: constant("https://rtc-example.org"), mediaDevices, - constant(member.rawDisplayName ?? "nodisplayname"), - constant(member.getMxcAvatarUrl()), - constant(null), - constant(null), - ); + displayName$: constant(member.rawDisplayName ?? "nodisplayname"), + mxcAvatarUrl$: constant(member.getMxcAvatarUrl()), + handRaised$: constant(null), + reaction$: constant(null), + }); } export function mockRemoteParticipant( @@ -364,7 +365,7 @@ export function mockRemoteParticipant( } as RemoteParticipant; } -export function createRemoteMedia( +export function mockRemoteMedia( rtcMember: CallMembership, roomMember: Partial, participant: RemoteParticipant | null, @@ -376,23 +377,20 @@ export function createRemoteMedia( ), ): RemoteUserMediaViewModel { const member = mockMatrixRoomMember(rtcMember, roomMember); - return new RemoteUserMediaViewModel( - testScope(), - "remote", - member.userId, - rtcMember.rtcBackendIdentity, - constant(participant), - { - kind: E2eeType.PER_PARTICIPANT, - }, - constant(livekitRoom), - constant("https://rtc-example.org"), - constant(false), - constant(member.rawDisplayName ?? "nodisplayname"), - constant(member.getMxcAvatarUrl()), - constant(null), - constant(null), - ); + return createRemoteUserMedia(testScope(), { + id: "remote", + userId: member.userId, + rtcBackendIdentity: rtcMember.rtcBackendIdentity, + participant$: constant(participant), + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + livekitRoom$: constant(livekitRoom), + focusUrl$: constant("https://rtc-example.org"), + pretendToBeDisconnected$: constant(false), + displayName$: constant(member.rawDisplayName ?? "nodisplayname"), + mxcAvatarUrl$: constant(member.getMxcAvatarUrl()), + handRaised$: constant(null), + reaction$: constant(null), + }); } export function mockConfig(