From 6995388a299ac04feaf2e241718b9820c2bc82b2 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 25 Feb 2026 14:02:59 +0100 Subject: [PATCH 1/4] Convert media view model classes to interfaces Timo and I agreed previously that we should ditch the class pattern for view models and instead have them be interfaces which are simply created by functions. They're more straightforward to write, mock, and instantiate this way. The code for media view models and media items is pretty much the last remaining instance of the class pattern. Since I was about to introduce a new media view model for ringing, I wanted to get this refactor out of the way first rather than add to the technical debt. This refactor also makes things a little easier for https://github.com/element-hq/element-call/pull/3747 by extracting volume controls into their own module. --- src/state/CallViewModel/CallViewModel.ts | 187 +++--- src/state/MediaItem.ts | 196 ++++++ src/state/MediaViewModel.test.ts | 44 +- src/state/MediaViewModel.ts | 795 ++++++++++------------- src/state/ScreenShare.ts | 53 -- src/state/UserMedia.ts | 209 ------ src/state/VolumeControls.ts | 101 +++ src/tile/GridTile.test.tsx | 4 +- src/tile/GridTile.tsx | 41 +- src/tile/MediaView.tsx | 8 +- src/tile/SpotlightTile.test.tsx | 8 +- src/tile/SpotlightTile.tsx | 46 +- src/utils/observable.ts | 14 + src/utils/test.ts | 70 +- 14 files changed, 862 insertions(+), 914 deletions(-) create mode 100644 src/state/MediaItem.ts delete mode 100644 src/state/ScreenShare.ts delete mode 100644 src/state/UserMedia.ts create mode 100644 src/state/VolumeControls.ts diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 61afb7b9..7f34d86f 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -51,14 +51,12 @@ import { v4 as uuidv4 } from "uuid"; import { type IMembershipManager } from "matrix-js-sdk/lib/matrixrtc/IMembershipManager"; import { - LocalUserMediaViewModel, type MediaViewModel, - type RemoteUserMediaViewModel, - ScreenShareViewModel, + type ScreenShareViewModel, type UserMediaViewModel, } from "../MediaViewModel"; import { - accumulate, + createToggle$, filterBehavior, generateItem, generateItems, @@ -92,8 +90,6 @@ import { type MuteStates } from "../MuteStates"; import { getUrlParams } from "../../UrlParams"; import { type ProcessorState } from "../../livekit/TrackProcessorContext"; import { ElementWidgetActions, widget } from "../../widget"; -import { UserMedia } from "../UserMedia.ts"; -import { ScreenShare } from "../ScreenShare.ts"; import { type GridLayoutMedia, type Layout, @@ -144,6 +140,11 @@ import { import { Publisher } from "./localMember/Publisher.ts"; import { type Connection } from "./remoteMembers/Connection.ts"; import { createLayoutModeSwitch } from "./LayoutSwitch.ts"; +import { + createWrappedUserMedia, + type WrappedUserMediaViewModel, + type MediaItem, +} from "../MediaItem.ts"; const logger = rootLogger.getChild("[CallViewModel]"); //TODO @@ -193,7 +194,6 @@ interface LayoutScanState { tiles: TileStore; } -type MediaItem = UserMedia | ScreenShare; export type LivekitRoomItem = { livekitRoom: LivekitRoom; participants: string[]; @@ -283,7 +283,6 @@ export interface CallViewModel { allConnections$: Behavior; /** Participants sorted by livekit room so they can be used in the audio rendering */ livekitRoomItems$: Behavior; - userMedia$: Behavior; /** use the layout instead, this is just for the sdk export. */ matrixLivekitMembers$: Behavior; localMatrixLivekitMember$: Behavior; @@ -334,10 +333,6 @@ export interface CallViewModel { gridMode$: Behavior; setGridMode: (value: GridMode) => void; - // media view models and layout - grid$: Behavior; - spotlight$: Behavior; - pip$: Behavior; /** * The layout of tiles in the call interface. */ @@ -721,7 +716,7 @@ export function createCallViewModel$( /** * List of user media (camera feeds) that we want tiles for. */ - const userMedia$ = scope.behavior( + const userMedia$ = scope.behavior( combineLatest([ localMatrixLivekitMember$, matrixLivekitMembers$, @@ -767,36 +762,35 @@ export function createCallViewModel$( } } }, - (scope, _, dup, mediaId, userId, participant, connection$, rtcId) => { - const livekitRoom$ = scope.behavior( - connection$.pipe(map((c) => c?.livekitRoom)), - ); - const focusUrl$ = scope.behavior( - connection$.pipe(map((c) => c?.transport.livekit_service_url)), - ); - const displayName$ = scope.behavior( - matrixMemberMetadataStore - .createDisplayNameBehavior$(userId) - .pipe(map((name) => name ?? userId)), - ); - - return new UserMedia( - scope, - `${mediaId}:${dup}`, + (scope, _, dup, mediaId, userId, participant, connection$, rtcId) => + createWrappedUserMedia(scope, { + id: `${mediaId}:${dup}`, userId, - rtcId, + rtcBackendIdentity: rtcId, participant, - options.encryptionSystem, - livekitRoom$, - focusUrl$, + encryptionSystem: options.encryptionSystem, + livekitRoom$: scope.behavior( + connection$.pipe(map((c) => c?.livekitRoom)), + ), + focusUrl$: scope.behavior( + connection$.pipe(map((c) => c?.transport.livekit_service_url)), + ), mediaDevices, - localMembership.reconnecting$, - displayName$, - matrixMemberMetadataStore.createAvatarUrlBehavior$(userId), - handsRaised$.pipe(map((v) => v[mediaId]?.time ?? null)), - reactions$.pipe(map((v) => v[mediaId] ?? undefined)), - ); - }, + pretendToBeDisconnected$: localMembership.reconnecting$, + displayName$: scope.behavior( + matrixMemberMetadataStore + .createDisplayNameBehavior$(userId) + .pipe(map((name) => name ?? userId)), + ), + mxcAvatarUrl$: + matrixMemberMetadataStore.createAvatarUrlBehavior$(userId), + handRaised$: scope.behavior( + handsRaised$.pipe(map((v) => v[mediaId]?.time ?? null)), + ), + reaction$: scope.behavior( + reactions$.pipe(map((v) => v[mediaId] ?? undefined)), + ), + }), ), ), ); @@ -821,11 +815,9 @@ export function createCallViewModel$( /** * List of MediaItems that we want to display, that are of type ScreenShare */ - const screenShares$ = scope.behavior( + const screenShares$ = scope.behavior( mediaItems$.pipe( - map((mediaItems) => - mediaItems.filter((m): m is ScreenShare => m instanceof ScreenShare), - ), + map((mediaItems) => mediaItems.filter((m) => m.type === "screen share")), ), ); @@ -888,39 +880,39 @@ export function createCallViewModel$( merge(userHangup$, widgetHangup$).pipe(map(() => "user" as const)), ).pipe(scope.share); - const spotlightSpeaker$ = scope.behavior( + const spotlightSpeaker$ = scope.behavior( userMedia$.pipe( switchMap((mediaItems) => mediaItems.length === 0 ? of([]) : combineLatest( mediaItems.map((m) => - m.vm.speaking$.pipe(map((s) => [m, s] as const)), + m.speaking$.pipe(map((s) => [m, s] as const)), ), ), ), - scan<(readonly [UserMedia, boolean])[], UserMedia | undefined, null>( - (prev, mediaItems) => { - // Only remote users that are still in the call should be sticky - const [stickyMedia, stickySpeaking] = - (!prev?.vm.local && mediaItems.find(([m]) => m === prev)) || []; - // Decide who to spotlight: - // If the previous speaker is still speaking, stick with them rather - // than switching eagerly to someone else - return stickySpeaking - ? stickyMedia! - : // Otherwise, select any remote user who is speaking - (mediaItems.find(([m, s]) => !m.vm.local && s)?.[0] ?? - // Otherwise, stick with the person who was last speaking - stickyMedia ?? - // Otherwise, spotlight an arbitrary remote user - mediaItems.find(([m]) => !m.vm.local)?.[0] ?? - // Otherwise, spotlight the local user - mediaItems.find(([m]) => m.vm.local)?.[0]); - }, - null, - ), - map((speaker) => speaker?.vm ?? null), + scan< + (readonly [UserMediaViewModel, boolean])[], + UserMediaViewModel | undefined, + undefined + >((prev, mediaItems) => { + // Only remote users that are still in the call should be sticky + const [stickyMedia, stickySpeaking] = + (!prev?.local && mediaItems.find(([m]) => m === prev)) || []; + // Decide who to spotlight: + // If the previous speaker is still speaking, stick with them rather + // than switching eagerly to someone else + return stickySpeaking + ? stickyMedia! + : // Otherwise, select any remote user who is speaking + (mediaItems.find(([m, s]) => !m.local && s)?.[0] ?? + // Otherwise, stick with the person who was last speaking + stickyMedia ?? + // Otherwise, spotlight an arbitrary remote user + mediaItems.find(([m]) => !m.local)?.[0] ?? + // Otherwise, spotlight the local user + mediaItems.find(([m]) => m.local)?.[0]); + }, undefined), ), ); @@ -934,7 +926,7 @@ export function createCallViewModel$( return bins.length === 0 ? of([]) : combineLatest(bins, (...bins) => - bins.sort(([, bin1], [, bin2]) => bin1 - bin2).map(([m]) => m.vm), + bins.sort(([, bin1], [, bin2]) => bin1 - bin2).map(([m]) => m), ); }), distinctUntilChanged(shallowEquals), @@ -944,9 +936,7 @@ export function createCallViewModel$( const spotlight$ = scope.behavior( screenShares$.pipe( switchMap((screenShares) => { - if (screenShares.length > 0) { - return of(screenShares.map((m) => m.vm)); - } + if (screenShares.length > 0) return of(screenShares); return spotlightSpeaker$.pipe( map((speaker) => (speaker ? [speaker] : [])), @@ -956,7 +946,7 @@ export function createCallViewModel$( ), ); - const pip$ = scope.behavior( + const pip$ = scope.behavior( combineLatest([ // TODO This also needs epoch logic to dedupe the screenshares and mediaItems emits screenShares$, @@ -968,28 +958,17 @@ export function createCallViewModel$( return spotlightSpeaker$; } if (!spotlight || spotlight.local) { - return of(null); + return of(undefined); } const localUserMedia = mediaItems.find( - (m) => m.vm instanceof LocalUserMediaViewModel, - ) as UserMedia | undefined; - - const localUserMediaViewModel = localUserMedia?.vm as - | LocalUserMediaViewModel - | undefined; - - if (!localUserMediaViewModel) { - return of(null); + (m) => m.type === "user" && m.local, + ); + if (!localUserMedia) { + return of(undefined); } - return localUserMediaViewModel.alwaysShow$.pipe( - map((alwaysShow) => { - if (alwaysShow) { - return localUserMediaViewModel; - } - - return null; - }), + return localUserMedia.alwaysShow$.pipe( + map((alwaysShow) => (alwaysShow ? localUserMedia : undefined)), ); }), ), @@ -998,7 +977,7 @@ export function createCallViewModel$( const hasRemoteScreenShares$ = scope.behavior( spotlight$.pipe( map((spotlight) => - spotlight.some((vm) => !vm.local && vm instanceof ScreenShareViewModel), + spotlight.some((vm) => vm.type === "screen share" && !vm.local), ), ), ); @@ -1039,8 +1018,10 @@ export function createCallViewModel$( ); const spotlightExpandedToggle$ = new Subject(); - const spotlightExpanded$ = scope.behavior( - spotlightExpandedToggle$.pipe(accumulate(false, (expanded) => !expanded)), + const spotlightExpanded$ = createToggle$( + scope, + false, + spotlightExpandedToggle$, ); const { setGridMode, gridMode$ } = createLayoutModeSwitch( @@ -1053,7 +1034,7 @@ export function createCallViewModel$( [grid$, spotlight$], (grid, spotlight) => ({ type: "grid", - spotlight: spotlight.some((vm) => vm instanceof ScreenShareViewModel) + spotlight: spotlight.some((vm) => vm.type === "screen share") ? spotlight : undefined, grid, @@ -1085,12 +1066,8 @@ export function createCallViewModel$( mediaItems$.pipe( map((mediaItems) => { if (mediaItems.length !== 2) return null; - const local = mediaItems.find((vm) => vm.vm.local)?.vm as - | LocalUserMediaViewModel - | undefined; - const remote = mediaItems.find((vm) => !vm.vm.local)?.vm as - | RemoteUserMediaViewModel - | undefined; + const local = mediaItems.find((vm) => vm.type === "user" && vm.local); + const remote = mediaItems.find((vm) => vm.type === "user" && !vm.local); // There might not be a remote tile if there are screen shares, or if // only the local user is in the call and they're using the duplicate // tiles option @@ -1138,7 +1115,7 @@ export function createCallViewModel$( oneOnOne === null ? combineLatest([grid$, spotlight$], (grid, spotlight) => grid.length > smallMobileCallThreshold || - spotlight.some((vm) => vm instanceof ScreenShareViewModel) + spotlight.some((vm) => vm.type === "screen share") ? spotlightPortraitLayoutMedia$ : gridLayoutMedia$, ).pipe(switchAll()) @@ -1245,7 +1222,7 @@ export function createCallViewModel$( // screen sharing feeds are in the spotlight we still need them. return l.spotlight.media$.pipe( map((models: MediaViewModel[]) => - models.some((m) => m instanceof ScreenShareViewModel), + models.some((m) => m.type === "screen share"), ), ); // In expanded spotlight layout, the active speaker is always shown in @@ -1552,11 +1529,7 @@ export function createCallViewModel$( toggleSpotlightExpanded$: toggleSpotlightExpanded$, gridMode$: gridMode$, setGridMode: setGridMode, - grid$: grid$, - spotlight$: spotlight$, - pip$: pip$, layout$: layout$, - userMedia$, localMatrixLivekitMember$, matrixLivekitMembers$: scope.behavior( matrixLivekitMembers$.pipe( diff --git a/src/state/MediaItem.ts b/src/state/MediaItem.ts new file mode 100644 index 00000000..947fdfd4 --- /dev/null +++ b/src/state/MediaItem.ts @@ -0,0 +1,196 @@ +/* +Copyright 2025-2026 Element Software Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { combineLatest, map, of, switchMap } from "rxjs"; +import { + type LocalParticipant, + ParticipantEvent, + type RemoteParticipant, +} from "livekit-client"; +import { observeParticipantEvents } from "@livekit/components-core"; + +import { type ObservableScope } from "./ObservableScope.ts"; +import { + createLocalScreenShare, + createLocalUserMedia, + createRemoteScreenShare, + createRemoteUserMedia, + type ScreenShareViewModel, + type UserMediaViewModel, + type LocalUserMediaInputs, + type RemoteUserMediaInputs, +} from "./MediaViewModel.ts"; +import type { Behavior } from "./Behavior.ts"; +import type { MediaDevices } from "./MediaDevices.ts"; +import { observeSpeaker$ } from "./observeSpeaker.ts"; +import { generateItems } from "../utils/observable.ts"; +import { type TaggedParticipant } from "./CallViewModel/remoteMembers/MatrixLivekitMembers.ts"; + +/** + * Sorting bins defining the order in which media tiles appear in the layout. + */ +enum SortingBin { + /** + * Yourself, when the "always show self" option is on. + */ + SelfAlwaysShown, + /** + * Participants that are sharing their screen. + */ + Presenters, + /** + * Participants that have been speaking recently. + */ + Speakers, + /** + * Participants that have their hand raised. + */ + HandRaised, + /** + * Participants with video. + */ + Video, + /** + * Participants not sharing any video. + */ + NoVideo, + /** + * Yourself, when the "always show self" option is off. + */ + SelfNotAlwaysShown, +} + +/** + * A user media item to be presented in a tile. This is a thin wrapper around + * UserMediaViewModel which additionally carries data relevant to the tile + * layout algorithms (data which the MediaView component should be ignorant of). + */ +export type WrappedUserMediaViewModel = UserMediaViewModel & { + /** + * All screen share media associated with this user media. + */ + screenShares$: Behavior; + /** + * Which sorting bin the media item should be placed in. + */ + bin$: Behavior; +}; + +interface WrappedUserMediaInputs extends Omit< + LocalUserMediaInputs & RemoteUserMediaInputs, + "participant$" +> { + participant: TaggedParticipant; + mediaDevices: MediaDevices; + pretendToBeDisconnected$: Behavior; +} + +export function createWrappedUserMedia( + scope: ObservableScope, + { + participant, + mediaDevices, + pretendToBeDisconnected$, + ...inputs + }: WrappedUserMediaInputs, +): WrappedUserMediaViewModel { + const userMedia = + participant.type === "local" + ? createLocalUserMedia(scope, { + participant$: participant.value$, + mediaDevices, + ...inputs, + }) + : createRemoteUserMedia(scope, { + participant$: participant.value$, + pretendToBeDisconnected$, + ...inputs, + }); + + // TypeScript needs this widening of the type to happen in a separate statement + const participant$: Behavior = + participant.value$; + + const screenShares$ = scope.behavior( + participant$.pipe( + switchMap((p) => + p === null + ? of([]) + : observeParticipantEvents( + p, + ParticipantEvent.TrackPublished, + ParticipantEvent.TrackUnpublished, + ParticipantEvent.LocalTrackPublished, + ParticipantEvent.LocalTrackUnpublished, + ).pipe( + // Technically more than one screen share might be possible... our + // MediaViewModels don't support it though since they look for a unique + // track for the given source. So generateItems here is a bit overkill. + generateItems( + `${inputs.id} screenShares$`, + function* (p) { + if (p.isScreenShareEnabled) + yield { + keys: ["screen-share"], + data: undefined, + }; + }, + (scope, _data$, key) => { + const id = `${inputs.id}:${key}`; + return participant.type === "local" + ? createLocalScreenShare(scope, { + ...inputs, + id, + participant$: participant.value$, + }) + : createRemoteScreenShare(scope, { + ...inputs, + id, + participant$: participant.value$, + pretendToBeDisconnected$, + }); + }, + ), + ), + ), + ), + ); + + const speaker$ = scope.behavior(observeSpeaker$(userMedia.speaking$)); + const presenter$ = scope.behavior( + screenShares$.pipe(map((screenShares) => screenShares.length > 0)), + ); + + return { + ...userMedia, + screenShares$, + bin$: scope.behavior( + combineLatest( + [ + speaker$, + presenter$, + userMedia.videoEnabled$, + userMedia.handRaised$, + userMedia.local ? userMedia.alwaysShow$ : of(null), + ], + (speaker, presenter, video, handRaised, alwaysShow) => { + if (alwaysShow !== null) + return alwaysShow + ? SortingBin.SelfAlwaysShown + : SortingBin.SelfNotAlwaysShown; + else if (presenter) return SortingBin.Presenters; + else if (speaker) return SortingBin.Speakers; + else if (handRaised) return SortingBin.HandRaised; + else if (video) return SortingBin.Video; + else return SortingBin.NoVideo; + }, + ), + ), + }; +} + +export type MediaItem = WrappedUserMediaViewModel | ScreenShareViewModel; diff --git a/src/state/MediaViewModel.test.ts b/src/state/MediaViewModel.test.ts index 92868216..73b396db 100644 --- a/src/state/MediaViewModel.test.ts +++ b/src/state/MediaViewModel.test.ts @@ -17,8 +17,8 @@ import { mockLocalParticipant, mockMediaDevices, mockRtcMembership, - createLocalMedia, - createRemoteMedia, + mockLocalMedia, + mockRemoteMedia, withTestScheduler, mockRemoteParticipant, } from "../utils/test"; @@ -45,7 +45,7 @@ const rtcMembership = mockRtcMembership("@alice:example.org", "AAAA"); test("control a participant's volume", () => { const setVolumeSpy = vi.fn(); - const vm = createRemoteMedia( + const vm = mockRemoteMedia( rtcMembership, {}, mockRemoteParticipant({ setVolume: setVolumeSpy }), @@ -54,33 +54,33 @@ test("control a participant's volume", () => { schedule("-ab---c---d|", { a() { // Try muting by toggling - vm.toggleLocallyMuted(); + vm.togglePlaybackMuted(); expect(setVolumeSpy).toHaveBeenLastCalledWith(0); }, b() { // Try unmuting by dragging the slider back up - vm.setLocalVolume(0.6); - vm.setLocalVolume(0.8); - vm.commitLocalVolume(); + vm.adjustPlaybackVolume(0.6); + vm.adjustPlaybackVolume(0.8); + vm.commitPlaybackVolume(); expect(setVolumeSpy).toHaveBeenCalledWith(0.6); expect(setVolumeSpy).toHaveBeenLastCalledWith(0.8); }, c() { // Try muting by dragging the slider back down - vm.setLocalVolume(0.2); - vm.setLocalVolume(0); - vm.commitLocalVolume(); + vm.adjustPlaybackVolume(0.2); + vm.adjustPlaybackVolume(0); + vm.commitPlaybackVolume(); expect(setVolumeSpy).toHaveBeenCalledWith(0.2); expect(setVolumeSpy).toHaveBeenLastCalledWith(0); }, d() { // Try unmuting by toggling - vm.toggleLocallyMuted(); + vm.togglePlaybackMuted(); // The volume should return to the last non-zero committed volume expect(setVolumeSpy).toHaveBeenLastCalledWith(0.8); }, }); - expectObservable(vm.localVolume$).toBe("ab(cd)(ef)g", { + expectObservable(vm.playbackVolume$).toBe("ab(cd)(ef)g", { a: 1, b: 0, c: 0.6, @@ -93,11 +93,11 @@ test("control a participant's volume", () => { }); test("toggle fit/contain for a participant's video", () => { - const vm = createRemoteMedia(rtcMembership, {}, mockRemoteParticipant({})); + const vm = mockRemoteMedia(rtcMembership, {}, mockRemoteParticipant({})); withTestScheduler(({ expectObservable, schedule }) => { schedule("-ab|", { - a: () => vm.toggleFitContain(), - b: () => vm.toggleFitContain(), + a: () => vm.toggleCropVideo(), + b: () => vm.toggleCropVideo(), }); expectObservable(vm.cropVideo$).toBe("abc", { a: true, @@ -108,7 +108,7 @@ test("toggle fit/contain for a participant's video", () => { }); test("local media remembers whether it should always be shown", () => { - const vm1 = createLocalMedia( + const vm1 = mockLocalMedia( rtcMembership, {}, mockLocalParticipant({}), @@ -120,7 +120,7 @@ test("local media remembers whether it should always be shown", () => { }); // Next local media should start out *not* always shown - const vm2 = createLocalMedia( + const vm2 = mockLocalMedia( rtcMembership, {}, mockLocalParticipant({}), @@ -166,7 +166,7 @@ test("switch cameras", async () => { const selectVideoInput = vi.fn(); - const vm = createLocalMedia( + const vm = mockLocalMedia( rtcMembership, {}, mockLocalParticipant({ @@ -206,17 +206,17 @@ test("switch cameras", async () => { }); test("remote media is in waiting state when participant has not yet connected", () => { - const vm = createRemoteMedia(rtcMembership, {}, null); // null participant + const vm = mockRemoteMedia(rtcMembership, {}, null); // null participant expect(vm.waitingForMedia$.value).toBe(true); }); test("remote media is not in waiting state when participant is connected", () => { - const vm = createRemoteMedia(rtcMembership, {}, mockRemoteParticipant({})); + const vm = mockRemoteMedia(rtcMembership, {}, mockRemoteParticipant({})); expect(vm.waitingForMedia$.value).toBe(false); }); test("remote media is not in waiting state when participant is connected with no publications", () => { - const vm = createRemoteMedia( + const vm = mockRemoteMedia( rtcMembership, {}, mockRemoteParticipant({ @@ -228,7 +228,7 @@ test("remote media is not in waiting state when participant is connected with no }); test("remote media is not in waiting state when user does not intend to publish anywhere", () => { - const vm = createRemoteMedia( + const vm = mockRemoteMedia( rtcMembership, {}, mockRemoteParticipant({}), diff --git a/src/state/MediaViewModel.ts b/src/state/MediaViewModel.ts index 7f806697..c8c468ac 100644 --- a/src/state/MediaViewModel.ts +++ b/src/state/MediaViewModel.ts @@ -29,7 +29,6 @@ import { } from "livekit-client"; import { logger } from "matrix-js-sdk/lib/logger"; import { - BehaviorSubject, type Observable, Subject, combineLatest, @@ -47,7 +46,7 @@ import { import { alwaysShowSelf } from "../settings/settings"; import { showConnectionStats } from "../settings/settings"; -import { accumulate } from "../utils/observable"; +import { createToggle$ } from "../utils/observable"; import { type EncryptionSystem } from "../e2ee/sharedKeyManagement"; import { E2eeType } from "../e2ee/e2eeType"; import { type ReactionOption } from "../reactions"; @@ -55,6 +54,7 @@ import { platform } from "../Platform"; import { type MediaDevices } from "./MediaDevices"; import { type Behavior } from "./Behavior"; import { type ObservableScope } from "./ObservableScope"; +import { createVolumeControls, type VolumeControls } from "./VolumeControls"; export function observeTrackReference$( participant: Participant, @@ -103,7 +103,7 @@ export function observeRtpStreamStats$( ); } -export function observeInboundRtpStreamStats$( +function observeInboundRtpStreamStats$( participant: Participant, source: Track.Source, ): Observable { @@ -112,15 +112,6 @@ export function observeInboundRtpStreamStats$( ); } -export function observeOutboundRtpStreamStats$( - participant: Participant, - source: Track.Source, -): Observable { - return observeRtpStreamStats$(participant, source, "outbound-rtp").pipe( - map((x) => x as RTCOutboundRtpStreamStats | undefined), - ); -} - function observeRemoteTrackReceivingOkay$( participant: Participant, source: Track.Source, @@ -218,74 +209,98 @@ export enum EncryptionStatus { PasswordInvalid, } -abstract class BaseMediaViewModel { +interface BaseMediaViewModel { + /** + * An opaque identifier for this media. + */ + id: string; + /** + * The Matrix user to which this media belongs. + */ + userId: string; + displayName$: Behavior; + mxcAvatarUrl$: Behavior; +} + +type BaseMediaInputs = BaseMediaViewModel; + +// This function exists to strip out superfluous data from the input object +function createBaseMedia({ + id, + userId, + displayName$, + mxcAvatarUrl$, +}: BaseMediaInputs): BaseMediaViewModel { + return { id, userId, displayName$, mxcAvatarUrl$ }; +} + +interface MemberMediaViewModel extends BaseMediaViewModel { /** * The LiveKit video track for this media. */ - public readonly video$: Behavior; + video$: Behavior; + /** + * The URL of the LiveKit focus on which this member should be publishing. + * Exposed for debugging. + */ + focusUrl$: Behavior; /** * Whether there should be a warning that this media is unencrypted. */ - public readonly unencryptedWarning$: Behavior; + unencryptedWarning$: Behavior; + encryptionStatus$: Behavior; +} - public readonly encryptionStatus$: Behavior; +interface MemberMediaInputs extends BaseMediaViewModel { + participant$: Behavior; + livekitRoom$: Behavior; + audioSource: AudioSource; + videoSource: VideoSource; + focusUrl$: Behavior; + encryptionSystem: EncryptionSystem; +} - /** - * Whether this media corresponds to the local participant. - */ - public abstract readonly local: boolean; - - private observeTrackReference$( +function createMemberMedia( + scope: ObservableScope, + { + participant$, + livekitRoom$, + audioSource, + videoSource, + focusUrl$, + encryptionSystem, + ...inputs + }: MemberMediaInputs, +): MemberMediaViewModel { + const trackBehavior$ = ( source: Track.Source, - ): Behavior { - return this.scope.behavior( - this.participant$.pipe( + ): Behavior => + scope.behavior( + participant$.pipe( switchMap((p) => !p ? of(undefined) : observeTrackReference$(p, source), ), ), ); - } - public constructor( - protected readonly scope: ObservableScope, - /** - * An opaque identifier for this media. - */ - public readonly id: string, - /** - * The Matrix user to which this media belongs. - */ - public readonly userId: string, - // We don't necessarily have a participant if a user connects via MatrixRTC but not (yet) through - // livekit. - protected readonly participant$: Behavior< - LocalParticipant | RemoteParticipant | null - >, + const audio$ = trackBehavior$(audioSource); + const video$ = trackBehavior$(videoSource); - encryptionSystem: EncryptionSystem, - audioSource: AudioSource, - videoSource: VideoSource, - protected readonly livekitRoom$: Behavior, - public readonly focusUrl$: Behavior, - public readonly displayName$: Behavior, - public readonly mxcAvatarUrl$: Behavior, - ) { - const audio$ = this.observeTrackReference$(audioSource); - this.video$ = this.observeTrackReference$(videoSource); - - this.unencryptedWarning$ = this.scope.behavior( + return { + ...createBaseMedia(inputs), + video$, + focusUrl$, + unencryptedWarning$: scope.behavior( combineLatest( - [audio$, this.video$], + [audio$, video$], (a, v) => encryptionSystem.kind !== E2eeType.NONE && (a?.publication.isEncrypted === false || v?.publication.isEncrypted === false), ), - ); - - this.encryptionStatus$ = this.scope.behavior( - this.participant$.pipe( + ), + encryptionStatus$: scope.behavior( + participant$.pipe( switchMap((participant): Observable => { if (!participant) { return of(EncryptionStatus.Connecting); @@ -346,132 +361,149 @@ abstract class BaseMediaViewModel { } }), ), - ); - } + ), + }; } -/** - * Some participant's media. - */ -export type MediaViewModel = UserMediaViewModel | ScreenShareViewModel; -export type UserMediaViewModel = - | LocalUserMediaViewModel - | RemoteUserMediaViewModel; +interface BaseUserMediaViewModel extends MemberMediaViewModel { + type: "user"; + speaking$: Behavior; + audioEnabled$: Behavior; + videoEnabled$: Behavior; + cropVideo$: Behavior; + toggleCropVideo: () => void; + /** + * The expected identity of the LiveKit participant. Exposed for debugging. + */ + rtcBackendIdentity: string; + handRaised$: Behavior; + reaction$: Behavior; + audioStreamStats$: Observable< + RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined + >; + videoStreamStats$: Observable< + RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined + >; +} -/** - * Some participant's user media. - */ -abstract class BaseUserMediaViewModel extends BaseMediaViewModel { - private readonly _speaking$ = this.scope.behavior( - this.participant$.pipe( - switchMap((p) => - p - ? observeParticipantEvents( - p, - ParticipantEvent.IsSpeakingChanged, - ).pipe(map((p) => p.isSpeaking)) - : of(false), - ), +interface BaseUserMediaInputs extends Omit< + MemberMediaInputs, + "audioSource" | "videoSource" +> { + rtcBackendIdentity: string; + handRaised$: Behavior; + reaction$: Behavior; + statsType: "inbound-rtp" | "outbound-rtp"; +} + +function createBaseUserMedia( + scope: ObservableScope, + { + rtcBackendIdentity, + handRaised$, + reaction$, + statsType, + ...inputs + }: BaseUserMediaInputs, +): BaseUserMediaViewModel { + const { participant$ } = inputs; + const media$ = scope.behavior( + participant$.pipe( + switchMap((p) => (p && observeParticipantMedia(p)) ?? of(undefined)), ), ); - /** - * Whether the participant is speaking. - */ - // Getter backed by a private field so that subclasses can override it - public get speaking$(): Behavior { - return this._speaking$; - } + const toggleCropVideo$ = new Subject(); - /** - * Whether this participant is sending audio (i.e. is unmuted on their side). - */ - public readonly audioEnabled$: Behavior; - - private readonly _videoEnabled$: Behavior; - /** - * Whether this participant is sending video. - */ - // Getter backed by a private field so that subclasses can override it - public get videoEnabled$(): Behavior { - return this._videoEnabled$; - } - - private readonly _cropVideo$ = new BehaviorSubject(true); - /** - * Whether the tile video should be contained inside the tile or be cropped to fit. - */ - public readonly cropVideo$: Behavior = this._cropVideo$; - - public constructor( - scope: ObservableScope, - id: string, - userId: string, - /** - * The expected identity of the LiveKit participant. Exposed for debugging. - */ - public readonly rtcBackendIdentity: string, - participant$: Behavior, - encryptionSystem: EncryptionSystem, - livekitRoom$: Behavior, - focusUrl$: Behavior, - displayName$: Behavior, - mxcAvatarUrl$: Behavior, - public readonly handRaised$: Behavior, - public readonly reaction$: Behavior, - ) { - super( - scope, - id, - userId, - participant$, - encryptionSystem, - Track.Source.Microphone, - Track.Source.Camera, - livekitRoom$, - focusUrl$, - displayName$, - mxcAvatarUrl$, - ); - - const media$ = this.scope.behavior( + return { + ...createMemberMedia(scope, { + ...inputs, + audioSource: Track.Source.Microphone, + videoSource: Track.Source.Camera, + }), + type: "user", + speaking$: scope.behavior( participant$.pipe( - switchMap((p) => (p && observeParticipantMedia(p)) ?? of(undefined)), + switchMap((p) => + p + ? observeParticipantEvents( + p, + ParticipantEvent.IsSpeakingChanged, + ).pipe(map((p) => p.isSpeaking)) + : of(false), + ), ), - ); - this.audioEnabled$ = this.scope.behavior( + ), + audioEnabled$: scope.behavior( media$.pipe(map((m) => m?.microphoneTrack?.isMuted === false)), - ); - this._videoEnabled$ = this.scope.behavior( + ), + videoEnabled$: scope.behavior( media$.pipe(map((m) => m?.cameraTrack?.isMuted === false)), - ); - } - - public toggleFitContain(): void { - this._cropVideo$.next(!this._cropVideo$.value); - } - - public get local(): boolean { - return this instanceof LocalUserMediaViewModel; - } - - public abstract get audioStreamStats$(): Observable< - RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined - >; - public abstract get videoStreamStats$(): Observable< - RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined - >; + ), + cropVideo$: createToggle$(scope, true, toggleCropVideo$), + toggleCropVideo: () => toggleCropVideo$.next(), + rtcBackendIdentity, + handRaised$, + reaction$, + audioStreamStats$: combineLatest([ + participant$, + showConnectionStats.value$, + ]).pipe( + switchMap(([p, showConnectionStats]) => { + // + if (!p || !showConnectionStats) return of(undefined); + return observeRtpStreamStats$(p, Track.Source.Microphone, statsType); + }), + ), + videoStreamStats$: combineLatest([ + participant$, + showConnectionStats.value$, + ]).pipe( + switchMap(([p, showConnectionStats]) => { + if (!p || !showConnectionStats) return of(undefined); + return observeRtpStreamStats$(p, Track.Source.Camera, statsType); + }), + ), + }; } -/** - * The local participant's user media. - */ -export class LocalUserMediaViewModel extends BaseUserMediaViewModel { +export interface LocalUserMediaViewModel extends BaseUserMediaViewModel { + local: true; + /** + * Whether the video should be mirrored. + */ + mirror$: Behavior; + /** + * Whether to show this tile in a highly visible location near the start of + * the grid. + */ + alwaysShow$: Behavior; + setAlwaysShow: (value: boolean) => void; + switchCamera$: Behavior<(() => void) | null>; +} + +export interface LocalUserMediaInputs extends Omit< + BaseUserMediaInputs, + "statsType" +> { + participant$: Behavior; + mediaDevices: MediaDevices; +} + +export function createLocalUserMedia( + scope: ObservableScope, + { mediaDevices, ...inputs }: LocalUserMediaInputs, +): LocalUserMediaViewModel { + const baseUserMedia = createBaseUserMedia(scope, { + ...inputs, + statsType: "outbound-rtp", + }); + /** * The local video track as an observable that emits whenever the track * changes, the camera is switched, or the track is muted. */ - private readonly videoTrack$: Observable = - this.video$.pipe( + const videoTrack$: Observable = + baseUserMedia.video$.pipe( switchMap((v) => { const track = v?.publication.track; if (!(track instanceof LocalVideoTrack)) return of(null); @@ -488,35 +520,25 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { }), ); - /** - * Whether the video should be mirrored. - */ - public readonly mirror$ = this.scope.behavior( - this.videoTrack$.pipe( - // Mirror only front-facing cameras (those that face the user) - map( - (track) => - track !== null && - facingModeFromLocalTrack(track).facingMode === "user", + return { + ...baseUserMedia, + local: true, + mirror$: scope.behavior( + videoTrack$.pipe( + // Mirror only front-facing cameras (those that face the user) + map( + (track) => + track !== null && + facingModeFromLocalTrack(track).facingMode === "user", + ), ), ), - ); - - /** - * Whether to show this tile in a highly visible location near the start of - * the grid. - */ - public readonly alwaysShow$ = alwaysShowSelf.value$; - public readonly setAlwaysShow = alwaysShowSelf.setValue; - - /** - * Callback for switching between the front and back cameras. - */ - public readonly switchCamera$: Behavior<(() => void) | null> = - this.scope.behavior( + alwaysShow$: alwaysShowSelf.value$, + setAlwaysShow: alwaysShowSelf.setValue, + switchCamera$: scope.behavior( platform === "desktop" ? of(null) - : this.videoTrack$.pipe( + : videoTrack$.pipe( map((track) => { if (track === null) return null; const facingMode = facingModeFromLocalTrack(track).facingMode; @@ -535,272 +557,157 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { const deviceId = track.mediaStreamTrack.getSettings().deviceId; if (deviceId !== undefined) - this.mediaDevices.videoInput.select(deviceId); + mediaDevices.videoInput.select(deviceId); }) .catch((e) => logger.error("Failed to switch camera", facingMode, e), ); }), ), - ); - - public constructor( - scope: ObservableScope, - id: string, - userId: string, - rtcBackendIdentity: string, - participant$: Behavior, - encryptionSystem: EncryptionSystem, - livekitRoom$: Behavior, - focusUrl$: Behavior, - private readonly mediaDevices: MediaDevices, - displayName$: Behavior, - mxcAvatarUrl$: Behavior, - handRaised$: Behavior, - reaction$: Behavior, - ) { - super( - scope, - id, - userId, - rtcBackendIdentity, - participant$, - encryptionSystem, - livekitRoom$, - focusUrl$, - displayName$, - mxcAvatarUrl$, - handRaised$, - reaction$, - ); - } - - public audioStreamStats$ = combineLatest([ - this.participant$, - showConnectionStats.value$, - ]).pipe( - switchMap(([p, showConnectionStats]) => { - if (!p || !showConnectionStats) return of(undefined); - return observeOutboundRtpStreamStats$(p, Track.Source.Microphone); - }), - ); - - public videoStreamStats$ = combineLatest([ - this.participant$, - showConnectionStats.value$, - ]).pipe( - switchMap(([p, showConnectionStats]) => { - if (!p || !showConnectionStats) return of(undefined); - return observeOutboundRtpStreamStats$(p, Track.Source.Camera); - }), - ); + ), + }; } -/** - * A remote participant's user media. - */ -export class RemoteUserMediaViewModel extends BaseUserMediaViewModel { +export interface RemoteUserMediaViewModel + extends BaseUserMediaViewModel, VolumeControls { + local: false; /** * Whether we are waiting for this user's LiveKit participant to exist. This * could be because either we or the remote party are still connecting. */ - public readonly waitingForMedia$ = this.scope.behavior( - combineLatest( - [this.livekitRoom$, this.participant$], - (livekitRoom, participant) => - // If livekitRoom is undefined, the user is not attempting to publish on - // any transport and so we shouldn't expect a participant. (They might - // be a subscribe-only bot for example.) - livekitRoom !== undefined && participant === null, - ), - ); - - // This private field is used to override the value from the superclass - private __speaking$: Behavior; - public get speaking$(): Behavior { - return this.__speaking$; - } - - private readonly locallyMutedToggle$ = new Subject(); - private readonly localVolumeAdjustment$ = new Subject(); - private readonly localVolumeCommit$ = new Subject(); - - /** - * The volume to which this participant's audio is set, as a scalar - * multiplier. - */ - public readonly localVolume$ = this.scope.behavior( - merge( - this.locallyMutedToggle$.pipe(map(() => "toggle mute" as const)), - this.localVolumeAdjustment$, - this.localVolumeCommit$.pipe(map(() => "commit" as const)), - ).pipe( - accumulate({ volume: 1, committedVolume: 1 }, (state, event) => { - switch (event) { - case "toggle mute": - return { - ...state, - volume: state.volume === 0 ? state.committedVolume : 0, - }; - case "commit": - // Dragging the slider to zero should have the same effect as - // muting: keep the original committed volume, as if it were never - // dragged - return { - ...state, - committedVolume: - state.volume === 0 ? state.committedVolume : state.volume, - }; - default: - // Volume adjustment - return { ...state, volume: event }; - } - }), - map(({ volume }) => volume), - ), - ); - - // This private field is used to override the value from the superclass - private __videoEnabled$: Behavior; - public get videoEnabled$(): Behavior { - return this.__videoEnabled$; - } - - /** - * Whether this participant's audio is disabled. - */ - public readonly locallyMuted$ = this.scope.behavior( - this.localVolume$.pipe(map((volume) => volume === 0)), - ); - - public constructor( - scope: ObservableScope, - id: string, - userId: string, - rtcBackendIdentity: string, - participant$: Behavior, - encryptionSystem: EncryptionSystem, - livekitRoom$: Behavior, - focusUrl$: Behavior, - private readonly pretendToBeDisconnected$: Behavior, - displayName$: Behavior, - mxcAvatarUrl$: Behavior, - handRaised$: Behavior, - reaction$: Behavior, - ) { - super( - scope, - id, - userId, - rtcBackendIdentity, - participant$, - encryptionSystem, - livekitRoom$, - focusUrl$, - displayName$, - mxcAvatarUrl$, - handRaised$, - reaction$, - ); - - this.__speaking$ = this.scope.behavior( - pretendToBeDisconnected$.pipe( - switchMap((disconnected) => - disconnected ? of(false) : super.speaking$, - ), - ), - ); - - this.__videoEnabled$ = this.scope.behavior( - pretendToBeDisconnected$.pipe( - switchMap((disconnected) => - disconnected ? of(false) : super.videoEnabled$, - ), - ), - ); - - // Sync the local volume with LiveKit - combineLatest([ - participant$, - // The local volume, taking into account whether we're supposed to pretend - // that the audio stream is disconnected (since we don't necessarily want - // that to modify the UI state). - this.pretendToBeDisconnected$.pipe( - switchMap((disconnected) => (disconnected ? of(0) : this.localVolume$)), - this.scope.bind(), - ), - ]).subscribe(([p, volume]) => p?.setVolume(volume)); - } - - public toggleLocallyMuted(): void { - this.locallyMutedToggle$.next(); - } - - public setLocalVolume(value: number): void { - this.localVolumeAdjustment$.next(value); - } - - public commitLocalVolume(): void { - this.localVolumeCommit$.next(); - } - - public audioStreamStats$ = combineLatest([ - this.participant$, - showConnectionStats.value$, - ]).pipe( - switchMap(([p, showConnectionStats]) => { - if (!p || !showConnectionStats) return of(undefined); - return observeInboundRtpStreamStats$(p, Track.Source.Microphone); - }), - ); - - public videoStreamStats$ = combineLatest([ - this.participant$, - showConnectionStats.value$, - ]).pipe( - switchMap(([p, showConnectionStats]) => { - if (!p || !showConnectionStats) return of(undefined); - return observeInboundRtpStreamStats$(p, Track.Source.Camera); - }), - ); + waitingForMedia$: Behavior; } -/** - * Some participant's screen share media. - */ -export class ScreenShareViewModel extends BaseMediaViewModel { +export interface RemoteUserMediaInputs extends Omit< + BaseUserMediaInputs, + "statsType" +> { + participant$: Behavior; + pretendToBeDisconnected$: Behavior; +} + +export function createRemoteUserMedia( + scope: ObservableScope, + { pretendToBeDisconnected$, ...inputs }: RemoteUserMediaInputs, +): RemoteUserMediaViewModel { + const baseUserMedia = createBaseUserMedia(scope, { + ...inputs, + statsType: "inbound-rtp", + }); + + return { + ...baseUserMedia, + ...createVolumeControls(scope, { + pretendToBeDisconnected$, + sink$: scope.behavior( + inputs.participant$.pipe(map((p) => (volume) => p?.setVolume(volume))), + ), + }), + local: false, + speaking$: scope.behavior( + pretendToBeDisconnected$.pipe( + switchMap((disconnected) => + disconnected ? of(false) : baseUserMedia.speaking$, + ), + ), + ), + videoEnabled$: scope.behavior( + pretendToBeDisconnected$.pipe( + switchMap((disconnected) => + disconnected ? of(false) : baseUserMedia.videoEnabled$, + ), + ), + ), + waitingForMedia$: scope.behavior( + combineLatest( + [inputs.livekitRoom$, inputs.participant$], + (livekitRoom, participant) => + // If livekitRoom is undefined, the user is not attempting to publish on + // any transport and so we shouldn't expect a participant. (They might + // be a subscribe-only bot for example.) + livekitRoom !== undefined && participant === null, + ), + ), + }; +} + +interface BaseScreenShareViewModel extends MemberMediaViewModel { + type: "screen share"; +} + +type BaseScreenShareInputs = Omit< + MemberMediaInputs, + "audioSource" | "videoSource" +>; + +function createBaseScreenShare( + scope: ObservableScope, + inputs: BaseScreenShareInputs, +): BaseScreenShareViewModel { + return { + ...createMemberMedia(scope, { + ...inputs, + audioSource: Track.Source.ScreenShareAudio, + videoSource: Track.Source.ScreenShare, + }), + type: "screen share", + }; +} + +export interface LocalScreenShareViewModel extends BaseScreenShareViewModel { + local: true; +} + +interface LocalScreenShareInputs extends BaseScreenShareInputs { + participant$: Behavior; +} + +export function createLocalScreenShare( + scope: ObservableScope, + inputs: LocalScreenShareInputs, +): LocalScreenShareViewModel { + return { ...createBaseScreenShare(scope, inputs), local: true }; +} + +export interface RemoteScreenShareViewModel extends BaseScreenShareViewModel { + local: false; /** * Whether this screen share's video should be displayed. */ - public readonly videoEnabled$ = this.scope.behavior( - this.pretendToBeDisconnected$.pipe(map((disconnected) => !disconnected)), - ); - - public constructor( - scope: ObservableScope, - id: string, - userId: string, - participant$: Behavior, - encryptionSystem: EncryptionSystem, - livekitRoom$: Behavior, - focusUrl$: Behavior, - private readonly pretendToBeDisconnected$: Behavior, - displayName$: Behavior, - mxcAvatarUrl$: Behavior, - public readonly local: boolean, - ) { - super( - scope, - id, - userId, - participant$, - encryptionSystem, - Track.Source.ScreenShareAudio, - Track.Source.ScreenShare, - livekitRoom$, - focusUrl$, - displayName$, - mxcAvatarUrl$, - ); - } + videoEnabled$: Behavior; } + +interface RemoteScreenShareInputs extends BaseScreenShareInputs { + participant$: Behavior; + pretendToBeDisconnected$: Behavior; +} + +export function createRemoteScreenShare( + scope: ObservableScope, + { pretendToBeDisconnected$, ...inputs }: RemoteScreenShareInputs, +): RemoteScreenShareViewModel { + return { + ...createBaseScreenShare(scope, inputs), + local: false, + videoEnabled$: scope.behavior( + pretendToBeDisconnected$.pipe(map((disconnected) => !disconnected)), + ), + }; +} + +/** + * Some participant's media. + */ +export type MediaViewModel = UserMediaViewModel | ScreenShareViewModel; +/** + * Some participant's user media (i.e. their microphone and camera feed). + */ +export type UserMediaViewModel = + | LocalUserMediaViewModel + | RemoteUserMediaViewModel; +/** + * Some participant's screen share media. + */ +export type ScreenShareViewModel = + | LocalScreenShareViewModel + | RemoteScreenShareViewModel; diff --git a/src/state/ScreenShare.ts b/src/state/ScreenShare.ts deleted file mode 100644 index 6c908b1f..00000000 --- a/src/state/ScreenShare.ts +++ /dev/null @@ -1,53 +0,0 @@ -/* -Copyright 2025 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - type LocalParticipant, - type RemoteParticipant, - type Room as LivekitRoom, -} from "livekit-client"; - -import { type ObservableScope } from "./ObservableScope.ts"; -import { ScreenShareViewModel } from "./MediaViewModel.ts"; -import type { EncryptionSystem } from "../e2ee/sharedKeyManagement.ts"; -import { constant, type Behavior } from "./Behavior.ts"; - -/** - * A screen share media item to be presented in a tile. This is a thin wrapper - * around ScreenShareViewModel which essentially just establishes an - * ObservableScope for behaviors that the view model depends on. - */ -export class ScreenShare { - public readonly vm: ScreenShareViewModel; - - public constructor( - private readonly scope: ObservableScope, - id: string, - userId: string, - participant: LocalParticipant | RemoteParticipant, - encryptionSystem: EncryptionSystem, - livekitRoom$: Behavior, - focusUrl$: Behavior, - pretendToBeDisconnected$: Behavior, - displayName$: Behavior, - mxcAvatarUrl$: Behavior, - ) { - this.vm = new ScreenShareViewModel( - this.scope, - id, - userId, - constant(participant), - encryptionSystem, - livekitRoom$, - focusUrl$, - pretendToBeDisconnected$, - displayName$, - mxcAvatarUrl$, - participant.isLocal, - ); - } -} diff --git a/src/state/UserMedia.ts b/src/state/UserMedia.ts deleted file mode 100644 index 2adc9134..00000000 --- a/src/state/UserMedia.ts +++ /dev/null @@ -1,209 +0,0 @@ -/* -Copyright 2025 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { combineLatest, map, type Observable, of, switchMap } from "rxjs"; -import { - type LocalParticipant, - ParticipantEvent, - type RemoteParticipant, - type Room as LivekitRoom, -} from "livekit-client"; -import { observeParticipantEvents } from "@livekit/components-core"; - -import { type ObservableScope } from "./ObservableScope.ts"; -import { - LocalUserMediaViewModel, - RemoteUserMediaViewModel, - type UserMediaViewModel, -} from "./MediaViewModel.ts"; -import type { Behavior } from "./Behavior.ts"; -import type { EncryptionSystem } from "../e2ee/sharedKeyManagement.ts"; -import type { MediaDevices } from "./MediaDevices.ts"; -import type { ReactionOption } from "../reactions"; -import { observeSpeaker$ } from "./observeSpeaker.ts"; -import { generateItems } from "../utils/observable.ts"; -import { ScreenShare } from "./ScreenShare.ts"; -import { type TaggedParticipant } from "./CallViewModel/remoteMembers/MatrixLivekitMembers.ts"; - -/** - * Sorting bins defining the order in which media tiles appear in the layout. - */ -enum SortingBin { - /** - * Yourself, when the "always show self" option is on. - */ - SelfAlwaysShown, - /** - * Participants that are sharing their screen. - */ - Presenters, - /** - * Participants that have been speaking recently. - */ - Speakers, - /** - * Participants that have their hand raised. - */ - HandRaised, - /** - * Participants with video. - */ - Video, - /** - * Participants not sharing any video. - */ - NoVideo, - /** - * Yourself, when the "always show self" option is off. - */ - SelfNotAlwaysShown, -} - -/** - * A user media item to be presented in a tile. This is a thin wrapper around - * UserMediaViewModel which additionally determines the media item's sorting bin - * for inclusion in the call layout and tracks associated screen shares. - */ -export class UserMedia { - public readonly vm: UserMediaViewModel = - this.participant.type === "local" - ? new LocalUserMediaViewModel( - this.scope, - this.id, - this.userId, - this.rtcBackendIdentity, - this.participant.value$, - this.encryptionSystem, - this.livekitRoom$, - this.focusUrl$, - this.mediaDevices, - this.displayName$, - this.mxcAvatarUrl$, - this.scope.behavior(this.handRaised$), - this.scope.behavior(this.reaction$), - ) - : new RemoteUserMediaViewModel( - this.scope, - this.id, - this.userId, - this.rtcBackendIdentity, - this.participant.value$, - this.encryptionSystem, - this.livekitRoom$, - this.focusUrl$, - this.pretendToBeDisconnected$, - this.displayName$, - this.mxcAvatarUrl$, - this.scope.behavior(this.handRaised$), - this.scope.behavior(this.reaction$), - ); - - private readonly speaker$ = this.scope.behavior( - observeSpeaker$(this.vm.speaking$), - ); - - // TypeScript needs this widening of the type to happen in a separate statement - private readonly participant$: Behavior< - LocalParticipant | RemoteParticipant | null - > = this.participant.value$; - - /** - * All screen share media associated with this user media. - */ - public readonly screenShares$ = this.scope.behavior( - this.participant$.pipe( - switchMap((p) => - p === null - ? of([]) - : observeParticipantEvents( - p, - ParticipantEvent.TrackPublished, - ParticipantEvent.TrackUnpublished, - ParticipantEvent.LocalTrackPublished, - ParticipantEvent.LocalTrackUnpublished, - ).pipe( - // Technically more than one screen share might be possible... our - // MediaViewModels don't support it though since they look for a unique - // track for the given source. So generateItems here is a bit overkill. - generateItems( - `${this.id} screenShares$`, - function* (p) { - if (p.isScreenShareEnabled) - yield { - keys: ["screen-share"], - data: undefined, - }; - }, - (scope, _data$, key) => - new ScreenShare( - scope, - `${this.id}:${key}`, - this.userId, - p, - this.encryptionSystem, - this.livekitRoom$, - this.focusUrl$, - this.pretendToBeDisconnected$, - this.displayName$, - this.mxcAvatarUrl$, - ), - ), - ), - ), - ), - ); - - private readonly presenter$ = this.scope.behavior( - this.screenShares$.pipe(map((screenShares) => screenShares.length > 0)), - ); - - /** - * Which sorting bin the media item should be placed in. - */ - // This is exposed here rather than by UserMediaViewModel because it's only - // relevant to the layout algorithms; the MediaView component should be - // ignorant of this value. - public readonly bin$ = combineLatest( - [ - this.speaker$, - this.presenter$, - this.vm.videoEnabled$, - this.vm.handRaised$, - this.vm instanceof LocalUserMediaViewModel - ? this.vm.alwaysShow$ - : of(false), - ], - (speaker, presenter, video, handRaised, alwaysShow) => { - if (this.vm.local) - return alwaysShow - ? SortingBin.SelfAlwaysShown - : SortingBin.SelfNotAlwaysShown; - else if (presenter) return SortingBin.Presenters; - else if (speaker) return SortingBin.Speakers; - else if (handRaised) return SortingBin.HandRaised; - else if (video) return SortingBin.Video; - else return SortingBin.NoVideo; - }, - ); - - public constructor( - private readonly scope: ObservableScope, - public readonly id: string, - private readonly userId: string, - private readonly rtcBackendIdentity: string, - private readonly participant: TaggedParticipant, - private readonly encryptionSystem: EncryptionSystem, - private readonly livekitRoom$: Behavior, - private readonly focusUrl$: Behavior, - private readonly mediaDevices: MediaDevices, - private readonly pretendToBeDisconnected$: Behavior, - private readonly displayName$: Behavior, - private readonly mxcAvatarUrl$: Behavior, - private readonly handRaised$: Observable, - private readonly reaction$: Observable, - ) {} -} diff --git a/src/state/VolumeControls.ts b/src/state/VolumeControls.ts new file mode 100644 index 00000000..beb7ae00 --- /dev/null +++ b/src/state/VolumeControls.ts @@ -0,0 +1,101 @@ +/* +Copyright 2026 Element Software Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { combineLatest, map, merge, of, Subject, switchMap } from "rxjs"; + +import { type Behavior } from "./Behavior"; +import { type ObservableScope } from "./ObservableScope"; +import { accumulate } from "../utils/observable"; + +/** + * Controls for audio playback volume. + */ +export interface VolumeControls { + /** + * The volume to which the audio is set, as a scalar multiplier. + */ + playbackVolume$: Behavior; + /** + * Whether playback of this audio is disabled. + */ + playbackMuted$: Behavior; + togglePlaybackMuted: () => void; + adjustPlaybackVolume: (value: number) => void; + commitPlaybackVolume: () => void; +} + +interface VolumeControlsInputs { + pretendToBeDisconnected$: Behavior; + /** + * The callback to run to notify the module performing audio playback of the + * requested volume. + */ + sink$: Behavior<(volume: number) => void>; +} + +/** + * Creates a set of controls for audio playback volume and syncs this with the + * audio playback module for the duration of the scope. + */ +export function createVolumeControls( + scope: ObservableScope, + { pretendToBeDisconnected$, sink$ }: VolumeControlsInputs, +): VolumeControls { + const toggleMuted$ = new Subject<"toggle mute">(); + const adjustVolume$ = new Subject(); + const commitVolume$ = new Subject<"commit">(); + + const playbackVolume$ = scope.behavior( + merge(toggleMuted$, adjustVolume$, commitVolume$).pipe( + accumulate({ volume: 1, committedVolume: 1 }, (state, event) => { + switch (event) { + case "toggle mute": + return { + ...state, + volume: state.volume === 0 ? state.committedVolume : 0, + }; + case "commit": + // Dragging the slider to zero should have the same effect as + // muting: keep the original committed volume, as if it were never + // dragged + return { + ...state, + committedVolume: + state.volume === 0 ? state.committedVolume : state.volume, + }; + default: + // Volume adjustment + return { ...state, volume: event }; + } + }), + map(({ volume }) => volume), + ), + ); + + // Sync the requested volume with the audio playback module + combineLatest([ + sink$, + // The playback volume, taking into account whether we're supposed to + // pretend that the audio stream is disconnected (since we don't necessarily + // want that to modify the UI state). + pretendToBeDisconnected$.pipe( + switchMap((disconnected) => (disconnected ? of(0) : playbackVolume$)), + ), + ]) + .pipe(scope.bind()) + .subscribe(([sink, volume]) => sink(volume)); + + return { + playbackVolume$, + playbackMuted$: scope.behavior( + playbackVolume$.pipe(map((volume) => volume === 0)), + ), + togglePlaybackMuted: () => toggleMuted$.next("toggle mute"), + adjustPlaybackVolume: (value: number) => adjustVolume$.next(value), + commitPlaybackVolume: () => commitVolume$.next("commit"), + }; +} diff --git a/src/tile/GridTile.test.tsx b/src/tile/GridTile.test.tsx index 9bc0efb2..02f09a17 100644 --- a/src/tile/GridTile.test.tsx +++ b/src/tile/GridTile.test.tsx @@ -14,7 +14,7 @@ import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; import { GridTile } from "./GridTile"; import { mockRtcMembership, - createRemoteMedia, + mockRemoteMedia, mockRemoteParticipant, } from "../utils/test"; import { GridTileViewModel } from "../state/TileViewModel"; @@ -29,7 +29,7 @@ global.IntersectionObserver = class MockIntersectionObserver { } as unknown as typeof IntersectionObserver; test("GridTile is accessible", async () => { - const vm = createRemoteMedia( + const vm = mockRemoteMedia( mockRtcMembership("@alice:example.org", "AAAA"), { rawDisplayName: "Alice", diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index 92262f05..370e0723 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -41,7 +41,7 @@ import { useObservableEagerState } from "observable-hooks"; import styles from "./GridTile.module.css"; import { type UserMediaViewModel, - LocalUserMediaViewModel, + type LocalUserMediaViewModel, type RemoteUserMediaViewModel, } from "../state/MediaViewModel"; import { Slider } from "../Slider"; @@ -68,7 +68,7 @@ interface TileProps { interface UserMediaTileProps extends TileProps { vm: UserMediaViewModel; mirror: boolean; - locallyMuted: boolean; + playbackMuted: boolean; waitingForMedia?: boolean; primaryButton?: ReactNode; menuStart?: ReactNode; @@ -79,7 +79,7 @@ const UserMediaTile: FC = ({ ref, vm, showSpeakingIndicators, - locallyMuted, + playbackMuted, waitingForMedia, primaryButton, menuStart, @@ -109,7 +109,7 @@ const UserMediaTile: FC = ({ const onSelectFitContain = useCallback( (e: Event) => { e.preventDefault(); - vm.toggleFitContain(); + vm.toggleCropVideo(); }, [vm], ); @@ -117,12 +117,12 @@ const UserMediaTile: FC = ({ const handRaised = useBehavior(vm.handRaised$); const reaction = useBehavior(vm.reaction$); - const AudioIcon = locallyMuted + const AudioIcon = playbackMuted ? VolumeOffSolidIcon : audioEnabled ? MicOnSolidIcon : MicOffSolidIcon; - const audioIconLabel = locallyMuted + const audioIconLabel = playbackMuted ? t("video_tile.muted_for_me") : audioEnabled ? t("microphone_on") @@ -166,7 +166,7 @@ const UserMediaTile: FC = ({ width={20} height={20} aria-label={audioIconLabel} - data-muted={locallyMuted || !audioEnabled} + data-muted={playbackMuted || !audioEnabled} className={styles.muteIcon} /> } @@ -245,7 +245,7 @@ const LocalUserMediaTile: FC = ({ = ({ }) => { const { t } = useTranslation(); const waitingForMedia = useBehavior(vm.waitingForMedia$); - const locallyMuted = useBehavior(vm.locallyMuted$); - const localVolume = useBehavior(vm.localVolume$); + const playbackMuted = useBehavior(vm.playbackMuted$); + const playbackVolume = useBehavior(vm.playbackVolume$); const onSelectMute = useCallback( (e: Event) => { e.preventDefault(); - vm.toggleLocallyMuted(); + vm.togglePlaybackMuted(); }, [vm], ); - const onChangeLocalVolume = useCallback( - (v: number) => vm.setLocalVolume(v), - [vm], - ); - const onCommitLocalVolume = useCallback(() => vm.commitLocalVolume(), [vm]); - const VolumeIcon = locallyMuted ? VolumeOffIcon : VolumeOnIcon; + const VolumeIcon = playbackMuted ? VolumeOffIcon : VolumeOnIcon; return ( {/* TODO: Figure out how to make this slider keyboard accessible */} @@ -332,9 +327,9 @@ const RemoteUserMediaTile: FC = ({ = ({ const displayName = useBehavior(media.displayName$); const mxcAvatarUrl = useBehavior(media.mxcAvatarUrl$); - if (media instanceof LocalUserMediaViewModel) { + if (media.local) { return ( = ({ }) => { const { t } = useTranslation(); const [handRaiseTimerVisible] = useSetting(showHandRaisedTimer); - const [showConnectioStats] = useSetting(showConnectionStats); + const [showConnectionStats] = useSetting(showConnectionStatsSetting); const avatarSize = Math.round(Math.min(targetWidth, targetHeight) / 2); @@ -139,10 +139,10 @@ export const MediaView: FC = ({ {waitingForMedia && (
{t("video_tile.waiting_for_media")} - {showConnectioStats ? " " + rtcBackendIdentity : ""} + {showConnectionStats ? " " + rtcBackendIdentity : ""}
)} - {(audioStreamStats || videoStreamStats) && ( + {showConnectionStats && ( <> { - const vm1 = createRemoteMedia( + const vm1 = mockRemoteMedia( mockRtcMembership("@alice:example.org", "AAAA"), { rawDisplayName: "Alice", @@ -37,7 +37,7 @@ test("SpotlightTile is accessible", async () => { mockRemoteParticipant({}), ); - const vm2 = createLocalMedia( + const vm2 = mockLocalMedia( mockRtcMembership("@bob:example.org", "BBBB"), { rawDisplayName: "Bob", diff --git a/src/tile/SpotlightTile.tsx b/src/tile/SpotlightTile.tsx index e685327e..c0f1135f 100644 --- a/src/tile/SpotlightTile.tsx +++ b/src/tile/SpotlightTile.tsx @@ -34,11 +34,12 @@ import { MediaView } from "./MediaView"; import styles from "./SpotlightTile.module.css"; import { type EncryptionStatus, - LocalUserMediaViewModel, + type LocalUserMediaViewModel, type MediaViewModel, - ScreenShareViewModel, type UserMediaViewModel, type RemoteUserMediaViewModel, + type ScreenShareViewModel, + type RemoteScreenShareViewModel, } from "../state/MediaViewModel"; import { useInitial } from "../useInitial"; import { useMergedRefs } from "../useMergedRefs"; @@ -54,7 +55,6 @@ interface SpotlightItemBaseProps { targetWidth: number; targetHeight: number; video: TrackReferenceOrPlaceholder | undefined; - videoEnabled: boolean; userId: string; unencryptedWarning: boolean; encryptionStatus: EncryptionStatus; @@ -67,6 +67,7 @@ interface SpotlightItemBaseProps { interface SpotlightUserMediaItemBaseProps extends SpotlightItemBaseProps { videoFit: "contain" | "cover"; + videoEnabled: boolean; } interface SpotlightLocalUserMediaItemProps extends SpotlightUserMediaItemBaseProps { @@ -106,14 +107,16 @@ const SpotlightUserMediaItem: FC = ({ ...props }) => { const cropVideo = useBehavior(vm.cropVideo$); + const videoEnabled = useBehavior(vm.videoEnabled$); const baseProps: SpotlightUserMediaItemBaseProps & RefAttributes = { videoFit: cropVideo ? "cover" : "contain", + videoEnabled, ...props, }; - return vm instanceof LocalUserMediaViewModel ? ( + return vm.local ? ( ) : ( @@ -122,6 +125,31 @@ const SpotlightUserMediaItem: FC = ({ SpotlightUserMediaItem.displayName = "SpotlightUserMediaItem"; +interface SpotlightScreenShareItemProps extends SpotlightItemBaseProps { + vm: ScreenShareViewModel; + videoEnabled: boolean; +} + +const SpotlightScreenShareItem: FC = ({ + vm, + ...props +}) => { + return ; +}; + +interface SpotlightRemoteScreenShareItemProps extends SpotlightItemBaseProps { + vm: RemoteScreenShareViewModel; +} + +const SpotlightRemoteScreenShareItem: FC< + SpotlightRemoteScreenShareItemProps +> = ({ vm, ...props }) => { + const videoEnabled = useBehavior(vm.videoEnabled$); + return ( + + ); +}; + interface SpotlightItemProps { ref?: Ref; vm: MediaViewModel; @@ -152,7 +180,6 @@ const SpotlightItem: FC = ({ const displayName = useBehavior(vm.displayName$); const mxcAvatarUrl = useBehavior(vm.mxcAvatarUrl$); const video = useBehavior(vm.video$); - const videoEnabled = useBehavior(vm.videoEnabled$); const unencryptedWarning = useBehavior(vm.unencryptedWarning$); const encryptionStatus = useBehavior(vm.encryptionStatus$); @@ -178,7 +205,6 @@ const SpotlightItem: FC = ({ targetWidth, targetHeight, video: video ?? undefined, - videoEnabled, userId: vm.userId, unencryptedWarning, focusUrl, @@ -189,10 +215,12 @@ const SpotlightItem: FC = ({ "aria-hidden": ariaHidden, }; - return vm instanceof ScreenShareViewModel ? ( - + if (vm.type === "user") + return ; + return vm.local ? ( + ) : ( - + ); }; diff --git a/src/utils/observable.ts b/src/utils/observable.ts index 2e19748b..353dc877 100644 --- a/src/utils/observable.ts +++ b/src/utils/observable.ts @@ -61,6 +61,20 @@ export function accumulate( events$.pipe(scan(update, initial), startWith(initial)); } +/** + * Given a source of toggle events, creates a Behavior whose value toggles + * between `true` and `false`. + */ +export function createToggle$( + scope: ObservableScope, + initialValue: boolean, + toggle$: Observable, +): Behavior { + return scope.behavior( + toggle$.pipe(accumulate(initialValue, (state) => !state)), + ); +} + const switchSymbol = Symbol("switch"); /** diff --git a/src/utils/test.ts b/src/utils/test.ts index d78bdf42..9231a3d1 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -53,8 +53,10 @@ import { import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; import { - LocalUserMediaViewModel, - RemoteUserMediaViewModel, + createLocalUserMedia, + createRemoteUserMedia, + type LocalUserMediaViewModel, + type RemoteUserMediaViewModel, } from "../state/MediaViewModel"; import { E2eeType } from "../e2ee/e2eeType"; import { @@ -323,30 +325,27 @@ export function mockLocalParticipant( } as Partial as LocalParticipant; } -export function createLocalMedia( +export function mockLocalMedia( rtcMember: CallMembership, roomMember: Partial, localParticipant: LocalParticipant, mediaDevices: MediaDevices, ): LocalUserMediaViewModel { const member = mockMatrixRoomMember(rtcMember, roomMember); - return new LocalUserMediaViewModel( - testScope(), - "local", - member.userId, - rtcMember.rtcBackendIdentity, - constant(localParticipant), - { - kind: E2eeType.PER_PARTICIPANT, - }, - constant(mockLivekitRoom({ localParticipant })), - constant("https://rtc-example.org"), + return createLocalUserMedia(testScope(), { + id: "local", + userId: member.userId, + rtcBackendIdentity: rtcMember.rtcBackendIdentity, + participant$: constant(localParticipant), + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + livekitRoom$: constant(mockLivekitRoom({ localParticipant })), + focusUrl$: constant("https://rtc-example.org"), mediaDevices, - constant(member.rawDisplayName ?? "nodisplayname"), - constant(member.getMxcAvatarUrl()), - constant(null), - constant(null), - ); + displayName$: constant(member.rawDisplayName ?? "nodisplayname"), + mxcAvatarUrl$: constant(member.getMxcAvatarUrl()), + handRaised$: constant(null), + reaction$: constant(null), + }); } export function mockRemoteParticipant( @@ -364,7 +363,7 @@ export function mockRemoteParticipant( } as RemoteParticipant; } -export function createRemoteMedia( +export function mockRemoteMedia( rtcMember: CallMembership, roomMember: Partial, participant: RemoteParticipant | null, @@ -376,23 +375,20 @@ export function createRemoteMedia( ), ): RemoteUserMediaViewModel { const member = mockMatrixRoomMember(rtcMember, roomMember); - return new RemoteUserMediaViewModel( - testScope(), - "remote", - member.userId, - rtcMember.rtcBackendIdentity, - constant(participant), - { - kind: E2eeType.PER_PARTICIPANT, - }, - constant(livekitRoom), - constant("https://rtc-example.org"), - constant(false), - constant(member.rawDisplayName ?? "nodisplayname"), - constant(member.getMxcAvatarUrl()), - constant(null), - constant(null), - ); + return createRemoteUserMedia(testScope(), { + id: "remote", + userId: member.userId, + rtcBackendIdentity: rtcMember.rtcBackendIdentity, + participant$: constant(participant), + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + livekitRoom$: constant(livekitRoom), + focusUrl$: constant("https://rtc-example.org"), + pretendToBeDisconnected$: constant(false), + displayName$: constant(member.rawDisplayName ?? "nodisplayname"), + mxcAvatarUrl$: constant(member.getMxcAvatarUrl()), + handRaised$: constant(null), + reaction$: constant(null), + }); } export function mockConfig( From 6b51b7dc58e534ca5f8fe611cfab03f9fe318e3f Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 25 Feb 2026 22:34:07 +0100 Subject: [PATCH 2/4] Split MediaViewModel into multiple files --- src/state/CallViewModel/CallViewModel.ts | 12 +- .../CallViewModel/localMember/Publisher.ts | 2 +- src/state/MediaViewModel.ts | 713 ------------------ src/state/TileStore.ts | 3 +- src/state/TileViewModel.ts | 3 +- src/state/layout-types.ts | 10 +- src/state/media/LocalScreenShareViewModel.ts | 32 + src/state/media/LocalUserMediaViewModel.ts | 137 ++++ src/state/{ => media}/MediaItem.ts | 32 +- src/state/{ => media}/MediaViewModel.test.ts | 11 +- src/state/media/MediaViewModel.ts | 44 ++ src/state/media/MemberMediaViewModel.ts | 280 +++++++ src/state/media/RemoteScreenShareViewModel.ts | 44 ++ src/state/media/RemoteUserMediaViewModel.ts | 82 ++ src/state/media/ScreenShareViewModel.ts | 51 ++ src/state/media/UserMediaViewModel.ts | 143 ++++ src/state/media/observeRtpStreamStats.ts | 59 ++ src/state/{ => media}/observeSpeaker.test.ts | 2 +- src/state/{ => media}/observeSpeaker.ts | 0 src/state/observeTrackReference.ts | 28 + src/tile/GridTile.tsx | 8 +- src/tile/MediaView.test.tsx | 2 +- src/tile/MediaView.tsx | 2 +- src/tile/SpotlightTile.tsx | 16 +- src/utils/test.ts | 14 +- 25 files changed, 957 insertions(+), 773 deletions(-) delete mode 100644 src/state/MediaViewModel.ts create mode 100644 src/state/media/LocalScreenShareViewModel.ts create mode 100644 src/state/media/LocalUserMediaViewModel.ts rename src/state/{ => media}/MediaItem.ts (88%) rename src/state/{ => media}/MediaViewModel.test.ts (96%) create mode 100644 src/state/media/MediaViewModel.ts create mode 100644 src/state/media/MemberMediaViewModel.ts create mode 100644 src/state/media/RemoteScreenShareViewModel.ts create mode 100644 src/state/media/RemoteUserMediaViewModel.ts create mode 100644 src/state/media/ScreenShareViewModel.ts create mode 100644 src/state/media/UserMediaViewModel.ts create mode 100644 src/state/media/observeRtpStreamStats.ts rename src/state/{ => media}/observeSpeaker.test.ts (98%) rename src/state/{ => media}/observeSpeaker.ts (100%) create mode 100644 src/state/observeTrackReference.ts diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 7f34d86f..c19c4818 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -50,11 +50,6 @@ import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/En import { v4 as uuidv4 } from "uuid"; import { type IMembershipManager } from "matrix-js-sdk/lib/matrixrtc/IMembershipManager"; -import { - type MediaViewModel, - type ScreenShareViewModel, - type UserMediaViewModel, -} from "../MediaViewModel"; import { createToggle$, filterBehavior, @@ -142,9 +137,12 @@ import { type Connection } from "./remoteMembers/Connection.ts"; import { createLayoutModeSwitch } from "./LayoutSwitch.ts"; import { createWrappedUserMedia, - type WrappedUserMediaViewModel, type MediaItem, -} from "../MediaItem.ts"; + type WrappedUserMediaViewModel, +} from "../media/MediaItem.ts"; +import { type ScreenShareViewModel } from "../media/ScreenShareViewModel.ts"; +import { type UserMediaViewModel } from "../media/UserMediaViewModel.ts"; +import { type MediaViewModel } from "../media/MediaViewModel.ts"; const logger = rootLogger.getChild("[CallViewModel]"); //TODO diff --git a/src/state/CallViewModel/localMember/Publisher.ts b/src/state/CallViewModel/localMember/Publisher.ts index 8df38743..b7841c49 100644 --- a/src/state/CallViewModel/localMember/Publisher.ts +++ b/src/state/CallViewModel/localMember/Publisher.ts @@ -30,7 +30,7 @@ import { trackProcessorSync, } from "../../../livekit/TrackProcessorContext.tsx"; import { getUrlParams } from "../../../UrlParams.ts"; -import { observeTrackReference$ } from "../../MediaViewModel.ts"; +import { observeTrackReference$ } from "../../observeTrackReference"; import { type Connection } from "../remoteMembers/Connection.ts"; import { ObservableScope } from "../../ObservableScope.ts"; diff --git a/src/state/MediaViewModel.ts b/src/state/MediaViewModel.ts deleted file mode 100644 index c8c468ac..00000000 --- a/src/state/MediaViewModel.ts +++ /dev/null @@ -1,713 +0,0 @@ -/* -Copyright 2023, 2024 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - type AudioSource, - type VideoSource, - type TrackReference, - observeParticipantEvents, - observeParticipantMedia, - roomEventSelector, -} from "@livekit/components-core"; -import { - type LocalParticipant, - LocalTrack, - LocalVideoTrack, - type Participant, - ParticipantEvent, - type RemoteParticipant, - Track, - TrackEvent, - facingModeFromLocalTrack, - type Room as LivekitRoom, - RoomEvent as LivekitRoomEvent, - RemoteTrack, -} from "livekit-client"; -import { logger } from "matrix-js-sdk/lib/logger"; -import { - type Observable, - Subject, - combineLatest, - filter, - fromEvent, - interval, - map, - merge, - of, - startWith, - switchMap, - throttleTime, - distinctUntilChanged, -} from "rxjs"; - -import { alwaysShowSelf } from "../settings/settings"; -import { showConnectionStats } from "../settings/settings"; -import { createToggle$ } from "../utils/observable"; -import { type EncryptionSystem } from "../e2ee/sharedKeyManagement"; -import { E2eeType } from "../e2ee/e2eeType"; -import { type ReactionOption } from "../reactions"; -import { platform } from "../Platform"; -import { type MediaDevices } from "./MediaDevices"; -import { type Behavior } from "./Behavior"; -import { type ObservableScope } from "./ObservableScope"; -import { createVolumeControls, type VolumeControls } from "./VolumeControls"; - -export function observeTrackReference$( - participant: Participant, - source: Track.Source, -): Observable { - return observeParticipantMedia(participant).pipe( - map(() => participant.getTrackPublication(source)), - distinctUntilChanged(), - map((publication) => publication && { participant, publication, source }), - ); -} - -export function observeRtpStreamStats$( - participant: Participant, - source: Track.Source, - type: "inbound-rtp" | "outbound-rtp", -): Observable< - RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined -> { - return combineLatest([ - observeTrackReference$(participant, source), - interval(1000).pipe(startWith(0)), - ]).pipe( - switchMap(async ([trackReference]) => { - const track = trackReference?.publication?.track; - if ( - !track || - !(track instanceof RemoteTrack || track instanceof LocalTrack) - ) { - return undefined; - } - const report = await track.getRTCStatsReport(); - if (!report) { - return undefined; - } - - for (const v of report.values()) { - if (v.type === type) { - return v; - } - } - - return undefined; - }), - startWith(undefined), - ); -} - -function observeInboundRtpStreamStats$( - participant: Participant, - source: Track.Source, -): Observable { - return observeRtpStreamStats$(participant, source, "inbound-rtp").pipe( - map((x) => x as RTCInboundRtpStreamStats | undefined), - ); -} - -function observeRemoteTrackReceivingOkay$( - participant: Participant, - source: Track.Source, -): Observable { - let lastStats: { - framesDecoded: number | undefined; - framesDropped: number | undefined; - framesReceived: number | undefined; - } = { - framesDecoded: undefined, - framesDropped: undefined, - framesReceived: undefined, - }; - - return observeInboundRtpStreamStats$(participant, source).pipe( - map((stats) => { - if (!stats) return undefined; - const { framesDecoded, framesDropped, framesReceived } = stats; - return { - framesDecoded, - framesDropped, - framesReceived, - }; - }), - filter((newStats) => !!newStats), - map((newStats): boolean | undefined => { - const oldStats = lastStats; - lastStats = newStats; - if ( - typeof newStats.framesReceived === "number" && - typeof oldStats.framesReceived === "number" && - typeof newStats.framesDecoded === "number" && - typeof oldStats.framesDecoded === "number" - ) { - const framesReceivedDelta = - newStats.framesReceived - oldStats.framesReceived; - const framesDecodedDelta = - newStats.framesDecoded - oldStats.framesDecoded; - - // if we received >0 frames and managed to decode >0 frames then we treat that as success - - if (framesReceivedDelta > 0) { - return framesDecodedDelta > 0; - } - } - - // no change - return undefined; - }), - filter((x) => typeof x === "boolean"), - startWith(undefined), - ); -} - -function encryptionErrorObservable$( - room$: Behavior, - participant: Participant, - encryptionSystem: EncryptionSystem, - criteria: string, -): Observable { - return room$.pipe( - switchMap((room) => { - if (room === undefined) return of(false); - return roomEventSelector(room, LivekitRoomEvent.EncryptionError).pipe( - map((e) => { - const [err] = e; - if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) { - return ( - // Ideally we would pull the participant identity from the field on the error. - // However, it gets lost in the serialization process between workers. - // So, instead we do a string match - (err?.message.includes(participant.identity) && - err?.message.includes(criteria)) ?? - false - ); - } else if (encryptionSystem.kind === E2eeType.SHARED_KEY) { - return !!err?.message.includes(criteria); - } - - return false; - }), - ); - }), - distinctUntilChanged(), - throttleTime(1000), // Throttle to avoid spamming the UI - startWith(false), - ); -} - -export enum EncryptionStatus { - Connecting, - Okay, - KeyMissing, - KeyInvalid, - PasswordInvalid, -} - -interface BaseMediaViewModel { - /** - * An opaque identifier for this media. - */ - id: string; - /** - * The Matrix user to which this media belongs. - */ - userId: string; - displayName$: Behavior; - mxcAvatarUrl$: Behavior; -} - -type BaseMediaInputs = BaseMediaViewModel; - -// This function exists to strip out superfluous data from the input object -function createBaseMedia({ - id, - userId, - displayName$, - mxcAvatarUrl$, -}: BaseMediaInputs): BaseMediaViewModel { - return { id, userId, displayName$, mxcAvatarUrl$ }; -} - -interface MemberMediaViewModel extends BaseMediaViewModel { - /** - * The LiveKit video track for this media. - */ - video$: Behavior; - /** - * The URL of the LiveKit focus on which this member should be publishing. - * Exposed for debugging. - */ - focusUrl$: Behavior; - /** - * Whether there should be a warning that this media is unencrypted. - */ - unencryptedWarning$: Behavior; - encryptionStatus$: Behavior; -} - -interface MemberMediaInputs extends BaseMediaViewModel { - participant$: Behavior; - livekitRoom$: Behavior; - audioSource: AudioSource; - videoSource: VideoSource; - focusUrl$: Behavior; - encryptionSystem: EncryptionSystem; -} - -function createMemberMedia( - scope: ObservableScope, - { - participant$, - livekitRoom$, - audioSource, - videoSource, - focusUrl$, - encryptionSystem, - ...inputs - }: MemberMediaInputs, -): MemberMediaViewModel { - const trackBehavior$ = ( - source: Track.Source, - ): Behavior => - scope.behavior( - participant$.pipe( - switchMap((p) => - !p ? of(undefined) : observeTrackReference$(p, source), - ), - ), - ); - - const audio$ = trackBehavior$(audioSource); - const video$ = trackBehavior$(videoSource); - - return { - ...createBaseMedia(inputs), - video$, - focusUrl$, - unencryptedWarning$: scope.behavior( - combineLatest( - [audio$, video$], - (a, v) => - encryptionSystem.kind !== E2eeType.NONE && - (a?.publication.isEncrypted === false || - v?.publication.isEncrypted === false), - ), - ), - encryptionStatus$: scope.behavior( - participant$.pipe( - switchMap((participant): Observable => { - if (!participant) { - return of(EncryptionStatus.Connecting); - } else if ( - participant.isLocal || - encryptionSystem.kind === E2eeType.NONE - ) { - return of(EncryptionStatus.Okay); - } else if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) { - return combineLatest([ - encryptionErrorObservable$( - livekitRoom$, - participant, - encryptionSystem, - "MissingKey", - ), - encryptionErrorObservable$( - livekitRoom$, - participant, - encryptionSystem, - "InvalidKey", - ), - observeRemoteTrackReceivingOkay$(participant, audioSource), - observeRemoteTrackReceivingOkay$(participant, videoSource), - ]).pipe( - map(([keyMissing, keyInvalid, audioOkay, videoOkay]) => { - if (keyMissing) return EncryptionStatus.KeyMissing; - if (keyInvalid) return EncryptionStatus.KeyInvalid; - if (audioOkay || videoOkay) return EncryptionStatus.Okay; - return undefined; // no change - }), - filter((x) => !!x), - startWith(EncryptionStatus.Connecting), - ); - } else { - return combineLatest([ - encryptionErrorObservable$( - livekitRoom$, - participant, - encryptionSystem, - "InvalidKey", - ), - observeRemoteTrackReceivingOkay$(participant, audioSource), - observeRemoteTrackReceivingOkay$(participant, videoSource), - ]).pipe( - map( - ([keyInvalid, audioOkay, videoOkay]): - | EncryptionStatus - | undefined => { - if (keyInvalid) return EncryptionStatus.PasswordInvalid; - if (audioOkay || videoOkay) return EncryptionStatus.Okay; - return undefined; // no change - }, - ), - filter((x) => !!x), - startWith(EncryptionStatus.Connecting), - ); - } - }), - ), - ), - }; -} - -interface BaseUserMediaViewModel extends MemberMediaViewModel { - type: "user"; - speaking$: Behavior; - audioEnabled$: Behavior; - videoEnabled$: Behavior; - cropVideo$: Behavior; - toggleCropVideo: () => void; - /** - * The expected identity of the LiveKit participant. Exposed for debugging. - */ - rtcBackendIdentity: string; - handRaised$: Behavior; - reaction$: Behavior; - audioStreamStats$: Observable< - RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined - >; - videoStreamStats$: Observable< - RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined - >; -} - -interface BaseUserMediaInputs extends Omit< - MemberMediaInputs, - "audioSource" | "videoSource" -> { - rtcBackendIdentity: string; - handRaised$: Behavior; - reaction$: Behavior; - statsType: "inbound-rtp" | "outbound-rtp"; -} - -function createBaseUserMedia( - scope: ObservableScope, - { - rtcBackendIdentity, - handRaised$, - reaction$, - statsType, - ...inputs - }: BaseUserMediaInputs, -): BaseUserMediaViewModel { - const { participant$ } = inputs; - const media$ = scope.behavior( - participant$.pipe( - switchMap((p) => (p && observeParticipantMedia(p)) ?? of(undefined)), - ), - ); - const toggleCropVideo$ = new Subject(); - - return { - ...createMemberMedia(scope, { - ...inputs, - audioSource: Track.Source.Microphone, - videoSource: Track.Source.Camera, - }), - type: "user", - speaking$: scope.behavior( - participant$.pipe( - switchMap((p) => - p - ? observeParticipantEvents( - p, - ParticipantEvent.IsSpeakingChanged, - ).pipe(map((p) => p.isSpeaking)) - : of(false), - ), - ), - ), - audioEnabled$: scope.behavior( - media$.pipe(map((m) => m?.microphoneTrack?.isMuted === false)), - ), - videoEnabled$: scope.behavior( - media$.pipe(map((m) => m?.cameraTrack?.isMuted === false)), - ), - cropVideo$: createToggle$(scope, true, toggleCropVideo$), - toggleCropVideo: () => toggleCropVideo$.next(), - rtcBackendIdentity, - handRaised$, - reaction$, - audioStreamStats$: combineLatest([ - participant$, - showConnectionStats.value$, - ]).pipe( - switchMap(([p, showConnectionStats]) => { - // - if (!p || !showConnectionStats) return of(undefined); - return observeRtpStreamStats$(p, Track.Source.Microphone, statsType); - }), - ), - videoStreamStats$: combineLatest([ - participant$, - showConnectionStats.value$, - ]).pipe( - switchMap(([p, showConnectionStats]) => { - if (!p || !showConnectionStats) return of(undefined); - return observeRtpStreamStats$(p, Track.Source.Camera, statsType); - }), - ), - }; -} - -export interface LocalUserMediaViewModel extends BaseUserMediaViewModel { - local: true; - /** - * Whether the video should be mirrored. - */ - mirror$: Behavior; - /** - * Whether to show this tile in a highly visible location near the start of - * the grid. - */ - alwaysShow$: Behavior; - setAlwaysShow: (value: boolean) => void; - switchCamera$: Behavior<(() => void) | null>; -} - -export interface LocalUserMediaInputs extends Omit< - BaseUserMediaInputs, - "statsType" -> { - participant$: Behavior; - mediaDevices: MediaDevices; -} - -export function createLocalUserMedia( - scope: ObservableScope, - { mediaDevices, ...inputs }: LocalUserMediaInputs, -): LocalUserMediaViewModel { - const baseUserMedia = createBaseUserMedia(scope, { - ...inputs, - statsType: "outbound-rtp", - }); - - /** - * The local video track as an observable that emits whenever the track - * changes, the camera is switched, or the track is muted. - */ - const videoTrack$: Observable = - baseUserMedia.video$.pipe( - switchMap((v) => { - const track = v?.publication.track; - if (!(track instanceof LocalVideoTrack)) return of(null); - return merge( - // Watch for track restarts because they indicate a camera switch. - // This event is also emitted when unmuting the track object. - fromEvent(track, TrackEvent.Restarted).pipe( - startWith(null), - map(() => track), - ), - // When the track object is muted, reset it to null. - fromEvent(track, TrackEvent.Muted).pipe(map(() => null)), - ); - }), - ); - - return { - ...baseUserMedia, - local: true, - mirror$: scope.behavior( - videoTrack$.pipe( - // Mirror only front-facing cameras (those that face the user) - map( - (track) => - track !== null && - facingModeFromLocalTrack(track).facingMode === "user", - ), - ), - ), - alwaysShow$: alwaysShowSelf.value$, - setAlwaysShow: alwaysShowSelf.setValue, - switchCamera$: scope.behavior( - platform === "desktop" - ? of(null) - : videoTrack$.pipe( - map((track) => { - if (track === null) return null; - const facingMode = facingModeFromLocalTrack(track).facingMode; - // If the camera isn't front or back-facing, don't provide a switch - // camera shortcut at all - if (facingMode !== "user" && facingMode !== "environment") - return null; - // Restart the track with a camera facing the opposite direction - return (): void => - void track - .restartTrack({ - facingMode: facingMode === "user" ? "environment" : "user", - }) - .then(() => { - // Inform the MediaDevices which camera was chosen - const deviceId = - track.mediaStreamTrack.getSettings().deviceId; - if (deviceId !== undefined) - mediaDevices.videoInput.select(deviceId); - }) - .catch((e) => - logger.error("Failed to switch camera", facingMode, e), - ); - }), - ), - ), - }; -} - -export interface RemoteUserMediaViewModel - extends BaseUserMediaViewModel, VolumeControls { - local: false; - /** - * Whether we are waiting for this user's LiveKit participant to exist. This - * could be because either we or the remote party are still connecting. - */ - waitingForMedia$: Behavior; -} - -export interface RemoteUserMediaInputs extends Omit< - BaseUserMediaInputs, - "statsType" -> { - participant$: Behavior; - pretendToBeDisconnected$: Behavior; -} - -export function createRemoteUserMedia( - scope: ObservableScope, - { pretendToBeDisconnected$, ...inputs }: RemoteUserMediaInputs, -): RemoteUserMediaViewModel { - const baseUserMedia = createBaseUserMedia(scope, { - ...inputs, - statsType: "inbound-rtp", - }); - - return { - ...baseUserMedia, - ...createVolumeControls(scope, { - pretendToBeDisconnected$, - sink$: scope.behavior( - inputs.participant$.pipe(map((p) => (volume) => p?.setVolume(volume))), - ), - }), - local: false, - speaking$: scope.behavior( - pretendToBeDisconnected$.pipe( - switchMap((disconnected) => - disconnected ? of(false) : baseUserMedia.speaking$, - ), - ), - ), - videoEnabled$: scope.behavior( - pretendToBeDisconnected$.pipe( - switchMap((disconnected) => - disconnected ? of(false) : baseUserMedia.videoEnabled$, - ), - ), - ), - waitingForMedia$: scope.behavior( - combineLatest( - [inputs.livekitRoom$, inputs.participant$], - (livekitRoom, participant) => - // If livekitRoom is undefined, the user is not attempting to publish on - // any transport and so we shouldn't expect a participant. (They might - // be a subscribe-only bot for example.) - livekitRoom !== undefined && participant === null, - ), - ), - }; -} - -interface BaseScreenShareViewModel extends MemberMediaViewModel { - type: "screen share"; -} - -type BaseScreenShareInputs = Omit< - MemberMediaInputs, - "audioSource" | "videoSource" ->; - -function createBaseScreenShare( - scope: ObservableScope, - inputs: BaseScreenShareInputs, -): BaseScreenShareViewModel { - return { - ...createMemberMedia(scope, { - ...inputs, - audioSource: Track.Source.ScreenShareAudio, - videoSource: Track.Source.ScreenShare, - }), - type: "screen share", - }; -} - -export interface LocalScreenShareViewModel extends BaseScreenShareViewModel { - local: true; -} - -interface LocalScreenShareInputs extends BaseScreenShareInputs { - participant$: Behavior; -} - -export function createLocalScreenShare( - scope: ObservableScope, - inputs: LocalScreenShareInputs, -): LocalScreenShareViewModel { - return { ...createBaseScreenShare(scope, inputs), local: true }; -} - -export interface RemoteScreenShareViewModel extends BaseScreenShareViewModel { - local: false; - /** - * Whether this screen share's video should be displayed. - */ - videoEnabled$: Behavior; -} - -interface RemoteScreenShareInputs extends BaseScreenShareInputs { - participant$: Behavior; - pretendToBeDisconnected$: Behavior; -} - -export function createRemoteScreenShare( - scope: ObservableScope, - { pretendToBeDisconnected$, ...inputs }: RemoteScreenShareInputs, -): RemoteScreenShareViewModel { - return { - ...createBaseScreenShare(scope, inputs), - local: false, - videoEnabled$: scope.behavior( - pretendToBeDisconnected$.pipe(map((disconnected) => !disconnected)), - ), - }; -} - -/** - * Some participant's media. - */ -export type MediaViewModel = UserMediaViewModel | ScreenShareViewModel; -/** - * Some participant's user media (i.e. their microphone and camera feed). - */ -export type UserMediaViewModel = - | LocalUserMediaViewModel - | RemoteUserMediaViewModel; -/** - * Some participant's screen share media. - */ -export type ScreenShareViewModel = - | LocalScreenShareViewModel - | RemoteScreenShareViewModel; diff --git a/src/state/TileStore.ts b/src/state/TileStore.ts index 7b95bd8e..a954eb4e 100644 --- a/src/state/TileStore.ts +++ b/src/state/TileStore.ts @@ -8,10 +8,11 @@ Please see LICENSE in the repository root for full details. import { BehaviorSubject } from "rxjs"; import { logger } from "matrix-js-sdk/lib/logger"; -import { type MediaViewModel, type UserMediaViewModel } from "./MediaViewModel"; import { GridTileViewModel, SpotlightTileViewModel } from "./TileViewModel"; import { fillGaps } from "../utils/iter"; import { debugTileLayout } from "../settings/settings"; +import { type MediaViewModel } from "./media/MediaViewModel"; +import { type UserMediaViewModel } from "./media/UserMediaViewModel"; function debugEntries(entries: GridTileData[]): string[] { return entries.map((e) => e.media.displayName$.value); diff --git a/src/state/TileViewModel.ts b/src/state/TileViewModel.ts index a645a0d1..8b13c685 100644 --- a/src/state/TileViewModel.ts +++ b/src/state/TileViewModel.ts @@ -5,8 +5,9 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type MediaViewModel, type UserMediaViewModel } from "./MediaViewModel"; import { type Behavior } from "./Behavior"; +import { type MediaViewModel } from "./media/MediaViewModel"; +import { type UserMediaViewModel } from "./media/UserMediaViewModel"; let nextId = 0; function createId(): string { diff --git a/src/state/layout-types.ts b/src/state/layout-types.ts index f32869df..33796f66 100644 --- a/src/state/layout-types.ts +++ b/src/state/layout-types.ts @@ -5,16 +5,14 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ +import { type LocalUserMediaViewModel } from "./media/LocalUserMediaViewModel.ts"; +import { type MediaViewModel } from "./media/MediaViewModel.ts"; +import { type RemoteUserMediaViewModel } from "./media/RemoteUserMediaViewModel.ts"; +import { type UserMediaViewModel } from "./media/UserMediaViewModel.ts"; import { type GridTileViewModel, type SpotlightTileViewModel, } from "./TileViewModel.ts"; -import { - type LocalUserMediaViewModel, - type RemoteUserMediaViewModel, - type MediaViewModel, - type UserMediaViewModel, -} from "./MediaViewModel.ts"; export interface GridLayoutMedia { type: "grid"; diff --git a/src/state/media/LocalScreenShareViewModel.ts b/src/state/media/LocalScreenShareViewModel.ts new file mode 100644 index 00000000..b31739d9 --- /dev/null +++ b/src/state/media/LocalScreenShareViewModel.ts @@ -0,0 +1,32 @@ +/* +Copyright 2023, 2024 New Vector Ltd. +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type LocalParticipant } from "livekit-client"; + +import { type Behavior } from "../Behavior"; +import { + type BaseScreenShareInputs, + type BaseScreenShareViewModel, + createBaseScreenShare, +} from "./ScreenShareViewModel"; +import { type ObservableScope } from "../ObservableScope"; + +export interface LocalScreenShareViewModel extends BaseScreenShareViewModel { + local: true; +} + +export interface LocalScreenShareInputs extends BaseScreenShareInputs { + participant$: Behavior; +} + +export function createLocalScreenShare( + scope: ObservableScope, + inputs: LocalScreenShareInputs, +): LocalScreenShareViewModel { + return { ...createBaseScreenShare(scope, inputs), local: true }; +} diff --git a/src/state/media/LocalUserMediaViewModel.ts b/src/state/media/LocalUserMediaViewModel.ts new file mode 100644 index 00000000..fd21428b --- /dev/null +++ b/src/state/media/LocalUserMediaViewModel.ts @@ -0,0 +1,137 @@ +/* +Copyright 2023, 2024 New Vector Ltd. +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + facingModeFromLocalTrack, + type LocalParticipant, + LocalVideoTrack, + TrackEvent, +} from "livekit-client"; +import { + fromEvent, + map, + merge, + type Observable, + of, + startWith, + switchMap, +} from "rxjs"; +import { logger } from "matrix-js-sdk/lib/logger"; + +import { type Behavior } from "../Behavior"; +import { + type BaseUserMediaInputs, + type BaseUserMediaViewModel, + createBaseUserMedia, +} from "./UserMediaViewModel"; +import { type ObservableScope } from "../ObservableScope"; +import { alwaysShowSelf } from "../../settings/settings"; +import { platform } from "../../Platform"; +import { type MediaDevices } from "../MediaDevices"; + +export interface LocalUserMediaViewModel extends BaseUserMediaViewModel { + local: true; + /** + * Whether the video should be mirrored. + */ + mirror$: Behavior; + /** + * Whether to show this tile in a highly visible location near the start of + * the grid. + */ + alwaysShow$: Behavior; + setAlwaysShow: (value: boolean) => void; + switchCamera$: Behavior<(() => void) | null>; +} + +export interface LocalUserMediaInputs extends Omit< + BaseUserMediaInputs, + "statsType" +> { + participant$: Behavior; + mediaDevices: MediaDevices; +} + +export function createLocalUserMedia( + scope: ObservableScope, + { mediaDevices, ...inputs }: LocalUserMediaInputs, +): LocalUserMediaViewModel { + const baseUserMedia = createBaseUserMedia(scope, { + ...inputs, + statsType: "outbound-rtp", + }); + + /** + * The local video track as an observable that emits whenever the track + * changes, the camera is switched, or the track is muted. + */ + const videoTrack$: Observable = + baseUserMedia.video$.pipe( + switchMap((v) => { + const track = v?.publication.track; + if (!(track instanceof LocalVideoTrack)) return of(null); + return merge( + // Watch for track restarts because they indicate a camera switch. + // This event is also emitted when unmuting the track object. + fromEvent(track, TrackEvent.Restarted).pipe( + startWith(null), + map(() => track), + ), + // When the track object is muted, reset it to null. + fromEvent(track, TrackEvent.Muted).pipe(map(() => null)), + ); + }), + ); + + return { + ...baseUserMedia, + local: true, + mirror$: scope.behavior( + videoTrack$.pipe( + // Mirror only front-facing cameras (those that face the user) + map( + (track) => + track !== null && + facingModeFromLocalTrack(track).facingMode === "user", + ), + ), + ), + alwaysShow$: alwaysShowSelf.value$, + setAlwaysShow: alwaysShowSelf.setValue, + switchCamera$: scope.behavior( + platform === "desktop" + ? of(null) + : videoTrack$.pipe( + map((track) => { + if (track === null) return null; + const facingMode = facingModeFromLocalTrack(track).facingMode; + // If the camera isn't front or back-facing, don't provide a switch + // camera shortcut at all + if (facingMode !== "user" && facingMode !== "environment") + return null; + // Restart the track with a camera facing the opposite direction + return (): void => + void track + .restartTrack({ + facingMode: facingMode === "user" ? "environment" : "user", + }) + .then(() => { + // Inform the MediaDevices which camera was chosen + const deviceId = + track.mediaStreamTrack.getSettings().deviceId; + if (deviceId !== undefined) + mediaDevices.videoInput.select(deviceId); + }) + .catch((e) => + logger.error("Failed to switch camera", facingMode, e), + ); + }), + ), + ), + }; +} diff --git a/src/state/MediaItem.ts b/src/state/media/MediaItem.ts similarity index 88% rename from src/state/MediaItem.ts rename to src/state/media/MediaItem.ts index 947fdfd4..6cd80045 100644 --- a/src/state/MediaItem.ts +++ b/src/state/media/MediaItem.ts @@ -13,22 +13,24 @@ import { } from "livekit-client"; import { observeParticipantEvents } from "@livekit/components-core"; -import { type ObservableScope } from "./ObservableScope.ts"; -import { - createLocalScreenShare, - createLocalUserMedia, - createRemoteScreenShare, - createRemoteUserMedia, - type ScreenShareViewModel, - type UserMediaViewModel, - type LocalUserMediaInputs, - type RemoteUserMediaInputs, -} from "./MediaViewModel.ts"; -import type { Behavior } from "./Behavior.ts"; -import type { MediaDevices } from "./MediaDevices.ts"; +import { type ObservableScope } from "../ObservableScope.ts"; +import type { Behavior } from "../Behavior.ts"; +import type { MediaDevices } from "../MediaDevices.ts"; import { observeSpeaker$ } from "./observeSpeaker.ts"; -import { generateItems } from "../utils/observable.ts"; -import { type TaggedParticipant } from "./CallViewModel/remoteMembers/MatrixLivekitMembers.ts"; +import { generateItems } from "../../utils/observable.ts"; +import { type TaggedParticipant } from "../CallViewModel/remoteMembers/MatrixLivekitMembers.ts"; +import { type UserMediaViewModel } from "./UserMediaViewModel.ts"; +import { type ScreenShareViewModel } from "./ScreenShareViewModel.ts"; +import { + createLocalUserMedia, + type LocalUserMediaInputs, +} from "./LocalUserMediaViewModel.ts"; +import { + createRemoteUserMedia, + type RemoteUserMediaInputs, +} from "./RemoteUserMediaViewModel.ts"; +import { createLocalScreenShare } from "./LocalScreenShareViewModel.ts"; +import { createRemoteScreenShare } from "./RemoteScreenShareViewModel.ts"; /** * Sorting bins defining the order in which media tiles appear in the layout. diff --git a/src/state/MediaViewModel.test.ts b/src/state/media/MediaViewModel.test.ts similarity index 96% rename from src/state/MediaViewModel.test.ts rename to src/state/media/MediaViewModel.test.ts index 73b396db..71475b8c 100644 --- a/src/state/MediaViewModel.test.ts +++ b/src/state/media/MediaViewModel.test.ts @@ -21,9 +21,8 @@ import { mockRemoteMedia, withTestScheduler, mockRemoteParticipant, -} from "../utils/test"; -import { getValue } from "../utils/observable"; -import { constant } from "./Behavior"; +} from "../../utils/test"; +import { constant } from "../Behavior"; global.MediaStreamTrack = class {} as unknown as { new (): MediaStreamTrack; @@ -35,7 +34,7 @@ global.MediaStream = class {} as unknown as { }; const platformMock = vi.hoisted(() => vi.fn(() => "desktop")); -vi.mock("../Platform", () => ({ +vi.mock("../../Platform", () => ({ get platform(): string { return platformMock(); }, @@ -184,7 +183,7 @@ test("switch cameras", async () => { ); // Switch to back camera - getValue(vm.switchCamera$)!(); + vm.switchCamera$.value!(); expect(restartTrack).toHaveBeenCalledExactlyOnceWith({ facingMode: "environment", }); @@ -195,7 +194,7 @@ test("switch cameras", async () => { expect(deviceId).toBe("back camera"); // Switch to front camera - getValue(vm.switchCamera$)!(); + vm.switchCamera$.value!(); expect(restartTrack).toHaveBeenCalledTimes(2); expect(restartTrack).toHaveBeenLastCalledWith({ facingMode: "user" }); await waitFor(() => { diff --git a/src/state/media/MediaViewModel.ts b/src/state/media/MediaViewModel.ts new file mode 100644 index 00000000..bdc4875b --- /dev/null +++ b/src/state/media/MediaViewModel.ts @@ -0,0 +1,44 @@ +/* +Copyright 2023, 2024 New Vector Ltd. +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type Behavior } from "../Behavior"; +import { type ScreenShareViewModel } from "./ScreenShareViewModel"; +import { type UserMediaViewModel } from "./UserMediaViewModel"; + +/** + * A participant's media. + */ +export type MediaViewModel = UserMediaViewModel | ScreenShareViewModel; + +/** + * Properties which are common to all MediaViewModels. + */ +export interface BaseMediaViewModel { + /** + * An opaque identifier for this media. + */ + id: string; + /** + * The Matrix user to which this media belongs. + */ + userId: string; + displayName$: Behavior; + mxcAvatarUrl$: Behavior; +} + +export type BaseMediaInputs = BaseMediaViewModel; + +// All this function does is strip out superfluous data from the input object +export function createBaseMedia({ + id, + userId, + displayName$, + mxcAvatarUrl$, +}: BaseMediaInputs): BaseMediaViewModel { + return { id, userId, displayName$, mxcAvatarUrl$ }; +} diff --git a/src/state/media/MemberMediaViewModel.ts b/src/state/media/MemberMediaViewModel.ts new file mode 100644 index 00000000..de64d93b --- /dev/null +++ b/src/state/media/MemberMediaViewModel.ts @@ -0,0 +1,280 @@ +/* +Copyright 2023, 2024 New Vector Ltd. +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + type Room as LivekitRoom, + RoomEvent as LivekitRoomEvent, + type Participant, + type Track, +} from "livekit-client"; +import { + type AudioSource, + roomEventSelector, + type TrackReference, + type VideoSource, +} from "@livekit/components-core"; +import { type LocalParticipant, type RemoteParticipant } from "livekit-client"; +import { + combineLatest, + distinctUntilChanged, + filter, + map, + type Observable, + of, + startWith, + switchMap, + throttleTime, +} from "rxjs"; + +import { type Behavior } from "../Behavior"; +import { type BaseMediaViewModel, createBaseMedia } from "./MediaViewModel"; +import { type EncryptionSystem } from "../../e2ee/sharedKeyManagement"; +import { type ObservableScope } from "../ObservableScope"; +import { observeTrackReference$ } from "../observeTrackReference"; +import { E2eeType } from "../../e2ee/e2eeType"; +import { observeRtpStreamStats$ } from "./observeRtpStreamStats"; + +export enum EncryptionStatus { + Connecting, + Okay, + KeyMissing, + KeyInvalid, + PasswordInvalid, +} + +/** + * Media belonging to an active member of the RTC session. + */ +export interface MemberMediaViewModel extends BaseMediaViewModel { + /** + * The LiveKit video track for this media. + */ + video$: Behavior; + /** + * The URL of the LiveKit focus on which this member should be publishing. + * Exposed for debugging. + */ + focusUrl$: Behavior; + /** + * Whether there should be a warning that this media is unencrypted. + */ + unencryptedWarning$: Behavior; + encryptionStatus$: Behavior; +} + +export interface MemberMediaInputs extends BaseMediaViewModel { + participant$: Behavior; + livekitRoom$: Behavior; + audioSource: AudioSource; + videoSource: VideoSource; + focusUrl$: Behavior; + encryptionSystem: EncryptionSystem; +} + +export function createMemberMedia( + scope: ObservableScope, + { + participant$, + livekitRoom$, + audioSource, + videoSource, + focusUrl$, + encryptionSystem, + ...inputs + }: MemberMediaInputs, +): MemberMediaViewModel { + const trackBehavior$ = ( + source: Track.Source, + ): Behavior => + scope.behavior( + participant$.pipe( + switchMap((p) => + !p ? of(undefined) : observeTrackReference$(p, source), + ), + ), + ); + + const audio$ = trackBehavior$(audioSource); + const video$ = trackBehavior$(videoSource); + + return { + ...createBaseMedia(inputs), + video$, + focusUrl$, + unencryptedWarning$: scope.behavior( + combineLatest( + [audio$, video$], + (a, v) => + encryptionSystem.kind !== E2eeType.NONE && + (a?.publication.isEncrypted === false || + v?.publication.isEncrypted === false), + ), + ), + encryptionStatus$: scope.behavior( + participant$.pipe( + switchMap((participant): Observable => { + if (!participant) { + return of(EncryptionStatus.Connecting); + } else if ( + participant.isLocal || + encryptionSystem.kind === E2eeType.NONE + ) { + return of(EncryptionStatus.Okay); + } else if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) { + return combineLatest([ + encryptionErrorObservable$( + livekitRoom$, + participant, + encryptionSystem, + "MissingKey", + ), + encryptionErrorObservable$( + livekitRoom$, + participant, + encryptionSystem, + "InvalidKey", + ), + observeRemoteTrackReceivingOkay$(participant, audioSource), + observeRemoteTrackReceivingOkay$(participant, videoSource), + ]).pipe( + map(([keyMissing, keyInvalid, audioOkay, videoOkay]) => { + if (keyMissing) return EncryptionStatus.KeyMissing; + if (keyInvalid) return EncryptionStatus.KeyInvalid; + if (audioOkay || videoOkay) return EncryptionStatus.Okay; + return undefined; // no change + }), + filter((x) => !!x), + startWith(EncryptionStatus.Connecting), + ); + } else { + return combineLatest([ + encryptionErrorObservable$( + livekitRoom$, + participant, + encryptionSystem, + "InvalidKey", + ), + observeRemoteTrackReceivingOkay$(participant, audioSource), + observeRemoteTrackReceivingOkay$(participant, videoSource), + ]).pipe( + map( + ([keyInvalid, audioOkay, videoOkay]): + | EncryptionStatus + | undefined => { + if (keyInvalid) return EncryptionStatus.PasswordInvalid; + if (audioOkay || videoOkay) return EncryptionStatus.Okay; + return undefined; // no change + }, + ), + filter((x) => !!x), + startWith(EncryptionStatus.Connecting), + ); + } + }), + ), + ), + }; +} + +function observeInboundRtpStreamStats$( + participant: Participant, + source: Track.Source, +): Observable { + return observeRtpStreamStats$(participant, source, "inbound-rtp").pipe( + map((x) => x as RTCInboundRtpStreamStats | undefined), + ); +} + +function encryptionErrorObservable$( + room$: Behavior, + participant: Participant, + encryptionSystem: EncryptionSystem, + criteria: string, +): Observable { + return room$.pipe( + switchMap((room) => { + if (room === undefined) return of(false); + return roomEventSelector(room, LivekitRoomEvent.EncryptionError).pipe( + map((e) => { + const [err] = e; + if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) { + return ( + // Ideally we would pull the participant identity from the field on the error. + // However, it gets lost in the serialization process between workers. + // So, instead we do a string match + (err?.message.includes(participant.identity) && + err?.message.includes(criteria)) ?? + false + ); + } else if (encryptionSystem.kind === E2eeType.SHARED_KEY) { + return !!err?.message.includes(criteria); + } + + return false; + }), + ); + }), + distinctUntilChanged(), + throttleTime(1000), // Throttle to avoid spamming the UI + startWith(false), + ); +} + +function observeRemoteTrackReceivingOkay$( + participant: Participant, + source: Track.Source, +): Observable { + let lastStats: { + framesDecoded: number | undefined; + framesDropped: number | undefined; + framesReceived: number | undefined; + } = { + framesDecoded: undefined, + framesDropped: undefined, + framesReceived: undefined, + }; + + return observeInboundRtpStreamStats$(participant, source).pipe( + map((stats) => { + if (!stats) return undefined; + const { framesDecoded, framesDropped, framesReceived } = stats; + return { + framesDecoded, + framesDropped, + framesReceived, + }; + }), + filter((newStats) => !!newStats), + map((newStats): boolean | undefined => { + const oldStats = lastStats; + lastStats = newStats; + if ( + typeof newStats.framesReceived === "number" && + typeof oldStats.framesReceived === "number" && + typeof newStats.framesDecoded === "number" && + typeof oldStats.framesDecoded === "number" + ) { + const framesReceivedDelta = + newStats.framesReceived - oldStats.framesReceived; + const framesDecodedDelta = + newStats.framesDecoded - oldStats.framesDecoded; + + // if we received >0 frames and managed to decode >0 frames then we treat that as success + + if (framesReceivedDelta > 0) { + return framesDecodedDelta > 0; + } + } + + // no change + return undefined; + }), + filter((x) => typeof x === "boolean"), + startWith(undefined), + ); +} diff --git a/src/state/media/RemoteScreenShareViewModel.ts b/src/state/media/RemoteScreenShareViewModel.ts new file mode 100644 index 00000000..eff6d9c1 --- /dev/null +++ b/src/state/media/RemoteScreenShareViewModel.ts @@ -0,0 +1,44 @@ +/* +Copyright 2023, 2024 New Vector Ltd. +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type RemoteParticipant } from "livekit-client"; +import { map } from "rxjs"; + +import { type Behavior } from "../Behavior"; +import { + type BaseScreenShareInputs, + type BaseScreenShareViewModel, + createBaseScreenShare, +} from "./ScreenShareViewModel"; +import { type ObservableScope } from "../ObservableScope"; + +export interface RemoteScreenShareViewModel extends BaseScreenShareViewModel { + local: false; + /** + * Whether this screen share's video should be displayed. + */ + videoEnabled$: Behavior; +} + +export interface RemoteScreenShareInputs extends BaseScreenShareInputs { + participant$: Behavior; + pretendToBeDisconnected$: Behavior; +} + +export function createRemoteScreenShare( + scope: ObservableScope, + { pretendToBeDisconnected$, ...inputs }: RemoteScreenShareInputs, +): RemoteScreenShareViewModel { + return { + ...createBaseScreenShare(scope, inputs), + local: false, + videoEnabled$: scope.behavior( + pretendToBeDisconnected$.pipe(map((disconnected) => !disconnected)), + ), + }; +} diff --git a/src/state/media/RemoteUserMediaViewModel.ts b/src/state/media/RemoteUserMediaViewModel.ts new file mode 100644 index 00000000..4307dea4 --- /dev/null +++ b/src/state/media/RemoteUserMediaViewModel.ts @@ -0,0 +1,82 @@ +/* +Copyright 2023, 2024 New Vector Ltd. +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type RemoteParticipant } from "livekit-client"; +import { combineLatest, map, of, switchMap } from "rxjs"; + +import { type Behavior } from "../Behavior"; +import { createVolumeControls, type VolumeControls } from "../VolumeControls"; +import { + type BaseUserMediaInputs, + type BaseUserMediaViewModel, + createBaseUserMedia, +} from "./UserMediaViewModel"; +import { type ObservableScope } from "../ObservableScope"; + +export interface RemoteUserMediaViewModel + extends BaseUserMediaViewModel, VolumeControls { + local: false; + /** + * Whether we are waiting for this user's LiveKit participant to exist. This + * could be because either we or the remote party are still connecting. + */ + waitingForMedia$: Behavior; +} + +export interface RemoteUserMediaInputs extends Omit< + BaseUserMediaInputs, + "statsType" +> { + participant$: Behavior; + pretendToBeDisconnected$: Behavior; +} + +export function createRemoteUserMedia( + scope: ObservableScope, + { pretendToBeDisconnected$, ...inputs }: RemoteUserMediaInputs, +): RemoteUserMediaViewModel { + const baseUserMedia = createBaseUserMedia(scope, { + ...inputs, + statsType: "inbound-rtp", + }); + + return { + ...baseUserMedia, + ...createVolumeControls(scope, { + pretendToBeDisconnected$, + sink$: scope.behavior( + inputs.participant$.pipe(map((p) => (volume) => p?.setVolume(volume))), + ), + }), + local: false, + speaking$: scope.behavior( + pretendToBeDisconnected$.pipe( + switchMap((disconnected) => + disconnected ? of(false) : baseUserMedia.speaking$, + ), + ), + ), + videoEnabled$: scope.behavior( + pretendToBeDisconnected$.pipe( + switchMap((disconnected) => + disconnected ? of(false) : baseUserMedia.videoEnabled$, + ), + ), + ), + waitingForMedia$: scope.behavior( + combineLatest( + [inputs.livekitRoom$, inputs.participant$], + (livekitRoom, participant) => + // If livekitRoom is undefined, the user is not attempting to publish on + // any transport and so we shouldn't expect a participant. (They might + // be a subscribe-only bot for example.) + livekitRoom !== undefined && participant === null, + ), + ), + }; +} diff --git a/src/state/media/ScreenShareViewModel.ts b/src/state/media/ScreenShareViewModel.ts new file mode 100644 index 00000000..36cd9440 --- /dev/null +++ b/src/state/media/ScreenShareViewModel.ts @@ -0,0 +1,51 @@ +/* +Copyright 2023, 2024 New Vector Ltd. +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { Track } from "livekit-client"; + +import { type ObservableScope } from "../ObservableScope"; +import { type LocalScreenShareViewModel } from "./LocalScreenShareViewModel"; +import { + createMemberMedia, + type MemberMediaInputs, + type MemberMediaViewModel, +} from "./MemberMediaViewModel"; +import { type RemoteScreenShareViewModel } from "./RemoteScreenShareViewModel"; + +/** + * A participant's screen share media. + */ +export type ScreenShareViewModel = + | LocalScreenShareViewModel + | RemoteScreenShareViewModel; + +/** + * Properties which are common to all ScreenShareViewModels. + */ +export interface BaseScreenShareViewModel extends MemberMediaViewModel { + type: "screen share"; +} + +export type BaseScreenShareInputs = Omit< + MemberMediaInputs, + "audioSource" | "videoSource" +>; + +export function createBaseScreenShare( + scope: ObservableScope, + inputs: BaseScreenShareInputs, +): BaseScreenShareViewModel { + return { + ...createMemberMedia(scope, { + ...inputs, + audioSource: Track.Source.ScreenShareAudio, + videoSource: Track.Source.ScreenShare, + }), + type: "screen share", + }; +} diff --git a/src/state/media/UserMediaViewModel.ts b/src/state/media/UserMediaViewModel.ts new file mode 100644 index 00000000..8da5e63a --- /dev/null +++ b/src/state/media/UserMediaViewModel.ts @@ -0,0 +1,143 @@ +/* +Copyright 2023, 2024 New Vector Ltd. +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + combineLatest, + map, + type Observable, + of, + Subject, + switchMap, +} from "rxjs"; +import { + observeParticipantEvents, + observeParticipantMedia, +} from "@livekit/components-core"; +import { ParticipantEvent, Track } from "livekit-client"; + +import { type ReactionOption } from "../../reactions"; +import { type Behavior } from "../Behavior"; +import { type LocalUserMediaViewModel } from "./LocalUserMediaViewModel"; +import { + createMemberMedia, + type MemberMediaInputs, + type MemberMediaViewModel, +} from "./MemberMediaViewModel"; +import { type RemoteUserMediaViewModel } from "./RemoteUserMediaViewModel"; +import { type ObservableScope } from "../ObservableScope"; +import { createToggle$ } from "../../utils/observable"; +import { showConnectionStats } from "../../settings/settings"; +import { observeRtpStreamStats$ } from "./observeRtpStreamStats"; + +/** + * A participant's user media (i.e. their microphone and camera feed). + */ +export type UserMediaViewModel = + | LocalUserMediaViewModel + | RemoteUserMediaViewModel; + +export interface BaseUserMediaViewModel extends MemberMediaViewModel { + type: "user"; + speaking$: Behavior; + audioEnabled$: Behavior; + videoEnabled$: Behavior; + cropVideo$: Behavior; + toggleCropVideo: () => void; + /** + * The expected identity of the LiveKit participant. Exposed for debugging. + */ + rtcBackendIdentity: string; + handRaised$: Behavior; + reaction$: Behavior; + audioStreamStats$: Observable< + RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined + >; + videoStreamStats$: Observable< + RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined + >; +} + +export interface BaseUserMediaInputs extends Omit< + MemberMediaInputs, + "audioSource" | "videoSource" +> { + rtcBackendIdentity: string; + handRaised$: Behavior; + reaction$: Behavior; + statsType: "inbound-rtp" | "outbound-rtp"; +} + +export function createBaseUserMedia( + scope: ObservableScope, + { + rtcBackendIdentity, + handRaised$, + reaction$, + statsType, + ...inputs + }: BaseUserMediaInputs, +): BaseUserMediaViewModel { + const { participant$ } = inputs; + const media$ = scope.behavior( + participant$.pipe( + switchMap((p) => (p && observeParticipantMedia(p)) ?? of(undefined)), + ), + ); + const toggleCropVideo$ = new Subject(); + + return { + ...createMemberMedia(scope, { + ...inputs, + audioSource: Track.Source.Microphone, + videoSource: Track.Source.Camera, + }), + type: "user", + speaking$: scope.behavior( + participant$.pipe( + switchMap((p) => + p + ? observeParticipantEvents( + p, + ParticipantEvent.IsSpeakingChanged, + ).pipe(map((p) => p.isSpeaking)) + : of(false), + ), + ), + ), + audioEnabled$: scope.behavior( + media$.pipe(map((m) => m?.microphoneTrack?.isMuted === false)), + ), + videoEnabled$: scope.behavior( + media$.pipe(map((m) => m?.cameraTrack?.isMuted === false)), + ), + cropVideo$: createToggle$(scope, true, toggleCropVideo$), + toggleCropVideo: () => toggleCropVideo$.next(), + rtcBackendIdentity, + handRaised$, + reaction$, + audioStreamStats$: combineLatest([ + participant$, + showConnectionStats.value$, + ]).pipe( + switchMap(([p, showConnectionStats]) => { + // + if (!p || !showConnectionStats) return of(undefined); + return observeRtpStreamStats$(p, Track.Source.Microphone, statsType); + }), + ), + videoStreamStats$: combineLatest([ + participant$, + showConnectionStats.value$, + ]).pipe( + switchMap(([p, showConnectionStats]) => { + if (!p || !showConnectionStats) return of(undefined); + return observeRtpStreamStats$(p, Track.Source.Camera, statsType); + }), + ), + }; +} diff --git a/src/state/media/observeRtpStreamStats.ts b/src/state/media/observeRtpStreamStats.ts new file mode 100644 index 00000000..695edc65 --- /dev/null +++ b/src/state/media/observeRtpStreamStats.ts @@ -0,0 +1,59 @@ +/* +Copyright 2023, 2024 New Vector Ltd. +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + LocalTrack, + type Participant, + RemoteTrack, + type Track, +} from "livekit-client"; +import { + combineLatest, + interval, + type Observable, + startWith, + switchMap, +} from "rxjs"; + +import { observeTrackReference$ } from "../observeTrackReference"; + +export function observeRtpStreamStats$( + participant: Participant, + source: Track.Source, + type: "inbound-rtp" | "outbound-rtp", +): Observable< + RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined +> { + return combineLatest([ + observeTrackReference$(participant, source), + interval(1000).pipe(startWith(0)), + ]).pipe( + switchMap(async ([trackReference]) => { + const track = trackReference?.publication?.track; + if ( + !track || + !(track instanceof RemoteTrack || track instanceof LocalTrack) + ) { + return undefined; + } + const report = await track.getRTCStatsReport(); + if (!report) { + return undefined; + } + + for (const v of report.values()) { + if (v.type === type) { + return v; + } + } + + return undefined; + }), + startWith(undefined), + ); +} diff --git a/src/state/observeSpeaker.test.ts b/src/state/media/observeSpeaker.test.ts similarity index 98% rename from src/state/observeSpeaker.test.ts rename to src/state/media/observeSpeaker.test.ts index 224916d2..18622fb8 100644 --- a/src/state/observeSpeaker.test.ts +++ b/src/state/media/observeSpeaker.test.ts @@ -7,7 +7,7 @@ Please see LICENSE in the repository root for full details. import { describe, test } from "vitest"; -import { withTestScheduler } from "../utils/test"; +import { withTestScheduler } from "../../utils/test"; import { observeSpeaker$ } from "./observeSpeaker"; const yesNo = { diff --git a/src/state/observeSpeaker.ts b/src/state/media/observeSpeaker.ts similarity index 100% rename from src/state/observeSpeaker.ts rename to src/state/media/observeSpeaker.ts diff --git a/src/state/observeTrackReference.ts b/src/state/observeTrackReference.ts new file mode 100644 index 00000000..8e295d05 --- /dev/null +++ b/src/state/observeTrackReference.ts @@ -0,0 +1,28 @@ +/* +Copyright 2023, 2024 New Vector Ltd. +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + observeParticipantMedia, + type TrackReference, +} from "@livekit/components-core"; +import { type Participant, type Track } from "livekit-client"; +import { distinctUntilChanged, map, type Observable } from "rxjs"; + +/** + * Reactively reads a participant's track reference for a given media source. + */ +export function observeTrackReference$( + participant: Participant, + source: Track.Source, +): Observable { + return observeParticipantMedia(participant).pipe( + map(() => participant.getTrackPublication(source)), + distinctUntilChanged(), + map((publication) => publication && { participant, publication, source }), + ); +} diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index 370e0723..9c3adea7 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -39,11 +39,6 @@ import { import { useObservableEagerState } from "observable-hooks"; import styles from "./GridTile.module.css"; -import { - type UserMediaViewModel, - type LocalUserMediaViewModel, - type RemoteUserMediaViewModel, -} from "../state/MediaViewModel"; import { Slider } from "../Slider"; import { MediaView } from "./MediaView"; import { useLatest } from "../useLatest"; @@ -51,6 +46,9 @@ import { type GridTileViewModel } from "../state/TileViewModel"; import { useMergedRefs } from "../useMergedRefs"; import { useReactionsSender } from "../reactions/useReactionsSender"; import { useBehavior } from "../useBehavior"; +import { type LocalUserMediaViewModel } from "../state/media/LocalUserMediaViewModel"; +import { type RemoteUserMediaViewModel } from "../state/media/RemoteUserMediaViewModel"; +import { type UserMediaViewModel } from "../state/media/UserMediaViewModel"; interface TileProps { ref?: Ref; diff --git a/src/tile/MediaView.test.tsx b/src/tile/MediaView.test.tsx index fdfdb631..a509d3a5 100644 --- a/src/tile/MediaView.test.tsx +++ b/src/tile/MediaView.test.tsx @@ -18,7 +18,7 @@ import { TrackInfo } from "@livekit/protocol"; import { type ComponentProps } from "react"; import { MediaView } from "./MediaView"; -import { EncryptionStatus } from "../state/MediaViewModel"; +import { EncryptionStatus } from "../state/media/MemberMediaViewModel"; import { mockLocalParticipant } from "../utils/test"; describe("MediaView", () => { diff --git a/src/tile/MediaView.tsx b/src/tile/MediaView.tsx index c349fc7e..f912c069 100644 --- a/src/tile/MediaView.tsx +++ b/src/tile/MediaView.tsx @@ -16,7 +16,7 @@ import { ErrorSolidIcon } from "@vector-im/compound-design-tokens/assets/web/ico import styles from "./MediaView.module.css"; import { Avatar } from "../Avatar"; -import { type EncryptionStatus } from "../state/MediaViewModel"; +import { type EncryptionStatus } from "../state/media/MemberMediaViewModel"; import { RaisedHandIndicator } from "../reactions/RaisedHandIndicator"; import { showConnectionStats as showConnectionStatsSetting, diff --git a/src/tile/SpotlightTile.tsx b/src/tile/SpotlightTile.tsx index c0f1135f..75c69479 100644 --- a/src/tile/SpotlightTile.tsx +++ b/src/tile/SpotlightTile.tsx @@ -32,21 +32,19 @@ import FullScreenMaximiseIcon from "../icons/FullScreenMaximise.svg?react"; import FullScreenMinimiseIcon from "../icons/FullScreenMinimise.svg?react"; import { MediaView } from "./MediaView"; import styles from "./SpotlightTile.module.css"; -import { - type EncryptionStatus, - type LocalUserMediaViewModel, - type MediaViewModel, - type UserMediaViewModel, - type RemoteUserMediaViewModel, - type ScreenShareViewModel, - type RemoteScreenShareViewModel, -} from "../state/MediaViewModel"; import { useInitial } from "../useInitial"; import { useMergedRefs } from "../useMergedRefs"; import { useReactiveState } from "../useReactiveState"; import { useLatest } from "../useLatest"; import { type SpotlightTileViewModel } from "../state/TileViewModel"; import { useBehavior } from "../useBehavior"; +import { type EncryptionStatus } from "../state/media/MemberMediaViewModel"; +import { type LocalUserMediaViewModel } from "../state/media/LocalUserMediaViewModel"; +import { type RemoteUserMediaViewModel } from "../state/media/RemoteUserMediaViewModel"; +import { type UserMediaViewModel } from "../state/media/UserMediaViewModel"; +import { type ScreenShareViewModel } from "../state/media/ScreenShareViewModel"; +import { type RemoteScreenShareViewModel } from "../state/media/RemoteScreenShareViewModel"; +import { type MediaViewModel } from "../state/media/MediaViewModel"; interface SpotlightItemBaseProps { ref?: Ref; diff --git a/src/utils/test.ts b/src/utils/test.ts index 9231a3d1..c1e67927 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -52,12 +52,6 @@ import { } from "matrix-js-sdk/lib/matrixrtc/IKeyTransport"; import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; -import { - createLocalUserMedia, - createRemoteUserMedia, - type LocalUserMediaViewModel, - type RemoteUserMediaViewModel, -} from "../state/MediaViewModel"; import { E2eeType } from "../e2ee/e2eeType"; import { DEFAULT_CONFIG, @@ -68,6 +62,14 @@ import { type MediaDevices } from "../state/MediaDevices"; import { type Behavior, constant } from "../state/Behavior"; import { ObservableScope } from "../state/ObservableScope"; import { MuteStates } from "../state/MuteStates"; +import { + createLocalUserMedia, + type LocalUserMediaViewModel, +} from "../state/media/LocalUserMediaViewModel"; +import { + createRemoteUserMedia, + type RemoteUserMediaViewModel, +} from "../state/media/RemoteUserMediaViewModel"; export function withFakeTimers(continuation: () => void): void { vi.useFakeTimers(); From 9930288d1f43254521a1e213de29d03929d37ab7 Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 27 Feb 2026 17:08:04 +0100 Subject: [PATCH 3/4] Move observeInboundRtpStreamStats$ into the appropriate file --- src/state/media/MemberMediaViewModel.ts | 11 +---------- src/state/media/observeRtpStreamStats.ts | 10 ++++++++++ 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/state/media/MemberMediaViewModel.ts b/src/state/media/MemberMediaViewModel.ts index de64d93b..b7c2549d 100644 --- a/src/state/media/MemberMediaViewModel.ts +++ b/src/state/media/MemberMediaViewModel.ts @@ -37,7 +37,7 @@ import { type EncryptionSystem } from "../../e2ee/sharedKeyManagement"; import { type ObservableScope } from "../ObservableScope"; import { observeTrackReference$ } from "../observeTrackReference"; import { E2eeType } from "../../e2ee/e2eeType"; -import { observeRtpStreamStats$ } from "./observeRtpStreamStats"; +import { observeInboundRtpStreamStats$ } from "./observeRtpStreamStats"; export enum EncryptionStatus { Connecting, @@ -181,15 +181,6 @@ export function createMemberMedia( }; } -function observeInboundRtpStreamStats$( - participant: Participant, - source: Track.Source, -): Observable { - return observeRtpStreamStats$(participant, source, "inbound-rtp").pipe( - map((x) => x as RTCInboundRtpStreamStats | undefined), - ); -} - function encryptionErrorObservable$( room$: Behavior, participant: Participant, diff --git a/src/state/media/observeRtpStreamStats.ts b/src/state/media/observeRtpStreamStats.ts index 695edc65..d1640382 100644 --- a/src/state/media/observeRtpStreamStats.ts +++ b/src/state/media/observeRtpStreamStats.ts @@ -18,6 +18,7 @@ import { type Observable, startWith, switchMap, + map, } from "rxjs"; import { observeTrackReference$ } from "../observeTrackReference"; @@ -57,3 +58,12 @@ export function observeRtpStreamStats$( startWith(undefined), ); } + +export function observeInboundRtpStreamStats$( + participant: Participant, + source: Track.Source, +): Observable { + return observeRtpStreamStats$(participant, source, "inbound-rtp").pipe( + map((x) => x as RTCInboundRtpStreamStats | undefined), + ); +} From 0ce24929dc38b7f2caf13fd4bd8d6789508fd9bc Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 27 Feb 2026 17:12:24 +0100 Subject: [PATCH 4/4] Add TODO to clean up encryption status code --- src/state/media/MemberMediaViewModel.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/state/media/MemberMediaViewModel.ts b/src/state/media/MemberMediaViewModel.ts index b7c2549d..e7f57b59 100644 --- a/src/state/media/MemberMediaViewModel.ts +++ b/src/state/media/MemberMediaViewModel.ts @@ -39,6 +39,7 @@ import { observeTrackReference$ } from "../observeTrackReference"; import { E2eeType } from "../../e2ee/e2eeType"; import { observeInboundRtpStreamStats$ } from "./observeRtpStreamStats"; +// TODO: Encryption status is kinda broken and thus unused right now. Remove? export enum EncryptionStatus { Connecting, Okay,