diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 61afb7b9..7f34d86f 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -51,14 +51,12 @@ import { v4 as uuidv4 } from "uuid"; import { type IMembershipManager } from "matrix-js-sdk/lib/matrixrtc/IMembershipManager"; import { - LocalUserMediaViewModel, type MediaViewModel, - type RemoteUserMediaViewModel, - ScreenShareViewModel, + type ScreenShareViewModel, type UserMediaViewModel, } from "../MediaViewModel"; import { - accumulate, + createToggle$, filterBehavior, generateItem, generateItems, @@ -92,8 +90,6 @@ import { type MuteStates } from "../MuteStates"; import { getUrlParams } from "../../UrlParams"; import { type ProcessorState } from "../../livekit/TrackProcessorContext"; import { ElementWidgetActions, widget } from "../../widget"; -import { UserMedia } from "../UserMedia.ts"; -import { ScreenShare } from "../ScreenShare.ts"; import { type GridLayoutMedia, type Layout, @@ -144,6 +140,11 @@ import { import { Publisher } from "./localMember/Publisher.ts"; import { type Connection } from "./remoteMembers/Connection.ts"; import { createLayoutModeSwitch } from "./LayoutSwitch.ts"; +import { + createWrappedUserMedia, + type WrappedUserMediaViewModel, + type MediaItem, +} from "../MediaItem.ts"; const logger = rootLogger.getChild("[CallViewModel]"); //TODO @@ -193,7 +194,6 @@ interface LayoutScanState { tiles: TileStore; } -type MediaItem = UserMedia | ScreenShare; export type LivekitRoomItem = { livekitRoom: LivekitRoom; participants: string[]; @@ -283,7 +283,6 @@ export interface CallViewModel { allConnections$: Behavior; /** Participants sorted by livekit room so they can be used in the audio rendering */ livekitRoomItems$: Behavior; - userMedia$: Behavior; /** use the layout instead, this is just for the sdk export. */ matrixLivekitMembers$: Behavior; localMatrixLivekitMember$: Behavior; @@ -334,10 +333,6 @@ export interface CallViewModel { gridMode$: Behavior; setGridMode: (value: GridMode) => void; - // media view models and layout - grid$: Behavior; - spotlight$: Behavior; - pip$: Behavior; /** * The layout of tiles in the call interface. */ @@ -721,7 +716,7 @@ export function createCallViewModel$( /** * List of user media (camera feeds) that we want tiles for. */ - const userMedia$ = scope.behavior( + const userMedia$ = scope.behavior( combineLatest([ localMatrixLivekitMember$, matrixLivekitMembers$, @@ -767,36 +762,35 @@ export function createCallViewModel$( } } }, - (scope, _, dup, mediaId, userId, participant, connection$, rtcId) => { - const livekitRoom$ = scope.behavior( - connection$.pipe(map((c) => c?.livekitRoom)), - ); - const focusUrl$ = scope.behavior( - connection$.pipe(map((c) => c?.transport.livekit_service_url)), - ); - const displayName$ = scope.behavior( - matrixMemberMetadataStore - .createDisplayNameBehavior$(userId) - .pipe(map((name) => name ?? userId)), - ); - - return new UserMedia( - scope, - `${mediaId}:${dup}`, + (scope, _, dup, mediaId, userId, participant, connection$, rtcId) => + createWrappedUserMedia(scope, { + id: `${mediaId}:${dup}`, userId, - rtcId, + rtcBackendIdentity: rtcId, participant, - options.encryptionSystem, - livekitRoom$, - focusUrl$, + encryptionSystem: options.encryptionSystem, + livekitRoom$: scope.behavior( + connection$.pipe(map((c) => c?.livekitRoom)), + ), + focusUrl$: scope.behavior( + connection$.pipe(map((c) => c?.transport.livekit_service_url)), + ), mediaDevices, - localMembership.reconnecting$, - displayName$, - matrixMemberMetadataStore.createAvatarUrlBehavior$(userId), - handsRaised$.pipe(map((v) => v[mediaId]?.time ?? null)), - reactions$.pipe(map((v) => v[mediaId] ?? undefined)), - ); - }, + pretendToBeDisconnected$: localMembership.reconnecting$, + displayName$: scope.behavior( + matrixMemberMetadataStore + .createDisplayNameBehavior$(userId) + .pipe(map((name) => name ?? userId)), + ), + mxcAvatarUrl$: + matrixMemberMetadataStore.createAvatarUrlBehavior$(userId), + handRaised$: scope.behavior( + handsRaised$.pipe(map((v) => v[mediaId]?.time ?? null)), + ), + reaction$: scope.behavior( + reactions$.pipe(map((v) => v[mediaId] ?? undefined)), + ), + }), ), ), ); @@ -821,11 +815,9 @@ export function createCallViewModel$( /** * List of MediaItems that we want to display, that are of type ScreenShare */ - const screenShares$ = scope.behavior( + const screenShares$ = scope.behavior( mediaItems$.pipe( - map((mediaItems) => - mediaItems.filter((m): m is ScreenShare => m instanceof ScreenShare), - ), + map((mediaItems) => mediaItems.filter((m) => m.type === "screen share")), ), ); @@ -888,39 +880,39 @@ export function createCallViewModel$( merge(userHangup$, widgetHangup$).pipe(map(() => "user" as const)), ).pipe(scope.share); - const spotlightSpeaker$ = scope.behavior( + const spotlightSpeaker$ = scope.behavior( userMedia$.pipe( switchMap((mediaItems) => mediaItems.length === 0 ? of([]) : combineLatest( mediaItems.map((m) => - m.vm.speaking$.pipe(map((s) => [m, s] as const)), + m.speaking$.pipe(map((s) => [m, s] as const)), ), ), ), - scan<(readonly [UserMedia, boolean])[], UserMedia | undefined, null>( - (prev, mediaItems) => { - // Only remote users that are still in the call should be sticky - const [stickyMedia, stickySpeaking] = - (!prev?.vm.local && mediaItems.find(([m]) => m === prev)) || []; - // Decide who to spotlight: - // If the previous speaker is still speaking, stick with them rather - // than switching eagerly to someone else - return stickySpeaking - ? stickyMedia! - : // Otherwise, select any remote user who is speaking - (mediaItems.find(([m, s]) => !m.vm.local && s)?.[0] ?? - // Otherwise, stick with the person who was last speaking - stickyMedia ?? - // Otherwise, spotlight an arbitrary remote user - mediaItems.find(([m]) => !m.vm.local)?.[0] ?? - // Otherwise, spotlight the local user - mediaItems.find(([m]) => m.vm.local)?.[0]); - }, - null, - ), - map((speaker) => speaker?.vm ?? null), + scan< + (readonly [UserMediaViewModel, boolean])[], + UserMediaViewModel | undefined, + undefined + >((prev, mediaItems) => { + // Only remote users that are still in the call should be sticky + const [stickyMedia, stickySpeaking] = + (!prev?.local && mediaItems.find(([m]) => m === prev)) || []; + // Decide who to spotlight: + // If the previous speaker is still speaking, stick with them rather + // than switching eagerly to someone else + return stickySpeaking + ? stickyMedia! + : // Otherwise, select any remote user who is speaking + (mediaItems.find(([m, s]) => !m.local && s)?.[0] ?? + // Otherwise, stick with the person who was last speaking + stickyMedia ?? + // Otherwise, spotlight an arbitrary remote user + mediaItems.find(([m]) => !m.local)?.[0] ?? + // Otherwise, spotlight the local user + mediaItems.find(([m]) => m.local)?.[0]); + }, undefined), ), ); @@ -934,7 +926,7 @@ export function createCallViewModel$( return bins.length === 0 ? of([]) : combineLatest(bins, (...bins) => - bins.sort(([, bin1], [, bin2]) => bin1 - bin2).map(([m]) => m.vm), + bins.sort(([, bin1], [, bin2]) => bin1 - bin2).map(([m]) => m), ); }), distinctUntilChanged(shallowEquals), @@ -944,9 +936,7 @@ export function createCallViewModel$( const spotlight$ = scope.behavior( screenShares$.pipe( switchMap((screenShares) => { - if (screenShares.length > 0) { - return of(screenShares.map((m) => m.vm)); - } + if (screenShares.length > 0) return of(screenShares); return spotlightSpeaker$.pipe( map((speaker) => (speaker ? [speaker] : [])), @@ -956,7 +946,7 @@ export function createCallViewModel$( ), ); - const pip$ = scope.behavior( + const pip$ = scope.behavior( combineLatest([ // TODO This also needs epoch logic to dedupe the screenshares and mediaItems emits screenShares$, @@ -968,28 +958,17 @@ export function createCallViewModel$( return spotlightSpeaker$; } if (!spotlight || spotlight.local) { - return of(null); + return of(undefined); } const localUserMedia = mediaItems.find( - (m) => m.vm instanceof LocalUserMediaViewModel, - ) as UserMedia | undefined; - - const localUserMediaViewModel = localUserMedia?.vm as - | LocalUserMediaViewModel - | undefined; - - if (!localUserMediaViewModel) { - return of(null); + (m) => m.type === "user" && m.local, + ); + if (!localUserMedia) { + return of(undefined); } - return localUserMediaViewModel.alwaysShow$.pipe( - map((alwaysShow) => { - if (alwaysShow) { - return localUserMediaViewModel; - } - - return null; - }), + return localUserMedia.alwaysShow$.pipe( + map((alwaysShow) => (alwaysShow ? localUserMedia : undefined)), ); }), ), @@ -998,7 +977,7 @@ export function createCallViewModel$( const hasRemoteScreenShares$ = scope.behavior( spotlight$.pipe( map((spotlight) => - spotlight.some((vm) => !vm.local && vm instanceof ScreenShareViewModel), + spotlight.some((vm) => vm.type === "screen share" && !vm.local), ), ), ); @@ -1039,8 +1018,10 @@ export function createCallViewModel$( ); const spotlightExpandedToggle$ = new Subject(); - const spotlightExpanded$ = scope.behavior( - spotlightExpandedToggle$.pipe(accumulate(false, (expanded) => !expanded)), + const spotlightExpanded$ = createToggle$( + scope, + false, + spotlightExpandedToggle$, ); const { setGridMode, gridMode$ } = createLayoutModeSwitch( @@ -1053,7 +1034,7 @@ export function createCallViewModel$( [grid$, spotlight$], (grid, spotlight) => ({ type: "grid", - spotlight: spotlight.some((vm) => vm instanceof ScreenShareViewModel) + spotlight: spotlight.some((vm) => vm.type === "screen share") ? spotlight : undefined, grid, @@ -1085,12 +1066,8 @@ export function createCallViewModel$( mediaItems$.pipe( map((mediaItems) => { if (mediaItems.length !== 2) return null; - const local = mediaItems.find((vm) => vm.vm.local)?.vm as - | LocalUserMediaViewModel - | undefined; - const remote = mediaItems.find((vm) => !vm.vm.local)?.vm as - | RemoteUserMediaViewModel - | undefined; + const local = mediaItems.find((vm) => vm.type === "user" && vm.local); + const remote = mediaItems.find((vm) => vm.type === "user" && !vm.local); // There might not be a remote tile if there are screen shares, or if // only the local user is in the call and they're using the duplicate // tiles option @@ -1138,7 +1115,7 @@ export function createCallViewModel$( oneOnOne === null ? combineLatest([grid$, spotlight$], (grid, spotlight) => grid.length > smallMobileCallThreshold || - spotlight.some((vm) => vm instanceof ScreenShareViewModel) + spotlight.some((vm) => vm.type === "screen share") ? spotlightPortraitLayoutMedia$ : gridLayoutMedia$, ).pipe(switchAll()) @@ -1245,7 +1222,7 @@ export function createCallViewModel$( // screen sharing feeds are in the spotlight we still need them. return l.spotlight.media$.pipe( map((models: MediaViewModel[]) => - models.some((m) => m instanceof ScreenShareViewModel), + models.some((m) => m.type === "screen share"), ), ); // In expanded spotlight layout, the active speaker is always shown in @@ -1552,11 +1529,7 @@ export function createCallViewModel$( toggleSpotlightExpanded$: toggleSpotlightExpanded$, gridMode$: gridMode$, setGridMode: setGridMode, - grid$: grid$, - spotlight$: spotlight$, - pip$: pip$, layout$: layout$, - userMedia$, localMatrixLivekitMember$, matrixLivekitMembers$: scope.behavior( matrixLivekitMembers$.pipe( diff --git a/src/state/MediaItem.ts b/src/state/MediaItem.ts new file mode 100644 index 00000000..947fdfd4 --- /dev/null +++ b/src/state/MediaItem.ts @@ -0,0 +1,196 @@ +/* +Copyright 2025-2026 Element Software Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { combineLatest, map, of, switchMap } from "rxjs"; +import { + type LocalParticipant, + ParticipantEvent, + type RemoteParticipant, +} from "livekit-client"; +import { observeParticipantEvents } from "@livekit/components-core"; + +import { type ObservableScope } from "./ObservableScope.ts"; +import { + createLocalScreenShare, + createLocalUserMedia, + createRemoteScreenShare, + createRemoteUserMedia, + type ScreenShareViewModel, + type UserMediaViewModel, + type LocalUserMediaInputs, + type RemoteUserMediaInputs, +} from "./MediaViewModel.ts"; +import type { Behavior } from "./Behavior.ts"; +import type { MediaDevices } from "./MediaDevices.ts"; +import { observeSpeaker$ } from "./observeSpeaker.ts"; +import { generateItems } from "../utils/observable.ts"; +import { type TaggedParticipant } from "./CallViewModel/remoteMembers/MatrixLivekitMembers.ts"; + +/** + * Sorting bins defining the order in which media tiles appear in the layout. + */ +enum SortingBin { + /** + * Yourself, when the "always show self" option is on. + */ + SelfAlwaysShown, + /** + * Participants that are sharing their screen. + */ + Presenters, + /** + * Participants that have been speaking recently. + */ + Speakers, + /** + * Participants that have their hand raised. + */ + HandRaised, + /** + * Participants with video. + */ + Video, + /** + * Participants not sharing any video. + */ + NoVideo, + /** + * Yourself, when the "always show self" option is off. + */ + SelfNotAlwaysShown, +} + +/** + * A user media item to be presented in a tile. This is a thin wrapper around + * UserMediaViewModel which additionally carries data relevant to the tile + * layout algorithms (data which the MediaView component should be ignorant of). + */ +export type WrappedUserMediaViewModel = UserMediaViewModel & { + /** + * All screen share media associated with this user media. + */ + screenShares$: Behavior; + /** + * Which sorting bin the media item should be placed in. + */ + bin$: Behavior; +}; + +interface WrappedUserMediaInputs extends Omit< + LocalUserMediaInputs & RemoteUserMediaInputs, + "participant$" +> { + participant: TaggedParticipant; + mediaDevices: MediaDevices; + pretendToBeDisconnected$: Behavior; +} + +export function createWrappedUserMedia( + scope: ObservableScope, + { + participant, + mediaDevices, + pretendToBeDisconnected$, + ...inputs + }: WrappedUserMediaInputs, +): WrappedUserMediaViewModel { + const userMedia = + participant.type === "local" + ? createLocalUserMedia(scope, { + participant$: participant.value$, + mediaDevices, + ...inputs, + }) + : createRemoteUserMedia(scope, { + participant$: participant.value$, + pretendToBeDisconnected$, + ...inputs, + }); + + // TypeScript needs this widening of the type to happen in a separate statement + const participant$: Behavior = + participant.value$; + + const screenShares$ = scope.behavior( + participant$.pipe( + switchMap((p) => + p === null + ? of([]) + : observeParticipantEvents( + p, + ParticipantEvent.TrackPublished, + ParticipantEvent.TrackUnpublished, + ParticipantEvent.LocalTrackPublished, + ParticipantEvent.LocalTrackUnpublished, + ).pipe( + // Technically more than one screen share might be possible... our + // MediaViewModels don't support it though since they look for a unique + // track for the given source. So generateItems here is a bit overkill. + generateItems( + `${inputs.id} screenShares$`, + function* (p) { + if (p.isScreenShareEnabled) + yield { + keys: ["screen-share"], + data: undefined, + }; + }, + (scope, _data$, key) => { + const id = `${inputs.id}:${key}`; + return participant.type === "local" + ? createLocalScreenShare(scope, { + ...inputs, + id, + participant$: participant.value$, + }) + : createRemoteScreenShare(scope, { + ...inputs, + id, + participant$: participant.value$, + pretendToBeDisconnected$, + }); + }, + ), + ), + ), + ), + ); + + const speaker$ = scope.behavior(observeSpeaker$(userMedia.speaking$)); + const presenter$ = scope.behavior( + screenShares$.pipe(map((screenShares) => screenShares.length > 0)), + ); + + return { + ...userMedia, + screenShares$, + bin$: scope.behavior( + combineLatest( + [ + speaker$, + presenter$, + userMedia.videoEnabled$, + userMedia.handRaised$, + userMedia.local ? userMedia.alwaysShow$ : of(null), + ], + (speaker, presenter, video, handRaised, alwaysShow) => { + if (alwaysShow !== null) + return alwaysShow + ? SortingBin.SelfAlwaysShown + : SortingBin.SelfNotAlwaysShown; + else if (presenter) return SortingBin.Presenters; + else if (speaker) return SortingBin.Speakers; + else if (handRaised) return SortingBin.HandRaised; + else if (video) return SortingBin.Video; + else return SortingBin.NoVideo; + }, + ), + ), + }; +} + +export type MediaItem = WrappedUserMediaViewModel | ScreenShareViewModel; diff --git a/src/state/MediaViewModel.test.ts b/src/state/MediaViewModel.test.ts index 92868216..73b396db 100644 --- a/src/state/MediaViewModel.test.ts +++ b/src/state/MediaViewModel.test.ts @@ -17,8 +17,8 @@ import { mockLocalParticipant, mockMediaDevices, mockRtcMembership, - createLocalMedia, - createRemoteMedia, + mockLocalMedia, + mockRemoteMedia, withTestScheduler, mockRemoteParticipant, } from "../utils/test"; @@ -45,7 +45,7 @@ const rtcMembership = mockRtcMembership("@alice:example.org", "AAAA"); test("control a participant's volume", () => { const setVolumeSpy = vi.fn(); - const vm = createRemoteMedia( + const vm = mockRemoteMedia( rtcMembership, {}, mockRemoteParticipant({ setVolume: setVolumeSpy }), @@ -54,33 +54,33 @@ test("control a participant's volume", () => { schedule("-ab---c---d|", { a() { // Try muting by toggling - vm.toggleLocallyMuted(); + vm.togglePlaybackMuted(); expect(setVolumeSpy).toHaveBeenLastCalledWith(0); }, b() { // Try unmuting by dragging the slider back up - vm.setLocalVolume(0.6); - vm.setLocalVolume(0.8); - vm.commitLocalVolume(); + vm.adjustPlaybackVolume(0.6); + vm.adjustPlaybackVolume(0.8); + vm.commitPlaybackVolume(); expect(setVolumeSpy).toHaveBeenCalledWith(0.6); expect(setVolumeSpy).toHaveBeenLastCalledWith(0.8); }, c() { // Try muting by dragging the slider back down - vm.setLocalVolume(0.2); - vm.setLocalVolume(0); - vm.commitLocalVolume(); + vm.adjustPlaybackVolume(0.2); + vm.adjustPlaybackVolume(0); + vm.commitPlaybackVolume(); expect(setVolumeSpy).toHaveBeenCalledWith(0.2); expect(setVolumeSpy).toHaveBeenLastCalledWith(0); }, d() { // Try unmuting by toggling - vm.toggleLocallyMuted(); + vm.togglePlaybackMuted(); // The volume should return to the last non-zero committed volume expect(setVolumeSpy).toHaveBeenLastCalledWith(0.8); }, }); - expectObservable(vm.localVolume$).toBe("ab(cd)(ef)g", { + expectObservable(vm.playbackVolume$).toBe("ab(cd)(ef)g", { a: 1, b: 0, c: 0.6, @@ -93,11 +93,11 @@ test("control a participant's volume", () => { }); test("toggle fit/contain for a participant's video", () => { - const vm = createRemoteMedia(rtcMembership, {}, mockRemoteParticipant({})); + const vm = mockRemoteMedia(rtcMembership, {}, mockRemoteParticipant({})); withTestScheduler(({ expectObservable, schedule }) => { schedule("-ab|", { - a: () => vm.toggleFitContain(), - b: () => vm.toggleFitContain(), + a: () => vm.toggleCropVideo(), + b: () => vm.toggleCropVideo(), }); expectObservable(vm.cropVideo$).toBe("abc", { a: true, @@ -108,7 +108,7 @@ test("toggle fit/contain for a participant's video", () => { }); test("local media remembers whether it should always be shown", () => { - const vm1 = createLocalMedia( + const vm1 = mockLocalMedia( rtcMembership, {}, mockLocalParticipant({}), @@ -120,7 +120,7 @@ test("local media remembers whether it should always be shown", () => { }); // Next local media should start out *not* always shown - const vm2 = createLocalMedia( + const vm2 = mockLocalMedia( rtcMembership, {}, mockLocalParticipant({}), @@ -166,7 +166,7 @@ test("switch cameras", async () => { const selectVideoInput = vi.fn(); - const vm = createLocalMedia( + const vm = mockLocalMedia( rtcMembership, {}, mockLocalParticipant({ @@ -206,17 +206,17 @@ test("switch cameras", async () => { }); test("remote media is in waiting state when participant has not yet connected", () => { - const vm = createRemoteMedia(rtcMembership, {}, null); // null participant + const vm = mockRemoteMedia(rtcMembership, {}, null); // null participant expect(vm.waitingForMedia$.value).toBe(true); }); test("remote media is not in waiting state when participant is connected", () => { - const vm = createRemoteMedia(rtcMembership, {}, mockRemoteParticipant({})); + const vm = mockRemoteMedia(rtcMembership, {}, mockRemoteParticipant({})); expect(vm.waitingForMedia$.value).toBe(false); }); test("remote media is not in waiting state when participant is connected with no publications", () => { - const vm = createRemoteMedia( + const vm = mockRemoteMedia( rtcMembership, {}, mockRemoteParticipant({ @@ -228,7 +228,7 @@ test("remote media is not in waiting state when participant is connected with no }); test("remote media is not in waiting state when user does not intend to publish anywhere", () => { - const vm = createRemoteMedia( + const vm = mockRemoteMedia( rtcMembership, {}, mockRemoteParticipant({}), diff --git a/src/state/MediaViewModel.ts b/src/state/MediaViewModel.ts index 7f806697..c8c468ac 100644 --- a/src/state/MediaViewModel.ts +++ b/src/state/MediaViewModel.ts @@ -29,7 +29,6 @@ import { } from "livekit-client"; import { logger } from "matrix-js-sdk/lib/logger"; import { - BehaviorSubject, type Observable, Subject, combineLatest, @@ -47,7 +46,7 @@ import { import { alwaysShowSelf } from "../settings/settings"; import { showConnectionStats } from "../settings/settings"; -import { accumulate } from "../utils/observable"; +import { createToggle$ } from "../utils/observable"; import { type EncryptionSystem } from "../e2ee/sharedKeyManagement"; import { E2eeType } from "../e2ee/e2eeType"; import { type ReactionOption } from "../reactions"; @@ -55,6 +54,7 @@ import { platform } from "../Platform"; import { type MediaDevices } from "./MediaDevices"; import { type Behavior } from "./Behavior"; import { type ObservableScope } from "./ObservableScope"; +import { createVolumeControls, type VolumeControls } from "./VolumeControls"; export function observeTrackReference$( participant: Participant, @@ -103,7 +103,7 @@ export function observeRtpStreamStats$( ); } -export function observeInboundRtpStreamStats$( +function observeInboundRtpStreamStats$( participant: Participant, source: Track.Source, ): Observable { @@ -112,15 +112,6 @@ export function observeInboundRtpStreamStats$( ); } -export function observeOutboundRtpStreamStats$( - participant: Participant, - source: Track.Source, -): Observable { - return observeRtpStreamStats$(participant, source, "outbound-rtp").pipe( - map((x) => x as RTCOutboundRtpStreamStats | undefined), - ); -} - function observeRemoteTrackReceivingOkay$( participant: Participant, source: Track.Source, @@ -218,74 +209,98 @@ export enum EncryptionStatus { PasswordInvalid, } -abstract class BaseMediaViewModel { +interface BaseMediaViewModel { + /** + * An opaque identifier for this media. + */ + id: string; + /** + * The Matrix user to which this media belongs. + */ + userId: string; + displayName$: Behavior; + mxcAvatarUrl$: Behavior; +} + +type BaseMediaInputs = BaseMediaViewModel; + +// This function exists to strip out superfluous data from the input object +function createBaseMedia({ + id, + userId, + displayName$, + mxcAvatarUrl$, +}: BaseMediaInputs): BaseMediaViewModel { + return { id, userId, displayName$, mxcAvatarUrl$ }; +} + +interface MemberMediaViewModel extends BaseMediaViewModel { /** * The LiveKit video track for this media. */ - public readonly video$: Behavior; + video$: Behavior; + /** + * The URL of the LiveKit focus on which this member should be publishing. + * Exposed for debugging. + */ + focusUrl$: Behavior; /** * Whether there should be a warning that this media is unencrypted. */ - public readonly unencryptedWarning$: Behavior; + unencryptedWarning$: Behavior; + encryptionStatus$: Behavior; +} - public readonly encryptionStatus$: Behavior; +interface MemberMediaInputs extends BaseMediaViewModel { + participant$: Behavior; + livekitRoom$: Behavior; + audioSource: AudioSource; + videoSource: VideoSource; + focusUrl$: Behavior; + encryptionSystem: EncryptionSystem; +} - /** - * Whether this media corresponds to the local participant. - */ - public abstract readonly local: boolean; - - private observeTrackReference$( +function createMemberMedia( + scope: ObservableScope, + { + participant$, + livekitRoom$, + audioSource, + videoSource, + focusUrl$, + encryptionSystem, + ...inputs + }: MemberMediaInputs, +): MemberMediaViewModel { + const trackBehavior$ = ( source: Track.Source, - ): Behavior { - return this.scope.behavior( - this.participant$.pipe( + ): Behavior => + scope.behavior( + participant$.pipe( switchMap((p) => !p ? of(undefined) : observeTrackReference$(p, source), ), ), ); - } - public constructor( - protected readonly scope: ObservableScope, - /** - * An opaque identifier for this media. - */ - public readonly id: string, - /** - * The Matrix user to which this media belongs. - */ - public readonly userId: string, - // We don't necessarily have a participant if a user connects via MatrixRTC but not (yet) through - // livekit. - protected readonly participant$: Behavior< - LocalParticipant | RemoteParticipant | null - >, + const audio$ = trackBehavior$(audioSource); + const video$ = trackBehavior$(videoSource); - encryptionSystem: EncryptionSystem, - audioSource: AudioSource, - videoSource: VideoSource, - protected readonly livekitRoom$: Behavior, - public readonly focusUrl$: Behavior, - public readonly displayName$: Behavior, - public readonly mxcAvatarUrl$: Behavior, - ) { - const audio$ = this.observeTrackReference$(audioSource); - this.video$ = this.observeTrackReference$(videoSource); - - this.unencryptedWarning$ = this.scope.behavior( + return { + ...createBaseMedia(inputs), + video$, + focusUrl$, + unencryptedWarning$: scope.behavior( combineLatest( - [audio$, this.video$], + [audio$, video$], (a, v) => encryptionSystem.kind !== E2eeType.NONE && (a?.publication.isEncrypted === false || v?.publication.isEncrypted === false), ), - ); - - this.encryptionStatus$ = this.scope.behavior( - this.participant$.pipe( + ), + encryptionStatus$: scope.behavior( + participant$.pipe( switchMap((participant): Observable => { if (!participant) { return of(EncryptionStatus.Connecting); @@ -346,132 +361,149 @@ abstract class BaseMediaViewModel { } }), ), - ); - } + ), + }; } -/** - * Some participant's media. - */ -export type MediaViewModel = UserMediaViewModel | ScreenShareViewModel; -export type UserMediaViewModel = - | LocalUserMediaViewModel - | RemoteUserMediaViewModel; +interface BaseUserMediaViewModel extends MemberMediaViewModel { + type: "user"; + speaking$: Behavior; + audioEnabled$: Behavior; + videoEnabled$: Behavior; + cropVideo$: Behavior; + toggleCropVideo: () => void; + /** + * The expected identity of the LiveKit participant. Exposed for debugging. + */ + rtcBackendIdentity: string; + handRaised$: Behavior; + reaction$: Behavior; + audioStreamStats$: Observable< + RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined + >; + videoStreamStats$: Observable< + RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined + >; +} -/** - * Some participant's user media. - */ -abstract class BaseUserMediaViewModel extends BaseMediaViewModel { - private readonly _speaking$ = this.scope.behavior( - this.participant$.pipe( - switchMap((p) => - p - ? observeParticipantEvents( - p, - ParticipantEvent.IsSpeakingChanged, - ).pipe(map((p) => p.isSpeaking)) - : of(false), - ), +interface BaseUserMediaInputs extends Omit< + MemberMediaInputs, + "audioSource" | "videoSource" +> { + rtcBackendIdentity: string; + handRaised$: Behavior; + reaction$: Behavior; + statsType: "inbound-rtp" | "outbound-rtp"; +} + +function createBaseUserMedia( + scope: ObservableScope, + { + rtcBackendIdentity, + handRaised$, + reaction$, + statsType, + ...inputs + }: BaseUserMediaInputs, +): BaseUserMediaViewModel { + const { participant$ } = inputs; + const media$ = scope.behavior( + participant$.pipe( + switchMap((p) => (p && observeParticipantMedia(p)) ?? of(undefined)), ), ); - /** - * Whether the participant is speaking. - */ - // Getter backed by a private field so that subclasses can override it - public get speaking$(): Behavior { - return this._speaking$; - } + const toggleCropVideo$ = new Subject(); - /** - * Whether this participant is sending audio (i.e. is unmuted on their side). - */ - public readonly audioEnabled$: Behavior; - - private readonly _videoEnabled$: Behavior; - /** - * Whether this participant is sending video. - */ - // Getter backed by a private field so that subclasses can override it - public get videoEnabled$(): Behavior { - return this._videoEnabled$; - } - - private readonly _cropVideo$ = new BehaviorSubject(true); - /** - * Whether the tile video should be contained inside the tile or be cropped to fit. - */ - public readonly cropVideo$: Behavior = this._cropVideo$; - - public constructor( - scope: ObservableScope, - id: string, - userId: string, - /** - * The expected identity of the LiveKit participant. Exposed for debugging. - */ - public readonly rtcBackendIdentity: string, - participant$: Behavior, - encryptionSystem: EncryptionSystem, - livekitRoom$: Behavior, - focusUrl$: Behavior, - displayName$: Behavior, - mxcAvatarUrl$: Behavior, - public readonly handRaised$: Behavior, - public readonly reaction$: Behavior, - ) { - super( - scope, - id, - userId, - participant$, - encryptionSystem, - Track.Source.Microphone, - Track.Source.Camera, - livekitRoom$, - focusUrl$, - displayName$, - mxcAvatarUrl$, - ); - - const media$ = this.scope.behavior( + return { + ...createMemberMedia(scope, { + ...inputs, + audioSource: Track.Source.Microphone, + videoSource: Track.Source.Camera, + }), + type: "user", + speaking$: scope.behavior( participant$.pipe( - switchMap((p) => (p && observeParticipantMedia(p)) ?? of(undefined)), + switchMap((p) => + p + ? observeParticipantEvents( + p, + ParticipantEvent.IsSpeakingChanged, + ).pipe(map((p) => p.isSpeaking)) + : of(false), + ), ), - ); - this.audioEnabled$ = this.scope.behavior( + ), + audioEnabled$: scope.behavior( media$.pipe(map((m) => m?.microphoneTrack?.isMuted === false)), - ); - this._videoEnabled$ = this.scope.behavior( + ), + videoEnabled$: scope.behavior( media$.pipe(map((m) => m?.cameraTrack?.isMuted === false)), - ); - } - - public toggleFitContain(): void { - this._cropVideo$.next(!this._cropVideo$.value); - } - - public get local(): boolean { - return this instanceof LocalUserMediaViewModel; - } - - public abstract get audioStreamStats$(): Observable< - RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined - >; - public abstract get videoStreamStats$(): Observable< - RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined - >; + ), + cropVideo$: createToggle$(scope, true, toggleCropVideo$), + toggleCropVideo: () => toggleCropVideo$.next(), + rtcBackendIdentity, + handRaised$, + reaction$, + audioStreamStats$: combineLatest([ + participant$, + showConnectionStats.value$, + ]).pipe( + switchMap(([p, showConnectionStats]) => { + // + if (!p || !showConnectionStats) return of(undefined); + return observeRtpStreamStats$(p, Track.Source.Microphone, statsType); + }), + ), + videoStreamStats$: combineLatest([ + participant$, + showConnectionStats.value$, + ]).pipe( + switchMap(([p, showConnectionStats]) => { + if (!p || !showConnectionStats) return of(undefined); + return observeRtpStreamStats$(p, Track.Source.Camera, statsType); + }), + ), + }; } -/** - * The local participant's user media. - */ -export class LocalUserMediaViewModel extends BaseUserMediaViewModel { +export interface LocalUserMediaViewModel extends BaseUserMediaViewModel { + local: true; + /** + * Whether the video should be mirrored. + */ + mirror$: Behavior; + /** + * Whether to show this tile in a highly visible location near the start of + * the grid. + */ + alwaysShow$: Behavior; + setAlwaysShow: (value: boolean) => void; + switchCamera$: Behavior<(() => void) | null>; +} + +export interface LocalUserMediaInputs extends Omit< + BaseUserMediaInputs, + "statsType" +> { + participant$: Behavior; + mediaDevices: MediaDevices; +} + +export function createLocalUserMedia( + scope: ObservableScope, + { mediaDevices, ...inputs }: LocalUserMediaInputs, +): LocalUserMediaViewModel { + const baseUserMedia = createBaseUserMedia(scope, { + ...inputs, + statsType: "outbound-rtp", + }); + /** * The local video track as an observable that emits whenever the track * changes, the camera is switched, or the track is muted. */ - private readonly videoTrack$: Observable = - this.video$.pipe( + const videoTrack$: Observable = + baseUserMedia.video$.pipe( switchMap((v) => { const track = v?.publication.track; if (!(track instanceof LocalVideoTrack)) return of(null); @@ -488,35 +520,25 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { }), ); - /** - * Whether the video should be mirrored. - */ - public readonly mirror$ = this.scope.behavior( - this.videoTrack$.pipe( - // Mirror only front-facing cameras (those that face the user) - map( - (track) => - track !== null && - facingModeFromLocalTrack(track).facingMode === "user", + return { + ...baseUserMedia, + local: true, + mirror$: scope.behavior( + videoTrack$.pipe( + // Mirror only front-facing cameras (those that face the user) + map( + (track) => + track !== null && + facingModeFromLocalTrack(track).facingMode === "user", + ), ), ), - ); - - /** - * Whether to show this tile in a highly visible location near the start of - * the grid. - */ - public readonly alwaysShow$ = alwaysShowSelf.value$; - public readonly setAlwaysShow = alwaysShowSelf.setValue; - - /** - * Callback for switching between the front and back cameras. - */ - public readonly switchCamera$: Behavior<(() => void) | null> = - this.scope.behavior( + alwaysShow$: alwaysShowSelf.value$, + setAlwaysShow: alwaysShowSelf.setValue, + switchCamera$: scope.behavior( platform === "desktop" ? of(null) - : this.videoTrack$.pipe( + : videoTrack$.pipe( map((track) => { if (track === null) return null; const facingMode = facingModeFromLocalTrack(track).facingMode; @@ -535,272 +557,157 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { const deviceId = track.mediaStreamTrack.getSettings().deviceId; if (deviceId !== undefined) - this.mediaDevices.videoInput.select(deviceId); + mediaDevices.videoInput.select(deviceId); }) .catch((e) => logger.error("Failed to switch camera", facingMode, e), ); }), ), - ); - - public constructor( - scope: ObservableScope, - id: string, - userId: string, - rtcBackendIdentity: string, - participant$: Behavior, - encryptionSystem: EncryptionSystem, - livekitRoom$: Behavior, - focusUrl$: Behavior, - private readonly mediaDevices: MediaDevices, - displayName$: Behavior, - mxcAvatarUrl$: Behavior, - handRaised$: Behavior, - reaction$: Behavior, - ) { - super( - scope, - id, - userId, - rtcBackendIdentity, - participant$, - encryptionSystem, - livekitRoom$, - focusUrl$, - displayName$, - mxcAvatarUrl$, - handRaised$, - reaction$, - ); - } - - public audioStreamStats$ = combineLatest([ - this.participant$, - showConnectionStats.value$, - ]).pipe( - switchMap(([p, showConnectionStats]) => { - if (!p || !showConnectionStats) return of(undefined); - return observeOutboundRtpStreamStats$(p, Track.Source.Microphone); - }), - ); - - public videoStreamStats$ = combineLatest([ - this.participant$, - showConnectionStats.value$, - ]).pipe( - switchMap(([p, showConnectionStats]) => { - if (!p || !showConnectionStats) return of(undefined); - return observeOutboundRtpStreamStats$(p, Track.Source.Camera); - }), - ); + ), + }; } -/** - * A remote participant's user media. - */ -export class RemoteUserMediaViewModel extends BaseUserMediaViewModel { +export interface RemoteUserMediaViewModel + extends BaseUserMediaViewModel, VolumeControls { + local: false; /** * Whether we are waiting for this user's LiveKit participant to exist. This * could be because either we or the remote party are still connecting. */ - public readonly waitingForMedia$ = this.scope.behavior( - combineLatest( - [this.livekitRoom$, this.participant$], - (livekitRoom, participant) => - // If livekitRoom is undefined, the user is not attempting to publish on - // any transport and so we shouldn't expect a participant. (They might - // be a subscribe-only bot for example.) - livekitRoom !== undefined && participant === null, - ), - ); - - // This private field is used to override the value from the superclass - private __speaking$: Behavior; - public get speaking$(): Behavior { - return this.__speaking$; - } - - private readonly locallyMutedToggle$ = new Subject(); - private readonly localVolumeAdjustment$ = new Subject(); - private readonly localVolumeCommit$ = new Subject(); - - /** - * The volume to which this participant's audio is set, as a scalar - * multiplier. - */ - public readonly localVolume$ = this.scope.behavior( - merge( - this.locallyMutedToggle$.pipe(map(() => "toggle mute" as const)), - this.localVolumeAdjustment$, - this.localVolumeCommit$.pipe(map(() => "commit" as const)), - ).pipe( - accumulate({ volume: 1, committedVolume: 1 }, (state, event) => { - switch (event) { - case "toggle mute": - return { - ...state, - volume: state.volume === 0 ? state.committedVolume : 0, - }; - case "commit": - // Dragging the slider to zero should have the same effect as - // muting: keep the original committed volume, as if it were never - // dragged - return { - ...state, - committedVolume: - state.volume === 0 ? state.committedVolume : state.volume, - }; - default: - // Volume adjustment - return { ...state, volume: event }; - } - }), - map(({ volume }) => volume), - ), - ); - - // This private field is used to override the value from the superclass - private __videoEnabled$: Behavior; - public get videoEnabled$(): Behavior { - return this.__videoEnabled$; - } - - /** - * Whether this participant's audio is disabled. - */ - public readonly locallyMuted$ = this.scope.behavior( - this.localVolume$.pipe(map((volume) => volume === 0)), - ); - - public constructor( - scope: ObservableScope, - id: string, - userId: string, - rtcBackendIdentity: string, - participant$: Behavior, - encryptionSystem: EncryptionSystem, - livekitRoom$: Behavior, - focusUrl$: Behavior, - private readonly pretendToBeDisconnected$: Behavior, - displayName$: Behavior, - mxcAvatarUrl$: Behavior, - handRaised$: Behavior, - reaction$: Behavior, - ) { - super( - scope, - id, - userId, - rtcBackendIdentity, - participant$, - encryptionSystem, - livekitRoom$, - focusUrl$, - displayName$, - mxcAvatarUrl$, - handRaised$, - reaction$, - ); - - this.__speaking$ = this.scope.behavior( - pretendToBeDisconnected$.pipe( - switchMap((disconnected) => - disconnected ? of(false) : super.speaking$, - ), - ), - ); - - this.__videoEnabled$ = this.scope.behavior( - pretendToBeDisconnected$.pipe( - switchMap((disconnected) => - disconnected ? of(false) : super.videoEnabled$, - ), - ), - ); - - // Sync the local volume with LiveKit - combineLatest([ - participant$, - // The local volume, taking into account whether we're supposed to pretend - // that the audio stream is disconnected (since we don't necessarily want - // that to modify the UI state). - this.pretendToBeDisconnected$.pipe( - switchMap((disconnected) => (disconnected ? of(0) : this.localVolume$)), - this.scope.bind(), - ), - ]).subscribe(([p, volume]) => p?.setVolume(volume)); - } - - public toggleLocallyMuted(): void { - this.locallyMutedToggle$.next(); - } - - public setLocalVolume(value: number): void { - this.localVolumeAdjustment$.next(value); - } - - public commitLocalVolume(): void { - this.localVolumeCommit$.next(); - } - - public audioStreamStats$ = combineLatest([ - this.participant$, - showConnectionStats.value$, - ]).pipe( - switchMap(([p, showConnectionStats]) => { - if (!p || !showConnectionStats) return of(undefined); - return observeInboundRtpStreamStats$(p, Track.Source.Microphone); - }), - ); - - public videoStreamStats$ = combineLatest([ - this.participant$, - showConnectionStats.value$, - ]).pipe( - switchMap(([p, showConnectionStats]) => { - if (!p || !showConnectionStats) return of(undefined); - return observeInboundRtpStreamStats$(p, Track.Source.Camera); - }), - ); + waitingForMedia$: Behavior; } -/** - * Some participant's screen share media. - */ -export class ScreenShareViewModel extends BaseMediaViewModel { +export interface RemoteUserMediaInputs extends Omit< + BaseUserMediaInputs, + "statsType" +> { + participant$: Behavior; + pretendToBeDisconnected$: Behavior; +} + +export function createRemoteUserMedia( + scope: ObservableScope, + { pretendToBeDisconnected$, ...inputs }: RemoteUserMediaInputs, +): RemoteUserMediaViewModel { + const baseUserMedia = createBaseUserMedia(scope, { + ...inputs, + statsType: "inbound-rtp", + }); + + return { + ...baseUserMedia, + ...createVolumeControls(scope, { + pretendToBeDisconnected$, + sink$: scope.behavior( + inputs.participant$.pipe(map((p) => (volume) => p?.setVolume(volume))), + ), + }), + local: false, + speaking$: scope.behavior( + pretendToBeDisconnected$.pipe( + switchMap((disconnected) => + disconnected ? of(false) : baseUserMedia.speaking$, + ), + ), + ), + videoEnabled$: scope.behavior( + pretendToBeDisconnected$.pipe( + switchMap((disconnected) => + disconnected ? of(false) : baseUserMedia.videoEnabled$, + ), + ), + ), + waitingForMedia$: scope.behavior( + combineLatest( + [inputs.livekitRoom$, inputs.participant$], + (livekitRoom, participant) => + // If livekitRoom is undefined, the user is not attempting to publish on + // any transport and so we shouldn't expect a participant. (They might + // be a subscribe-only bot for example.) + livekitRoom !== undefined && participant === null, + ), + ), + }; +} + +interface BaseScreenShareViewModel extends MemberMediaViewModel { + type: "screen share"; +} + +type BaseScreenShareInputs = Omit< + MemberMediaInputs, + "audioSource" | "videoSource" +>; + +function createBaseScreenShare( + scope: ObservableScope, + inputs: BaseScreenShareInputs, +): BaseScreenShareViewModel { + return { + ...createMemberMedia(scope, { + ...inputs, + audioSource: Track.Source.ScreenShareAudio, + videoSource: Track.Source.ScreenShare, + }), + type: "screen share", + }; +} + +export interface LocalScreenShareViewModel extends BaseScreenShareViewModel { + local: true; +} + +interface LocalScreenShareInputs extends BaseScreenShareInputs { + participant$: Behavior; +} + +export function createLocalScreenShare( + scope: ObservableScope, + inputs: LocalScreenShareInputs, +): LocalScreenShareViewModel { + return { ...createBaseScreenShare(scope, inputs), local: true }; +} + +export interface RemoteScreenShareViewModel extends BaseScreenShareViewModel { + local: false; /** * Whether this screen share's video should be displayed. */ - public readonly videoEnabled$ = this.scope.behavior( - this.pretendToBeDisconnected$.pipe(map((disconnected) => !disconnected)), - ); - - public constructor( - scope: ObservableScope, - id: string, - userId: string, - participant$: Behavior, - encryptionSystem: EncryptionSystem, - livekitRoom$: Behavior, - focusUrl$: Behavior, - private readonly pretendToBeDisconnected$: Behavior, - displayName$: Behavior, - mxcAvatarUrl$: Behavior, - public readonly local: boolean, - ) { - super( - scope, - id, - userId, - participant$, - encryptionSystem, - Track.Source.ScreenShareAudio, - Track.Source.ScreenShare, - livekitRoom$, - focusUrl$, - displayName$, - mxcAvatarUrl$, - ); - } + videoEnabled$: Behavior; } + +interface RemoteScreenShareInputs extends BaseScreenShareInputs { + participant$: Behavior; + pretendToBeDisconnected$: Behavior; +} + +export function createRemoteScreenShare( + scope: ObservableScope, + { pretendToBeDisconnected$, ...inputs }: RemoteScreenShareInputs, +): RemoteScreenShareViewModel { + return { + ...createBaseScreenShare(scope, inputs), + local: false, + videoEnabled$: scope.behavior( + pretendToBeDisconnected$.pipe(map((disconnected) => !disconnected)), + ), + }; +} + +/** + * Some participant's media. + */ +export type MediaViewModel = UserMediaViewModel | ScreenShareViewModel; +/** + * Some participant's user media (i.e. their microphone and camera feed). + */ +export type UserMediaViewModel = + | LocalUserMediaViewModel + | RemoteUserMediaViewModel; +/** + * Some participant's screen share media. + */ +export type ScreenShareViewModel = + | LocalScreenShareViewModel + | RemoteScreenShareViewModel; diff --git a/src/state/ScreenShare.ts b/src/state/ScreenShare.ts deleted file mode 100644 index 6c908b1f..00000000 --- a/src/state/ScreenShare.ts +++ /dev/null @@ -1,53 +0,0 @@ -/* -Copyright 2025 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - type LocalParticipant, - type RemoteParticipant, - type Room as LivekitRoom, -} from "livekit-client"; - -import { type ObservableScope } from "./ObservableScope.ts"; -import { ScreenShareViewModel } from "./MediaViewModel.ts"; -import type { EncryptionSystem } from "../e2ee/sharedKeyManagement.ts"; -import { constant, type Behavior } from "./Behavior.ts"; - -/** - * A screen share media item to be presented in a tile. This is a thin wrapper - * around ScreenShareViewModel which essentially just establishes an - * ObservableScope for behaviors that the view model depends on. - */ -export class ScreenShare { - public readonly vm: ScreenShareViewModel; - - public constructor( - private readonly scope: ObservableScope, - id: string, - userId: string, - participant: LocalParticipant | RemoteParticipant, - encryptionSystem: EncryptionSystem, - livekitRoom$: Behavior, - focusUrl$: Behavior, - pretendToBeDisconnected$: Behavior, - displayName$: Behavior, - mxcAvatarUrl$: Behavior, - ) { - this.vm = new ScreenShareViewModel( - this.scope, - id, - userId, - constant(participant), - encryptionSystem, - livekitRoom$, - focusUrl$, - pretendToBeDisconnected$, - displayName$, - mxcAvatarUrl$, - participant.isLocal, - ); - } -} diff --git a/src/state/UserMedia.ts b/src/state/UserMedia.ts deleted file mode 100644 index 2adc9134..00000000 --- a/src/state/UserMedia.ts +++ /dev/null @@ -1,209 +0,0 @@ -/* -Copyright 2025 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { combineLatest, map, type Observable, of, switchMap } from "rxjs"; -import { - type LocalParticipant, - ParticipantEvent, - type RemoteParticipant, - type Room as LivekitRoom, -} from "livekit-client"; -import { observeParticipantEvents } from "@livekit/components-core"; - -import { type ObservableScope } from "./ObservableScope.ts"; -import { - LocalUserMediaViewModel, - RemoteUserMediaViewModel, - type UserMediaViewModel, -} from "./MediaViewModel.ts"; -import type { Behavior } from "./Behavior.ts"; -import type { EncryptionSystem } from "../e2ee/sharedKeyManagement.ts"; -import type { MediaDevices } from "./MediaDevices.ts"; -import type { ReactionOption } from "../reactions"; -import { observeSpeaker$ } from "./observeSpeaker.ts"; -import { generateItems } from "../utils/observable.ts"; -import { ScreenShare } from "./ScreenShare.ts"; -import { type TaggedParticipant } from "./CallViewModel/remoteMembers/MatrixLivekitMembers.ts"; - -/** - * Sorting bins defining the order in which media tiles appear in the layout. - */ -enum SortingBin { - /** - * Yourself, when the "always show self" option is on. - */ - SelfAlwaysShown, - /** - * Participants that are sharing their screen. - */ - Presenters, - /** - * Participants that have been speaking recently. - */ - Speakers, - /** - * Participants that have their hand raised. - */ - HandRaised, - /** - * Participants with video. - */ - Video, - /** - * Participants not sharing any video. - */ - NoVideo, - /** - * Yourself, when the "always show self" option is off. - */ - SelfNotAlwaysShown, -} - -/** - * A user media item to be presented in a tile. This is a thin wrapper around - * UserMediaViewModel which additionally determines the media item's sorting bin - * for inclusion in the call layout and tracks associated screen shares. - */ -export class UserMedia { - public readonly vm: UserMediaViewModel = - this.participant.type === "local" - ? new LocalUserMediaViewModel( - this.scope, - this.id, - this.userId, - this.rtcBackendIdentity, - this.participant.value$, - this.encryptionSystem, - this.livekitRoom$, - this.focusUrl$, - this.mediaDevices, - this.displayName$, - this.mxcAvatarUrl$, - this.scope.behavior(this.handRaised$), - this.scope.behavior(this.reaction$), - ) - : new RemoteUserMediaViewModel( - this.scope, - this.id, - this.userId, - this.rtcBackendIdentity, - this.participant.value$, - this.encryptionSystem, - this.livekitRoom$, - this.focusUrl$, - this.pretendToBeDisconnected$, - this.displayName$, - this.mxcAvatarUrl$, - this.scope.behavior(this.handRaised$), - this.scope.behavior(this.reaction$), - ); - - private readonly speaker$ = this.scope.behavior( - observeSpeaker$(this.vm.speaking$), - ); - - // TypeScript needs this widening of the type to happen in a separate statement - private readonly participant$: Behavior< - LocalParticipant | RemoteParticipant | null - > = this.participant.value$; - - /** - * All screen share media associated with this user media. - */ - public readonly screenShares$ = this.scope.behavior( - this.participant$.pipe( - switchMap((p) => - p === null - ? of([]) - : observeParticipantEvents( - p, - ParticipantEvent.TrackPublished, - ParticipantEvent.TrackUnpublished, - ParticipantEvent.LocalTrackPublished, - ParticipantEvent.LocalTrackUnpublished, - ).pipe( - // Technically more than one screen share might be possible... our - // MediaViewModels don't support it though since they look for a unique - // track for the given source. So generateItems here is a bit overkill. - generateItems( - `${this.id} screenShares$`, - function* (p) { - if (p.isScreenShareEnabled) - yield { - keys: ["screen-share"], - data: undefined, - }; - }, - (scope, _data$, key) => - new ScreenShare( - scope, - `${this.id}:${key}`, - this.userId, - p, - this.encryptionSystem, - this.livekitRoom$, - this.focusUrl$, - this.pretendToBeDisconnected$, - this.displayName$, - this.mxcAvatarUrl$, - ), - ), - ), - ), - ), - ); - - private readonly presenter$ = this.scope.behavior( - this.screenShares$.pipe(map((screenShares) => screenShares.length > 0)), - ); - - /** - * Which sorting bin the media item should be placed in. - */ - // This is exposed here rather than by UserMediaViewModel because it's only - // relevant to the layout algorithms; the MediaView component should be - // ignorant of this value. - public readonly bin$ = combineLatest( - [ - this.speaker$, - this.presenter$, - this.vm.videoEnabled$, - this.vm.handRaised$, - this.vm instanceof LocalUserMediaViewModel - ? this.vm.alwaysShow$ - : of(false), - ], - (speaker, presenter, video, handRaised, alwaysShow) => { - if (this.vm.local) - return alwaysShow - ? SortingBin.SelfAlwaysShown - : SortingBin.SelfNotAlwaysShown; - else if (presenter) return SortingBin.Presenters; - else if (speaker) return SortingBin.Speakers; - else if (handRaised) return SortingBin.HandRaised; - else if (video) return SortingBin.Video; - else return SortingBin.NoVideo; - }, - ); - - public constructor( - private readonly scope: ObservableScope, - public readonly id: string, - private readonly userId: string, - private readonly rtcBackendIdentity: string, - private readonly participant: TaggedParticipant, - private readonly encryptionSystem: EncryptionSystem, - private readonly livekitRoom$: Behavior, - private readonly focusUrl$: Behavior, - private readonly mediaDevices: MediaDevices, - private readonly pretendToBeDisconnected$: Behavior, - private readonly displayName$: Behavior, - private readonly mxcAvatarUrl$: Behavior, - private readonly handRaised$: Observable, - private readonly reaction$: Observable, - ) {} -} diff --git a/src/state/VolumeControls.ts b/src/state/VolumeControls.ts new file mode 100644 index 00000000..beb7ae00 --- /dev/null +++ b/src/state/VolumeControls.ts @@ -0,0 +1,101 @@ +/* +Copyright 2026 Element Software Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { combineLatest, map, merge, of, Subject, switchMap } from "rxjs"; + +import { type Behavior } from "./Behavior"; +import { type ObservableScope } from "./ObservableScope"; +import { accumulate } from "../utils/observable"; + +/** + * Controls for audio playback volume. + */ +export interface VolumeControls { + /** + * The volume to which the audio is set, as a scalar multiplier. + */ + playbackVolume$: Behavior; + /** + * Whether playback of this audio is disabled. + */ + playbackMuted$: Behavior; + togglePlaybackMuted: () => void; + adjustPlaybackVolume: (value: number) => void; + commitPlaybackVolume: () => void; +} + +interface VolumeControlsInputs { + pretendToBeDisconnected$: Behavior; + /** + * The callback to run to notify the module performing audio playback of the + * requested volume. + */ + sink$: Behavior<(volume: number) => void>; +} + +/** + * Creates a set of controls for audio playback volume and syncs this with the + * audio playback module for the duration of the scope. + */ +export function createVolumeControls( + scope: ObservableScope, + { pretendToBeDisconnected$, sink$ }: VolumeControlsInputs, +): VolumeControls { + const toggleMuted$ = new Subject<"toggle mute">(); + const adjustVolume$ = new Subject(); + const commitVolume$ = new Subject<"commit">(); + + const playbackVolume$ = scope.behavior( + merge(toggleMuted$, adjustVolume$, commitVolume$).pipe( + accumulate({ volume: 1, committedVolume: 1 }, (state, event) => { + switch (event) { + case "toggle mute": + return { + ...state, + volume: state.volume === 0 ? state.committedVolume : 0, + }; + case "commit": + // Dragging the slider to zero should have the same effect as + // muting: keep the original committed volume, as if it were never + // dragged + return { + ...state, + committedVolume: + state.volume === 0 ? state.committedVolume : state.volume, + }; + default: + // Volume adjustment + return { ...state, volume: event }; + } + }), + map(({ volume }) => volume), + ), + ); + + // Sync the requested volume with the audio playback module + combineLatest([ + sink$, + // The playback volume, taking into account whether we're supposed to + // pretend that the audio stream is disconnected (since we don't necessarily + // want that to modify the UI state). + pretendToBeDisconnected$.pipe( + switchMap((disconnected) => (disconnected ? of(0) : playbackVolume$)), + ), + ]) + .pipe(scope.bind()) + .subscribe(([sink, volume]) => sink(volume)); + + return { + playbackVolume$, + playbackMuted$: scope.behavior( + playbackVolume$.pipe(map((volume) => volume === 0)), + ), + togglePlaybackMuted: () => toggleMuted$.next("toggle mute"), + adjustPlaybackVolume: (value: number) => adjustVolume$.next(value), + commitPlaybackVolume: () => commitVolume$.next("commit"), + }; +} diff --git a/src/tile/GridTile.test.tsx b/src/tile/GridTile.test.tsx index 9bc0efb2..02f09a17 100644 --- a/src/tile/GridTile.test.tsx +++ b/src/tile/GridTile.test.tsx @@ -14,7 +14,7 @@ import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; import { GridTile } from "./GridTile"; import { mockRtcMembership, - createRemoteMedia, + mockRemoteMedia, mockRemoteParticipant, } from "../utils/test"; import { GridTileViewModel } from "../state/TileViewModel"; @@ -29,7 +29,7 @@ global.IntersectionObserver = class MockIntersectionObserver { } as unknown as typeof IntersectionObserver; test("GridTile is accessible", async () => { - const vm = createRemoteMedia( + const vm = mockRemoteMedia( mockRtcMembership("@alice:example.org", "AAAA"), { rawDisplayName: "Alice", diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index 92262f05..370e0723 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -41,7 +41,7 @@ import { useObservableEagerState } from "observable-hooks"; import styles from "./GridTile.module.css"; import { type UserMediaViewModel, - LocalUserMediaViewModel, + type LocalUserMediaViewModel, type RemoteUserMediaViewModel, } from "../state/MediaViewModel"; import { Slider } from "../Slider"; @@ -68,7 +68,7 @@ interface TileProps { interface UserMediaTileProps extends TileProps { vm: UserMediaViewModel; mirror: boolean; - locallyMuted: boolean; + playbackMuted: boolean; waitingForMedia?: boolean; primaryButton?: ReactNode; menuStart?: ReactNode; @@ -79,7 +79,7 @@ const UserMediaTile: FC = ({ ref, vm, showSpeakingIndicators, - locallyMuted, + playbackMuted, waitingForMedia, primaryButton, menuStart, @@ -109,7 +109,7 @@ const UserMediaTile: FC = ({ const onSelectFitContain = useCallback( (e: Event) => { e.preventDefault(); - vm.toggleFitContain(); + vm.toggleCropVideo(); }, [vm], ); @@ -117,12 +117,12 @@ const UserMediaTile: FC = ({ const handRaised = useBehavior(vm.handRaised$); const reaction = useBehavior(vm.reaction$); - const AudioIcon = locallyMuted + const AudioIcon = playbackMuted ? VolumeOffSolidIcon : audioEnabled ? MicOnSolidIcon : MicOffSolidIcon; - const audioIconLabel = locallyMuted + const audioIconLabel = playbackMuted ? t("video_tile.muted_for_me") : audioEnabled ? t("microphone_on") @@ -166,7 +166,7 @@ const UserMediaTile: FC = ({ width={20} height={20} aria-label={audioIconLabel} - data-muted={locallyMuted || !audioEnabled} + data-muted={playbackMuted || !audioEnabled} className={styles.muteIcon} /> } @@ -245,7 +245,7 @@ const LocalUserMediaTile: FC = ({ = ({ }) => { const { t } = useTranslation(); const waitingForMedia = useBehavior(vm.waitingForMedia$); - const locallyMuted = useBehavior(vm.locallyMuted$); - const localVolume = useBehavior(vm.localVolume$); + const playbackMuted = useBehavior(vm.playbackMuted$); + const playbackVolume = useBehavior(vm.playbackVolume$); const onSelectMute = useCallback( (e: Event) => { e.preventDefault(); - vm.toggleLocallyMuted(); + vm.togglePlaybackMuted(); }, [vm], ); - const onChangeLocalVolume = useCallback( - (v: number) => vm.setLocalVolume(v), - [vm], - ); - const onCommitLocalVolume = useCallback(() => vm.commitLocalVolume(), [vm]); - const VolumeIcon = locallyMuted ? VolumeOffIcon : VolumeOnIcon; + const VolumeIcon = playbackMuted ? VolumeOffIcon : VolumeOnIcon; return ( {/* TODO: Figure out how to make this slider keyboard accessible */} @@ -332,9 +327,9 @@ const RemoteUserMediaTile: FC = ({ = ({ const displayName = useBehavior(media.displayName$); const mxcAvatarUrl = useBehavior(media.mxcAvatarUrl$); - if (media instanceof LocalUserMediaViewModel) { + if (media.local) { return ( = ({ }) => { const { t } = useTranslation(); const [handRaiseTimerVisible] = useSetting(showHandRaisedTimer); - const [showConnectioStats] = useSetting(showConnectionStats); + const [showConnectionStats] = useSetting(showConnectionStatsSetting); const avatarSize = Math.round(Math.min(targetWidth, targetHeight) / 2); @@ -139,10 +139,10 @@ export const MediaView: FC = ({ {waitingForMedia && (
{t("video_tile.waiting_for_media")} - {showConnectioStats ? " " + rtcBackendIdentity : ""} + {showConnectionStats ? " " + rtcBackendIdentity : ""}
)} - {(audioStreamStats || videoStreamStats) && ( + {showConnectionStats && ( <> { - const vm1 = createRemoteMedia( + const vm1 = mockRemoteMedia( mockRtcMembership("@alice:example.org", "AAAA"), { rawDisplayName: "Alice", @@ -37,7 +37,7 @@ test("SpotlightTile is accessible", async () => { mockRemoteParticipant({}), ); - const vm2 = createLocalMedia( + const vm2 = mockLocalMedia( mockRtcMembership("@bob:example.org", "BBBB"), { rawDisplayName: "Bob", diff --git a/src/tile/SpotlightTile.tsx b/src/tile/SpotlightTile.tsx index e685327e..c0f1135f 100644 --- a/src/tile/SpotlightTile.tsx +++ b/src/tile/SpotlightTile.tsx @@ -34,11 +34,12 @@ import { MediaView } from "./MediaView"; import styles from "./SpotlightTile.module.css"; import { type EncryptionStatus, - LocalUserMediaViewModel, + type LocalUserMediaViewModel, type MediaViewModel, - ScreenShareViewModel, type UserMediaViewModel, type RemoteUserMediaViewModel, + type ScreenShareViewModel, + type RemoteScreenShareViewModel, } from "../state/MediaViewModel"; import { useInitial } from "../useInitial"; import { useMergedRefs } from "../useMergedRefs"; @@ -54,7 +55,6 @@ interface SpotlightItemBaseProps { targetWidth: number; targetHeight: number; video: TrackReferenceOrPlaceholder | undefined; - videoEnabled: boolean; userId: string; unencryptedWarning: boolean; encryptionStatus: EncryptionStatus; @@ -67,6 +67,7 @@ interface SpotlightItemBaseProps { interface SpotlightUserMediaItemBaseProps extends SpotlightItemBaseProps { videoFit: "contain" | "cover"; + videoEnabled: boolean; } interface SpotlightLocalUserMediaItemProps extends SpotlightUserMediaItemBaseProps { @@ -106,14 +107,16 @@ const SpotlightUserMediaItem: FC = ({ ...props }) => { const cropVideo = useBehavior(vm.cropVideo$); + const videoEnabled = useBehavior(vm.videoEnabled$); const baseProps: SpotlightUserMediaItemBaseProps & RefAttributes = { videoFit: cropVideo ? "cover" : "contain", + videoEnabled, ...props, }; - return vm instanceof LocalUserMediaViewModel ? ( + return vm.local ? ( ) : ( @@ -122,6 +125,31 @@ const SpotlightUserMediaItem: FC = ({ SpotlightUserMediaItem.displayName = "SpotlightUserMediaItem"; +interface SpotlightScreenShareItemProps extends SpotlightItemBaseProps { + vm: ScreenShareViewModel; + videoEnabled: boolean; +} + +const SpotlightScreenShareItem: FC = ({ + vm, + ...props +}) => { + return ; +}; + +interface SpotlightRemoteScreenShareItemProps extends SpotlightItemBaseProps { + vm: RemoteScreenShareViewModel; +} + +const SpotlightRemoteScreenShareItem: FC< + SpotlightRemoteScreenShareItemProps +> = ({ vm, ...props }) => { + const videoEnabled = useBehavior(vm.videoEnabled$); + return ( + + ); +}; + interface SpotlightItemProps { ref?: Ref; vm: MediaViewModel; @@ -152,7 +180,6 @@ const SpotlightItem: FC = ({ const displayName = useBehavior(vm.displayName$); const mxcAvatarUrl = useBehavior(vm.mxcAvatarUrl$); const video = useBehavior(vm.video$); - const videoEnabled = useBehavior(vm.videoEnabled$); const unencryptedWarning = useBehavior(vm.unencryptedWarning$); const encryptionStatus = useBehavior(vm.encryptionStatus$); @@ -178,7 +205,6 @@ const SpotlightItem: FC = ({ targetWidth, targetHeight, video: video ?? undefined, - videoEnabled, userId: vm.userId, unencryptedWarning, focusUrl, @@ -189,10 +215,12 @@ const SpotlightItem: FC = ({ "aria-hidden": ariaHidden, }; - return vm instanceof ScreenShareViewModel ? ( - + if (vm.type === "user") + return ; + return vm.local ? ( + ) : ( - + ); }; diff --git a/src/utils/observable.ts b/src/utils/observable.ts index 2e19748b..353dc877 100644 --- a/src/utils/observable.ts +++ b/src/utils/observable.ts @@ -61,6 +61,20 @@ export function accumulate( events$.pipe(scan(update, initial), startWith(initial)); } +/** + * Given a source of toggle events, creates a Behavior whose value toggles + * between `true` and `false`. + */ +export function createToggle$( + scope: ObservableScope, + initialValue: boolean, + toggle$: Observable, +): Behavior { + return scope.behavior( + toggle$.pipe(accumulate(initialValue, (state) => !state)), + ); +} + const switchSymbol = Symbol("switch"); /** diff --git a/src/utils/test.ts b/src/utils/test.ts index d78bdf42..9231a3d1 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -53,8 +53,10 @@ import { import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; import { - LocalUserMediaViewModel, - RemoteUserMediaViewModel, + createLocalUserMedia, + createRemoteUserMedia, + type LocalUserMediaViewModel, + type RemoteUserMediaViewModel, } from "../state/MediaViewModel"; import { E2eeType } from "../e2ee/e2eeType"; import { @@ -323,30 +325,27 @@ export function mockLocalParticipant( } as Partial as LocalParticipant; } -export function createLocalMedia( +export function mockLocalMedia( rtcMember: CallMembership, roomMember: Partial, localParticipant: LocalParticipant, mediaDevices: MediaDevices, ): LocalUserMediaViewModel { const member = mockMatrixRoomMember(rtcMember, roomMember); - return new LocalUserMediaViewModel( - testScope(), - "local", - member.userId, - rtcMember.rtcBackendIdentity, - constant(localParticipant), - { - kind: E2eeType.PER_PARTICIPANT, - }, - constant(mockLivekitRoom({ localParticipant })), - constant("https://rtc-example.org"), + return createLocalUserMedia(testScope(), { + id: "local", + userId: member.userId, + rtcBackendIdentity: rtcMember.rtcBackendIdentity, + participant$: constant(localParticipant), + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + livekitRoom$: constant(mockLivekitRoom({ localParticipant })), + focusUrl$: constant("https://rtc-example.org"), mediaDevices, - constant(member.rawDisplayName ?? "nodisplayname"), - constant(member.getMxcAvatarUrl()), - constant(null), - constant(null), - ); + displayName$: constant(member.rawDisplayName ?? "nodisplayname"), + mxcAvatarUrl$: constant(member.getMxcAvatarUrl()), + handRaised$: constant(null), + reaction$: constant(null), + }); } export function mockRemoteParticipant( @@ -364,7 +363,7 @@ export function mockRemoteParticipant( } as RemoteParticipant; } -export function createRemoteMedia( +export function mockRemoteMedia( rtcMember: CallMembership, roomMember: Partial, participant: RemoteParticipant | null, @@ -376,23 +375,20 @@ export function createRemoteMedia( ), ): RemoteUserMediaViewModel { const member = mockMatrixRoomMember(rtcMember, roomMember); - return new RemoteUserMediaViewModel( - testScope(), - "remote", - member.userId, - rtcMember.rtcBackendIdentity, - constant(participant), - { - kind: E2eeType.PER_PARTICIPANT, - }, - constant(livekitRoom), - constant("https://rtc-example.org"), - constant(false), - constant(member.rawDisplayName ?? "nodisplayname"), - constant(member.getMxcAvatarUrl()), - constant(null), - constant(null), - ); + return createRemoteUserMedia(testScope(), { + id: "remote", + userId: member.userId, + rtcBackendIdentity: rtcMember.rtcBackendIdentity, + participant$: constant(participant), + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + livekitRoom$: constant(livekitRoom), + focusUrl$: constant("https://rtc-example.org"), + pretendToBeDisconnected$: constant(false), + displayName$: constant(member.rawDisplayName ?? "nodisplayname"), + mxcAvatarUrl$: constant(member.getMxcAvatarUrl()), + handRaised$: constant(null), + reaction$: constant(null), + }); } export function mockConfig(