From 214bb75d270fb6c7ba7829f772de4b6876acfd78 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 1 Oct 2025 02:29:37 +0000 Subject: [PATCH 01/75] Update embedded package dependencies to v8.13.0 --- embedded/android/gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/embedded/android/gradle/libs.versions.toml b/embedded/android/gradle/libs.versions.toml index 9982f14d..8ec7801a 100644 --- a/embedded/android/gradle/libs.versions.toml +++ b/embedded/android/gradle/libs.versions.toml @@ -2,7 +2,7 @@ # https://docs.gradle.org/current/userguide/platforms.html#sub::toml-dependencies-format [versions] -android_gradle_plugin = "8.11.1" +android_gradle_plugin = "8.13.0" [libraries] android_gradle_plugin = { module = "com.android.tools.build:gradle", version.ref = "android_gradle_plugin" } From 9cdbb1135f7de4c467104917b6dd4a826321ada1 Mon Sep 17 00:00:00 2001 From: Timo K Date: Tue, 28 Oct 2025 21:18:47 +0100 Subject: [PATCH 02/75] temp --- src/state/CallViewModel.ts | 652 ++++++++---------- src/state/ownMember/OwnMembership.ts | 85 +++ .../{ => ownMember}/PublishConnection.ts | 20 +- .../{ => remoteMembers}/Connection.test.ts | 38 +- src/state/{ => remoteMembers}/Connection.ts | 58 +- src/state/remoteMembers/displayname.ts | 78 +++ .../remoteMembers/matrixLivekitMerger.ts | 199 ++++++ 7 files changed, 720 insertions(+), 410 deletions(-) create mode 100644 src/state/ownMember/OwnMembership.ts rename src/state/{ => ownMember}/PublishConnection.ts (94%) rename src/state/{ => remoteMembers}/Connection.test.ts (94%) rename src/state/{ => remoteMembers}/Connection.ts (84%) create mode 100644 src/state/remoteMembers/displayname.ts create mode 100644 src/state/remoteMembers/matrixLivekitMerger.ts diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index d7735b26..1977bf4a 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -118,12 +118,12 @@ import { } from "../rtcSessionHelpers"; import { E2eeType } from "../e2ee/e2eeType"; import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider"; -import { type Connection, RemoteConnection } from "./Connection"; +import { type Connection, RemoteConnection } from "./remoteMembers/Connection.ts"; import { type MuteStates } from "./MuteStates"; import { getUrlParams } from "../UrlParams"; import { type ProcessorState } from "../livekit/TrackProcessorContext"; import { ElementWidgetActions, widget } from "../widget"; -import { PublishConnection } from "./PublishConnection.ts"; +import { PublishConnection } from "./ownMember/PublishConnection.ts"; import { type Async, async$, mapAsync, ready } from "./Async"; import { sharingScreen$, UserMedia } from "./UserMedia.ts"; import { ScreenShare } from "./ScreenShare.ts"; @@ -138,6 +138,7 @@ import { } from "./layout-types.ts"; import { ElementCallError, UnknownCallError } from "../utils/errors.ts"; import { ObservableScope } from "./ObservableScope.ts"; +import { memberDisplaynames$ } from "./remoteMembers/displayname.ts"; export interface CallViewModelOptions { encryptionSystem: EncryptionSystem; @@ -217,10 +218,12 @@ export class CallViewModel { private readonly join$ = new Subject(); + // DISCUSS BAD ? public join(): void { this.join$.next(); } + // CODESMALL // This is functionally the same Observable as leave$, except here it's // hoisted to the top of the class. This enables the cyclic dependency between // leave$ -> autoLeave$ -> callPickupState$ -> livekitConnectionState$ -> @@ -233,6 +236,7 @@ export class CallViewModel { * Whether we are joined to the call. This reflects our local state rather * than whether all connections are truly up and running. */ + // DISCUSS ? lets think why we need joined and how to do it better private readonly joined$ = this.scope.behavior( this.join$.pipe( map(() => true), @@ -246,26 +250,290 @@ export class CallViewModel { ); /** - * The MatrixRTC session participants. + * The transport that we would personally prefer to publish on (if not for the + * transport preferences of others, perhaps). */ - // Note that MatrixRTCSession already filters the call memberships by users - // that are joined to the room; we don't need to perform extra filtering here. - private readonly memberships$ = this.scope.behavior( - fromEvent( - this.matrixRTCSession, - MatrixRTCSessionEvent.MembershipsChanged, - ).pipe( - startWith(null), - map(() => this.matrixRTCSession.memberships), + // DISCUSS move to ownMembership + private readonly preferredTransport$ = this.scope.behavior( + async$(makeTransport(this.matrixRTCSession)), + ); + + /** + * The transport over which we should be actively publishing our media. + * null when not joined. + */ + // DISCUSSION ownMembershipManager + private readonly localTransport$: Behavior | null> = + this.scope.behavior( + this.transports$.pipe( + map((transports) => transports?.local ?? null), + distinctUntilChanged | null>(deepCompare), + ), + ); + + /** + * The transport we should advertise in our MatrixRTC membership (plus whether + * it is a multi-SFU transport and whether we should use sticky events). + */ + // DISCUSSION ownMembershipManager + private readonly advertisedTransport$: Behavior<{ + multiSfu: boolean; + preferStickyEvents: boolean; + transport: LivekitTransport; + } | null> = this.scope.behavior( + this.transports$.pipe( + map((transports) => + transports?.local.state === "ready" && + transports.preferred.state === "ready" + ? { + multiSfu: transports.multiSfu, + preferStickyEvents: transports.preferStickyEvents, + // In non-multi-SFU mode we should always advertise the preferred + // SFU to minimize the number of membership updates + transport: transports.multiSfu + ? transports.local.value + : transports.preferred.value, + } + : null, + ), + distinctUntilChanged<{ + multiSfu: boolean; + preferStickyEvents: boolean; + transport: LivekitTransport; + } | null>(deepCompare), + ), + ); + + // DISCUSSION move to ConnectionManager + /** + * The local connection over which we will publish our media. It could + * possibly also have some remote users' media available on it. + * null when not joined. + */ + private readonly localConnection$: Behavior | null> = + this.scope.behavior( + generateKeyed$< + Async | null, + PublishConnection, + Async | null + >( + this.localTransport$, + (transport, createOrGet) => + transport && + mapAsync(transport, (transport) => + createOrGet( + // Stable key that uniquely idenifies the transport + JSON.stringify({ + url: transport.livekit_service_url, + alias: transport.livekit_alias, + }), + (scope) => + new PublishConnection( + { + transport, + client: this.matrixRoom.client, + scope, + remoteTransports$: this.remoteTransports$, + livekitRoomFactory: this.options.livekitRoomFactory, + }, + this.mediaDevices, + this.muteStates, + this.e2eeLivekitOptions(), + this.scope.behavior(this.trackProcessorState$), + ), + ), + ), + ), + ); + + // DISCUSSION move to ConnectionManager + public readonly livekitConnectionState$ = + // TODO: This options.connectionState$ behavior is a small hack inserted + // here to facilitate testing. This would likely be better served by + // breaking CallViewModel down into more naturally testable components. + this.options.connectionState$ ?? + this.scope.behavior( + this.localConnection$.pipe( + switchMap((c) => + c?.state === "ready" + ? // TODO mapping to ConnectionState for compatibility, but we should use the full state? + c.value.state$.pipe( + switchMap((s) => { + if (s.state === "ConnectedToLkRoom") + return s.connectionState$; + return of(ConnectionState.Disconnected); + }), + ) + : of(ConnectionState.Disconnected), + ), + ), + ); + + /** + * Connections for each transport in use by one or more session members that + * is *distinct* from the local transport. + */ + // DISCUSSION move to ConnectionManager + private readonly remoteConnections$ = this.scope.behavior( + generateKeyed$( + this.transports$, + (transports, createOrGet) => { + const connections: Connection[] = []; + + // Until the local transport becomes ready we have no idea which + // transports will actually need a dedicated remote connection + if (transports?.local.state === "ready") { + // TODO: Handle custom transport.livekit_alias values here + const localServiceUrl = transports.local.value.livekit_service_url; + const remoteServiceUrls = new Set( + transports.remote.map( + ({ transport }) => transport.livekit_service_url, + ), + ); + remoteServiceUrls.delete(localServiceUrl); + + for (const remoteServiceUrl of remoteServiceUrls) + connections.push( + createOrGet( + remoteServiceUrl, + (scope) => + new RemoteConnection( + { + transport: { + type: "livekit", + livekit_service_url: remoteServiceUrl, + livekit_alias: this.livekitAlias, + }, + client: this.matrixRoom.client, + scope, + remoteTransports$: this.remoteTransports$, + livekitRoomFactory: this.options.livekitRoomFactory, + }, + this.e2eeLivekitOptions(), + ), + ), + ); + } + + return connections; + }, ), ); /** - * The transport that we would personally prefer to publish on (if not for the - * transport preferences of others, perhaps). + * A list of the connections that should be active at any given time. */ - private readonly preferredTransport$ = this.scope.behavior( - async$(makeTransport(this.matrixRTCSession)), + // DISCUSSION move to ConnectionManager + private readonly connections$ = this.scope.behavior( + combineLatest( + [this.localConnection$, this.remoteConnections$], + (local, remote) => [ + ...(local?.state === "ready" ? [local.value] : []), + ...remote.values(), + ], + ), + ); + + /** + * Emits with connections whenever they should be started or stopped. + */ + // DISCUSSION move to ConnectionManager + private readonly connectionInstructions$ = this.connections$.pipe( + pairwise(), + map(([prev, next]) => { + const start = new Set(next.values()); + for (const connection of prev) start.delete(connection); + const stop = new Set(prev.values()); + for (const connection of next) stop.delete(connection); + + return { start, stop }; + }), + ); + + public readonly allLivekitRooms$ = this.scope.behavior( + this.connections$.pipe( + map((connections) => + [...connections.values()].map((c) => ({ + room: c.livekitRoom, + url: c.transport.livekit_service_url, + isLocal: c instanceof PublishConnection, + })), + ), + ), + ); + + private readonly userId = this.matrixRoom.client.getUserId()!; + private readonly deviceId = this.matrixRoom.client.getDeviceId()!; + + /** + * Whether we are connected to the MatrixRTC session. + */ + // DISCUSSION own membership manager + private readonly matrixConnected$ = this.scope.behavior( + // To consider ourselves connected to MatrixRTC, we check the following: + and$( + // The client is connected to the sync loop + ( + fromEvent(this.matrixRoom.client, ClientEvent.Sync) as Observable< + [SyncState] + > + ).pipe( + startWith([this.matrixRoom.client.getSyncState()]), + map(([state]) => state === SyncState.Syncing), + ), + // Room state observed by session says we're connected + fromEvent( + this.matrixRTCSession, + MembershipManagerEvent.StatusChanged, + ).pipe( + startWith(null), + map(() => this.matrixRTCSession.membershipStatus === Status.Connected), + ), + // Also watch out for warnings that we've likely hit a timeout and our + // delayed leave event is being sent (this condition is here because it + // provides an earlier warning than the sync loop timeout, and we wouldn't + // see the actual leave event until we reconnect to the sync loop) + fromEvent( + this.matrixRTCSession, + MembershipManagerEvent.ProbablyLeft, + ).pipe( + startWith(null), + map(() => this.matrixRTCSession.probablyLeft !== true), + ), + ), + ); + + /** + * Whether we are "fully" connected to the call. Accounts for both the + * connection to the MatrixRTC session and the LiveKit publish connection. + */ + // DISCUSSION own membership manager + private readonly connected$ = this.scope.behavior( + and$( + this.matrixConnected$, + this.livekitConnectionState$.pipe( + map((state) => state === ConnectionState.Connected), + ), + ), + ); + + /** + * Whether we should tell the user that we're reconnecting to the call. + */ + // DISCUSSION own membership manager + public readonly reconnecting$ = this.scope.behavior( + this.connected$.pipe( + // We are reconnecting if we previously had some successful initial + // connection but are now disconnected + scan( + ({ connectedPreviously }, connectedNow) => ({ + connectedPreviously: connectedPreviously || connectedNow, + reconnecting: connectedPreviously && !connectedNow, + }), + { connectedPreviously: false, reconnecting: false }, + ), + map(({ reconnecting }) => reconnecting), + ), ); /** @@ -276,7 +544,8 @@ export class CallViewModel { * together when it might change together is what you have to do in RxJS to * avoid reading inconsistent state or observing too many changes.) */ - // TODO-MULTI-SFU find a better name for this. with the addition of sticky events it's no longer just about transports. + // TODO-MULTI-SFU find a better name for this. With the addition of sticky events it's no longer just about transports. + // DISCUSS move the local part to the own membership file private readonly transports$: Behavior<{ local: Async; remote: { membership: CallMembership; transport: LivekitTransport }[]; @@ -342,282 +611,6 @@ export class CallViewModel { ), ); - /** - * Lists the transports used by each MatrixRTC session member other than - * ourselves. - */ - private readonly remoteTransports$ = this.scope.behavior( - this.transports$.pipe(map((transports) => transports?.remote ?? [])), - ); - - /** - * The transport over which we should be actively publishing our media. - * null when not joined. - */ - private readonly localTransport$: Behavior | null> = - this.scope.behavior( - this.transports$.pipe( - map((transports) => transports?.local ?? null), - distinctUntilChanged | null>(deepCompare), - ), - ); - - /** - * The transport we should advertise in our MatrixRTC membership (plus whether - * it is a multi-SFU transport and whether we should use sticky events). - */ - private readonly advertisedTransport$: Behavior<{ - multiSfu: boolean; - preferStickyEvents: boolean; - transport: LivekitTransport; - } | null> = this.scope.behavior( - this.transports$.pipe( - map((transports) => - transports?.local.state === "ready" && - transports.preferred.state === "ready" - ? { - multiSfu: transports.multiSfu, - preferStickyEvents: transports.preferStickyEvents, - // In non-multi-SFU mode we should always advertise the preferred - // SFU to minimize the number of membership updates - transport: transports.multiSfu - ? transports.local.value - : transports.preferred.value, - } - : null, - ), - distinctUntilChanged<{ - multiSfu: boolean; - preferStickyEvents: boolean; - transport: LivekitTransport; - } | null>(deepCompare), - ), - ); - - /** - * The local connection over which we will publish our media. It could - * possibly also have some remote users' media available on it. - * null when not joined. - */ - private readonly localConnection$: Behavior | null> = - this.scope.behavior( - generateKeyed$< - Async | null, - PublishConnection, - Async | null - >( - this.localTransport$, - (transport, createOrGet) => - transport && - mapAsync(transport, (transport) => - createOrGet( - // Stable key that uniquely idenifies the transport - JSON.stringify({ - url: transport.livekit_service_url, - alias: transport.livekit_alias, - }), - (scope) => - new PublishConnection( - { - transport, - client: this.matrixRoom.client, - scope, - remoteTransports$: this.remoteTransports$, - livekitRoomFactory: this.options.livekitRoomFactory, - }, - this.mediaDevices, - this.muteStates, - this.e2eeLivekitOptions(), - this.scope.behavior(this.trackProcessorState$), - ), - ), - ), - ), - ); - - public readonly livekitConnectionState$ = - // TODO: This options.connectionState$ behavior is a small hack inserted - // here to facilitate testing. This would likely be better served by - // breaking CallViewModel down into more naturally testable components. - this.options.connectionState$ ?? - this.scope.behavior( - this.localConnection$.pipe( - switchMap((c) => - c?.state === "ready" - ? // TODO mapping to ConnectionState for compatibility, but we should use the full state? - c.value.transportState$.pipe( - switchMap((s) => { - if (s.state === "ConnectedToLkRoom") - return s.connectionState$; - return of(ConnectionState.Disconnected); - }), - ) - : of(ConnectionState.Disconnected), - ), - ), - ); - - /** - * Connections for each transport in use by one or more session members that - * is *distinct* from the local transport. - */ - private readonly remoteConnections$ = this.scope.behavior( - generateKeyed$( - this.transports$, - (transports, createOrGet) => { - const connections: Connection[] = []; - - // Until the local transport becomes ready we have no idea which - // transports will actually need a dedicated remote connection - if (transports?.local.state === "ready") { - // TODO: Handle custom transport.livekit_alias values here - const localServiceUrl = transports.local.value.livekit_service_url; - const remoteServiceUrls = new Set( - transports.remote.map( - ({ transport }) => transport.livekit_service_url, - ), - ); - remoteServiceUrls.delete(localServiceUrl); - - for (const remoteServiceUrl of remoteServiceUrls) - connections.push( - createOrGet( - remoteServiceUrl, - (scope) => - new RemoteConnection( - { - transport: { - type: "livekit", - livekit_service_url: remoteServiceUrl, - livekit_alias: this.livekitAlias, - }, - client: this.matrixRoom.client, - scope, - remoteTransports$: this.remoteTransports$, - livekitRoomFactory: this.options.livekitRoomFactory, - }, - this.e2eeLivekitOptions(), - ), - ), - ); - } - - return connections; - }, - ), - ); - - /** - * A list of the connections that should be active at any given time. - */ - private readonly connections$ = this.scope.behavior( - combineLatest( - [this.localConnection$, this.remoteConnections$], - (local, remote) => [ - ...(local?.state === "ready" ? [local.value] : []), - ...remote.values(), - ], - ), - ); - - /** - * Emits with connections whenever they should be started or stopped. - */ - private readonly connectionInstructions$ = this.connections$.pipe( - pairwise(), - map(([prev, next]) => { - const start = new Set(next.values()); - for (const connection of prev) start.delete(connection); - const stop = new Set(prev.values()); - for (const connection of next) stop.delete(connection); - - return { start, stop }; - }), - ); - - public readonly allLivekitRooms$ = this.scope.behavior( - this.connections$.pipe( - map((connections) => - [...connections.values()].map((c) => ({ - room: c.livekitRoom, - url: c.transport.livekit_service_url, - isLocal: c instanceof PublishConnection, - })), - ), - ), - ); - - private readonly userId = this.matrixRoom.client.getUserId()!; - private readonly deviceId = this.matrixRoom.client.getDeviceId()!; - - /** - * Whether we are connected to the MatrixRTC session. - */ - private readonly matrixConnected$ = this.scope.behavior( - // To consider ourselves connected to MatrixRTC, we check the following: - and$( - // The client is connected to the sync loop - ( - fromEvent(this.matrixRoom.client, ClientEvent.Sync) as Observable< - [SyncState] - > - ).pipe( - startWith([this.matrixRoom.client.getSyncState()]), - map(([state]) => state === SyncState.Syncing), - ), - // Room state observed by session says we're connected - fromEvent( - this.matrixRTCSession, - MembershipManagerEvent.StatusChanged, - ).pipe( - startWith(null), - map(() => this.matrixRTCSession.membershipStatus === Status.Connected), - ), - // Also watch out for warnings that we've likely hit a timeout and our - // delayed leave event is being sent (this condition is here because it - // provides an earlier warning than the sync loop timeout, and we wouldn't - // see the actual leave event until we reconnect to the sync loop) - fromEvent( - this.matrixRTCSession, - MembershipManagerEvent.ProbablyLeft, - ).pipe( - startWith(null), - map(() => this.matrixRTCSession.probablyLeft !== true), - ), - ), - ); - - /** - * Whether we are "fully" connected to the call. Accounts for both the - * connection to the MatrixRTC session and the LiveKit publish connection. - */ - private readonly connected$ = this.scope.behavior( - and$( - this.matrixConnected$, - this.livekitConnectionState$.pipe( - map((state) => state === ConnectionState.Connected), - ), - ), - ); - - /** - * Whether we should tell the user that we're reconnecting to the call. - */ - public readonly reconnecting$ = this.scope.behavior( - this.connected$.pipe( - // We are reconnecting if we previously had some successful initial - // connection but are now disconnected - scan( - ({ connectedPreviously }, connectedNow) => ({ - connectedPreviously: connectedPreviously || connectedNow, - reconnecting: connectedPreviously && !connectedNow, - }), - { connectedPreviously: false, reconnecting: false }, - ), - map(({ reconnecting }) => reconnecting), - ), - ); - /** * Whether various media/event sources should pretend to be disconnected from * all network input, even if their connection still technically works. @@ -626,6 +619,7 @@ export class CallViewModel { // that the LiveKit connection is still functional while the homeserver is // down, for example, and we want to avoid making people worry that the app is // in a split-brained state. + // DISCUSSION own membership manager ALSO this probably can be simplifis private readonly pretendToBeDisconnected$ = this.reconnecting$; /** @@ -718,57 +712,6 @@ export class CallViewModel { ), ); - /** - * Displaynames for each member of the call. This will disambiguate - * any displaynames that clashes with another member. Only members - * joined to the call are considered here. - */ - // It turns out that doing the disambiguation above is rather expensive on Safari (10x slower - // than on Chrome/Firefox). This means it is important that we multicast the result so that we - // don't do this work more times than we need to. This is achieved by converting to a behavior: - public readonly memberDisplaynames$ = this.scope.behavior( - combineLatest( - [ - // Handle call membership changes - this.memberships$, - // Additionally handle display name changes (implicitly reacting to them) - fromEvent(this.matrixRoom, RoomStateEvent.Members).pipe( - startWith(null), - ), - // TODO: do we need: pauseWhen(this.pretendToBeDisconnected$), - ], - (memberships, _displaynames) => { - const displaynameMap = new Map([ - [ - `${this.userId}:${this.deviceId}`, - this.matrixRoom.getMember(this.userId)?.rawDisplayName ?? - this.userId, - ], - ]); - const room = this.matrixRoom; - - // We only consider RTC members for disambiguation as they are the only visible members. - for (const rtcMember of memberships) { - const matrixIdentifier = `${rtcMember.userId}:${rtcMember.deviceId}`; - const { member } = getRoomMemberFromRtcMember(rtcMember, room); - if (!member) { - logger.error( - "Could not find member for media id:", - matrixIdentifier, - ); - continue; - } - const disambiguate = shouldDisambiguate(member, memberships, room); - displaynameMap.set( - matrixIdentifier, - calculateDisplayName(member, disambiguate), - ); - } - return displaynameMap; - }, - ), - ); - public readonly handsRaised$ = this.scope.behavior( this.handsRaisedSubject$.pipe(pauseWhen(this.pretendToBeDisconnected$)), ); @@ -787,6 +730,14 @@ export class CallViewModel { ), ); + memberDisplaynames$ = memberDisplaynames$( + this.matrixRoom, + this.memberships$, + this.scope, + this.userId, + this.deviceId, + ); + /** * List of MediaItems that we want to have tiles for. */ @@ -1655,6 +1606,8 @@ export class CallViewModel { /** * Emits an array of reactions that should be visible on the screen. */ + // DISCUSSION move this into a reaction file + // const {visibleReactions$, audibleReactions$} = reactionsObservables$(showReactionSetting$, ) public readonly visibleReactions$ = this.scope.behavior( showReactions.value$.pipe( switchMap((show) => (show ? this.reactions$ : of({}))), @@ -1790,6 +1743,7 @@ export class CallViewModel { private readonly trackProcessorState$: Observable, ) { // Start and stop local and remote connections as needed + // DISCUSSION connection manager this.connectionInstructions$ .pipe(this.scope.bind()) .subscribe(({ start, stop }) => { @@ -1947,13 +1901,3 @@ function getE2eeKeyProvider( return keyProvider; } } - -function getRoomMemberFromRtcMember( - rtcMember: CallMembership, - room: MatrixRoom, -): { id: string; member: RoomMember | undefined } { - return { - id: rtcMember.userId + ":" + rtcMember.deviceId, - member: room.getMember(rtcMember.userId) ?? undefined, - }; -} diff --git a/src/state/ownMember/OwnMembership.ts b/src/state/ownMember/OwnMembership.ts new file mode 100644 index 00000000..1a5c1b24 --- /dev/null +++ b/src/state/ownMember/OwnMembership.ts @@ -0,0 +1,85 @@ +import { Behavior } from "../Behavior"; + +const ownMembership$ = ( + multiSfu: boolean, + preferStickyEvents: boolean, +): { + connected: Behavior; + transport: Behavior; +} => { + /** + * Lists the transports used by ourselves, plus all other MatrixRTC session + * members. For completeness this also lists the preferred transport and + * whether we are in multi-SFU mode or sticky events mode (because + * advertisedTransport$ wants to read them at the same time, and bundling data + * together when it might change together is what you have to do in RxJS to + * avoid reading inconsistent state or observing too many changes.) + */ + // TODO-MULTI-SFU find a better name for this. With the addition of sticky events it's no longer just about transports. + // DISCUSS move to MatrixLivekitMerger + const transport$: Behavior<{ + local: Async; + preferred: Async; + multiSfu: boolean; + preferStickyEvents: boolean; + } | null> = this.scope.behavior( + this.joined$.pipe( + switchMap((joined) => + joined + ? combineLatest( + [ + this.preferredTransport$, + this.memberships$, + multiSfu.value$, + preferStickyEvents.value$, + ], + (preferred, memberships, preferMultiSfu, preferStickyEvents) => { + // Multi-SFU must be implicitly enabled when using sticky events + const multiSfu = preferStickyEvents || preferMultiSfu; + + const oldestMembership = + this.matrixRTCSession.getOldestMembership(); + const remote = memberships.flatMap((m) => { + if (m.userId === this.userId && m.deviceId === this.deviceId) + return []; + const t = m.getTransport(oldestMembership ?? m); + return t && isLivekitTransport(t) + ? [{ membership: m, transport: t }] + : []; + }); + + let local = preferred; + if (!multiSfu) { + const oldest = this.matrixRTCSession.getOldestMembership(); + if (oldest !== undefined) { + const selection = oldest.getTransport(oldest); + // TODO selection can be null if no transport is configured should we report an error? + if (selection && isLivekitTransport(selection)) + local = ready(selection); + } + } + + if (local.state === "error") { + this._configError$.next( + local.value instanceof ElementCallError + ? local.value + : new UnknownCallError(local.value), + ); + } + + return { + local, + remote, + preferred, + multiSfu, + preferStickyEvents, + }; + }, + ) + : of(null), + ), + ), + ); + + return { connected: true, transport$ }; +}; diff --git a/src/state/PublishConnection.ts b/src/state/ownMember/PublishConnection.ts similarity index 94% rename from src/state/PublishConnection.ts rename to src/state/ownMember/PublishConnection.ts index cfbcba90..3feb8a52 100644 --- a/src/state/PublishConnection.ts +++ b/src/state/ownMember/PublishConnection.ts @@ -21,19 +21,19 @@ import { } from "rxjs"; import { logger } from "matrix-js-sdk/lib/logger"; -import type { Behavior } from "./Behavior.ts"; -import type { MediaDevices, SelectedDevice } from "./MediaDevices.ts"; -import type { MuteStates } from "./MuteStates.ts"; +import type { Behavior } from "../Behavior.ts"; +import type { MediaDevices, SelectedDevice } from "../MediaDevices.ts"; +import type { MuteStates } from "../MuteStates.ts"; import { type ProcessorState, trackProcessorSync, -} from "../livekit/TrackProcessorContext.tsx"; -import { getUrlParams } from "../UrlParams.ts"; -import { defaultLiveKitOptions } from "../livekit/options.ts"; -import { getValue } from "../utils/observable.ts"; -import { observeTrackReference$ } from "./MediaViewModel.ts"; -import { Connection, type ConnectionOpts } from "./Connection.ts"; -import { type ObservableScope } from "./ObservableScope.ts"; +} from "../../livekit/TrackProcessorContext.tsx"; +import { getUrlParams } from "../../UrlParams.ts"; +import { defaultLiveKitOptions } from "../../livekit/options.ts"; +import { getValue } from "../../utils/observable.ts"; +import { observeTrackReference$ } from "../MediaViewModel.ts"; +import { Connection, type ConnectionOpts } from "../remoteMembers/Connection.ts"; +import { type ObservableScope } from "../ObservableScope.ts"; /** * A connection to the local LiveKit room, the one the user is publishing to. diff --git a/src/state/Connection.test.ts b/src/state/remoteMembers/Connection.test.ts similarity index 94% rename from src/state/Connection.test.ts rename to src/state/remoteMembers/Connection.test.ts index b5389db4..3b0f42ee 100644 --- a/src/state/Connection.test.ts +++ b/src/state/remoteMembers/Connection.test.ts @@ -34,17 +34,17 @@ import type { } from "matrix-js-sdk/lib/matrixrtc"; import { type ConnectionOpts, - type TransportState, + type ConnectionState, type PublishingParticipant, RemoteConnection, } from "./Connection.ts"; -import { ObservableScope } from "./ObservableScope.ts"; -import { type OpenIDClientParts } from "../livekit/openIDSFU.ts"; -import { FailToGetOpenIdToken } from "../utils/errors.ts"; -import { PublishConnection } from "./PublishConnection.ts"; -import { mockMediaDevices, mockMuteStates } from "../utils/test.ts"; -import type { ProcessorState } from "../livekit/TrackProcessorContext.tsx"; -import { type MuteStates } from "./MuteStates.ts"; +import { ObservableScope } from "../ObservableScope.ts"; +import { type OpenIDClientParts } from "../../livekit/openIDSFU.ts"; +import { FailToGetOpenIdToken } from "../../utils/errors.ts"; +import { PublishConnection } from "../ownMember/PublishConnection.ts"; +import { mockMediaDevices, mockMuteStates } from "../../utils/test.ts"; +import type { ProcessorState } from "../../livekit/TrackProcessorContext.tsx"; +import { type MuteStates } from "../MuteStates.ts"; let testScope: ObservableScope; @@ -161,7 +161,7 @@ describe("Start connection states", () => { }; const connection = new RemoteConnection(opts, undefined); - expect(connection.transportState$.getValue().state).toEqual("Initialized"); + expect(connection.state$.getValue().state).toEqual("Initialized"); }); it("fail to getOpenId token then error state", async () => { @@ -178,8 +178,8 @@ describe("Start connection states", () => { const connection = new RemoteConnection(opts, undefined); - const capturedStates: TransportState[] = []; - const s = connection.transportState$.subscribe((value) => { + const capturedStates: ConnectionState[] = []; + const s = connection.state$.subscribe((value) => { capturedStates.push(value); }); onTestFinished(() => s.unsubscribe()); @@ -231,8 +231,8 @@ describe("Start connection states", () => { const connection = new RemoteConnection(opts, undefined); - const capturedStates: TransportState[] = []; - const s = connection.transportState$.subscribe((value) => { + const capturedStates: ConnectionState[] = []; + const s = connection.state$.subscribe((value) => { capturedStates.push(value); }); onTestFinished(() => s.unsubscribe()); @@ -288,8 +288,8 @@ describe("Start connection states", () => { const connection = new RemoteConnection(opts, undefined); - const capturedStates: TransportState[] = []; - const s = connection.transportState$.subscribe((value) => { + const capturedStates: ConnectionState[] = []; + const s = connection.state$.subscribe((value) => { capturedStates.push(value); }); onTestFinished(() => s.unsubscribe()); @@ -345,8 +345,8 @@ describe("Start connection states", () => { const connection = setupRemoteConnection(); - const capturedStates: TransportState[] = []; - const s = connection.transportState$.subscribe((value) => { + const capturedStates: ConnectionState[] = []; + const s = connection.state$.subscribe((value) => { capturedStates.push(value); }); onTestFinished(() => s.unsubscribe()); @@ -401,7 +401,7 @@ describe("Publishing participants observations", () => { const bobIsAPublisher = Promise.withResolvers(); const danIsAPublisher = Promise.withResolvers(); const observedPublishers: PublishingParticipant[][] = []; - const s = connection.publishingParticipants$.subscribe((publishers) => { + const s = connection.allLivekitParticipants$.subscribe((publishers) => { observedPublishers.push(publishers); if ( publishers.some( @@ -538,7 +538,7 @@ describe("Publishing participants observations", () => { const connection = setupRemoteConnection(); let observedPublishers: PublishingParticipant[][] = []; - const s = connection.publishingParticipants$.subscribe((publishers) => { + const s = connection.allLivekitParticipants$.subscribe((publishers) => { observedPublishers.push(publishers); }); onTestFinished(() => s.unsubscribe()); diff --git a/src/state/Connection.ts b/src/state/remoteMembers/Connection.ts similarity index 84% rename from src/state/Connection.ts rename to src/state/remoteMembers/Connection.ts index 005c1359..72239de0 100644 --- a/src/state/Connection.ts +++ b/src/state/remoteMembers/Connection.ts @@ -11,7 +11,7 @@ import { } from "@livekit/components-core"; import { ConnectionError, - type ConnectionState, + type ConnectionState as LivekitConenctionState, type E2EEOptions, type RemoteParticipant, Room as LivekitRoom, @@ -21,21 +21,21 @@ import { type CallMembership, type LivekitTransport, } from "matrix-js-sdk/lib/matrixrtc"; -import { logger } from "matrix-js-sdk/lib/logger"; import { BehaviorSubject, combineLatest, type Observable } from "rxjs"; +import { type Logger } from "matrix-js-sdk/lib/logger"; import { getSFUConfigWithOpenID, type OpenIDClientParts, type SFUConfig, -} from "../livekit/openIDSFU"; -import { type Behavior } from "./Behavior"; -import { type ObservableScope } from "./ObservableScope"; -import { defaultLiveKitOptions } from "../livekit/options"; +} from "../../livekit/openIDSFU.ts"; +import { type Behavior } from "../Behavior.ts"; +import { type ObservableScope } from "../ObservableScope.ts"; +import { defaultLiveKitOptions } from "../../livekit/options.ts"; import { InsufficientCapacityError, SFURoomCreationRestrictedError, -} from "../utils/errors.ts"; +} from "../../utils/errors.ts"; export interface ConnectionOpts { /** The media transport to connect to. */ @@ -44,8 +44,14 @@ export interface ConnectionOpts { client: OpenIDClientParts; /** The observable scope to use for this connection. */ scope: ObservableScope; - /** An observable of the current RTC call memberships and their associated transports. */ - remoteTransports$: Behavior< + /** + * An observable of the current RTC call memberships and their associated transports. + * Used to differentiate between publishing and subscribging participants on each connection. + * Used to find out which rtc member should upload to this connection (publishingParticipants$). + * The livekit room gives access to all the users subscribing to this connection, we need + * to filter out the ones that are uploading to this connection. + */ + membershipsWithTransport$: Behavior< { membership: CallMembership; transport: LivekitTransport }[] >; @@ -53,7 +59,7 @@ export interface ConnectionOpts { livekitRoomFactory?: (options?: RoomOptions) => LivekitRoom; } -export type TransportState = +export type ConnectionState = | { state: "Initialized" } | { state: "FetchingConfig"; transport: LivekitTransport } | { state: "ConnectingToLkRoom"; transport: LivekitTransport } @@ -61,7 +67,7 @@ export type TransportState = | { state: "FailedToStart"; error: Error; transport: LivekitTransport } | { state: "ConnectedToLkRoom"; - connectionState$: Observable; + livekitConnectionState$: Observable; transport: LivekitTransport; } | { state: "Stopped"; transport: LivekitTransport }; @@ -88,15 +94,14 @@ export type PublishingParticipant = { */ export class Connection { // Private Behavior - private readonly _transportState$ = new BehaviorSubject({ + private readonly _state$ = new BehaviorSubject({ state: "Initialized", }); /** * The current state of the connection to the media transport. */ - public readonly transportState$: Behavior = - this._transportState$; + public readonly state$: Behavior = this._state$; /** * Whether the connection has been stopped. @@ -118,7 +123,7 @@ export class Connection { public async start(): Promise { this.stopped = false; try { - this._transportState$.next({ + this._state$.next({ state: "FetchingConfig", transport: this.transport, }); @@ -126,7 +131,7 @@ export class Connection { // If we were stopped while fetching the config, don't proceed to connect if (this.stopped) return; - this._transportState$.next({ + this._state$.next({ state: "ConnectingToLkRoom", transport: this.transport, }); @@ -157,13 +162,13 @@ export class Connection { // If we were stopped while connecting, don't proceed to update state. if (this.stopped) return; - this._transportState$.next({ + this._state$.next({ state: "ConnectedToLkRoom", transport: this.transport, - connectionState$: connectionStateObserver(this.livekitRoom), + livekitConnectionState$: connectionStateObserver(this.livekitRoom), }); } catch (error) { - this._transportState$.next({ + this._state$.next({ state: "FailedToStart", error: error instanceof Error ? error : new Error(`${error}`), transport: this.transport, @@ -188,7 +193,7 @@ export class Connection { public async stop(): Promise { if (this.stopped) return; await this.livekitRoom.disconnect(); - this._transportState$.next({ + this._state$.next({ state: "Stopped", transport: this.transport, }); @@ -218,23 +223,22 @@ export class Connection { protected constructor( public readonly livekitRoom: LivekitRoom, opts: ConnectionOpts, + logger?: Logger, ) { - logger.log( + logger?.info( `[Connection] Creating new connection to ${opts.transport.livekit_service_url} ${opts.transport.livekit_alias}`, ); - const { transport, client, scope, remoteTransports$ } = opts; + const { transport, client, scope, membershipsWithTransport$ } = opts; this.transport = transport; this.client = client; - const participantsIncludingSubscribers$ = scope.behavior( - connectedParticipantsObserver(this.livekitRoom), - [], - ); + const participantsIncludingSubscribers$: Behavior = + scope.behavior(connectedParticipantsObserver(this.livekitRoom), []); this.publishingParticipants$ = scope.behavior( combineLatest( - [participantsIncludingSubscribers$, remoteTransports$], + [participantsIncludingSubscribers$, membershipsWithTransport$], (participants, remoteTransports) => remoteTransports // Find all members that claim to publish on this connection diff --git a/src/state/remoteMembers/displayname.ts b/src/state/remoteMembers/displayname.ts new file mode 100644 index 00000000..ec3231c6 --- /dev/null +++ b/src/state/remoteMembers/displayname.ts @@ -0,0 +1,78 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type Room, type RoomMember, RoomStateEvent } from "matrix-js-sdk"; +import { combineLatest, fromEvent, type Observable, startWith } from "rxjs"; +import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; +import { logger } from "matrix-js-sdk/lib/logger"; +import { type Room as MatrixRoom } from "matrix-js-sdk/lib/matrix"; + +import { type ObservableScope } from "../ObservableScope"; +import { calculateDisplayName, shouldDisambiguate } from "../../utils/displayname"; + +/** + * Displayname for each member of the call. This will disambiguate + * any displayname that clashes with another member. Only members + * joined to the call are considered here. + */ +// don't do this work more times than we need to. This is achieved by converting to a behavior: +export const memberDisplaynames$ = ( + matrixRoom: Room, + memberships$: Observable, + scope: ObservableScope, + userId: string, + deviceId: string, +) => + scope.behavior( + combineLatest( + [ + // Handle call membership changes + memberships$, + // Additionally handle display name changes (implicitly reacting to them) + fromEvent(matrixRoom, RoomStateEvent.Members).pipe(startWith(null)), + // TODO: do we need: pauseWhen(this.pretendToBeDisconnected$), + ], + (memberships, _displaynames) => { + const displaynameMap = new Map([ + [ + `${userId}:${deviceId}`, + matrixRoom.getMember(userId)?.rawDisplayName ?? userId, + ], + ]); + const room = matrixRoom; + + // We only consider RTC members for disambiguation as they are the only visible members. + for (const rtcMember of memberships) { + const matrixIdentifier = `${rtcMember.userId}:${rtcMember.deviceId}`; + const { member } = getRoomMemberFromRtcMember(rtcMember, room); + if (!member) { + logger.error( + "Could not find member for media id:", + matrixIdentifier, + ); + continue; + } + const disambiguate = shouldDisambiguate(member, memberships, room); + displaynameMap.set( + matrixIdentifier, + calculateDisplayName(member, disambiguate), + ); + } + return displaynameMap; + }, + ), + ); + +export function getRoomMemberFromRtcMember( + rtcMember: CallMembership, + room: MatrixRoom, +): { id: string; member: RoomMember | undefined } { + return { + id: rtcMember.userId + ":" + rtcMember.deviceId, + member: room.getMember(rtcMember.userId) ?? undefined, + }; +} diff --git a/src/state/remoteMembers/matrixLivekitMerger.ts b/src/state/remoteMembers/matrixLivekitMerger.ts new file mode 100644 index 00000000..935f36cb --- /dev/null +++ b/src/state/remoteMembers/matrixLivekitMerger.ts @@ -0,0 +1,199 @@ +/* +Copyright 2025 Element c. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + LocalParticipant, + Participant, + RemoteParticipant, + type Participant as LivekitParticipant, + type Room as LivekitRoom, +} from "livekit-client"; +import { + type MatrixRTCSession, + MatrixRTCSessionEvent, + type CallMembership, + type Transport, + LivekitTransport, + isLivekitTransport, +} from "matrix-js-sdk/lib/matrixrtc"; +import { + combineLatest, + fromEvent, + map, + startWith, + switchMap, + type Observable, +} from "rxjs"; + +import { type ObservableScope } from "../ObservableScope"; +import { type Connection } from "./Connection"; +import { Behavior } from "../Behavior"; +import { RoomMember } from "matrix-js-sdk"; +import { getRoomMemberFromRtcMember } from "./displayname"; + + +// TODOs: +// - make ConnectionManager its own actual class +// - write test for scopes (do we really need to bind scope) +class ConnectionManager { + constructor(transports$: Observable) {} + public readonly connections$: Observable; +} + +/** + * Represent a matrix call member and his associated livekit participation. + * `livekitParticipant` can be undefined if the member is not yet connected to the livekit room + * or if it has no livekit transport at all. + */ + +export interface MatrixLivekitItem { + callMembership: CallMembership; + livekitParticipant?: LivekitParticipant; +} + +// Alternative structure idea: +// const livekitMatrixItems$ = (callMemberships$,connectionManager,scope): Observable => { + + // Map of Connection -> to (callMembership, LivekitParticipant?)) +type participants = {participant: LocalParticipant | RemoteParticipant}[] + +interface LivekitRoomWithParticipants { + livekitRoom: LivekitRoom; + url: string; // Included for use as a React key + participants: { + // What id is that?? + // Looks like it userId:Deviceid? + id: string; + participant: LocalParticipant | RemoteParticipant | undefined; + // Why do we fetch a full room member here? + // looks like it is only for avatars? + // TODO: Remove that. have some Avatar Provider that can fetch avatar for user ids. + member: RoomMember; + }[]; +} + +/** + * Combines MatrixRtc and Livekit worlds. + * + * It has a small public interface: + * - in (via constructor): + * - an observable of CallMembership[] to track the call members (The matrix side) + * - a `ConnectionManager` for the lk rooms (The livekit side) + * - out (via public Observable): + * - `remoteMatrixLivekitItems` an observable of MatrixLivekitItem[] to track the remote members and associated livekit data. + */ +export class MatrixLivekitMerger { + public remoteMatrixLivekitItems$: Observable; + + /** + * The MatrixRTC session participants. + */ + // Note that MatrixRTCSession already filters the call memberships by users + // that are joined to the room; we don't need to perform extra filtering here. + private readonly memberships$ = this.scope.behavior( + fromEvent( + this.matrixRTCSession, + MatrixRTCSessionEvent.MembershipsChanged, + ).pipe( + startWith(null), + map(() => this.matrixRTCSession.memberships), + ), + ); + + public constructor( + private matrixRTCSession: MatrixRTCSession, + private connectionManager: ConnectionManager, + private scope: ObservableScope, + ) { + const publishingParticipants$ = combineLatest([ + this.memberships$, + connectionManager.connections$, + ]).pipe(map(), this.scope.bind()); + this.remoteMatrixLivekitItems$ = combineLatest([ + callMemberships$, + connectionManager.connections$, + ]).pipe(this.scope.bind()); + // Implementation goes here + } + + /** + * Lists the transports used by ourselves, plus all other MatrixRTC session + * members. For completeness this also lists the preferred transport and + * whether we are in multi-SFU mode or sticky events mode (because + * advertisedTransport$ wants to read them at the same time, and bundling data + * together when it might change together is what you have to do in RxJS to + * avoid reading inconsistent state or observing too many changes.) + */ + private readonly membershipsWithTransport$: Behavior<{ + membership: CallMembership; + transport?: LivekitTransport; + } | null> = this.scope.behavior( + this.memberships$.pipe( + map((memberships) => { + const oldestMembership = this.matrixRTCSession.getOldestMembership(); + + memberships.map((membership) => { + let transport = membership.getTransport(oldestMembership ?? membership) + return { membership, transport: isLivekitTransport(transport) ? transport : undefined }; + }) + }), + ), + ); + + /** + * Lists the transports used by each MatrixRTC session member other than + * ourselves. + */ + // private readonly remoteTransports$ = this.scope.behavior( + // this.membershipsWithTransport$.pipe( + // map((transports) => transports?.remote ?? []), + // ), + // ); + + /** + * Lists, for each LiveKit room, the LiveKit participants whose media should + * be presented. + */ + private readonly participantsByRoom$ = this.scope.behavior( + // TODO: Move this logic into Connection/PublishConnection if possible + + this.connectionManager.connections$.pipe( + switchMap((connections) => { + connections.map((c)=>c.publishingParticipants$.pipe( + map((publishingParticipants) => { + const participants: { + id: string; + participant: LivekitParticipant | undefined; + member: RoomMember; + }[] = publishingParticipants.map(({ participant, membership }) => ({ + // TODO update to UUID + id: `${membership.userId}:${membership.deviceId}`, + participant, + // This makes sense to add the the js-sdk callMembership (we only need the avatar so probably the call memberhsip just should aquire the avatar) + member: + getRoomMemberFromRtcMember( + membership, + this.matrixRoom, + )?.member ?? memberError(), + })); + + return { + livekitRoom: c.livekitRoom, + url: c.transport.livekit_service_url, + participants, + }; + }), + ), + ), + ), + ), + ); + }), + ) + .pipe(startWith([]), pauseWhen(this.pretendToBeDisconnected$)), + ); +} From cfe05f1ed9d6279aaf1062ac878007d546aa344d Mon Sep 17 00:00:00 2001 From: Timo K Date: Tue, 28 Oct 2025 21:58:10 +0100 Subject: [PATCH 03/75] more temp --- .../remoteMembers/matrixLivekitMerger.ts | 123 +++++++----------- 1 file changed, 49 insertions(+), 74 deletions(-) diff --git a/src/state/remoteMembers/matrixLivekitMerger.ts b/src/state/remoteMembers/matrixLivekitMerger.ts index 935f36cb..ef2fb852 100644 --- a/src/state/remoteMembers/matrixLivekitMerger.ts +++ b/src/state/remoteMembers/matrixLivekitMerger.ts @@ -31,17 +31,18 @@ import { import { type ObservableScope } from "../ObservableScope"; import { type Connection } from "./Connection"; -import { Behavior } from "../Behavior"; -import { RoomMember } from "matrix-js-sdk"; +import { Behavior, constant } from "../Behavior"; +import { Room as MatrixRoom, RoomMember } from "matrix-js-sdk"; import { getRoomMemberFromRtcMember } from "./displayname"; - +import { pauseWhen } from "../../utils/observable"; // TODOs: // - make ConnectionManager its own actual class // - write test for scopes (do we really need to bind scope) class ConnectionManager { constructor(transports$: Observable) {} - public readonly connections$: Observable; + public startWithMemberships(memberships$: Behavior) {} + public readonly connections$: Observable = constant([]); } /** @@ -51,16 +52,14 @@ class ConnectionManager { */ export interface MatrixLivekitItem { - callMembership: CallMembership; + membership: CallMembership; livekitParticipant?: LivekitParticipant; + member?: RoomMember; } // Alternative structure idea: // const livekitMatrixItems$ = (callMemberships$,connectionManager,scope): Observable => { - // Map of Connection -> to (callMembership, LivekitParticipant?)) -type participants = {participant: LocalParticipant | RemoteParticipant}[] - interface LivekitRoomWithParticipants { livekitRoom: LivekitRoom; url: string; // Included for use as a React key @@ -87,14 +86,12 @@ interface LivekitRoomWithParticipants { * - `remoteMatrixLivekitItems` an observable of MatrixLivekitItem[] to track the remote members and associated livekit data. */ export class MatrixLivekitMerger { - public remoteMatrixLivekitItems$: Observable; - /** * The MatrixRTC session participants. */ // Note that MatrixRTCSession already filters the call memberships by users // that are joined to the room; we don't need to perform extra filtering here. - private readonly memberships$ = this.scope.behavior( + public readonly memberships$ = this.scope.behavior( fromEvent( this.matrixRTCSession, MatrixRTCSessionEvent.MembershipsChanged, @@ -108,16 +105,9 @@ export class MatrixLivekitMerger { private matrixRTCSession: MatrixRTCSession, private connectionManager: ConnectionManager, private scope: ObservableScope, + private matrixRoom: MatrixRoom, ) { - const publishingParticipants$ = combineLatest([ - this.memberships$, - connectionManager.connections$, - ]).pipe(map(), this.scope.bind()); - this.remoteMatrixLivekitItems$ = combineLatest([ - callMemberships$, - connectionManager.connections$, - ]).pipe(this.scope.bind()); - // Implementation goes here + connectionManager.startWithMemberships(this.memberships$); } /** @@ -128,6 +118,7 @@ export class MatrixLivekitMerger { * together when it might change together is what you have to do in RxJS to * avoid reading inconsistent state or observing too many changes.) */ + // TODO pass this over to our conncetions private readonly membershipsWithTransport$: Behavior<{ membership: CallMembership; transport?: LivekitTransport; @@ -137,63 +128,47 @@ export class MatrixLivekitMerger { const oldestMembership = this.matrixRTCSession.getOldestMembership(); memberships.map((membership) => { - let transport = membership.getTransport(oldestMembership ?? membership) - return { membership, transport: isLivekitTransport(transport) ? transport : undefined }; - }) + let transport = membership.getTransport( + oldestMembership ?? membership, + ); + return { + membership, + transport: isLivekitTransport(transport) ? transport : undefined, + }; + }); }), ), ); - /** - * Lists the transports used by each MatrixRTC session member other than - * ourselves. - */ - // private readonly remoteTransports$ = this.scope.behavior( - // this.membershipsWithTransport$.pipe( - // map((transports) => transports?.remote ?? []), - // ), - // ); - - /** - * Lists, for each LiveKit room, the LiveKit participants whose media should - * be presented. - */ - private readonly participantsByRoom$ = this.scope.behavior( - // TODO: Move this logic into Connection/PublishConnection if possible - - this.connectionManager.connections$.pipe( - switchMap((connections) => { - connections.map((c)=>c.publishingParticipants$.pipe( - map((publishingParticipants) => { - const participants: { - id: string; - participant: LivekitParticipant | undefined; - member: RoomMember; - }[] = publishingParticipants.map(({ participant, membership }) => ({ - // TODO update to UUID - id: `${membership.userId}:${membership.deviceId}`, - participant, - // This makes sense to add the the js-sdk callMembership (we only need the avatar so probably the call memberhsip just should aquire the avatar) - member: - getRoomMemberFromRtcMember( - membership, - this.matrixRoom, - )?.member ?? memberError(), - })); - - return { - livekitRoom: c.livekitRoom, - url: c.transport.livekit_service_url, - participants, - }; - }), - ), - ), - ), - ), - ); - }), - ) - .pipe(startWith([]), pauseWhen(this.pretendToBeDisconnected$)), + private allPublishingParticipants$ = this.connectionManager.connections$.pipe( + switchMap((connections) => { + const listOfPublishingParticipants = connections.map( + (connection) => connection.publishingParticipants$, + ); + return combineLatest(listOfPublishingParticipants).pipe( + map((list) => list.flatMap((innerList) => innerList)), + ); + }), ); + + public readonly matrixLivekitItems$ = this.scope + .behavior( + this.allPublishingParticipants$.pipe( + map((participants) => { + const matrixLivekitItems: MatrixLivekitItem[] = participants.map( + ({ participant, membership }) => ({ + participant, + membership, + id: `${membership.userId}:${membership.deviceId}`, + // This makes sense to add the the js-sdk callMembership (we only need the avatar so probably the call memberhsip just should aquire the avatar) + member: + getRoomMemberFromRtcMember(membership, this.matrixRoom) + ?.member ?? memberError(), + }), + ); + return matrixLivekitItems; + }), + ), + ) + .pipe(startWith([]), pauseWhen(this.pretendToBeDisconnected$)); } From 3de0bbcfc960a2c9c4b2a3328db1c4bc07ce0a1c Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 29 Oct 2025 12:37:14 +0100 Subject: [PATCH 04/75] temp Co-authored-by: Valere --- src/state/remoteMembers/Connection.ts | 32 ++--- .../remoteMembers/matrixLivekitMerger.ts | 124 +++++++++++++++--- 2 files changed, 118 insertions(+), 38 deletions(-) diff --git a/src/state/remoteMembers/Connection.ts b/src/state/remoteMembers/Connection.ts index 72239de0..97127a48 100644 --- a/src/state/remoteMembers/Connection.ts +++ b/src/state/remoteMembers/Connection.ts @@ -16,8 +16,10 @@ import { type RemoteParticipant, Room as LivekitRoom, type RoomOptions, + Participant, } from "livekit-client"; import { + ParticipantId, type CallMembership, type LivekitTransport, } from "matrix-js-sdk/lib/matrixrtc"; @@ -37,6 +39,8 @@ import { SFURoomCreationRestrictedError, } from "../../utils/errors.ts"; +export type PublishingParticipant = Participant; + export interface ConnectionOpts { /** The media transport to connect to. */ transport: LivekitTransport; @@ -72,21 +76,6 @@ export type ConnectionState = } | { state: "Stopped"; transport: LivekitTransport }; -/** - * Represents participant publishing or expected to publish on the connection. - * It is paired with its associated rtc membership. - */ -export type PublishingParticipant = { - /** - * The LiveKit participant publishing on this connection, or undefined if the participant is not currently (yet) connected to the livekit room. - */ - participant: RemoteParticipant | undefined; - /** - * The rtc call membership associated with this participant. - */ - membership: CallMembership; -}; - /** * A connection to a Matrix RTC LiveKit backend. * @@ -205,7 +194,11 @@ export class Connection { * This is derived from `participantsIncludingSubscribers$` and `remoteTransports$`. * It filters the participants to only those that are associated with a membership that claims to publish on this connection. */ + public readonly publishingParticipants$: Behavior; + public readonly participantsWithPublishTrack$: Behavior< + PublishingParticipant[] + >; /** * The media transport to connect to. @@ -233,12 +226,15 @@ export class Connection { this.transport = transport; this.client = client; - const participantsIncludingSubscribers$: Behavior = - scope.behavior(connectedParticipantsObserver(this.livekitRoom), []); + this.participantsWithPublishTrack$ = scope.behavior( + connectedParticipantsObserver(this.livekitRoom), + [], + ); + // Legacy using callMemberships this.publishingParticipants$ = scope.behavior( combineLatest( - [participantsIncludingSubscribers$, membershipsWithTransport$], + [this.participantsIncludingSubscribers$, membershipsWithTransport$], (participants, remoteTransports) => remoteTransports // Find all members that claim to publish on this connection diff --git a/src/state/remoteMembers/matrixLivekitMerger.ts b/src/state/remoteMembers/matrixLivekitMerger.ts index ef2fb852..37a13f5f 100644 --- a/src/state/remoteMembers/matrixLivekitMerger.ts +++ b/src/state/remoteMembers/matrixLivekitMerger.ts @@ -19,6 +19,7 @@ import { type Transport, LivekitTransport, isLivekitTransport, + ParticipantId, } from "matrix-js-sdk/lib/matrixrtc"; import { combineLatest, @@ -40,20 +41,44 @@ import { pauseWhen } from "../../utils/observable"; // - make ConnectionManager its own actual class // - write test for scopes (do we really need to bind scope) class ConnectionManager { - constructor(transports$: Observable) {} - public startWithMemberships(memberships$: Behavior) {} + public setTansports(transports$: Behavior): void {} public readonly connections$: Observable = constant([]); + // connection is used to find the transport (to find matching callmembership) & for the livekitRoom + public readonly participantsByMemberId$: Behavior< + Map< + ParticipantId, + // It can be an array because a bad behaving client could be publishingParticipants$ + // multiple times to several livekit rooms. + { participant: LivekitParticipant; connection: Connection }[] + > + > = constant(new Map()); } +/** + * Represents participant publishing or expected to publish on the connection. + * It is paired with its associated rtc membership. + */ +export type PublishingParticipant = { + /** + * The LiveKit participant publishing on this connection, or undefined if the participant is not currently (yet) connected to the livekit room. + */ + participant: RemoteParticipant | undefined; + /** + * The rtc call membership associated with this participant. + */ + membership: CallMembership; +}; + /** * Represent a matrix call member and his associated livekit participation. * `livekitParticipant` can be undefined if the member is not yet connected to the livekit room * or if it has no livekit transport at all. */ - export interface MatrixLivekitItem { membership: CallMembership; livekitParticipant?: LivekitParticipant; + //TODO Try to remove this! Its waaay to much information + // Just use to get the member's avatar member?: RoomMember; } @@ -107,7 +132,7 @@ export class MatrixLivekitMerger { private scope: ObservableScope, private matrixRoom: MatrixRoom, ) { - connectionManager.startWithMemberships(this.memberships$); + connectionManager.setTansports(this.transports$); } /** @@ -118,17 +143,12 @@ export class MatrixLivekitMerger { * together when it might change together is what you have to do in RxJS to * avoid reading inconsistent state or observing too many changes.) */ - // TODO pass this over to our conncetions - private readonly membershipsWithTransport$: Behavior<{ - membership: CallMembership; - transport?: LivekitTransport; - } | null> = this.scope.behavior( + private readonly membershipsWithTransport$ = this.scope.behavior( this.memberships$.pipe( map((memberships) => { const oldestMembership = this.matrixRTCSession.getOldestMembership(); - - memberships.map((membership) => { - let transport = membership.getTransport( + return memberships.map((membership) => { + const transport = membership.getTransport( oldestMembership ?? membership, ); return { @@ -140,14 +160,65 @@ export class MatrixLivekitMerger { ), ); - private allPublishingParticipants$ = this.connectionManager.connections$.pipe( - switchMap((connections) => { - const listOfPublishingParticipants = connections.map( - (connection) => connection.publishingParticipants$, - ); - return combineLatest(listOfPublishingParticipants).pipe( - map((list) => list.flatMap((innerList) => innerList)), - ); + private readonly transports$ = this.scope.behavior( + this.membershipsWithTransport$.pipe( + map((membershipsWithTransport) => + membershipsWithTransport.reduce((acc, { transport }) => { + if ( + transport && + !acc.some((t) => areLivekitTransportsEqual(t, transport)) + ) { + acc.push(transport); + } + return acc; + }, [] as LivekitTransport[]), + ), + ), + ); + + // TODO move this over this the connection manager + // We have a lost of connections, for each of these these + // connection we create a stream of (participant, connection) tuples. + // Then we combine the several streams (1 per Connection) into a single stream of tuples. + private participantsWithConnection$ = + this.connectionManager.connections$.pipe( + switchMap((connections) => { + const listsOfParticipantWithConnection = connections.map( + (connection) => { + return connection.participantsWithPublishTrack$.pipe( + map((participants) => + participants.map((p) => ({ + participant: p, + connection, + })), + ), + ); + }, + ); + return combineLatest(listsOfParticipantWithConnection).pipe( + map((lists) => lists.flatMap((list) => list)), + ); + }), + ); + + // TODO move this over this the connection manager + // Filters the livekit partic + private participantsByMemberId$ = this.participantsWithConnection$.pipe( + map((participantsWithConnections) => { + const participantsByMemberId = new Map(); + participantsWithConnections.forEach(({ participant, connection }) => { + if (participant.getTrackPublications().length > 0) { + const currentVal = participantsByMemberId.get(participant.identity); + participantsByMemberId.set(participant.identity, { + connection, + participants: + currentVal === undefined + ? [participant] + : ([...currentVal, participant] as Participant[]), + }); + } + }); + return participantsByMemberId; }), ); @@ -172,3 +243,16 @@ export class MatrixLivekitMerger { ) .pipe(startWith([]), pauseWhen(this.pretendToBeDisconnected$)); } + +// TODO add this to the JS-SDK +function areLivekitTransportsEqual( + t1: LivekitTransport, + t2: LivekitTransport, +): boolean { + return ( + t1.livekit_service_url === t2.livekit_service_url && + // In case we have different lk rooms in the same SFU (depends on the livekit authorization service) + // It is only needed in case the livekit authorization service is not behaving as expected (or custom implementation) + t1.livekit_alias === t2.livekit_alias + ); +} From 62ef49ca0561ae7fc509f84ab01484635b7df33a Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 29 Oct 2025 15:20:06 +0100 Subject: [PATCH 05/75] temp Co-authored-by: Valere --- .../remoteMembers/MatrixLivekitMerger.test.ts | 30 +++++ .../remoteMembers/matrixLivekitMerger.ts | 120 ++++++++++-------- 2 files changed, 96 insertions(+), 54 deletions(-) create mode 100644 src/state/remoteMembers/MatrixLivekitMerger.test.ts diff --git a/src/state/remoteMembers/MatrixLivekitMerger.test.ts b/src/state/remoteMembers/MatrixLivekitMerger.test.ts new file mode 100644 index 00000000..df7aca0d --- /dev/null +++ b/src/state/remoteMembers/MatrixLivekitMerger.test.ts @@ -0,0 +1,30 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + test, + vi, + onTestFinished, + it, + describe, + expect, + beforeEach, + afterEach, +} from "vitest"; + +import { MatrixLivekitMerger } from "./matrixLivekitMerger"; +import { ObservableScope } from "../ObservableScope"; + +let testScope: ObservableScope; + +beforeEach(() => { + testScope = new ObservableScope(); +}); + +afterEach(() => { + testScope.end(); +}); diff --git a/src/state/remoteMembers/matrixLivekitMerger.ts b/src/state/remoteMembers/matrixLivekitMerger.ts index 37a13f5f..4cd68663 100644 --- a/src/state/remoteMembers/matrixLivekitMerger.ts +++ b/src/state/remoteMembers/matrixLivekitMerger.ts @@ -1,5 +1,5 @@ /* -Copyright 2025 Element c. +Copyright 2025 Element Creations Ltd. SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. @@ -23,7 +23,6 @@ import { } from "matrix-js-sdk/lib/matrixrtc"; import { combineLatest, - fromEvent, map, startWith, switchMap, @@ -36,6 +35,7 @@ import { Behavior, constant } from "../Behavior"; import { Room as MatrixRoom, RoomMember } from "matrix-js-sdk"; import { getRoomMemberFromRtcMember } from "./displayname"; import { pauseWhen } from "../../utils/observable"; +import { Logger } from "matrix-js-sdk/lib/logger"; // TODOs: // - make ConnectionManager its own actual class @@ -44,16 +44,17 @@ class ConnectionManager { public setTansports(transports$: Behavior): void {} public readonly connections$: Observable = constant([]); // connection is used to find the transport (to find matching callmembership) & for the livekitRoom - public readonly participantsByMemberId$: Behavior< - Map< - ParticipantId, - // It can be an array because a bad behaving client could be publishingParticipants$ - // multiple times to several livekit rooms. - { participant: LivekitParticipant; connection: Connection }[] - > - > = constant(new Map()); + public readonly participantsByMemberId$: Behavior = + constant(new Map()); } +export type ParticipantByMemberIdMap = Map< + ParticipantId, + // It can be an array because a bad behaving client could be publishingParticipants$ + // multiple times to several livekit rooms. + { participant: LivekitParticipant; connection: Connection }[] +>; + /** * Represents participant publishing or expected to publish on the connection. * It is paired with its associated rtc membership. @@ -111,27 +112,20 @@ interface LivekitRoomWithParticipants { * - `remoteMatrixLivekitItems` an observable of MatrixLivekitItem[] to track the remote members and associated livekit data. */ export class MatrixLivekitMerger { - /** - * The MatrixRTC session participants. - */ - // Note that MatrixRTCSession already filters the call memberships by users - // that are joined to the room; we don't need to perform extra filtering here. - public readonly memberships$ = this.scope.behavior( - fromEvent( - this.matrixRTCSession, - MatrixRTCSessionEvent.MembershipsChanged, - ).pipe( - startWith(null), - map(() => this.matrixRTCSession.memberships), - ), - ); + private readonly logger: Logger; + public constructor( - private matrixRTCSession: MatrixRTCSession, + private memberships$: Observable, private connectionManager: ConnectionManager, private scope: ObservableScope, + // TODO this is too much information for that class, + // apparently needed to get a room member to later get the Avatar + // => Extract an AvatarService instead? private matrixRoom: MatrixRoom, + parentLogger: Logger, ) { + this.logger = parentLogger.createChildLogger("MatrixLivekitMerger"); connectionManager.setTansports(this.transports$); } @@ -146,11 +140,9 @@ export class MatrixLivekitMerger { private readonly membershipsWithTransport$ = this.scope.behavior( this.memberships$.pipe( map((memberships) => { - const oldestMembership = this.matrixRTCSession.getOldestMembership(); return memberships.map((membership) => { - const transport = membership.getTransport( - oldestMembership ?? membership, - ); + const oldestMembership = memberships[0] ?? membership; + const transport = membership.getTransport(oldestMembership); return { membership, transport: isLivekitTransport(transport) ? transport : undefined, @@ -205,45 +197,65 @@ export class MatrixLivekitMerger { // Filters the livekit partic private participantsByMemberId$ = this.participantsWithConnection$.pipe( map((participantsWithConnections) => { - const participantsByMemberId = new Map(); - participantsWithConnections.forEach(({ participant, connection }) => { - if (participant.getTrackPublications().length > 0) { - const currentVal = participantsByMemberId.get(participant.identity); - participantsByMemberId.set(participant.identity, { - connection, - participants: - currentVal === undefined - ? [participant] - : ([...currentVal, participant] as Participant[]), - }); - } - }); + const participantsByMemberId = participantsWithConnections.reduce( + (acc, test) => { + const { participant, connection } = test; + if (participant.getTrackPublications().length > 0) { + const currentVal = acc.get(participant.identity); + if (!currentVal) { + acc.set(participant.identity, [{ connection, participant }]); + } else { + // already known + // This is user is publishing on several SFUs + currentVal.push({ connection, participant }); + this.logger.info( + `Participant ${participant.identity} is publishing on several SFUs ${currentVal.join()}`, + ); + } + } + return acc; + }, + new Map() as ParticipantByMemberIdMap, + ); + return participantsByMemberId; }), ); public readonly matrixLivekitItems$ = this.scope .behavior( - this.allPublishingParticipants$.pipe( - map((participants) => { - const matrixLivekitItems: MatrixLivekitItem[] = participants.map( - ({ participant, membership }) => ({ - participant, + combineLatest([ + this.membershipsWithTransport$, + this.participantsByMemberId$, + ]).pipe( + map(([memberships, participantsByMemberId]) => { + const items = memberships.map(({ membership, transport }) => { + const participantsWithConnection = participantsByMemberId.get( + membership.membershipID, + ); + const participant = + transport && + participantsWithConnection?.find((p) => + areLivekitTransportsEqual(p.connection.transport, transport), + ); + return { + livekitParticipant: participant, membership, - id: `${membership.userId}:${membership.deviceId}`, // This makes sense to add the the js-sdk callMembership (we only need the avatar so probably the call memberhsip just should aquire the avatar) member: - getRoomMemberFromRtcMember(membership, this.matrixRoom) - ?.member ?? memberError(), - }), - ); - return matrixLivekitItems; + // Why a member error? if we have a call membership there is a room member + getRoomMemberFromRtcMember(membership, this.matrixRoom)?.member, + } as MatrixLivekitItem; + }); + return items; }), ), ) - .pipe(startWith([]), pauseWhen(this.pretendToBeDisconnected$)); + .pipe(startWith([])); } +// TODO add back in the callviewmodel pauseWhen(this.pretendToBeDisconnected$) + // TODO add this to the JS-SDK function areLivekitTransportsEqual( t1: LivekitTransport, From 633a0f92903c56a8604a105bb796c8b245723b93 Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 29 Oct 2025 18:31:58 +0100 Subject: [PATCH 06/75] connection manager --- src/state/CallViewModel.ts | 57 +--- src/state/ownMember/OwnMembership.ts | 19 +- .../{PublishConnection.ts => Publisher.ts} | 0 src/state/remoteMembers/Connection.ts | 54 ++-- src/state/remoteMembers/ConnectionManager.ts | 219 +++++++++++++++ .../remoteMembers/matrixLivekitMerger.ts | 256 ++++++------------ 6 files changed, 344 insertions(+), 261 deletions(-) rename src/state/ownMember/{PublishConnection.ts => Publisher.ts} (100%) create mode 100644 src/state/remoteMembers/ConnectionManager.ts diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 1977bf4a..90a1f682 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -23,7 +23,6 @@ import { type Room as MatrixRoom, RoomEvent, type RoomMember, - RoomStateEvent, SyncState, } from "matrix-js-sdk"; import { deepCompare } from "matrix-js-sdk/lib/utils"; @@ -108,7 +107,6 @@ import { type ReactionOption, } from "../reactions"; import { shallowEquals } from "../utils/array"; -import { calculateDisplayName, shouldDisambiguate } from "../utils/displayname"; import { type MediaDevices } from "./MediaDevices"; import { type Behavior, constant } from "./Behavior"; import { @@ -118,12 +116,12 @@ import { } from "../rtcSessionHelpers"; import { E2eeType } from "../e2ee/e2eeType"; import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider"; -import { type Connection, RemoteConnection } from "./remoteMembers/Connection.ts"; +import { type Connection } from "./remoteMembers/Connection.ts"; import { type MuteStates } from "./MuteStates"; import { getUrlParams } from "../UrlParams"; import { type ProcessorState } from "../livekit/TrackProcessorContext"; import { ElementWidgetActions, widget } from "../widget"; -import { PublishConnection } from "./ownMember/PublishConnection.ts"; +import { PublishConnection } from "./ownMember/Publisher.ts"; import { type Async, async$, mapAsync, ready } from "./Async"; import { sharingScreen$, UserMedia } from "./UserMedia.ts"; import { ScreenShare } from "./ScreenShare.ts"; @@ -369,57 +367,6 @@ export class CallViewModel { ), ); - /** - * Connections for each transport in use by one or more session members that - * is *distinct* from the local transport. - */ - // DISCUSSION move to ConnectionManager - private readonly remoteConnections$ = this.scope.behavior( - generateKeyed$( - this.transports$, - (transports, createOrGet) => { - const connections: Connection[] = []; - - // Until the local transport becomes ready we have no idea which - // transports will actually need a dedicated remote connection - if (transports?.local.state === "ready") { - // TODO: Handle custom transport.livekit_alias values here - const localServiceUrl = transports.local.value.livekit_service_url; - const remoteServiceUrls = new Set( - transports.remote.map( - ({ transport }) => transport.livekit_service_url, - ), - ); - remoteServiceUrls.delete(localServiceUrl); - - for (const remoteServiceUrl of remoteServiceUrls) - connections.push( - createOrGet( - remoteServiceUrl, - (scope) => - new RemoteConnection( - { - transport: { - type: "livekit", - livekit_service_url: remoteServiceUrl, - livekit_alias: this.livekitAlias, - }, - client: this.matrixRoom.client, - scope, - remoteTransports$: this.remoteTransports$, - livekitRoomFactory: this.options.livekitRoomFactory, - }, - this.e2eeLivekitOptions(), - ), - ), - ); - } - - return connections; - }, - ), - ); - /** * A list of the connections that should be active at any given time. */ diff --git a/src/state/ownMember/OwnMembership.ts b/src/state/ownMember/OwnMembership.ts index 1a5c1b24..4ba4c380 100644 --- a/src/state/ownMember/OwnMembership.ts +++ b/src/state/ownMember/OwnMembership.ts @@ -1,12 +1,29 @@ -import { Behavior } from "../Behavior"; +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { LiveKitReactNativeInfo } from "livekit-client"; +import { Behavior, constant } from "../Behavior"; +import { LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { ConnectionManager } from "../remoteMembers/ConnectionManager"; const ownMembership$ = ( multiSfu: boolean, preferStickyEvents: boolean, + connectionManager: ConnectionManager, + transport: LivekitTransport, ): { connected: Behavior; transport: Behavior; } => { + const connection = connectionManager.registerTransports( + constant([transport]), + ); + const publisher = new Publisher(connection); + /** * Lists the transports used by ourselves, plus all other MatrixRTC session * members. For completeness this also lists the preferred transport and diff --git a/src/state/ownMember/PublishConnection.ts b/src/state/ownMember/Publisher.ts similarity index 100% rename from src/state/ownMember/PublishConnection.ts rename to src/state/ownMember/Publisher.ts diff --git a/src/state/remoteMembers/Connection.ts b/src/state/remoteMembers/Connection.ts index 97127a48..e815ea55 100644 --- a/src/state/remoteMembers/Connection.ts +++ b/src/state/remoteMembers/Connection.ts @@ -13,16 +13,12 @@ import { ConnectionError, type ConnectionState as LivekitConenctionState, type E2EEOptions, - type RemoteParticipant, Room as LivekitRoom, type RoomOptions, - Participant, + type Participant, + RoomEvent, } from "livekit-client"; -import { - ParticipantId, - type CallMembership, - type LivekitTransport, -} from "matrix-js-sdk/lib/matrixrtc"; +import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { BehaviorSubject, combineLatest, type Observable } from "rxjs"; import { type Logger } from "matrix-js-sdk/lib/logger"; @@ -55,9 +51,9 @@ export interface ConnectionOpts { * The livekit room gives access to all the users subscribing to this connection, we need * to filter out the ones that are uploading to this connection. */ - membershipsWithTransport$: Behavior< - { membership: CallMembership; transport: LivekitTransport }[] - >; + // membershipsWithTransport$: Behavior< + // { membership: CallMembership; transport: LivekitTransport }[] + // >; /** Optional factory to create the LiveKit room, mainly for testing purposes. */ livekitRoomFactory?: (options?: RoomOptions) => LivekitRoom; @@ -106,9 +102,13 @@ export class Connection { * 2. Use this token to request the SFU config to the MatrixRtc authentication service. * 3. Connect to the configured LiveKit room. * + * The errors are also represented as a state in the `state$` observable. + * It is safe to ignore those errors and handle them accordingly via the `state$` observable. * @throws {InsufficientCapacityError} if the LiveKit server indicates that it has insufficient capacity to accept the connection. * @throws {SFURoomCreationRestrictedError} if the LiveKit server indicates that the room does not exist and cannot be created. */ + // TODO dont make this throw and instead store a connection error state in this class? + // TODO consider an autostart pattern... public async start(): Promise { this.stopped = false; try { @@ -221,35 +221,21 @@ export class Connection { logger?.info( `[Connection] Creating new connection to ${opts.transport.livekit_service_url} ${opts.transport.livekit_alias}`, ); - const { transport, client, scope, membershipsWithTransport$ } = opts; + const { transport, client, scope } = opts; this.transport = transport; this.client = client; this.participantsWithPublishTrack$ = scope.behavior( - connectedParticipantsObserver(this.livekitRoom), - [], - ); - - // Legacy using callMemberships - this.publishingParticipants$ = scope.behavior( - combineLatest( - [this.participantsIncludingSubscribers$, membershipsWithTransport$], - (participants, remoteTransports) => - remoteTransports - // Find all members that claim to publish on this connection - .flatMap(({ membership, transport }) => - transport.livekit_service_url === - this.transport.livekit_service_url - ? [membership] - : [], - ) - // Pair with their associated LiveKit participant (if any) - .map((membership) => { - const id = `${membership.userId}:${membership.deviceId}`; - const participant = participants.find((p) => p.identity === id); - return { participant, membership }; - }), + connectedParticipantsObserver( + this.livekitRoom, + // VALR: added that while I think about it + { + additionalRoomEvents: [ + RoomEvent.TrackPublished, + RoomEvent.TrackUnpublished, + ], + }, ), [], ); diff --git a/src/state/remoteMembers/ConnectionManager.ts b/src/state/remoteMembers/ConnectionManager.ts new file mode 100644 index 00000000..311e621e --- /dev/null +++ b/src/state/remoteMembers/ConnectionManager.ts @@ -0,0 +1,219 @@ +// TODOs: +// - make ConnectionManager its own actual class + +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + type LivekitTransport, + type ParticipantId, +} from "matrix-js-sdk/lib/matrixrtc"; +import { BehaviorSubject, combineLatest, map, switchMap } from "rxjs"; +import { type Logger } from "matrix-js-sdk/lib/logger"; +import { + type E2EEOptions, + type Room as LivekitRoom, + type Participant as LivekitParticipant, +} from "livekit-client"; +import { type MatrixClient } from "matrix-js-sdk"; + +import { type Behavior } from "../Behavior"; +import { type Connection, RemoteConnection } from "./Connection"; +import { type ObservableScope } from "../ObservableScope"; +import { generateKeyed$ } from "../../utils/observable"; +import { areLivekitTransportsEqual } from "./matrixLivekitMerger"; + +export type ParticipantByMemberIdMap = Map< + ParticipantId, + // It can be an array because a bad behaving client could be publishingParticipants$ + // multiple times to several livekit rooms. + { participant: LivekitParticipant; connection: Connection }[] +>; + +// - write test for scopes (do we really need to bind scope) +export class ConnectionManager { + /** + * The transport to use for publishing. + * This extends the list of tranports + */ + private publishTransport$ = new BehaviorSubject( + undefined, + ); + + private transportSubscriptions$ = new BehaviorSubject< + Behavior[] + >([]); + + private transports$ = this.scope.behavior( + this.transportSubscriptions$.pipe( + switchMap((subscriptions) => + combineLatest(subscriptions.map((s) => s.transports)).pipe( + map((transportsNested) => transportsNested.flat()), + map(removeDuplicateTransports), + ), + ), + ), + ); + + public constructor( + private client: MatrixClient, + private e2eeLivekitOptions: () => E2EEOptions | undefined, + private scope: ObservableScope, + private logger?: Logger, + private livekitRoomFactory?: () => LivekitRoom, + ) { + this.scope = scope; + } + + public getOrCreatePublishConnection( + transport: LivekitTransport, + ): Connection | undefined { + this.publishTransport$.next(transport); + const equalsRequestedTransport = (c: Connection): boolean => + areLivekitTransportsEqual(c.transport, transport); + return this.connections$.value.find(equalsRequestedTransport); + } + /** + * Connections for each transport in use by one or more session members. + */ + private readonly connections$ = this.scope.behavior( + generateKeyed$( + this.transports$, + (transports, createOrGet) => { + const createConnection = + ( + transport: LivekitTransport, + ): ((scope: ObservableScope) => RemoteConnection) => + (scope) => { + const connection = new RemoteConnection( + { + transport, + client: this.client, + scope: scope, + livekitRoomFactory: this.livekitRoomFactory, + }, + this.e2eeLivekitOptions(), + ); + void connection.start(); + return connection; + }; + + const connections = transports.map((transport) => { + const key = + transport.livekit_service_url + "|" + transport.livekit_alias; + return createOrGet(key, createConnection(transport)); + }); + + return connections; + }, + ), + ); + + /** + * + * @param transports$ + */ + public registerTransports( + transports$: Behavior, + ): Connection[] { + if (!this.transportSubscriptions$.value.some((t$) => t$ === transports$)) { + this.transportSubscriptions$.next( + this.transportSubscriptions$.value.concat(transports$), + ); + } + // After updating the subscriptions our connection list is also updated. + return transports$.value + .map((transport) => { + const isConnectionForTransport = (connection: Connection): boolean => + areLivekitTransportsEqual(connection.transport, transport); + return this.connections$.value.find(isConnectionForTransport); + }) + .filter((c) => c !== undefined); + } + + public unregisterTransports( + transports$: Behavior, + ): boolean { + const subscriptions = this.transportSubscriptions$.value; + const subscriptionsUnregistered = subscriptions.filter( + (t$) => t$ !== transports$, + ); + const canUnregister = + subscriptions.length !== subscriptionsUnregistered.length; + if (canUnregister) + this.transportSubscriptions$.next(subscriptionsUnregistered); + return canUnregister; + } + + public unregisterAllTransports(): void { + this.transportSubscriptions$.next([]); + } + + // We have a lost of connections, for each of these these + // connection we create a stream of (participant, connection) tuples. + // Then we combine the several streams (1 per Connection) into a single stream of tuples. + private allParticipantsWithConnection$ = this.scope.behavior( + this.connections$.pipe( + switchMap((connections) => { + const listsOfParticipantWithConnection = connections.map( + (connection) => { + return connection.participantsWithPublishTrack$.pipe( + map((participants) => + participants.map((p) => ({ + participant: p, + connection, + })), + ), + ); + }, + ); + return combineLatest(listsOfParticipantWithConnection).pipe( + map((lists) => lists.flatMap((list) => list)), + ); + }), + ), + ); + + // Filters the livekit participants + public allParticipantsByMemberId$ = this.scope.behavior( + this.allParticipantsWithConnection$.pipe( + map((participantsWithConnections) => { + const participantsByMemberId = participantsWithConnections.reduce( + (acc, test) => { + const { participant, connection } = test; + if (participant.getTrackPublications().length > 0) { + const currentVal = acc.get(participant.identity); + if (!currentVal) { + acc.set(participant.identity, [{ connection, participant }]); + } else { + // already known + // This is user is publishing on several SFUs + currentVal.push({ connection, participant }); + this.logger?.info( + `Participant ${participant.identity} is publishing on several SFUs ${currentVal.join()}`, + ); + } + } + return acc; + }, + new Map() as ParticipantByMemberIdMap, + ); + + return participantsByMemberId; + }), + ), + ); +} +function removeDuplicateTransports( + transports: LivekitTransport[], +): LivekitTransport[] { + return transports.reduce((acc, transport) => { + if (!acc.some((t) => areLivekitTransportsEqual(t, transport))) + acc.push(transport); + return acc; + }, [] as LivekitTransport[]); +} diff --git a/src/state/remoteMembers/matrixLivekitMerger.ts b/src/state/remoteMembers/matrixLivekitMerger.ts index 4cd68663..eb33f5a5 100644 --- a/src/state/remoteMembers/matrixLivekitMerger.ts +++ b/src/state/remoteMembers/matrixLivekitMerger.ts @@ -6,54 +6,22 @@ Please see LICENSE in the repository root for full details. */ import { - LocalParticipant, - Participant, - RemoteParticipant, + type RemoteParticipant, type Participant as LivekitParticipant, - type Room as LivekitRoom, } from "livekit-client"; import { - type MatrixRTCSession, - MatrixRTCSessionEvent, - type CallMembership, - type Transport, - LivekitTransport, isLivekitTransport, - ParticipantId, + type LivekitTransport, + type CallMembership, } from "matrix-js-sdk/lib/matrixrtc"; -import { - combineLatest, - map, - startWith, - switchMap, - type Observable, -} from "rxjs"; +import { combineLatest, map, startWith, type Observable } from "rxjs"; +import type { Room as MatrixRoom, RoomMember } from "matrix-js-sdk"; +// import type { Logger } from "matrix-js-sdk/lib/logger"; +import { type Behavior } from "../Behavior"; import { type ObservableScope } from "../ObservableScope"; -import { type Connection } from "./Connection"; -import { Behavior, constant } from "../Behavior"; -import { Room as MatrixRoom, RoomMember } from "matrix-js-sdk"; +import { type ConnectionManager } from "./ConnectionManager"; import { getRoomMemberFromRtcMember } from "./displayname"; -import { pauseWhen } from "../../utils/observable"; -import { Logger } from "matrix-js-sdk/lib/logger"; - -// TODOs: -// - make ConnectionManager its own actual class -// - write test for scopes (do we really need to bind scope) -class ConnectionManager { - public setTansports(transports$: Behavior): void {} - public readonly connections$: Observable = constant([]); - // connection is used to find the transport (to find matching callmembership) & for the livekitRoom - public readonly participantsByMemberId$: Behavior = - constant(new Map()); -} - -export type ParticipantByMemberIdMap = Map< - ParticipantId, - // It can be an array because a bad behaving client could be publishingParticipants$ - // multiple times to several livekit rooms. - { participant: LivekitParticipant; connection: Connection }[] ->; /** * Represents participant publishing or expected to publish on the connection. @@ -86,21 +54,6 @@ export interface MatrixLivekitItem { // Alternative structure idea: // const livekitMatrixItems$ = (callMemberships$,connectionManager,scope): Observable => { -interface LivekitRoomWithParticipants { - livekitRoom: LivekitRoom; - url: string; // Included for use as a React key - participants: { - // What id is that?? - // Looks like it userId:Deviceid? - id: string; - participant: LocalParticipant | RemoteParticipant | undefined; - // Why do we fetch a full room member here? - // looks like it is only for avatars? - // TODO: Remove that. have some Avatar Provider that can fetch avatar for user ids. - member: RoomMember; - }[]; -} - /** * Combines MatrixRtc and Livekit worlds. * @@ -112,9 +65,13 @@ interface LivekitRoomWithParticipants { * - `remoteMatrixLivekitItems` an observable of MatrixLivekitItem[] to track the remote members and associated livekit data. */ export class MatrixLivekitMerger { - private readonly logger: Logger; + /** + * Stream of all the call members and their associated livekit data (if available). + */ + public matrixLivekitItems$: Behavior; + + // private readonly logger: Logger; - public constructor( private memberships$: Observable, private connectionManager: ConnectionManager, @@ -123,10 +80,64 @@ export class MatrixLivekitMerger { // apparently needed to get a room member to later get the Avatar // => Extract an AvatarService instead? private matrixRoom: MatrixRoom, - parentLogger: Logger, + // parentLogger: Logger, ) { - this.logger = parentLogger.createChildLogger("MatrixLivekitMerger"); - connectionManager.setTansports(this.transports$); + // this.logger = parentLogger.getChild("MatrixLivekitMerger"); + + this.matrixLivekitItems$ = this.scope.behavior( + this.start$().pipe(startWith([])), + ); + } + + // ======================================= + /// PRIVATES + // ======================================= + private start$(): Observable { + const membershipsWithTransport$ = + this.mapMembershipsToMembershipWithTransport$(); + + this.startFeedingConnectionManager(membershipsWithTransport$); + + return combineLatest([ + membershipsWithTransport$, + this.connectionManager.allParticipantsByMemberId$, + ]).pipe( + map(([memberships, participantsByMemberId]) => { + const items = memberships.map(({ membership, transport }) => { + const participantsWithConnection = participantsByMemberId.get( + membership.membershipID, + ); + const participant = + transport && + participantsWithConnection?.find((p) => + areLivekitTransportsEqual(p.connection.transport, transport), + ); + return { + livekitParticipant: participant, + membership, + // This makes sense to add the the js-sdk callMembership (we only need the avatar so probably the call memberhsip just should aquire the avatar) + member: + // Why a member error? if we have a call membership there is a room member + getRoomMemberFromRtcMember(membership, this.matrixRoom)?.member, + } as MatrixLivekitItem; + }); + return items; + }), + ); + } + + private startFeedingConnectionManager( + membershipsWithTransport$: Behavior< + { membership: CallMembership; transport?: LivekitTransport }[] + >, + ): void { + const transports$ = this.scope.behavior( + membershipsWithTransport$.pipe( + map((mts) => mts.flatMap(({ transport: t }) => (t ? [t] : []))), + ), + ); + // duplicated transports will be elimiated by the connection manager + this.connectionManager.registerTransports(transports$); } /** @@ -137,127 +148,30 @@ export class MatrixLivekitMerger { * together when it might change together is what you have to do in RxJS to * avoid reading inconsistent state or observing too many changes.) */ - private readonly membershipsWithTransport$ = this.scope.behavior( - this.memberships$.pipe( - map((memberships) => { - return memberships.map((membership) => { - const oldestMembership = memberships[0] ?? membership; - const transport = membership.getTransport(oldestMembership); - return { - membership, - transport: isLivekitTransport(transport) ? transport : undefined, - }; - }); - }), - ), - ); - - private readonly transports$ = this.scope.behavior( - this.membershipsWithTransport$.pipe( - map((membershipsWithTransport) => - membershipsWithTransport.reduce((acc, { transport }) => { - if ( - transport && - !acc.some((t) => areLivekitTransportsEqual(t, transport)) - ) { - acc.push(transport); - } - return acc; - }, [] as LivekitTransport[]), - ), - ), - ); - - // TODO move this over this the connection manager - // We have a lost of connections, for each of these these - // connection we create a stream of (participant, connection) tuples. - // Then we combine the several streams (1 per Connection) into a single stream of tuples. - private participantsWithConnection$ = - this.connectionManager.connections$.pipe( - switchMap((connections) => { - const listsOfParticipantWithConnection = connections.map( - (connection) => { - return connection.participantsWithPublishTrack$.pipe( - map((participants) => - participants.map((p) => ({ - participant: p, - connection, - })), - ), - ); - }, - ); - return combineLatest(listsOfParticipantWithConnection).pipe( - map((lists) => lists.flatMap((list) => list)), - ); - }), - ); - - // TODO move this over this the connection manager - // Filters the livekit partic - private participantsByMemberId$ = this.participantsWithConnection$.pipe( - map((participantsWithConnections) => { - const participantsByMemberId = participantsWithConnections.reduce( - (acc, test) => { - const { participant, connection } = test; - if (participant.getTrackPublications().length > 0) { - const currentVal = acc.get(participant.identity); - if (!currentVal) { - acc.set(participant.identity, [{ connection, participant }]); - } else { - // already known - // This is user is publishing on several SFUs - currentVal.push({ connection, participant }); - this.logger.info( - `Participant ${participant.identity} is publishing on several SFUs ${currentVal.join()}`, - ); - } - } - return acc; - }, - new Map() as ParticipantByMemberIdMap, - ); - - return participantsByMemberId; - }), - ); - - public readonly matrixLivekitItems$ = this.scope - .behavior( - combineLatest([ - this.membershipsWithTransport$, - this.participantsByMemberId$, - ]).pipe( - map(([memberships, participantsByMemberId]) => { - const items = memberships.map(({ membership, transport }) => { - const participantsWithConnection = participantsByMemberId.get( - membership.membershipID, - ); - const participant = - transport && - participantsWithConnection?.find((p) => - areLivekitTransportsEqual(p.connection.transport, transport), - ); + private mapMembershipsToMembershipWithTransport$(): Observable< + { membership: CallMembership; transport?: LivekitTransport }[] + > { + return this.scope.behavior( + this.memberships$.pipe( + map((memberships) => { + return memberships.map((membership) => { + const oldestMembership = memberships[0] ?? membership; + const transport = membership.getTransport(oldestMembership); return { - livekitParticipant: participant, membership, - // This makes sense to add the the js-sdk callMembership (we only need the avatar so probably the call memberhsip just should aquire the avatar) - member: - // Why a member error? if we have a call membership there is a room member - getRoomMemberFromRtcMember(membership, this.matrixRoom)?.member, - } as MatrixLivekitItem; + transport: isLivekitTransport(transport) ? transport : undefined, + }; }); - return items; }), ), - ) - .pipe(startWith([])); + ); + } } // TODO add back in the callviewmodel pauseWhen(this.pretendToBeDisconnected$) // TODO add this to the JS-SDK -function areLivekitTransportsEqual( +export function areLivekitTransportsEqual( t1: LivekitTransport, t2: LivekitTransport, ): boolean { From 6b513534f10e7e25c85dab63384eb33b1046222d Mon Sep 17 00:00:00 2001 From: Timo K Date: Thu, 30 Oct 2025 00:09:07 +0100 Subject: [PATCH 07/75] lots of fixup in the new classes --- src/state/ownMember/Publisher.ts | 216 ++++++++++-------- src/state/remoteMembers/Connection.test.ts | 2 +- src/state/remoteMembers/Connection.ts | 63 +---- src/state/remoteMembers/ConnectionManager.ts | 169 ++++++++++---- src/state/remoteMembers/displayname.ts | 10 +- .../remoteMembers/matrixLivekitMerger.ts | 2 +- 6 files changed, 256 insertions(+), 206 deletions(-) diff --git a/src/state/ownMember/Publisher.ts b/src/state/ownMember/Publisher.ts index 3feb8a52..c37445b0 100644 --- a/src/state/ownMember/Publisher.ts +++ b/src/state/ownMember/Publisher.ts @@ -5,12 +5,13 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ import { - ConnectionState, type E2EEOptions, LocalVideoTrack, - Room as LivekitRoom, - type RoomOptions, + type Room as LivekitRoom, Track, + type LocalTrack, + type LocalTrackPublication, + ConnectionState as LivekitConnectionState, } from "livekit-client"; import { map, @@ -19,7 +20,7 @@ import { type Subscription, switchMap, } from "rxjs"; -import { logger } from "matrix-js-sdk/lib/logger"; +import { type Logger } from "matrix-js-sdk/lib/logger"; import type { Behavior } from "../Behavior.ts"; import type { MediaDevices, SelectedDevice } from "../MediaDevices.ts"; @@ -29,55 +30,43 @@ import { trackProcessorSync, } from "../../livekit/TrackProcessorContext.tsx"; import { getUrlParams } from "../../UrlParams.ts"; -import { defaultLiveKitOptions } from "../../livekit/options.ts"; -import { getValue } from "../../utils/observable.ts"; import { observeTrackReference$ } from "../MediaViewModel.ts"; -import { Connection, type ConnectionOpts } from "../remoteMembers/Connection.ts"; +import { type Connection } from "../remoteMembers/Connection.ts"; import { type ObservableScope } from "../ObservableScope.ts"; /** - * A connection to the local LiveKit room, the one the user is publishing to. - * This connection will publish the local user's audio and video tracks. + * A wrapper for a Connection object. + * This wrapper will manage the connection used to publish to the LiveKit room. + * The Publisher is also responsible for creating the media tracks. */ -export class PublishConnection extends Connection { - private readonly scope: ObservableScope; - +export class Publisher { + public tracks: LocalTrack[] = []; /** - * Creates a new PublishConnection. - * @param args - The connection options. {@link ConnectionOpts} + * Creates a new Publisher. + * @param scope - The observable scope to use for managing the publisher. + * @param connection - The connection to use for publishing. * @param devices - The media devices to use for audio and video input. * @param muteStates - The mute states for audio and video. * @param e2eeLivekitOptions - The E2EE options to use for the LiveKit room. Use to share the same key provider across connections!. * @param trackerProcessorState$ - The processor state for the video track processor (e.g. background blur). */ public constructor( - args: ConnectionOpts, + private scope: ObservableScope, + private connection: Connection, devices: MediaDevices, private readonly muteStates: MuteStates, e2eeLivekitOptions: E2EEOptions | undefined, trackerProcessorState$: Behavior, + private logger?: Logger, ) { - const { scope } = args; - logger.info("[PublishConnection] Create LiveKit room"); + this.logger?.info("[PublishConnection] Create LiveKit room"); const { controlledAudioDevices } = getUrlParams(); - const factory = - args.livekitRoomFactory ?? - ((options: RoomOptions): LivekitRoom => new LivekitRoom(options)); - const room = factory( - generateRoomOption( - devices, - trackerProcessorState$.value, - controlledAudioDevices, - e2eeLivekitOptions, - ), - ); - room.setE2EEEnabled(e2eeLivekitOptions !== undefined)?.catch((e) => { - logger.error("Failed to set E2EE enabled on room", e); - }); + const room = connection.livekitRoom; - super(room, args); - this.scope = scope; + room.setE2EEEnabled(e2eeLivekitOptions !== undefined)?.catch((e) => { + this.logger?.error("Failed to set E2EE enabled on room", e); + }); // Setup track processor syncing (blur) this.observeTrackProcessors(scope, room, trackerProcessorState$); @@ -91,55 +80,98 @@ export class PublishConnection extends Connection { * Start the connection to LiveKit and publish local tracks. * * This will: - * 1. Request an OpenId token `request_token` (allows matrix users to verify their identity with a third-party service.) - * 2. Use this token to request the SFU config to the MatrixRtc authentication service. - * 3. Connect to the configured LiveKit room. - * 4. Create local audio and video tracks based on the current mute states and publish them to the room. + * wait for the connection to be ready. + // * 1. Request an OpenId token `request_token` (allows matrix users to verify their identity with a third-party service.) + // * 2. Use this token to request the SFU config to the MatrixRtc authentication service. + // * 3. Connect to the configured LiveKit room. + // * 4. Create local audio and video tracks based on the current mute states and publish them to the room. * * @throws {InsufficientCapacityError} if the LiveKit server indicates that it has insufficient capacity to accept the connection. * @throws {SFURoomCreationRestrictedError} if the LiveKit server indicates that the room does not exist and cannot be created. */ - public async start(): Promise { - this.stopped = false; - + public async createAndSetupTracks(): Promise { + const lkRoom = this.connection.livekitRoom; // Observe mute state changes and update LiveKit microphone/camera states accordingly this.observeMuteStates(this.scope); + // TODO: This should be an autostarted connection no need to start here. just check the connection state. // TODO: This will fetch the JWT token. Perhaps we could keep it preloaded // instead? This optimization would only be safe for a publish connection, // because we don't want to leak the user's intent to perhaps join a call to // remote servers before they actually commit to it. - await super.start(); - - if (this.stopped) return; - + const { promise, resolve, reject } = Promise.withResolvers(); + const sub = this.connection.state$.subscribe((s) => { + if (s.state !== "FailedToStart") { + reject(new Error("Disconnected from LiveKit server")); + } else { + resolve(); + } + }); + try { + await promise; + } catch (e) { + throw e; + } finally { + sub.unsubscribe(); + } // TODO-MULTI-SFU: Prepublish a microphone track const audio = this.muteStates.audio.enabled$.value; const video = this.muteStates.video.enabled$.value; // createTracks throws if called with audio=false and video=false if (audio || video) { // TODO this can still throw errors? It will also prompt for permissions if not already granted - const tracks = await this.livekitRoom.localParticipant.createTracks({ + this.tracks = await lkRoom.localParticipant.createTracks({ audio, video, }); - if (this.stopped) return; - for (const track of tracks) { - // TODO: handle errors? Needs the signaling connection to be up, but it has some retries internally - // with a timeout. - await this.livekitRoom.localParticipant.publishTrack(track); - if (this.stopped) return; - // TODO: check if the connection is still active? and break the loop if not? - } } } - public async stop(): Promise { + public async startPublishing(): Promise { + const lkRoom = this.connection.livekitRoom; + const { promise, resolve, reject } = Promise.withResolvers(); + const sub = this.connection.state$.subscribe((s) => { + switch (s.state) { + case "ConnectedToLkRoom": + resolve(); + break; + case "FailedToStart": + reject(new Error("Failed to connect to LiveKit server")); + break; + default: + this.logger?.info("waiting for connection: ", s.state); + } + }); + try { + await promise; + } catch (e) { + throw e; + } finally { + sub.unsubscribe(); + } + for (const track of this.tracks) { + // TODO: handle errors? Needs the signaling connection to be up, but it has some retries internally + // with a timeout. + await lkRoom.localParticipant.publishTrack(track); + + // TODO: check if the connection is still active? and break the loop if not? + } + return this.tracks; + } + + public async stopPublishing(): Promise { // TODO-MULTI-SFU: Move these calls back to ObservableScope.onEnd once scope // actually has the right lifetime this.muteStates.audio.unsetHandler(); this.muteStates.video.unsetHandler(); - await super.stop(); + + const localParticipant = this.connection.livekitRoom.localParticipant; + const tracks: LocalTrack[] = []; + const addToTracksIfDefined = (p: LocalTrackPublication): void => { + if (p.track !== undefined) tracks.push(p.track); + }; + localParticipant.trackPublications.forEach(addToTracksIfDefined); + await localParticipant.unpublishTracks(tracks); } /// Private methods @@ -156,15 +188,16 @@ export class PublishConnection extends Connection { devices: MediaDevices, scope: ObservableScope, ): void { + const lkRoom = this.connection.livekitRoom; devices.audioInput.selected$ .pipe( switchMap((device) => device?.hardwareDeviceChange$ ?? NEVER), scope.bind(), ) .subscribe(() => { - if (this.livekitRoom.state != ConnectionState.Connected) return; + if (lkRoom.state != LivekitConnectionState.Connected) return; const activeMicTrack = Array.from( - this.livekitRoom.localParticipant.audioTrackPublications.values(), + lkRoom.localParticipant.audioTrackPublications.values(), ).find((d) => d.source === Track.Source.Microphone)?.track; if ( @@ -179,11 +212,11 @@ export class PublishConnection extends Connection { // getUserMedia() call with deviceId: default to get the *new* default device. // Note that room.switchActiveDevice() won't work: Livekit will ignore it because // the deviceId hasn't changed (was & still is default). - this.livekitRoom.localParticipant + lkRoom.localParticipant .getTrackPublication(Track.Source.Microphone) ?.audioTrack?.restartTrack() .catch((e) => { - logger.error(`Failed to restart audio device track`, e); + this.logger?.error(`Failed to restart audio device track`, e); }); } }); @@ -195,27 +228,31 @@ export class PublishConnection extends Connection { devices: MediaDevices, controlledAudioDevices: boolean, ): void { + const lkRoom = this.connection.livekitRoom; const syncDevice = ( kind: MediaDeviceKind, selected$: Observable, ): Subscription => selected$.pipe(scope.bind()).subscribe((device) => { - if (this.livekitRoom.state != ConnectionState.Connected) return; + if (lkRoom.state != LivekitConnectionState.Connected) return; // if (this.connectionState$.value !== ConnectionState.Connected) return; - logger.info( + this.logger?.info( "[LivekitRoom] syncDevice room.getActiveDevice(kind) !== d.id :", - this.livekitRoom.getActiveDevice(kind), + lkRoom.getActiveDevice(kind), " !== ", device?.id, ); if ( device !== undefined && - this.livekitRoom.getActiveDevice(kind) !== device.id + lkRoom.getActiveDevice(kind) !== device.id ) { - this.livekitRoom + lkRoom .switchActiveDevice(kind, device.id) .catch((e) => - logger.error(`Failed to sync ${kind} device with LiveKit`, e), + this.logger?.error( + `Failed to sync ${kind} device with LiveKit`, + e, + ), ); } }); @@ -232,21 +269,28 @@ export class PublishConnection extends Connection { * @private */ private observeMuteStates(scope: ObservableScope): void { + const lkRoom = this.connection.livekitRoom; this.muteStates.audio.setHandler(async (desired) => { try { - await this.livekitRoom.localParticipant.setMicrophoneEnabled(desired); + await lkRoom.localParticipant.setMicrophoneEnabled(desired); } catch (e) { - logger.error("Failed to update LiveKit audio input mute state", e); + this.logger?.error( + "Failed to update LiveKit audio input mute state", + e, + ); } - return this.livekitRoom.localParticipant.isMicrophoneEnabled; + return lkRoom.localParticipant.isMicrophoneEnabled; }); this.muteStates.video.setHandler(async (desired) => { try { - await this.livekitRoom.localParticipant.setCameraEnabled(desired); + await lkRoom.localParticipant.setCameraEnabled(desired); } catch (e) { - logger.error("Failed to update LiveKit video input mute state", e); + this.logger?.error( + "Failed to update LiveKit video input mute state", + e, + ); } - return this.livekitRoom.localParticipant.isCameraEnabled; + return lkRoom.localParticipant.isCameraEnabled; }); } @@ -266,33 +310,3 @@ export class PublishConnection extends Connection { trackProcessorSync(track$, trackerProcessorState$); } } - -// Generate the initial LiveKit RoomOptions based on the current media devices and processor state. -function generateRoomOption( - devices: MediaDevices, - processorState: ProcessorState, - controlledAudioDevices: boolean, - e2eeLivekitOptions: E2EEOptions | undefined, -): RoomOptions { - return { - ...defaultLiveKitOptions, - videoCaptureDefaults: { - ...defaultLiveKitOptions.videoCaptureDefaults, - deviceId: devices.videoInput.selected$.value?.id, - processor: processorState.processor, - }, - audioCaptureDefaults: { - ...defaultLiveKitOptions.audioCaptureDefaults, - deviceId: devices.audioInput.selected$.value?.id, - }, - audioOutput: { - // When using controlled audio devices, we don't want to set the - // deviceId here, because it will be set by the native app. - // (also the id does not need to match a browser device id) - deviceId: controlledAudioDevices - ? undefined - : getValue(devices.audioOutput.selected$)?.id, - }, - e2ee: e2eeLivekitOptions, - }; -} diff --git a/src/state/remoteMembers/Connection.test.ts b/src/state/remoteMembers/Connection.test.ts index 3b0f42ee..0719e2c5 100644 --- a/src/state/remoteMembers/Connection.test.ts +++ b/src/state/remoteMembers/Connection.test.ts @@ -41,7 +41,7 @@ import { import { ObservableScope } from "../ObservableScope.ts"; import { type OpenIDClientParts } from "../../livekit/openIDSFU.ts"; import { FailToGetOpenIdToken } from "../../utils/errors.ts"; -import { PublishConnection } from "../ownMember/PublishConnection.ts"; +import { PublishConnection } from "../ownMember/Publisher.ts"; import { mockMediaDevices, mockMuteStates } from "../../utils/test.ts"; import type { ProcessorState } from "../../livekit/TrackProcessorContext.tsx"; import { type MuteStates } from "../MuteStates.ts"; diff --git a/src/state/remoteMembers/Connection.ts b/src/state/remoteMembers/Connection.ts index e815ea55..67b2dc8e 100644 --- a/src/state/remoteMembers/Connection.ts +++ b/src/state/remoteMembers/Connection.ts @@ -12,14 +12,12 @@ import { import { ConnectionError, type ConnectionState as LivekitConenctionState, - type E2EEOptions, - Room as LivekitRoom, - type RoomOptions, + type Room as LivekitRoom, type Participant, RoomEvent, } from "livekit-client"; import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; -import { BehaviorSubject, combineLatest, type Observable } from "rxjs"; +import { BehaviorSubject, type Observable } from "rxjs"; import { type Logger } from "matrix-js-sdk/lib/logger"; import { @@ -29,7 +27,6 @@ import { } from "../../livekit/openIDSFU.ts"; import { type Behavior } from "../Behavior.ts"; import { type ObservableScope } from "../ObservableScope.ts"; -import { defaultLiveKitOptions } from "../../livekit/options.ts"; import { InsufficientCapacityError, SFURoomCreationRestrictedError, @@ -44,19 +41,9 @@ export interface ConnectionOpts { client: OpenIDClientParts; /** The observable scope to use for this connection. */ scope: ObservableScope; - /** - * An observable of the current RTC call memberships and their associated transports. - * Used to differentiate between publishing and subscribging participants on each connection. - * Used to find out which rtc member should upload to this connection (publishingParticipants$). - * The livekit room gives access to all the users subscribing to this connection, we need - * to filter out the ones that are uploading to this connection. - */ - // membershipsWithTransport$: Behavior< - // { membership: CallMembership; transport: LivekitTransport }[] - // >; /** Optional factory to create the LiveKit room, mainly for testing purposes. */ - livekitRoomFactory?: (options?: RoomOptions) => LivekitRoom; + livekitRoomFactory: () => LivekitRoom; } export type ConnectionState = @@ -173,6 +160,7 @@ export class Connection { this.transport.livekit_alias, ); } + /** * Stops the connection. * @@ -195,10 +183,7 @@ export class Connection { * It filters the participants to only those that are associated with a membership that claims to publish on this connection. */ - public readonly publishingParticipants$: Behavior; - public readonly participantsWithPublishTrack$: Behavior< - PublishingParticipant[] - >; + public readonly participantsWithTrack$: Behavior; /** * The media transport to connect to. @@ -206,6 +191,8 @@ export class Connection { public readonly transport: LivekitTransport; private readonly client: OpenIDClientParts; + public readonly livekitRoom: LivekitRoom; + /** * Creates a new connection to a matrix RTC LiveKit backend. * @@ -213,20 +200,17 @@ export class Connection { * @param opts - Connection options {@link ConnectionOpts}. * */ - protected constructor( - public readonly livekitRoom: LivekitRoom, - opts: ConnectionOpts, - logger?: Logger, - ) { + public constructor(opts: ConnectionOpts, logger?: Logger) { logger?.info( `[Connection] Creating new connection to ${opts.transport.livekit_service_url} ${opts.transport.livekit_alias}`, ); const { transport, client, scope } = opts; + this.livekitRoom = opts.livekitRoomFactory(); this.transport = transport; this.client = client; - this.participantsWithPublishTrack$ = scope.behavior( + this.participantsWithTrack$ = scope.behavior( connectedParticipantsObserver( this.livekitRoom, // VALR: added that while I think about it @@ -243,30 +227,3 @@ export class Connection { scope.onEnd(() => void this.stop()); } } - -/** - * A remote connection to the Matrix RTC LiveKit backend. - * - * This connection is used for subscribing to remote participants. - * It does not publish any local tracks. - */ -export class RemoteConnection extends Connection { - /** - * Creates a new remote connection to a matrix RTC LiveKit backend. - * @param opts - * @param sharedE2eeOption - The shared E2EE options to use for the connection. - */ - public constructor( - opts: ConnectionOpts, - sharedE2eeOption: E2EEOptions | undefined, - ) { - const factory = - opts.livekitRoomFactory ?? - ((options: RoomOptions): LivekitRoom => new LivekitRoom(options)); - const livekitRoom = factory({ - ...defaultLiveKitOptions, - e2ee: sharedE2eeOption, - }); - super(livekitRoom, opts); - } -} diff --git a/src/state/remoteMembers/ConnectionManager.ts b/src/state/remoteMembers/ConnectionManager.ts index 311e621e..b7a37b11 100644 --- a/src/state/remoteMembers/ConnectionManager.ts +++ b/src/state/remoteMembers/ConnectionManager.ts @@ -16,16 +16,21 @@ import { BehaviorSubject, combineLatest, map, switchMap } from "rxjs"; import { type Logger } from "matrix-js-sdk/lib/logger"; import { type E2EEOptions, - type Room as LivekitRoom, + Room as LivekitRoom, type Participant as LivekitParticipant, + type RoomOptions, } from "livekit-client"; import { type MatrixClient } from "matrix-js-sdk"; import { type Behavior } from "../Behavior"; -import { type Connection, RemoteConnection } from "./Connection"; +import { Connection } from "./Connection"; import { type ObservableScope } from "../ObservableScope"; import { generateKeyed$ } from "../../utils/observable"; import { areLivekitTransportsEqual } from "./matrixLivekitMerger"; +import { getUrlParams } from "../../UrlParams"; +import { type ProcessorState } from "../../livekit/TrackProcessorContext"; +import { type MediaDevices } from "../MediaDevices"; +import { defaultLiveKitOptions } from "../../livekit/options"; export type ParticipantByMemberIdMap = Map< ParticipantId, @@ -33,25 +38,57 @@ export type ParticipantByMemberIdMap = Map< // multiple times to several livekit rooms. { participant: LivekitParticipant; connection: Connection }[] >; - -// - write test for scopes (do we really need to bind scope) +// TODO - write test for scopes (do we really need to bind scope) export class ConnectionManager { - /** - * The transport to use for publishing. - * This extends the list of tranports - */ - private publishTransport$ = new BehaviorSubject( - undefined, - ); + private livekitRoomFactory: () => LivekitRoom; + public constructor( + private client: MatrixClient, + private scope: ObservableScope, + private devices: MediaDevices, + private processorState: ProcessorState, + private e2eeLivekitOptions$: Behavior, + private logger?: Logger, + livekitRoomFactory?: () => LivekitRoom, + ) { + this.scope = scope; + const defaultFactory = (): LivekitRoom => + new LivekitRoom( + generateRoomOption( + this.devices, + this.processorState, + this.e2eeLivekitOptions$.value, + ), + ); + this.livekitRoomFactory = livekitRoomFactory ?? defaultFactory; + } - private transportSubscriptions$ = new BehaviorSubject< + /** + * A list of Behaviors each containing a LIST of LivekitTransport. + * Each of these behaviors can be interpreted as subscribed list of transports. + * + * Using `registerTransports` independent external modules can control what connections + * are created by the ConnectionManager. + * + * The connection manager will remove all duplicate transports in each subscibed list. + * + * See `unregisterAllTransports` and `unregisterTransport` for details on how to unsubscribe. + */ + private readonly transportsSubscriptions$ = new BehaviorSubject< Behavior[] >([]); - private transports$ = this.scope.behavior( - this.transportSubscriptions$.pipe( + /** + * All transports currently managed by the ConnectionManager. + * + * This list does not include duplicate transports. + * + * It is build based on the list of subscribed transports (`transportsSubscriptions$`). + * externally this is modified via `registerTransports()`. + */ + private readonly transports$ = this.scope.behavior( + this.transportsSubscriptions$.pipe( switchMap((subscriptions) => - combineLatest(subscriptions.map((s) => s.transports)).pipe( + combineLatest(subscriptions).pipe( map((transportsNested) => transportsNested.flat()), map(removeDuplicateTransports), ), @@ -59,24 +96,6 @@ export class ConnectionManager { ), ); - public constructor( - private client: MatrixClient, - private e2eeLivekitOptions: () => E2EEOptions | undefined, - private scope: ObservableScope, - private logger?: Logger, - private livekitRoomFactory?: () => LivekitRoom, - ) { - this.scope = scope; - } - - public getOrCreatePublishConnection( - transport: LivekitTransport, - ): Connection | undefined { - this.publishTransport$.next(transport); - const equalsRequestedTransport = (c: Connection): boolean => - areLivekitTransportsEqual(c.transport, transport); - return this.connections$.value.find(equalsRequestedTransport); - } /** * Connections for each transport in use by one or more session members. */ @@ -87,16 +106,16 @@ export class ConnectionManager { const createConnection = ( transport: LivekitTransport, - ): ((scope: ObservableScope) => RemoteConnection) => + ): ((scope: ObservableScope) => Connection) => (scope) => { - const connection = new RemoteConnection( + const connection = new Connection( { transport, client: this.client, scope: scope, livekitRoomFactory: this.livekitRoomFactory, }, - this.e2eeLivekitOptions(), + this.logger, ); void connection.start(); return connection; @@ -114,15 +133,23 @@ export class ConnectionManager { ); /** + * Add an a Behavior containing a list of transports to this ConnectionManager. * - * @param transports$ + * The intended usage is: + * - create a ConnectionManager + * - register one `transports$` behavior using registerTransports + * - add new connections to the `ConnectionManager` by updating the `transports$` behavior + * - remove a single connection by removing the transport. + * - remove this subscription by calling `unregisterTransports` and passing + * the same `transports$` behavior reference. + * @param transports$ The Behavior containing a list of transports to subscribe to. */ public registerTransports( transports$: Behavior, ): Connection[] { - if (!this.transportSubscriptions$.value.some((t$) => t$ === transports$)) { - this.transportSubscriptions$.next( - this.transportSubscriptions$.value.concat(transports$), + if (!this.transportsSubscriptions$.value.some((t$) => t$ === transports$)) { + this.transportsSubscriptions$.next( + this.transportsSubscriptions$.value.concat(transports$), ); } // After updating the subscriptions our connection list is also updated. @@ -135,22 +162,30 @@ export class ConnectionManager { .filter((c) => c !== undefined); } + /** + * Unsubscribe from the given transports. + * @param transports$ The behavior to unsubscribe from + * @returns + */ public unregisterTransports( transports$: Behavior, ): boolean { - const subscriptions = this.transportSubscriptions$.value; + const subscriptions = this.transportsSubscriptions$.value; const subscriptionsUnregistered = subscriptions.filter( (t$) => t$ !== transports$, ); const canUnregister = subscriptions.length !== subscriptionsUnregistered.length; if (canUnregister) - this.transportSubscriptions$.next(subscriptionsUnregistered); + this.transportsSubscriptions$.next(subscriptionsUnregistered); return canUnregister; } + /** + * Unsubscribe from all transports. + */ public unregisterAllTransports(): void { - this.transportSubscriptions$.next([]); + this.transportsSubscriptions$.next([]); } // We have a lost of connections, for each of these these @@ -161,7 +196,7 @@ export class ConnectionManager { switchMap((connections) => { const listsOfParticipantWithConnection = connections.map( (connection) => { - return connection.participantsWithPublishTrack$.pipe( + return connection.participantsWithTrack$.pipe( map((participants) => participants.map((p) => ({ participant: p, @@ -178,7 +213,13 @@ export class ConnectionManager { ), ); - // Filters the livekit participants + /** + * This field makes the connection manager to behave as close to a single SFU as possible. + * Each participant that is found on all connections managed by the manager will be listed. + * + * They are stored an a map keyed by `participant.identity` + * (which is equivalent to the `member.id` field in the `m.rtc.member` event) + */ public allParticipantsByMemberId$ = this.scope.behavior( this.allParticipantsWithConnection$.pipe( map((participantsWithConnections) => { @@ -191,10 +232,10 @@ export class ConnectionManager { acc.set(participant.identity, [{ connection, participant }]); } else { // already known - // This is user is publishing on several SFUs + // This is for users publishing on several SFUs currentVal.push({ connection, participant }); this.logger?.info( - `Participant ${participant.identity} is publishing on several SFUs ${currentVal.join()}`, + `Participant ${participant.identity} is publishing on several SFUs ${currentVal.map((v) => v.connection.transport.livekit_service_url).join(", ")}`, ); } } @@ -217,3 +258,37 @@ function removeDuplicateTransports( return acc; }, [] as LivekitTransport[]); } + +/** + * Generate the initial LiveKit RoomOptions based on the current media devices and processor state. + */ +function generateRoomOption( + devices: MediaDevices, + processorState: ProcessorState, + e2eeLivekitOptions: E2EEOptions | undefined, +): RoomOptions { + const { controlledAudioDevices } = getUrlParams(); + return { + ...defaultLiveKitOptions, + videoCaptureDefaults: { + ...defaultLiveKitOptions.videoCaptureDefaults, + deviceId: devices.videoInput.selected$.value?.id, + processor: processorState.processor, + }, + audioCaptureDefaults: { + ...defaultLiveKitOptions.audioCaptureDefaults, + deviceId: devices.audioInput.selected$.value?.id, + }, + audioOutput: { + // When using controlled audio devices, we don't want to set the + // deviceId here, because it will be set by the native app. + // (also the id does not need to match a browser device id) + deviceId: controlledAudioDevices + ? undefined + : devices.audioOutput.selected$.value?.id, + }, + e2ee: e2eeLivekitOptions, + // TODO test and consider this: + // webAudioMix: true, + }; +} diff --git a/src/state/remoteMembers/displayname.ts b/src/state/remoteMembers/displayname.ts index ec3231c6..f288e2d0 100644 --- a/src/state/remoteMembers/displayname.ts +++ b/src/state/remoteMembers/displayname.ts @@ -12,7 +12,11 @@ import { logger } from "matrix-js-sdk/lib/logger"; import { type Room as MatrixRoom } from "matrix-js-sdk/lib/matrix"; import { type ObservableScope } from "../ObservableScope"; -import { calculateDisplayName, shouldDisambiguate } from "../../utils/displayname"; +import { + calculateDisplayName, + shouldDisambiguate, +} from "../../utils/displayname"; +import { type Behavior } from "../Behavior"; /** * Displayname for each member of the call. This will disambiguate @@ -21,12 +25,12 @@ import { calculateDisplayName, shouldDisambiguate } from "../../utils/displaynam */ // don't do this work more times than we need to. This is achieved by converting to a behavior: export const memberDisplaynames$ = ( + scope: ObservableScope, matrixRoom: Room, memberships$: Observable, - scope: ObservableScope, userId: string, deviceId: string, -) => +): Behavior> => scope.behavior( combineLatest( [ diff --git a/src/state/remoteMembers/matrixLivekitMerger.ts b/src/state/remoteMembers/matrixLivekitMerger.ts index eb33f5a5..e77306c1 100644 --- a/src/state/remoteMembers/matrixLivekitMerger.ts +++ b/src/state/remoteMembers/matrixLivekitMerger.ts @@ -148,7 +148,7 @@ export class MatrixLivekitMerger { * together when it might change together is what you have to do in RxJS to * avoid reading inconsistent state or observing too many changes.) */ - private mapMembershipsToMembershipWithTransport$(): Observable< + private mapMembershipsToMembershipWithTransport$(): Behavior< { membership: CallMembership; transport?: LivekitTransport }[] > { return this.scope.behavior( From c8ef8d6a246beb97d1f8e6a80e54f25420752a33 Mon Sep 17 00:00:00 2001 From: Timo K Date: Thu, 30 Oct 2025 01:13:06 +0100 Subject: [PATCH 08/75] start moving over/removing things from the CallViewModel --- src/state/CallViewModel.ts | 418 +++--------------- src/state/ownMember/OwnMembership.ts | 160 ++++--- src/state/remoteMembers/Connection.test.ts | 5 +- src/state/remoteMembers/ConnectionManager.ts | 10 +- src/state/remoteMembers/displayname.ts | 10 +- .../remoteMembers/matrixLivekitMerger.ts | 95 ++-- src/utils/displayname.ts | 2 +- 7 files changed, 231 insertions(+), 469 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 90a1f682..c8f68cbb 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -137,7 +137,16 @@ import { import { ElementCallError, UnknownCallError } from "../utils/errors.ts"; import { ObservableScope } from "./ObservableScope.ts"; import { memberDisplaynames$ } from "./remoteMembers/displayname.ts"; +import { ConnectionManager } from "./remoteMembers/ConnectionManager.ts"; +import { MatrixLivekitMerger } from "./remoteMembers/matrixLivekitMerger.ts"; +//TODO +// Larger rename +// member,membership -> rtcMember +// participant -> livekitParticipant +// matrixLivekitItem -> callMember +// js-sdk +// callMembership -> rtcMembership export interface CallViewModelOptions { encryptionSystem: EncryptionSystem; autoLeaveWhenOthersLeft?: boolean; @@ -205,6 +214,29 @@ export class CallViewModel { null, ); + private memberships$ = this.scope.behavior( + fromEvent( + this.matrixRTCSession, + MatrixRTCSessionEvent.MembershipsChanged, + (_, memberships: CallMembership[]) => memberships, + ), + ); + + private connectionManager = new ConnectionManager( + this.scope, + this.matrixRoom.client, + this.mediaDevices, + this.trackProcessorState$, + this.e2eeLivekitOptions(), + ); + + private matrixLivekitMerger = new MatrixLivekitMerger( + this.scope, + this.memberships$, + this.connectionManager, + this.matrixRoom, + ); + /** * If there is a configuration error with the call (e.g. misconfigured E2EE). * This is a fatal error that prevents the call from being created/joined. @@ -221,7 +253,7 @@ export class CallViewModel { this.join$.next(); } - // CODESMALL + // CODESMELL? // This is functionally the same Observable as leave$, except here it's // hoisted to the top of the class. This enables the cyclic dependency between // leave$ -> autoLeave$ -> callPickupState$ -> livekitConnectionState$ -> @@ -302,112 +334,28 @@ export class CallViewModel { ), ); - // DISCUSSION move to ConnectionManager - /** - * The local connection over which we will publish our media. It could - * possibly also have some remote users' media available on it. - * null when not joined. - */ - private readonly localConnection$: Behavior | null> = - this.scope.behavior( - generateKeyed$< - Async | null, - PublishConnection, - Async | null - >( - this.localTransport$, - (transport, createOrGet) => - transport && - mapAsync(transport, (transport) => - createOrGet( - // Stable key that uniquely idenifies the transport - JSON.stringify({ - url: transport.livekit_service_url, - alias: transport.livekit_alias, - }), - (scope) => - new PublishConnection( - { - transport, - client: this.matrixRoom.client, - scope, - remoteTransports$: this.remoteTransports$, - livekitRoomFactory: this.options.livekitRoomFactory, - }, - this.mediaDevices, - this.muteStates, - this.e2eeLivekitOptions(), - this.scope.behavior(this.trackProcessorState$), - ), - ), - ), - ), - ); - - // DISCUSSION move to ConnectionManager - public readonly livekitConnectionState$ = - // TODO: This options.connectionState$ behavior is a small hack inserted - // here to facilitate testing. This would likely be better served by - // breaking CallViewModel down into more naturally testable components. - this.options.connectionState$ ?? - this.scope.behavior( - this.localConnection$.pipe( - switchMap((c) => - c?.state === "ready" - ? // TODO mapping to ConnectionState for compatibility, but we should use the full state? - c.value.state$.pipe( - switchMap((s) => { - if (s.state === "ConnectedToLkRoom") - return s.connectionState$; - return of(ConnectionState.Disconnected); - }), - ) - : of(ConnectionState.Disconnected), - ), - ), - ); - - /** - * A list of the connections that should be active at any given time. - */ - // DISCUSSION move to ConnectionManager - private readonly connections$ = this.scope.behavior( - combineLatest( - [this.localConnection$, this.remoteConnections$], - (local, remote) => [ - ...(local?.state === "ready" ? [local.value] : []), - ...remote.values(), - ], - ), - ); - - /** - * Emits with connections whenever they should be started or stopped. - */ - // DISCUSSION move to ConnectionManager - private readonly connectionInstructions$ = this.connections$.pipe( - pairwise(), - map(([prev, next]) => { - const start = new Set(next.values()); - for (const connection of prev) start.delete(connection); - const stop = new Set(prev.values()); - for (const connection of next) stop.delete(connection); - - return { start, stop }; - }), - ); - - public readonly allLivekitRooms$ = this.scope.behavior( - this.connections$.pipe( - map((connections) => - [...connections.values()].map((c) => ({ - room: c.livekitRoom, - url: c.transport.livekit_service_url, - isLocal: c instanceof PublishConnection, - })), - ), - ), - ); + // // DISCUSSION move to ConnectionManager + // public readonly livekitConnectionState$ = + // // TODO: This options.connectionState$ behavior is a small hack inserted + // // here to facilitate testing. This would likely be better served by + // // breaking CallViewModel down into more naturally testable components. + // this.options.connectionState$ ?? + // this.scope.behavior( + // this.localConnection$.pipe( + // switchMap((c) => + // c?.state === "ready" + // ? // TODO mapping to ConnectionState for compatibility, but we should use the full state? + // c.value.state$.pipe( + // switchMap((s) => { + // if (s.state === "ConnectedToLkRoom") + // return s.connectionState$; + // return of(ConnectionState.Disconnected); + // }), + // ) + // : of(ConnectionState.Disconnected), + // ), + // ), + // ); private readonly userId = this.matrixRoom.client.getUserId()!; private readonly deviceId = this.matrixRoom.client.getDeviceId()!; @@ -450,114 +398,6 @@ export class CallViewModel { ), ); - /** - * Whether we are "fully" connected to the call. Accounts for both the - * connection to the MatrixRTC session and the LiveKit publish connection. - */ - // DISCUSSION own membership manager - private readonly connected$ = this.scope.behavior( - and$( - this.matrixConnected$, - this.livekitConnectionState$.pipe( - map((state) => state === ConnectionState.Connected), - ), - ), - ); - - /** - * Whether we should tell the user that we're reconnecting to the call. - */ - // DISCUSSION own membership manager - public readonly reconnecting$ = this.scope.behavior( - this.connected$.pipe( - // We are reconnecting if we previously had some successful initial - // connection but are now disconnected - scan( - ({ connectedPreviously }, connectedNow) => ({ - connectedPreviously: connectedPreviously || connectedNow, - reconnecting: connectedPreviously && !connectedNow, - }), - { connectedPreviously: false, reconnecting: false }, - ), - map(({ reconnecting }) => reconnecting), - ), - ); - - /** - * Lists the transports used by ourselves, plus all other MatrixRTC session - * members. For completeness this also lists the preferred transport and - * whether we are in multi-SFU mode or sticky events mode (because - * advertisedTransport$ wants to read them at the same time, and bundling data - * together when it might change together is what you have to do in RxJS to - * avoid reading inconsistent state or observing too many changes.) - */ - // TODO-MULTI-SFU find a better name for this. With the addition of sticky events it's no longer just about transports. - // DISCUSS move the local part to the own membership file - private readonly transports$: Behavior<{ - local: Async; - remote: { membership: CallMembership; transport: LivekitTransport }[]; - preferred: Async; - multiSfu: boolean; - preferStickyEvents: boolean; - } | null> = this.scope.behavior( - this.joined$.pipe( - switchMap((joined) => - joined - ? combineLatest( - [ - this.preferredTransport$, - this.memberships$, - multiSfu.value$, - preferStickyEvents.value$, - ], - (preferred, memberships, preferMultiSfu, preferStickyEvents) => { - // Multi-SFU must be implicitly enabled when using sticky events - const multiSfu = preferStickyEvents || preferMultiSfu; - - const oldestMembership = - this.matrixRTCSession.getOldestMembership(); - const remote = memberships.flatMap((m) => { - if (m.userId === this.userId && m.deviceId === this.deviceId) - return []; - const t = m.getTransport(oldestMembership ?? m); - return t && isLivekitTransport(t) - ? [{ membership: m, transport: t }] - : []; - }); - - let local = preferred; - if (!multiSfu) { - const oldest = this.matrixRTCSession.getOldestMembership(); - if (oldest !== undefined) { - const selection = oldest.getTransport(oldest); - // TODO selection can be null if no transport is configured should we report an error? - if (selection && isLivekitTransport(selection)) - local = ready(selection); - } - } - - if (local.state === "error") { - this._configError$.next( - local.value instanceof ElementCallError - ? local.value - : new UnknownCallError(local.value), - ); - } - - return { - local, - remote, - preferred, - multiSfu, - preferStickyEvents, - }; - }, - ) - : of(null), - ), - ), - ); - /** * Whether various media/event sources should pretend to be disconnected from * all network input, even if their connection still technically works. @@ -569,95 +409,7 @@ export class CallViewModel { // DISCUSSION own membership manager ALSO this probably can be simplifis private readonly pretendToBeDisconnected$ = this.reconnecting$; - /** - * Lists, for each LiveKit room, the LiveKit participants whose media should - * be presented. - */ - private readonly participantsByRoom$ = this.scope.behavior< - { - livekitRoom: LivekitRoom; - url: string; // Included for use as a React key - participants: { - id: string; - participant: LocalParticipant | RemoteParticipant | undefined; - member: RoomMember; - }[]; - }[] - >( - // TODO: Move this logic into Connection/PublishConnection if possible - this.localConnection$ - .pipe( - switchMap((localConnection) => { - if (localConnection?.state !== "ready") return []; - const memberError = (): never => { - throw new Error("No room member for call membership"); - }; - const localParticipant = { - id: `${this.userId}:${this.deviceId}`, - participant: localConnection.value.livekitRoom.localParticipant, - member: - this.matrixRoom.getMember(this.userId ?? "") ?? memberError(), - }; - - return this.remoteConnections$.pipe( - switchMap((remoteConnections) => - combineLatest( - [localConnection.value, ...remoteConnections].map((c) => - c.publishingParticipants$.pipe( - map((ps) => { - const participants: { - id: string; - participant: - | LocalParticipant - | RemoteParticipant - | undefined; - member: RoomMember; - }[] = ps.map(({ participant, membership }) => ({ - id: `${membership.userId}:${membership.deviceId}`, - participant, - member: - getRoomMemberFromRtcMember( - membership, - this.matrixRoom, - )?.member ?? memberError(), - })); - if (c === localConnection.value) - participants.push(localParticipant); - - return { - livekitRoom: c.livekitRoom, - url: c.transport.livekit_service_url, - participants, - }; - }), - ), - ), - ), - ), - ); - }), - ) - .pipe(startWith([]), pauseWhen(this.pretendToBeDisconnected$)), - ); - - /** - * Lists, for each LiveKit room, the LiveKit participants whose audio should - * be rendered. - */ - // (This is effectively just participantsByRoom$ with a stricter type) - public readonly audioParticipants$ = this.scope.behavior( - this.participantsByRoom$.pipe( - map((data) => - data.map(({ livekitRoom, url, participants }) => ({ - livekitRoom, - url, - participants: participants.flatMap(({ participant }) => - participant instanceof RemoteParticipant ? [participant] : [], - ), - })), - ), - ), - ); + public readonly audioParticipants$; // now will be created based on the connectionmanager public readonly handsRaised$ = this.scope.behavior( this.handsRaisedSubject$.pipe(pauseWhen(this.pretendToBeDisconnected$)), @@ -677,17 +429,19 @@ export class CallViewModel { ), ); - memberDisplaynames$ = memberDisplaynames$( - this.matrixRoom, - this.memberships$, - this.scope, - this.userId, - this.deviceId, - ); + // Now will be added to the matricLivekitMerger + // memberDisplaynames$ = memberDisplaynames$( + // this.matrixRoom, + // this.memberships$, + // this.scope, + // this.userId, + // this.deviceId, + // ); /** * List of MediaItems that we want to have tiles for. */ + // TODO KEEP THIS!! and adapt it to what our membershipManger returns private readonly mediaItems$ = this.scope.behavior( generateKeyed$< [typeof this.participantsByRoom$.value, number], @@ -790,10 +544,12 @@ export class CallViewModel { * - There can be multiple participants for one Matrix user if they join from * multiple devices. */ + // TODO KEEP THIS!! and adapt it to what our membershipManger returns public readonly participantCount$ = this.scope.behavior( this.memberships$.pipe(map((ms) => ms.length)), ); + // TODO convert all ring and all others left logic into one callLifecycleTracker$(didSendCallNotification$,matrixLivekitItem$): {autoLeave$,callPickupState$} private readonly allOthersLeft$ = this.memberships$.pipe( pairwise(), filter( @@ -1687,46 +1443,8 @@ export class CallViewModel { private readonly reactionsSubject$: Observable< Record >, - private readonly trackProcessorState$: Observable, + private readonly trackProcessorState$: Behavior, ) { - // Start and stop local and remote connections as needed - // DISCUSSION connection manager - this.connectionInstructions$ - .pipe(this.scope.bind()) - .subscribe(({ start, stop }) => { - for (const c of stop) { - logger.info(`Disconnecting from ${c.transport.livekit_service_url}`); - c.stop().catch((err) => { - // TODO: better error handling - logger.error( - `Fail to stop connection to ${c.transport.livekit_service_url}`, - err, - ); - }); - } - for (const c of start) { - c.start().then( - () => - logger.info(`Connected to ${c.transport.livekit_service_url}`), - (e) => { - // We only want to report fatal errors `_configError$` for the publish connection. - // If there is an error with another connection, it will not terminate the call and will be displayed - // on eacn tile. - if ( - c instanceof PublishConnection && - e instanceof ElementCallError - ) { - this._configError$.next(e); - } - logger.error( - `Failed to start connection to ${c.transport.livekit_service_url}`, - e, - ); - }, - ); - } - }); - // Start and stop session membership as needed this.scope.reconcile(this.advertisedTransport$, async (advertised) => { if (advertised !== null) { diff --git a/src/state/ownMember/OwnMembership.ts b/src/state/ownMember/OwnMembership.ts index 4ba4c380..56d40b3e 100644 --- a/src/state/ownMember/OwnMembership.ts +++ b/src/state/ownMember/OwnMembership.ts @@ -19,84 +19,116 @@ const ownMembership$ = ( connected: Behavior; transport: Behavior; } => { + const userId = this.matrixRoom.client.getUserId()!; + const deviceId = this.matrixRoom.client.getDeviceId()!; + const connection = connectionManager.registerTransports( constant([transport]), ); const publisher = new Publisher(connection); + // HOW IT WAS PREVIEOUSLY CREATED + // new PublishConnection( + // { + // transport, + // client: this.matrixRoom.client, + // scope, + // remoteTransports$: this.remoteTransports$, + // livekitRoomFactory: this.options.livekitRoomFactory, + // }, + // this.mediaDevices, + // this.muteStates, + // this.e2eeLivekitOptions(), + // this.scope.behavior(this.trackProcessorState$), + // ), /** - * Lists the transports used by ourselves, plus all other MatrixRTC session - * members. For completeness this also lists the preferred transport and - * whether we are in multi-SFU mode or sticky events mode (because - * advertisedTransport$ wants to read them at the same time, and bundling data - * together when it might change together is what you have to do in RxJS to - * avoid reading inconsistent state or observing too many changes.) + * The transport that we would personally prefer to publish on (if not for the + * transport preferences of others, perhaps). */ - // TODO-MULTI-SFU find a better name for this. With the addition of sticky events it's no longer just about transports. - // DISCUSS move to MatrixLivekitMerger - const transport$: Behavior<{ - local: Async; - preferred: Async; + // DISCUSS move to ownMembership + private readonly preferredTransport$ = this.scope.behavior( + async$(makeTransport(this.matrixRTCSession)), + ); + + /** + * The transport over which we should be actively publishing our media. + * null when not joined. + */ + // DISCUSSION ownMembershipManager + private readonly localTransport$: Behavior | null> = + this.scope.behavior( + this.transports$.pipe( + map((transports) => transports?.local ?? null), + distinctUntilChanged | null>(deepCompare), + ), + ); + + /** + * The transport we should advertise in our MatrixRTC membership (plus whether + * it is a multi-SFU transport and whether we should use sticky events). + */ + // DISCUSSION ownMembershipManager + private readonly advertisedTransport$: Behavior<{ multiSfu: boolean; preferStickyEvents: boolean; + transport: LivekitTransport; } | null> = this.scope.behavior( - this.joined$.pipe( - switchMap((joined) => - joined - ? combineLatest( - [ - this.preferredTransport$, - this.memberships$, - multiSfu.value$, - preferStickyEvents.value$, - ], - (preferred, memberships, preferMultiSfu, preferStickyEvents) => { - // Multi-SFU must be implicitly enabled when using sticky events - const multiSfu = preferStickyEvents || preferMultiSfu; + this.transports$.pipe( + map((transports) => + transports?.local.state === "ready" && + transports.preferred.state === "ready" + ? { + multiSfu: transports.multiSfu, + preferStickyEvents: transports.preferStickyEvents, + // In non-multi-SFU mode we should always advertise the preferred + // SFU to minimize the number of membership updates + transport: transports.multiSfu + ? transports.local.value + : transports.preferred.value, + } + : null, + ), + distinctUntilChanged<{ + multiSfu: boolean; + preferStickyEvents: boolean; + transport: LivekitTransport; + } | null>(deepCompare), + ), + ); - const oldestMembership = - this.matrixRTCSession.getOldestMembership(); - const remote = memberships.flatMap((m) => { - if (m.userId === this.userId && m.deviceId === this.deviceId) - return []; - const t = m.getTransport(oldestMembership ?? m); - return t && isLivekitTransport(t) - ? [{ membership: m, transport: t }] - : []; - }); - - let local = preferred; - if (!multiSfu) { - const oldest = this.matrixRTCSession.getOldestMembership(); - if (oldest !== undefined) { - const selection = oldest.getTransport(oldest); - // TODO selection can be null if no transport is configured should we report an error? - if (selection && isLivekitTransport(selection)) - local = ready(selection); - } - } - - if (local.state === "error") { - this._configError$.next( - local.value instanceof ElementCallError - ? local.value - : new UnknownCallError(local.value), - ); - } - - return { - local, - remote, - preferred, - multiSfu, - preferStickyEvents, - }; - }, - ) - : of(null), + // MATRIX RELATED + // + /** + * Whether we are "fully" connected to the call. Accounts for both the + * connection to the MatrixRTC session and the LiveKit publish connection. + */ + // DISCUSSION own membership manager + private readonly connected$ = this.scope.behavior( + and$( + this.matrixConnected$, + this.livekitConnectionState$.pipe( + map((state) => state === ConnectionState.Connected), ), ), ); + /** + * Whether we should tell the user that we're reconnecting to the call. + */ + // DISCUSSION own membership manager + public readonly reconnecting$ = this.scope.behavior( + this.connected$.pipe( + // We are reconnecting if we previously had some successful initial + // connection but are now disconnected + scan( + ({ connectedPreviously }, connectedNow) => ({ + connectedPreviously: connectedPreviously || connectedNow, + reconnecting: connectedPreviously && !connectedNow, + }), + { connectedPreviously: false, reconnecting: false }, + ), + map(({ reconnecting }) => reconnecting), + ), + ); return { connected: true, transport$ }; }; diff --git a/src/state/remoteMembers/Connection.test.ts b/src/state/remoteMembers/Connection.test.ts index 0719e2c5..4af64578 100644 --- a/src/state/remoteMembers/Connection.test.ts +++ b/src/state/remoteMembers/Connection.test.ts @@ -17,7 +17,7 @@ import { } from "vitest"; import { BehaviorSubject, of } from "rxjs"; import { - ConnectionState, + ConnectionState as LivekitConnectionState, type LocalParticipant, type RemoteParticipant, type Room as LivekitRoom, @@ -36,12 +36,11 @@ import { type ConnectionOpts, type ConnectionState, type PublishingParticipant, - RemoteConnection, + Connection, } from "./Connection.ts"; import { ObservableScope } from "../ObservableScope.ts"; import { type OpenIDClientParts } from "../../livekit/openIDSFU.ts"; import { FailToGetOpenIdToken } from "../../utils/errors.ts"; -import { PublishConnection } from "../ownMember/Publisher.ts"; import { mockMediaDevices, mockMuteStates } from "../../utils/test.ts"; import type { ProcessorState } from "../../livekit/TrackProcessorContext.tsx"; import { type MuteStates } from "../MuteStates.ts"; diff --git a/src/state/remoteMembers/ConnectionManager.ts b/src/state/remoteMembers/ConnectionManager.ts index b7a37b11..f15bee10 100644 --- a/src/state/remoteMembers/ConnectionManager.ts +++ b/src/state/remoteMembers/ConnectionManager.ts @@ -42,11 +42,11 @@ export type ParticipantByMemberIdMap = Map< export class ConnectionManager { private livekitRoomFactory: () => LivekitRoom; public constructor( - private client: MatrixClient, private scope: ObservableScope, + private client: MatrixClient, private devices: MediaDevices, - private processorState: ProcessorState, - private e2eeLivekitOptions$: Behavior, + private processorState$: Behavior, + private e2eeLivekitOptions: E2EEOptions | undefined, private logger?: Logger, livekitRoomFactory?: () => LivekitRoom, ) { @@ -55,8 +55,8 @@ export class ConnectionManager { new LivekitRoom( generateRoomOption( this.devices, - this.processorState, - this.e2eeLivekitOptions$.value, + this.processorState$.value, + this.e2eeLivekitOptions, ), ); this.livekitRoomFactory = livekitRoomFactory ?? defaultFactory; diff --git a/src/state/remoteMembers/displayname.ts b/src/state/remoteMembers/displayname.ts index f288e2d0..67e11f99 100644 --- a/src/state/remoteMembers/displayname.ts +++ b/src/state/remoteMembers/displayname.ts @@ -5,11 +5,13 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type Room, type RoomMember, RoomStateEvent } from "matrix-js-sdk"; +import { type RoomMember, RoomStateEvent } from "matrix-js-sdk"; import { combineLatest, fromEvent, type Observable, startWith } from "rxjs"; import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; import { logger } from "matrix-js-sdk/lib/logger"; import { type Room as MatrixRoom } from "matrix-js-sdk/lib/matrix"; +// eslint-disable-next-line rxjs/no-internal +import { type HasEventTargetAddRemove } from "rxjs/internal/observable/fromEvent"; import { type ObservableScope } from "../ObservableScope"; import { @@ -22,11 +24,13 @@ import { type Behavior } from "../Behavior"; * Displayname for each member of the call. This will disambiguate * any displayname that clashes with another member. Only members * joined to the call are considered here. + * + * @returns Map uses the rtc member idenitfier as the key. */ // don't do this work more times than we need to. This is achieved by converting to a behavior: export const memberDisplaynames$ = ( scope: ObservableScope, - matrixRoom: Room, + matrixRoom: Pick & HasEventTargetAddRemove, memberships$: Observable, userId: string, deviceId: string, @@ -73,7 +77,7 @@ export const memberDisplaynames$ = ( export function getRoomMemberFromRtcMember( rtcMember: CallMembership, - room: MatrixRoom, + room: Pick, ): { id: string; member: RoomMember | undefined } { return { id: rtcMember.userId + ":" + rtcMember.deviceId, diff --git a/src/state/remoteMembers/matrixLivekitMerger.ts b/src/state/remoteMembers/matrixLivekitMerger.ts index e77306c1..0411a0ca 100644 --- a/src/state/remoteMembers/matrixLivekitMerger.ts +++ b/src/state/remoteMembers/matrixLivekitMerger.ts @@ -5,38 +5,23 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { - type RemoteParticipant, - type Participant as LivekitParticipant, -} from "livekit-client"; +import { type Participant as LivekitParticipant } from "livekit-client"; import { isLivekitTransport, type LivekitTransport, type CallMembership, } from "matrix-js-sdk/lib/matrixrtc"; import { combineLatest, map, startWith, type Observable } from "rxjs"; +// eslint-disable-next-line rxjs/no-internal +import { type HasEventTargetAddRemove } from "rxjs/internal/observable/fromEvent"; import type { Room as MatrixRoom, RoomMember } from "matrix-js-sdk"; // import type { Logger } from "matrix-js-sdk/lib/logger"; import { type Behavior } from "../Behavior"; import { type ObservableScope } from "../ObservableScope"; import { type ConnectionManager } from "./ConnectionManager"; -import { getRoomMemberFromRtcMember } from "./displayname"; - -/** - * Represents participant publishing or expected to publish on the connection. - * It is paired with its associated rtc membership. - */ -export type PublishingParticipant = { - /** - * The LiveKit participant publishing on this connection, or undefined if the participant is not currently (yet) connected to the livekit room. - */ - participant: RemoteParticipant | undefined; - /** - * The rtc call membership associated with this participant. - */ - membership: CallMembership; -}; +import { getRoomMemberFromRtcMember, memberDisplaynames$ } from "./displayname"; +import { type Connection } from "./Connection"; /** * Represent a matrix call member and his associated livekit participation. @@ -45,10 +30,16 @@ export type PublishingParticipant = { */ export interface MatrixLivekitItem { membership: CallMembership; - livekitParticipant?: LivekitParticipant; - //TODO Try to remove this! Its waaay to much information - // Just use to get the member's avatar + displayName: string; + participant?: LivekitParticipant; + connection?: Connection; + /** + * TODO Try to remove this! Its waaay to much information. + * Just get the member's avatar + * @deprecated + */ member?: RoomMember; + mxcAvatarUrl?: string; } // Alternative structure idea: @@ -73,13 +64,17 @@ export class MatrixLivekitMerger { // private readonly logger: Logger; public constructor( + private scope: ObservableScope, private memberships$: Observable, private connectionManager: ConnectionManager, - private scope: ObservableScope, // TODO this is too much information for that class, // apparently needed to get a room member to later get the Avatar // => Extract an AvatarService instead? - private matrixRoom: MatrixRoom, + // Better with just `getMember` + private matrixRoom: Pick & + HasEventTargetAddRemove, + private userId: string, + private deviceId: string, // parentLogger: Logger, ) { // this.logger = parentLogger.getChild("MatrixLivekitMerger"); @@ -93,6 +88,13 @@ export class MatrixLivekitMerger { /// PRIVATES // ======================================= private start$(): Observable { + const displaynameMap$ = memberDisplaynames$( + this.scope, + this.matrixRoom, + this.memberships$, + this.userId, + this.deviceId, + ); const membershipsWithTransport$ = this.mapMembershipsToMembershipWithTransport$(); @@ -101,26 +103,33 @@ export class MatrixLivekitMerger { return combineLatest([ membershipsWithTransport$, this.connectionManager.allParticipantsByMemberId$, + displaynameMap$, ]).pipe( - map(([memberships, participantsByMemberId]) => { - const items = memberships.map(({ membership, transport }) => { - const participantsWithConnection = participantsByMemberId.get( - membership.membershipID, - ); - const participant = - transport && - participantsWithConnection?.find((p) => - areLivekitTransportsEqual(p.connection.transport, transport), + map(([memberships, participantsByMemberId, displayNameMap]) => { + const items: MatrixLivekitItem[] = memberships.map( + ({ membership, transport }) => { + const participantsWithConnection = participantsByMemberId.get( + membership.membershipID, ); - return { - livekitParticipant: participant, - membership, - // This makes sense to add the the js-sdk callMembership (we only need the avatar so probably the call memberhsip just should aquire the avatar) - member: - // Why a member error? if we have a call membership there is a room member - getRoomMemberFromRtcMember(membership, this.matrixRoom)?.member, - } as MatrixLivekitItem; - }); + const participant = + transport && + participantsWithConnection?.find((p) => + areLivekitTransportsEqual(p.connection.transport, transport), + ); + const member = getRoomMemberFromRtcMember( + membership, + this.matrixRoom, + )?.member; + return { + ...participant, + membership, + // This makes sense to add the the js-sdk callMembership (we only need the avatar so probably the call memberhsip just should aquire the avatar) + member, + displayName: displayNameMap.get(membership.membershipID) ?? "---", + mxcAvatarUrl: member?.getMxcAvatarUrl(), + }; + }, + ); return items; }), ); diff --git a/src/utils/displayname.ts b/src/utils/displayname.ts index 8e989d3b..1e141255 100644 --- a/src/utils/displayname.ts +++ b/src/utils/displayname.ts @@ -41,7 +41,7 @@ function removeHiddenChars(str: string): string { export function shouldDisambiguate( member: { rawDisplayName?: string; userId: string }, memberships: CallMembership[], - room: Room, + room: Pick, ): boolean { const { rawDisplayName: displayName, userId } = member; if (!displayName || displayName === userId) return false; From 4f892e358a1d1f2cd07311c30b2c18cb67dc9c83 Mon Sep 17 00:00:00 2001 From: Timo K Date: Thu, 30 Oct 2025 15:15:49 +0100 Subject: [PATCH 09/75] start onwMemberhsip.ts --- src/livekit/openIDSFU.ts | 4 +- src/rtcSessionHelpers.ts | 21 ++- src/state/Async.ts | 7 + src/state/CallViewModel.ts | 49 ++----- src/state/ownMember/OwnMembership.ts | 190 ++++++++++++++++++++------- 5 files changed, 175 insertions(+), 96 deletions(-) diff --git a/src/livekit/openIDSFU.ts b/src/livekit/openIDSFU.ts index 70d1786d..073f6c75 100644 --- a/src/livekit/openIDSFU.ts +++ b/src/livekit/openIDSFU.ts @@ -25,7 +25,7 @@ export type OpenIDClientParts = Pick< export async function getSFUConfigWithOpenID( client: OpenIDClientParts, serviceUrl: string, - livekitAlias: string, + matrixRoomId: string, ): Promise { let openIdToken: IOpenIDToken; try { @@ -43,7 +43,7 @@ export async function getSFUConfigWithOpenID( const sfuConfig = await getLiveKitJWT( client, serviceUrl, - livekitAlias, + matrixRoomId, openIdToken, ); logger.info(`Got JWT from call's active focus URL.`); diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index fadc7b37..74023c22 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -13,6 +13,7 @@ import { } from "matrix-js-sdk/lib/matrixrtc"; import { logger } from "matrix-js-sdk/lib/logger"; import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery"; +import { type MatrixClient } from "matrix-js-sdk"; import { PosthogAnalytics } from "./analytics/PosthogAnalytics"; import { Config } from "./config/Config"; @@ -23,16 +24,13 @@ import { getSFUConfigWithOpenID } from "./livekit/openIDSFU.ts"; const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci"; -export function getLivekitAlias(rtcSession: MatrixRTCSession): string { - // For now we assume everything is a room-scoped call - return rtcSession.room.roomId; -} - async function makeTransportInternal( - rtcSession: MatrixRTCSession, + client: MatrixClient, + roomId: string, ): Promise { logger.log("Searching for a preferred transport"); - const livekitAlias = getLivekitAlias(rtcSession); + //TODO refactor this to use the jwt service returned alias. + const livekitAlias = roomId; // TODO-MULTI-SFU: Either remove this dev tool or make it more official const urlFromStorage = @@ -52,7 +50,7 @@ async function makeTransportInternal( } // Prioritize the .well-known/matrix/client, if available, over the configured SFU - const domain = rtcSession.room.client.getDomain(); + const domain = client.getDomain(); if (domain) { // we use AutoDiscovery instead of relying on the MatrixClient having already // been fully configured and started @@ -85,12 +83,13 @@ async function makeTransportInternal( } export async function makeTransport( - rtcSession: MatrixRTCSession, + client: MatrixClient, + roomId: string, ): Promise { - const transport = await makeTransportInternal(rtcSession); + const transport = await makeTransportInternal(client, roomId); // this will call the jwt/sfu/get endpoint to pre create the livekit room. await getSFUConfigWithOpenID( - rtcSession.room.client, + client, transport.livekit_service_url, transport.livekit_alias, ); diff --git a/src/state/Async.ts b/src/state/Async.ts index 61871f78..f2e0376b 100644 --- a/src/state/Async.ts +++ b/src/state/Async.ts @@ -6,6 +6,7 @@ Please see LICENSE in the repository root for full details. */ import { catchError, from, map, type Observable, of, startWith } from "rxjs"; +import { Behavior } from "./Behavior"; /** * Data that may need to be loaded asynchronously. @@ -51,3 +52,9 @@ export function mapAsync( ): Async { return async.state === "ready" ? ready(project(async.value)) : async; } + +export function unwrapAsync(fallback: A): (async: Async) => A { + return (async: Async) => { + return async.state === "ready" ? async.value : fallback; + }; +} diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index c8f68cbb..31aa2533 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -139,6 +139,7 @@ import { ObservableScope } from "./ObservableScope.ts"; import { memberDisplaynames$ } from "./remoteMembers/displayname.ts"; import { ConnectionManager } from "./remoteMembers/ConnectionManager.ts"; import { MatrixLivekitMerger } from "./remoteMembers/matrixLivekitMerger.ts"; +import { ownMembership$ } from "./ownMember/OwnMembership.ts"; //TODO // Larger rename @@ -237,6 +238,15 @@ export class CallViewModel { this.matrixRoom, ); + private ownMembership = ownMembership$({ + scope: this.scope, + muteStates: this.muteStates, + multiSfu: this.multiSfu, + mediaDevices: this.mediaDevices, + trackProcessorState$: this.trackProcessorState$, + e2eeLivekitOptions: this.e2eeLivekitOptions, + }); + /** * If there is a configuration error with the call (e.g. misconfigured E2EE). * This is a fatal error that prevents the call from being created/joined. @@ -358,45 +368,6 @@ export class CallViewModel { // ); private readonly userId = this.matrixRoom.client.getUserId()!; - private readonly deviceId = this.matrixRoom.client.getDeviceId()!; - - /** - * Whether we are connected to the MatrixRTC session. - */ - // DISCUSSION own membership manager - private readonly matrixConnected$ = this.scope.behavior( - // To consider ourselves connected to MatrixRTC, we check the following: - and$( - // The client is connected to the sync loop - ( - fromEvent(this.matrixRoom.client, ClientEvent.Sync) as Observable< - [SyncState] - > - ).pipe( - startWith([this.matrixRoom.client.getSyncState()]), - map(([state]) => state === SyncState.Syncing), - ), - // Room state observed by session says we're connected - fromEvent( - this.matrixRTCSession, - MembershipManagerEvent.StatusChanged, - ).pipe( - startWith(null), - map(() => this.matrixRTCSession.membershipStatus === Status.Connected), - ), - // Also watch out for warnings that we've likely hit a timeout and our - // delayed leave event is being sent (this condition is here because it - // provides an earlier warning than the sync loop timeout, and we wouldn't - // see the actual leave event until we reconnect to the sync loop) - fromEvent( - this.matrixRTCSession, - MembershipManagerEvent.ProbablyLeft, - ).pipe( - startWith(null), - map(() => this.matrixRTCSession.probablyLeft !== true), - ), - ), - ); /** * Whether various media/event sources should pretend to be disconnected from diff --git a/src/state/ownMember/OwnMembership.ts b/src/state/ownMember/OwnMembership.ts index 56d40b3e..52a09033 100644 --- a/src/state/ownMember/OwnMembership.ts +++ b/src/state/ownMember/OwnMembership.ts @@ -1,31 +1,155 @@ /* Copyright 2025 New Vector Ltd. -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +SPDX-License-IdFentifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { LiveKitReactNativeInfo } from "livekit-client"; -import { Behavior, constant } from "../Behavior"; -import { LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; -import { ConnectionManager } from "../remoteMembers/ConnectionManager"; +import { type E2EEOptions } from "livekit-client"; +import { logger } from "matrix-js-sdk/lib/logger"; +import { + type LivekitTransport, + type MatrixRTCSession, + MembershipManagerEvent, + Status, +} from "matrix-js-sdk/lib/matrixrtc"; +import { + ClientEvent, + type MatrixClient, + SyncState, + type Room as MatrixRoom, +} from "matrix-js-sdk"; +import { fromEvent, map, type Observable, scan, startWith } from "rxjs"; +import { multiSfu } from "../../settings/settings"; +import { type Behavior } from "../Behavior"; +import { type ConnectionManager } from "../remoteMembers/ConnectionManager"; +import { makeTransport } from "../../rtcSessionHelpers"; +import { type ObservableScope } from "../ObservableScope"; +import { async$, unwrapAsync } from "../Async"; +import { Publisher } from "./Publisher"; +import { type MuteStates } from "../MuteStates"; +import { type ProcessorState } from "../../livekit/TrackProcessorContext"; +import { type MediaDevices } from "../../state/MediaDevices"; +import { and$ } from "../../utils/observable"; -const ownMembership$ = ( - multiSfu: boolean, - preferStickyEvents: boolean, - connectionManager: ConnectionManager, - transport: LivekitTransport, -): { - connected: Behavior; - transport: Behavior; +interface Props { + scope: ObservableScope; + mediaDevices: MediaDevices; + muteStates: MuteStates; + connectionManager: ConnectionManager; + matrixRTCSession: MatrixRTCSession; + matrixRoom: MatrixRoom; + client: MatrixClient; + preferStickyEvents: boolean; + roomId: string; + e2eeLivekitOptions: E2EEOptions | undefined; + trackerProcessorState$: Behavior; +} + +/** + * This class is responsible for managing the own membership in a room. + * We want + * - a publisher + * - + * @param param0 + * @returns + * - publisher: The handle to create tracks and publish them to the room. + * - connected$: the current connection state. Including matrix server and livekit server connection. (only the livekit server relevant for our own participation) + * - transport$: the transport object the ownMembership$ ended up using. + * + */ +export const ownMembership$ = ({ + scope, + muteStates, + mediaDevices, + preferStickyEvents, + connectionManager, + matrixRTCSession, + matrixRoom, + e2eeLivekitOptions, + client, + roomId, + trackerProcessorState$, +}: Props): { + connected$: Behavior; + transport$: Behavior; + publisher: Publisher; } => { - const userId = this.matrixRoom.client.getUserId()!; - const deviceId = this.matrixRoom.client.getDeviceId()!; + const userId = client.getUserId()!; + const deviceId = client.getDeviceId()!; + const multiSfu$ = multiSfu.value$; + /** + * The transport that we would personally prefer to publish on (if not for the + * transport preferences of others, perhaps). + */ + const preferredTransport$ = scope.behavior( + async$(makeTransport(client, roomId)).pipe( + map(unwrapAsync(null)), + ), + ); const connection = connectionManager.registerTransports( - constant([transport]), + scope.behavior(preferredTransport$.pipe(map((t) => (t ? [t] : [])))), + )[0]; + if (!connection) { + logger.warn( + "No connection found when passing transport to connectionManager. transport:", + preferredTransport$.value, + ); + } + + /** + * Whether we are connected to the MatrixRTC session. + */ + // DISCUSSION own membership manager + const matrixConnected$ = scope.behavior( + // To consider ourselves connected to MatrixRTC, we check the following: + and$( + // The client is connected to the sync loop + ( + fromEvent(matrixRoom.client, ClientEvent.Sync) as Observable< + [SyncState] + > + ).pipe( + startWith([matrixRoom.client.getSyncState()]), + map(([state]) => state === SyncState.Syncing), + ), + // Room state observed by session says we're connected + fromEvent(matrixRTCSession, MembershipManagerEvent.StatusChanged).pipe( + startWith(null), + map(() => matrixRTCSession.membershipStatus === Status.Connected), + ), + // Also watch out for warnings that we've likely hit a timeout and our + // delayed leave event is being sent (this condition is here because it + // provides an earlier warning than the sync loop timeout, and we wouldn't + // see the actual leave event until we reconnect to the sync loop) + fromEvent(matrixRTCSession, MembershipManagerEvent.ProbablyLeft).pipe( + startWith(null), + map(() => matrixRTCSession.probablyLeft !== true), + ), + ), + ); + /** + * Whether we are "fully" connected to the call. Accounts for both the + * connection to the MatrixRTC session and the LiveKit publish connection. + */ + const connected$ = scope.behavior( + and$( + matrixConnected$, + connection.state$.pipe( + map((state) => state.state === "ConnectedToLkRoom"), + ), + ), + ); + + const publisher = new Publisher( + scope, + connection, + mediaDevices, + muteStates, + e2eeLivekitOptions, + trackerProcessorState$, ); - const publisher = new Publisher(connection); // HOW IT WAS PREVIEOUSLY CREATED // new PublishConnection( @@ -41,21 +165,13 @@ const ownMembership$ = ( // this.e2eeLivekitOptions(), // this.scope.behavior(this.trackProcessorState$), // ), - /** - * The transport that we would personally prefer to publish on (if not for the - * transport preferences of others, perhaps). - */ - // DISCUSS move to ownMembership - private readonly preferredTransport$ = this.scope.behavior( - async$(makeTransport(this.matrixRTCSession)), - ); /** * The transport over which we should be actively publishing our media. * null when not joined. */ // DISCUSSION ownMembershipManager - private readonly localTransport$: Behavior | null> = + const localTransport$: Behavior | null> = this.scope.behavior( this.transports$.pipe( map((transports) => transports?.local ?? null), @@ -68,7 +184,7 @@ const ownMembership$ = ( * it is a multi-SFU transport and whether we should use sticky events). */ // DISCUSSION ownMembershipManager - private readonly advertisedTransport$: Behavior<{ + const advertisedTransport$: Behavior<{ multiSfu: boolean; preferStickyEvents: boolean; transport: LivekitTransport; @@ -97,27 +213,13 @@ const ownMembership$ = ( ); // MATRIX RELATED - // - /** - * Whether we are "fully" connected to the call. Accounts for both the - * connection to the MatrixRTC session and the LiveKit publish connection. - */ - // DISCUSSION own membership manager - private readonly connected$ = this.scope.behavior( - and$( - this.matrixConnected$, - this.livekitConnectionState$.pipe( - map((state) => state === ConnectionState.Connected), - ), - ), - ); /** * Whether we should tell the user that we're reconnecting to the call. */ // DISCUSSION own membership manager - public readonly reconnecting$ = this.scope.behavior( - this.connected$.pipe( + const reconnecting$ = scope.behavior( + connected$.pipe( // We are reconnecting if we previously had some successful initial // connection but are now disconnected scan( @@ -130,5 +232,5 @@ const ownMembership$ = ( map(({ reconnecting }) => reconnecting), ), ); - return { connected: true, transport$ }; + return { connected$, transport$: preferredTransport$, publisher }; }; From a44171da1c8fd1fd6472ee7d30a17baae403eccb Mon Sep 17 00:00:00 2001 From: Timo K Date: Thu, 30 Oct 2025 22:15:35 +0100 Subject: [PATCH 10/75] changes summary valere timo --- src/state/ownMember/OwnMembership.ts | 101 +++++++++++++----- src/state/remoteMembers/ConnectionManager.ts | 24 ++--- .../remoteMembers/matrixLivekitMerger.ts | 3 +- 3 files changed, 86 insertions(+), 42 deletions(-) diff --git a/src/state/ownMember/OwnMembership.ts b/src/state/ownMember/OwnMembership.ts index 52a09033..6ffdb5b5 100644 --- a/src/state/ownMember/OwnMembership.ts +++ b/src/state/ownMember/OwnMembership.ts @@ -19,10 +19,21 @@ import { SyncState, type Room as MatrixRoom, } from "matrix-js-sdk"; -import { fromEvent, map, type Observable, scan, startWith } from "rxjs"; +import { + BehaviorSubject, + combineLatest, + from, + fromEvent, + map, + type Observable, + of, + scan, + startWith, + switchMap, +} from "rxjs"; import { multiSfu } from "../../settings/settings"; import { type Behavior } from "../Behavior"; -import { type ConnectionManager } from "../remoteMembers/ConnectionManager"; +import { ConnectionManager } from "../remoteMembers/ConnectionManager"; import { makeTransport } from "../../rtcSessionHelpers"; import { type ObservableScope } from "../ObservableScope"; import { async$, unwrapAsync } from "../Async"; @@ -31,7 +42,19 @@ import { type MuteStates } from "../MuteStates"; import { type ProcessorState } from "../../livekit/TrackProcessorContext"; import { type MediaDevices } from "../../state/MediaDevices"; import { and$ } from "../../utils/observable"; +import { areLivekitTransportsEqual } from "../remoteMembers/matrixLivekitMerger"; +/* + * - get well known + * - get oldest membership + * - get transport to use + * - get openId + jwt token + * - wait for createTrack() call + * - create tracks + * - wait for join() call + * - Publisher.publishTracks() + * - send join state/sticky event + */ interface Props { scope: ObservableScope; mediaDevices: MediaDevices; @@ -71,10 +94,18 @@ export const ownMembership$ = ({ roomId, trackerProcessorState$, }: Props): { - connected$: Behavior; - transport$: Behavior; - publisher: Publisher; + // publisher: Publisher + requestJoin(): Observable; + startTracks(): Track[]; } => { + // This should be used in a combineLatest with publisher$ to connect. + const shouldStartTracks$ = BehaviorSubject(false); + + // to make it possible to call startTracks before the preferredTransport$ has resolved. + const startTracks = () => { + shouldStartTracks$.next(true); + }; + const userId = client.getUserId()!; const deviceId = client.getDeviceId()!; const multiSfu$ = multiSfu.value$; @@ -82,22 +113,23 @@ export const ownMembership$ = ({ * The transport that we would personally prefer to publish on (if not for the * transport preferences of others, perhaps). */ - const preferredTransport$ = scope.behavior( - async$(makeTransport(client, roomId)).pipe( - map(unwrapAsync(null)), - ), + const preferredTransport$: Behavior = scope.behavior( + from(makeTransport(client, roomId)), ); - const connection = connectionManager.registerTransports( + connectionManager.registerTransports( scope.behavior(preferredTransport$.pipe(map((t) => (t ? [t] : [])))), - )[0]; - if (!connection) { - logger.warn( - "No connection found when passing transport to connectionManager. transport:", - preferredTransport$.value, - ); - } + ); + const connection$ = scope.behavior( + combineLatest([connectionManager.connections$, preferredTransport$]).pipe( + map(([connections, transport]) => + connections.find((connection) => + areLivekitTransportsEqual(connection.transport, transport), + ), + ), + ), + ); /** * Whether we are connected to the MatrixRTC session. */ @@ -129,6 +161,7 @@ export const ownMembership$ = ({ ), ), ); + /** * Whether we are "fully" connected to the call. Accounts for both the * connection to the MatrixRTC session and the LiveKit publish connection. @@ -136,19 +169,31 @@ export const ownMembership$ = ({ const connected$ = scope.behavior( and$( matrixConnected$, - connection.state$.pipe( - map((state) => state.state === "ConnectedToLkRoom"), + connection$.pipe( + switchMap((c) => + c + ? c.state$.pipe(map((state) => state.state === "ConnectedToLkRoom")) + : of(false), + ), ), ), ); - const publisher = new Publisher( - scope, - connection, - mediaDevices, - muteStates, - e2eeLivekitOptions, - trackerProcessorState$, + const publisher = scope.behavior( + connection$.pipe( + map((c) => + c + ? new Publisher( + scope, + c, + mediaDevices, + muteStates, + e2eeLivekitOptions, + trackerProcessorState$, + ) + : null, + ), + ), ); // HOW IT WAS PREVIEOUSLY CREATED @@ -171,11 +216,11 @@ export const ownMembership$ = ({ * null when not joined. */ // DISCUSSION ownMembershipManager - const localTransport$: Behavior | null> = + const localTransport$: Behavior = this.scope.behavior( this.transports$.pipe( map((transports) => transports?.local ?? null), - distinctUntilChanged | null>(deepCompare), + distinctUntilChanged(deepCompare), ), ); diff --git a/src/state/remoteMembers/ConnectionManager.ts b/src/state/remoteMembers/ConnectionManager.ts index f15bee10..845c2af0 100644 --- a/src/state/remoteMembers/ConnectionManager.ts +++ b/src/state/remoteMembers/ConnectionManager.ts @@ -99,7 +99,7 @@ export class ConnectionManager { /** * Connections for each transport in use by one or more session members. */ - private readonly connections$ = this.scope.behavior( + public readonly connections$ = this.scope.behavior( generateKeyed$( this.transports$, (transports, createOrGet) => { @@ -144,22 +144,20 @@ export class ConnectionManager { * the same `transports$` behavior reference. * @param transports$ The Behavior containing a list of transports to subscribe to. */ - public registerTransports( - transports$: Behavior, - ): Connection[] { + public registerTransports(transports$: Behavior): void { if (!this.transportsSubscriptions$.value.some((t$) => t$ === transports$)) { this.transportsSubscriptions$.next( this.transportsSubscriptions$.value.concat(transports$), ); } - // After updating the subscriptions our connection list is also updated. - return transports$.value - .map((transport) => { - const isConnectionForTransport = (connection: Connection): boolean => - areLivekitTransportsEqual(connection.transport, transport); - return this.connections$.value.find(isConnectionForTransport); - }) - .filter((c) => c !== undefined); + // // After updating the subscriptions our connection list is also updated. + // return transports$.value + // .map((transport) => { + // const isConnectionForTransport = (connection: Connection): boolean => + // areLivekitTransportsEqual(connection.transport, transport); + // return this.connections$.value.find(isConnectionForTransport); + // }) + // .filter((c) => c !== undefined); } /** @@ -218,7 +216,7 @@ export class ConnectionManager { * Each participant that is found on all connections managed by the manager will be listed. * * They are stored an a map keyed by `participant.identity` - * (which is equivalent to the `member.id` field in the `m.rtc.member` event) + * TODO (which is equivalent to the `member.id` field in the `m.rtc.member` event) right now its userId:deviceId */ public allParticipantsByMemberId$ = this.scope.behavior( this.allParticipantsWithConnection$.pipe( diff --git a/src/state/remoteMembers/matrixLivekitMerger.ts b/src/state/remoteMembers/matrixLivekitMerger.ts index 0411a0ca..bd6ed353 100644 --- a/src/state/remoteMembers/matrixLivekitMerger.ts +++ b/src/state/remoteMembers/matrixLivekitMerger.ts @@ -109,7 +109,8 @@ export class MatrixLivekitMerger { const items: MatrixLivekitItem[] = memberships.map( ({ membership, transport }) => { const participantsWithConnection = participantsByMemberId.get( - membership.membershipID, + // membership.membershipID, Currently its hardcoded by the jwt service to + `${membership.userId}:${membership.deviceId}`, ); const participant = transport && From b22a937e7c97cc3a3eb91ebd26a0473ddec3cfe2 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 1 Nov 2025 02:02:21 +0000 Subject: [PATCH 11/75] Update softprops/action-gh-release digest to 6da8fa9 --- .github/workflows/publish-embedded-packages.yaml | 4 ++-- .github/workflows/publish.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish-embedded-packages.yaml b/.github/workflows/publish-embedded-packages.yaml index 256e440e..db6d8f0d 100644 --- a/.github/workflows/publish-embedded-packages.yaml +++ b/.github/workflows/publish-embedded-packages.yaml @@ -85,7 +85,7 @@ jobs: run: find ${{ env.FILENAME_PREFIX }} -type f -print0 | sort -z | xargs -0 sha256sum | tee ${{ env.FILENAME_PREFIX }}.sha256 - name: Upload if: ${{ needs.versioning.outputs.DRY_RUN == 'false' }} - uses: softprops/action-gh-release@aec2ec56f94eb8180ceec724245f64ef008b89f5 # v2 + uses: softprops/action-gh-release@6da8fa9354ddfdc4aeace5fc48d7f679b5214090 # v2 with: files: | ${{ env.FILENAME_PREFIX }}.tar.gz @@ -264,7 +264,7 @@ jobs: echo "iOS: ${{ needs.publish_ios.outputs.ARTIFACT_VERSION }}" - name: Add release notes if: ${{ needs.versioning.outputs.DRY_RUN == 'false' }} - uses: softprops/action-gh-release@aec2ec56f94eb8180ceec724245f64ef008b89f5 # v2 + uses: softprops/action-gh-release@6da8fa9354ddfdc4aeace5fc48d7f679b5214090 # v2 with: append_body: true body: | diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 5b4b7936..34835635 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -42,7 +42,7 @@ jobs: - name: Create Checksum run: find ${{ env.FILENAME_PREFIX }} -type f -print0 | sort -z | xargs -0 sha256sum | tee ${{ env.FILENAME_PREFIX }}.sha256 - name: Upload - uses: softprops/action-gh-release@aec2ec56f94eb8180ceec724245f64ef008b89f5 # v2 + uses: softprops/action-gh-release@6da8fa9354ddfdc4aeace5fc48d7f679b5214090 # v2 with: files: | ${{ env.FILENAME_PREFIX }}.tar.gz @@ -68,7 +68,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Add release note - uses: softprops/action-gh-release@aec2ec56f94eb8180ceec724245f64ef008b89f5 # v2 + uses: softprops/action-gh-release@6da8fa9354ddfdc4aeace5fc48d7f679b5214090 # v2 with: append_body: true body: | From a4b1ce52cd46897d31117814ad544df6e8a7d897 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 3 Nov 2025 01:02:47 +0000 Subject: [PATCH 12/75] Update dependency livekit-client to v2.15.14 --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index e78dbbf2..9813bc66 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10110,8 +10110,8 @@ __metadata: linkType: hard "livekit-client@npm:^2.13.0": - version: 2.15.13 - resolution: "livekit-client@npm:2.15.13" + version: 2.15.14 + resolution: "livekit-client@npm:2.15.14" dependencies: "@livekit/mutex": "npm:1.1.1" "@livekit/protocol": "npm:1.42.2" @@ -10125,7 +10125,7 @@ __metadata: webrtc-adapter: "npm:^9.0.1" peerDependencies: "@types/dom-mediacapture-record": ^1 - checksum: 10c0/5a061df9000461a6d40ef8aa1e72e8aedc640181cc57fe6f2c48c5c7f90ce96a735b125aede377fc43f4692a685e098f17eeae0f42c5b2fed473305867bf2789 + checksum: 10c0/619a5932cbb2a7e797344f800db76d529e33f35029e48e0c50a4bccaf7e912fed617360ad745b30b6870b3de4318d714f26b02b7f61a0f0acd442d21f67fb417 languageName: node linkType: hard From 4c5f06a8a9d2b27a730f58e4ecfbb410b9f88c5a Mon Sep 17 00:00:00 2001 From: Valere Date: Mon, 3 Nov 2025 13:18:21 +0100 Subject: [PATCH 13/75] Refactoring to ease testing of ConnectionManager - Extract a ConnectionFactory - Change Connection manager allPariticipantWithConnection$ for structure that supports members with no participant --- src/state/remoteMembers/ConnectionFactory.ts | 114 +++++++ .../remoteMembers/ConnectionManager.test.ts | 307 ++++++++++++++++++ src/state/remoteMembers/ConnectionManager.ts | 247 ++++++-------- .../remoteMembers/MatrixLivekitMerger.test.ts | 236 +++++++++++++- .../remoteMembers/matrixLivekitMerger.ts | 31 +- 5 files changed, 778 insertions(+), 157 deletions(-) create mode 100644 src/state/remoteMembers/ConnectionFactory.ts create mode 100644 src/state/remoteMembers/ConnectionManager.test.ts diff --git a/src/state/remoteMembers/ConnectionFactory.ts b/src/state/remoteMembers/ConnectionFactory.ts new file mode 100644 index 00000000..a2a02e3e --- /dev/null +++ b/src/state/remoteMembers/ConnectionFactory.ts @@ -0,0 +1,114 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { + type E2EEOptions, + Room as LivekitRoom, + type RoomOptions, +} from "livekit-client"; +import { type Logger } from "matrix-js-sdk/lib/logger"; + +import { type ObservableScope } from "../ObservableScope.ts"; +import { Connection } from "./Connection.ts"; +import type { OpenIDClientParts } from "../../livekit/openIDSFU.ts"; +import type { MediaDevices } from "../MediaDevices.ts"; +import type { Behavior } from "../Behavior.ts"; +import type { ProcessorState } from "../../livekit/TrackProcessorContext.tsx"; +import { defaultLiveKitOptions } from "../../livekit/options.ts"; + +export interface ConnectionFactory { + createConnection( + transport: LivekitTransport, + scope: ObservableScope, + logger: Logger, + ): Connection; +} + +export class ECConnectionFactory implements ConnectionFactory { + private readonly livekitRoomFactory: () => LivekitRoom; + + /** + * Creates a ConnectionFactory for LiveKit connections. + * + * @param client - The OpenID client parts for authentication, needed to get openID and JWT tokens. + * @param devices - Used for video/audio out/in capture options. + * @param processorState$ - Effects like background blur (only for publishing connection?) + * @param e2eeLivekitOptions - The E2EE options to use for the LiveKit Room. + * @param controlledAudioDevices - Option to indicate whether audio output device is controlled externally (native mobile app). + * @param livekitRoomFactory - Optional factory function (for testing) to create LivekitRoom instances. If not provided, a default factory is used. + */ + public constructor( + private client: OpenIDClientParts, + private devices: MediaDevices, + private processorState$: Behavior, + private e2eeLivekitOptions: E2EEOptions | undefined, + private controlledAudioDevices: boolean, + livekitRoomFactory?: () => LivekitRoom, + ) { + const defaultFactory = (): LivekitRoom => + new LivekitRoom( + generateRoomOption( + this.devices, + this.processorState$.value, + this.e2eeLivekitOptions, + this.controlledAudioDevices, + ), + ); + this.livekitRoomFactory = livekitRoomFactory ?? defaultFactory; + } + + public createConnection( + transport: LivekitTransport, + scope: ObservableScope, + logger: Logger, + ): Connection { + return new Connection( + { + transport, + client: this.client, + scope: scope, + livekitRoomFactory: this.livekitRoomFactory, + }, + logger, + ); + } +} + +/** + * Generate the initial LiveKit RoomOptions based on the current media devices and processor state. + */ +function generateRoomOption( + devices: MediaDevices, + processorState: ProcessorState, + e2eeLivekitOptions: E2EEOptions | undefined, + controlledAudioDevices: boolean, +): RoomOptions { + return { + ...defaultLiveKitOptions, + videoCaptureDefaults: { + ...defaultLiveKitOptions.videoCaptureDefaults, + deviceId: devices.videoInput.selected$.value?.id, + processor: processorState.processor, + }, + audioCaptureDefaults: { + ...defaultLiveKitOptions.audioCaptureDefaults, + deviceId: devices.audioInput.selected$.value?.id, + }, + audioOutput: { + // When using controlled audio devices, we don't want to set the + // deviceId here, because it will be set by the native app. + // (also the id does not need to match a browser device id) + deviceId: controlledAudioDevices + ? undefined + : devices.audioOutput.selected$.value?.id, + }, + e2ee: e2eeLivekitOptions, + // TODO test and consider this: + // webAudioMix: true, + }; +} diff --git a/src/state/remoteMembers/ConnectionManager.test.ts b/src/state/remoteMembers/ConnectionManager.test.ts new file mode 100644 index 00000000..a0203840 --- /dev/null +++ b/src/state/remoteMembers/ConnectionManager.test.ts @@ -0,0 +1,307 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; +import { logger } from "matrix-js-sdk/lib/logger"; +import { BehaviorSubject } from "rxjs"; +import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { type Participant as LivekitParticipant } from "livekit-client"; + +import { ObservableScope } from "../ObservableScope.ts"; +import { + ConnectionManager, + type ConnectionManagerData, +} from "./ConnectionManager.ts"; +import { type ConnectionFactory } from "./ConnectionFactory.ts"; +import { type Connection } from "./Connection.ts"; +import { areLivekitTransportsEqual } from "./matrixLivekitMerger.ts"; +import { flushPromises } from "../../utils/test.ts"; + +// Some test constants + +const TRANSPORT_1: LivekitTransport = { + type: "livekit", + livekit_service_url: "https://lk.example.org", + livekit_alias: "!alias:example.org", +}; + +const TRANSPORT_2: LivekitTransport = { + type: "livekit", + livekit_service_url: "https://lk.sample.com", + livekit_alias: "!alias:sample.com", +}; + +const TRANSPORT_3: LivekitTransport = { + type: "livekit", + livekit_service_url: "https://lk-other.sample.com", + livekit_alias: "!alias:sample.com", +}; + +let testScope: ObservableScope; +let fakeConnectionFactory: ConnectionFactory; + +let testTransportStream$: BehaviorSubject; + +// The connection manager under test +let manager: ConnectionManager; + +beforeEach(() => { + testScope = new ObservableScope(); + + fakeConnectionFactory = {} as unknown as ConnectionFactory; + vi.mocked(fakeConnectionFactory).createConnection = vi + .fn() + .mockImplementation( + (transport: LivekitTransport, scope: ObservableScope) => { + const mockConnection = { + transport, + } as unknown as Connection; + vi.mocked(mockConnection).start = vi.fn(); + vi.mocked(mockConnection).stop = vi.fn(); + // Tie the connection's lifecycle to the scope to test scope lifecycle management + scope.onEnd(() => { + void mockConnection.stop(); + }); + return mockConnection; + }, + ); + + testTransportStream$ = new BehaviorSubject([]); + + manager = new ConnectionManager(testScope, fakeConnectionFactory, logger); + manager.registerTransports(testTransportStream$); +}); + +afterEach(() => { + testScope.end(); +}); + +describe("connections$ stream", () => { + test("Should create and start new connections for each transports", async () => { + const managedConnections = Promise.withResolvers(); + manager.connections$.subscribe((connections) => { + if (connections.length > 0) managedConnections.resolve(connections); + }); + + testTransportStream$.next([TRANSPORT_1, TRANSPORT_2]); + + const connections = await managedConnections.promise; + + expect(connections.length).toBe(2); + + expect( + vi.mocked(fakeConnectionFactory).createConnection, + ).toHaveBeenCalledTimes(2); + + const conn1 = connections.find((c) => + areLivekitTransportsEqual(c.transport, TRANSPORT_1), + ); + expect(conn1).toBeDefined(); + expect(conn1!.start).toHaveBeenCalled(); + + const conn2 = connections.find((c) => + areLivekitTransportsEqual(c.transport, TRANSPORT_2), + ); + expect(conn2).toBeDefined(); + expect(conn2!.start).toHaveBeenCalled(); + }); + + test("Should start connection only once", async () => { + const observedConnections: Connection[][] = []; + manager.connections$.subscribe((connections) => { + observedConnections.push(connections); + }); + + testTransportStream$.next([TRANSPORT_1]); + testTransportStream$.next([TRANSPORT_1]); + testTransportStream$.next([TRANSPORT_1]); + testTransportStream$.next([TRANSPORT_1]); + testTransportStream$.next([TRANSPORT_1]); + testTransportStream$.next([TRANSPORT_1, TRANSPORT_2]); + + await flushPromises(); + const connections = observedConnections.pop()!; + + expect(connections.length).toBe(2); + expect( + vi.mocked(fakeConnectionFactory).createConnection, + ).toHaveBeenCalledTimes(2); + + const conn2 = connections.find((c) => + areLivekitTransportsEqual(c.transport, TRANSPORT_2), + ); + expect(conn2).toBeDefined(); + + const conn1 = connections.find((c) => + areLivekitTransportsEqual(c.transport, TRANSPORT_1), + ); + expect(conn1).toBeDefined(); + expect(conn1!.start).toHaveBeenCalledOnce(); + }); + + test("Should cleanup connections when not needed anymore", async () => { + const observedConnections: Connection[][] = []; + manager.connections$.subscribe((connections) => { + observedConnections.push(connections); + }); + + testTransportStream$.next([TRANSPORT_1]); + testTransportStream$.next([TRANSPORT_1, TRANSPORT_2]); + + await flushPromises(); + + const conn2 = observedConnections + .pop()! + .find((c) => areLivekitTransportsEqual(c.transport, TRANSPORT_2))!; + + testTransportStream$.next([TRANSPORT_1]); + + await flushPromises(); + + // The second connection should have been stopped has it is no longer needed + expect(conn2.stop).toHaveBeenCalled(); + + // The first connection should still be active + const conn1 = observedConnections.pop()![0]; + expect(conn1.stop).not.toHaveBeenCalledOnce(); + }); +}); + +describe("connectionManagerData$ stream", () => { + // Used in test to control fake connections' participantsWithTrack$ streams + let fakePublishingParticipantsStreams: Map< + string, + BehaviorSubject + >; + + function keyForTransport(transport: LivekitTransport): string { + return `${transport.livekit_service_url}|${transport.livekit_alias}`; + } + + beforeEach(() => { + fakePublishingParticipantsStreams = new Map(); + // need a more advanced fake connection factory + vi.mocked(fakeConnectionFactory).createConnection = vi + .fn() + .mockImplementation( + (transport: LivekitTransport, scope: ObservableScope) => { + const fakePublishingParticipants$ = new BehaviorSubject< + LivekitParticipant[] + >([]); + const mockConnection = { + transport, + participantsWithTrack$: fakePublishingParticipants$, + } as unknown as Connection; + vi.mocked(mockConnection).start = vi.fn(); + vi.mocked(mockConnection).stop = vi.fn(); + // Tie the connection's lifecycle to the scope to test scope lifecycle management + scope.onEnd(() => { + void mockConnection.stop(); + }); + + fakePublishingParticipantsStreams.set( + keyForTransport(transport), + fakePublishingParticipants$, + ); + return mockConnection; + }, + ); + }); + + test("Should report connections with the publishing participants", async () => { + const managerDataUpdates: ConnectionManagerData[] = []; + manager.connectionManagerData$.subscribe((data) => { + managerDataUpdates.push(data); + }); + + testTransportStream$.next([TRANSPORT_1, TRANSPORT_2]); + await flushPromises(); + + const conn1Participants$ = fakePublishingParticipantsStreams.get( + keyForTransport(TRANSPORT_1), + )!; + + conn1Participants$.next([{ identity: "user1A" } as LivekitParticipant]); + + const conn2Participants$ = fakePublishingParticipantsStreams.get( + keyForTransport(TRANSPORT_2), + )!; + conn2Participants$.next([{ identity: "user2A" } as LivekitParticipant]); + + conn1Participants$.next([ + { identity: "user1A" } as LivekitParticipant, + { identity: "user1B" } as LivekitParticipant, + ]); + + testTransportStream$.next([TRANSPORT_1, TRANSPORT_2, TRANSPORT_3]); + + expect(managerDataUpdates[0].getConnections().length).toEqual(0); + + { + const data = managerDataUpdates[1]; + expect(data.getConnections().length).toEqual(2); + expect(data.getParticipantForTransport(TRANSPORT_1).length).toEqual(0); + expect(data.getParticipantForTransport(TRANSPORT_1).length).toEqual(0); + } + + { + const data = managerDataUpdates[2]; + expect(data.getConnections().length).toEqual(2); + expect(data.getParticipantForTransport(TRANSPORT_1).length).toEqual(1); + expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toEqual( + "user1A", + ); + expect(data.getParticipantForTransport(TRANSPORT_2).length).toEqual(0); + } + { + const data = managerDataUpdates[3]; + expect(data.getConnections().length).toEqual(2); + expect(data.getParticipantForTransport(TRANSPORT_1).length).toEqual(1); + expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toEqual( + "user1A", + ); + expect(data.getParticipantForTransport(TRANSPORT_2).length).toEqual(1); + expect(data.getParticipantForTransport(TRANSPORT_2)[0].identity).toEqual( + "user2A", + ); + } + + { + const data = managerDataUpdates[4]; + expect(data.getConnections().length).toEqual(2); + expect(data.getParticipantForTransport(TRANSPORT_1).length).toEqual(2); + expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toEqual( + "user1A", + ); + expect(data.getParticipantForTransport(TRANSPORT_1)[1].identity).toEqual( + "user1B", + ); + expect(data.getParticipantForTransport(TRANSPORT_2).length).toEqual(1); + expect(data.getParticipantForTransport(TRANSPORT_2)[0].identity).toEqual( + "user2A", + ); + } + + { + const data = managerDataUpdates[5]; + expect(data.getConnections().length).toEqual(3); + expect(data.getParticipantForTransport(TRANSPORT_1).length).toEqual(2); + expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toEqual( + "user1A", + ); + expect(data.getParticipantForTransport(TRANSPORT_1)[1].identity).toEqual( + "user1B", + ); + expect(data.getParticipantForTransport(TRANSPORT_2).length).toEqual(1); + expect(data.getParticipantForTransport(TRANSPORT_2)[0].identity).toEqual( + "user2A", + ); + + expect(data.getParticipantForTransport(TRANSPORT_3).length).toEqual(0); + } + }); +}); diff --git a/src/state/remoteMembers/ConnectionManager.ts b/src/state/remoteMembers/ConnectionManager.ts index 845c2af0..1f4b3a90 100644 --- a/src/state/remoteMembers/ConnectionManager.ts +++ b/src/state/remoteMembers/ConnectionManager.ts @@ -2,6 +2,7 @@ // - make ConnectionManager its own actual class /* +Copyright 2025 Element Creations Ltd. Copyright 2025 New Vector Ltd. SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial @@ -14,52 +15,84 @@ import { } from "matrix-js-sdk/lib/matrixrtc"; import { BehaviorSubject, combineLatest, map, switchMap } from "rxjs"; import { type Logger } from "matrix-js-sdk/lib/logger"; -import { - type E2EEOptions, - Room as LivekitRoom, - type Participant as LivekitParticipant, - type RoomOptions, -} from "livekit-client"; -import { type MatrixClient } from "matrix-js-sdk"; +import { type Participant as LivekitParticipant } from "livekit-client"; import { type Behavior } from "../Behavior"; -import { Connection } from "./Connection"; +import { type Connection } from "./Connection"; import { type ObservableScope } from "../ObservableScope"; import { generateKeyed$ } from "../../utils/observable"; import { areLivekitTransportsEqual } from "./matrixLivekitMerger"; -import { getUrlParams } from "../../UrlParams"; -import { type ProcessorState } from "../../livekit/TrackProcessorContext"; -import { type MediaDevices } from "../MediaDevices"; -import { defaultLiveKitOptions } from "../../livekit/options"; +import { type ConnectionFactory } from "./ConnectionFactory.ts"; + +export class ConnectionManagerData { + private readonly store: Map = + new Map(); + + public constructor() {} + + public add(connection: Connection, participants: LivekitParticipant[]): void { + const key = this.getKey(connection.transport); + const existing = this.store.get(key); + if (!existing) { + this.store.set(key, [connection, participants]); + } else { + existing[1].push(...participants); + } + } + + private getKey(transport: LivekitTransport): string { + return transport.livekit_service_url + "|" + transport.livekit_alias; + } + + public getConnections(): Connection[] { + return Array.from(this.store.values()).map(([connection]) => connection); + } + + public getConnectionForTransport( + transport: LivekitTransport, + ): Connection | undefined { + return this.store.get(this.getKey(transport))?.[0]; + } + + public getParticipantForTransport( + transport: LivekitTransport, + ): LivekitParticipant[] { + const key = transport.livekit_service_url + "|" + transport.livekit_alias; + const existing = this.store.get(key); + if (existing) { + return existing[1]; + } + return []; + } + /** + * Get all connections where the given participant is publishing. + * In theory, there could be several connections where the same participant is publishing but with + * only well behaving clients a participant should only be publishing on a single connection. + * @param participantId + */ + public getConnectionsForParticipant( + participantId: ParticipantId, + ): Connection[] { + const connections: Connection[] = []; + for (const [connection, participants] of this.store.values()) { + if (participants.some((p) => p.identity === participantId)) { + connections.push(connection); + } + } + return connections; + } +} -export type ParticipantByMemberIdMap = Map< - ParticipantId, - // It can be an array because a bad behaving client could be publishingParticipants$ - // multiple times to several livekit rooms. - { participant: LivekitParticipant; connection: Connection }[] ->; // TODO - write test for scopes (do we really need to bind scope) export class ConnectionManager { - private livekitRoomFactory: () => LivekitRoom; + private readonly logger: Logger; + public constructor( - private scope: ObservableScope, - private client: MatrixClient, - private devices: MediaDevices, - private processorState$: Behavior, - private e2eeLivekitOptions: E2EEOptions | undefined, - private logger?: Logger, - livekitRoomFactory?: () => LivekitRoom, + private readonly scope: ObservableScope, + private readonly connectionFactory: ConnectionFactory, + logger: Logger, ) { - this.scope = scope; - const defaultFactory = (): LivekitRoom => - new LivekitRoom( - generateRoomOption( - this.devices, - this.processorState$.value, - this.e2eeLivekitOptions, - ), - ); - this.livekitRoomFactory = livekitRoomFactory ?? defaultFactory; + this.logger = logger.getChild("ConnectionManager"); } /** @@ -94,6 +127,7 @@ export class ConnectionManager { ), ), ), + [], ); /** @@ -108,26 +142,23 @@ export class ConnectionManager { transport: LivekitTransport, ): ((scope: ObservableScope) => Connection) => (scope) => { - const connection = new Connection( - { - transport, - client: this.client, - scope: scope, - livekitRoomFactory: this.livekitRoomFactory, - }, + const connection = this.connectionFactory.createConnection( + transport, + scope, this.logger, ); + // Start the connection immediately + // Use connection state to track connection progress void connection.start(); + // TODO subscribe to connection state to retry or log issues? return connection; }; - const connections = transports.map((transport) => { + return transports.map((transport) => { const key = transport.livekit_service_url + "|" + transport.livekit_alias; return createOrGet(key, createConnection(transport)); }); - - return connections; }, ), ); @@ -186,67 +217,39 @@ export class ConnectionManager { this.transportsSubscriptions$.next([]); } - // We have a lost of connections, for each of these these - // connection we create a stream of (participant, connection) tuples. - // Then we combine the several streams (1 per Connection) into a single stream of tuples. - private allParticipantsWithConnection$ = this.scope.behavior( - this.connections$.pipe( - switchMap((connections) => { - const listsOfParticipantWithConnection = connections.map( - (connection) => { - return connection.participantsWithTrack$.pipe( - map((participants) => - participants.map((p) => ({ - participant: p, + public connectionManagerData$: Behavior = + this.scope.behavior( + this.connections$.pipe( + switchMap((connections) => { + // Map the connections to list of (connection, participant[])[] tuples + const listOfConnectionsWithPublishingParticipants = connections.map( + (connection) => { + return connection.participantsWithTrack$.pipe( + map((participants): [Connection, LivekitParticipant[]] => [ connection, - })), - ), - ); - }, - ); - return combineLatest(listsOfParticipantWithConnection).pipe( - map((lists) => lists.flatMap((list) => list)), - ); - }), - ), - ); - - /** - * This field makes the connection manager to behave as close to a single SFU as possible. - * Each participant that is found on all connections managed by the manager will be listed. - * - * They are stored an a map keyed by `participant.identity` - * TODO (which is equivalent to the `member.id` field in the `m.rtc.member` event) right now its userId:deviceId - */ - public allParticipantsByMemberId$ = this.scope.behavior( - this.allParticipantsWithConnection$.pipe( - map((participantsWithConnections) => { - const participantsByMemberId = participantsWithConnections.reduce( - (acc, test) => { - const { participant, connection } = test; - if (participant.getTrackPublications().length > 0) { - const currentVal = acc.get(participant.identity); - if (!currentVal) { - acc.set(participant.identity, [{ connection, participant }]); - } else { - // already known - // This is for users publishing on several SFUs - currentVal.push({ connection, participant }); - this.logger?.info( - `Participant ${participant.identity} is publishing on several SFUs ${currentVal.map((v) => v.connection.transport.livekit_service_url).join(", ")}`, - ); - } - } - return acc; - }, - new Map() as ParticipantByMemberIdMap, - ); - - return participantsByMemberId; - }), - ), - ); + participants, + ]), + ); + }, + ); + // combineLatest the several streams into a single stream with the ConnectionManagerData + return combineLatest( + listOfConnectionsWithPublishingParticipants, + ).pipe( + map((lists) => + lists.reduce((data, [connection, participants]) => { + data.add(connection, participants); + return data; + }, new ConnectionManagerData()), + ), + ); + }), + ), + // start empty + new ConnectionManagerData(), + ); } + function removeDuplicateTransports( transports: LivekitTransport[], ): LivekitTransport[] { @@ -256,37 +259,3 @@ function removeDuplicateTransports( return acc; }, [] as LivekitTransport[]); } - -/** - * Generate the initial LiveKit RoomOptions based on the current media devices and processor state. - */ -function generateRoomOption( - devices: MediaDevices, - processorState: ProcessorState, - e2eeLivekitOptions: E2EEOptions | undefined, -): RoomOptions { - const { controlledAudioDevices } = getUrlParams(); - return { - ...defaultLiveKitOptions, - videoCaptureDefaults: { - ...defaultLiveKitOptions.videoCaptureDefaults, - deviceId: devices.videoInput.selected$.value?.id, - processor: processorState.processor, - }, - audioCaptureDefaults: { - ...defaultLiveKitOptions.audioCaptureDefaults, - deviceId: devices.audioInput.selected$.value?.id, - }, - audioOutput: { - // When using controlled audio devices, we don't want to set the - // deviceId here, because it will be set by the native app. - // (also the id does not need to match a browser device id) - deviceId: controlledAudioDevices - ? undefined - : devices.audioOutput.selected$.value?.id, - }, - e2ee: e2eeLivekitOptions, - // TODO test and consider this: - // webAudioMix: true, - }; -} diff --git a/src/state/remoteMembers/MatrixLivekitMerger.test.ts b/src/state/remoteMembers/MatrixLivekitMerger.test.ts index df7aca0d..e3f08405 100644 --- a/src/state/remoteMembers/MatrixLivekitMerger.test.ts +++ b/src/state/remoteMembers/MatrixLivekitMerger.test.ts @@ -6,25 +6,253 @@ Please see LICENSE in the repository root for full details. */ import { + describe, test, vi, - onTestFinished, - it, - describe, expect, beforeEach, afterEach, + type MockedObject, } from "vitest"; +import { BehaviorSubject, take } from "rxjs"; +import { + type CallMembership, + type LivekitTransport, +} from "matrix-js-sdk/lib/matrixrtc"; +import { type Room as MatrixRoom } from "matrix-js-sdk"; +import { getParticipantId } from "matrix-js-sdk/lib/matrixrtc/utils"; -import { MatrixLivekitMerger } from "./matrixLivekitMerger"; +import { + type MatrixLivekitItem, + MatrixLivekitMerger, +} from "./matrixLivekitMerger"; import { ObservableScope } from "../ObservableScope"; +import { + type ConnectionManager, + ConnectionManagerData, +} from "./ConnectionManager"; +import { aliceRtcMember } from "../../utils/test-fixtures"; +import { mockRemoteParticipant } from "../../utils/test.ts"; +import { type Connection } from "./Connection.ts"; let testScope: ObservableScope; +let fakeManagerData$: BehaviorSubject; +let fakeMemberships$: BehaviorSubject; +let mockConnectionManager: MockedObject; +let mockMatrixRoom: MatrixRoom; +const userId = "@local:example.com"; +const deviceId = "DEVICE000"; + +// The merger beeing tested +let matrixLivekitMerger: MatrixLivekitMerger; beforeEach(() => { testScope = new ObservableScope(); + fakeMemberships$ = new BehaviorSubject([]); + fakeManagerData$ = new BehaviorSubject( + new ConnectionManagerData(), + ); + mockConnectionManager = vi.mocked({ + registerTransports: vi.fn(), + connectionManagerData$: fakeManagerData$, + } as unknown as ConnectionManager); + mockMatrixRoom = vi.mocked({ + getMember: vi.fn().mockReturnValue(null), + addEventListener: vi.fn(), + removeEventListener: vi.fn(), + } as unknown as MatrixRoom); + + matrixLivekitMerger = new MatrixLivekitMerger( + testScope, + fakeMemberships$, + mockConnectionManager, + mockMatrixRoom, + userId, + deviceId, + ); }); afterEach(() => { testScope.end(); }); + +test("should signal participant not yet connected to livekit", () => { + fakeMemberships$.next([aliceRtcMember]); + + let items: MatrixLivekitItem[] = []; + matrixLivekitMerger.matrixLivekitItems$.pipe(take(1)).subscribe((emitted) => { + items = emitted; + }); + + expect(items).toHaveLength(1); + const item = items[0]; + + // Assert the expected membership + expect(item.membership).toBe(aliceRtcMember); + + // Assert participant & connection are absent (not just `undefined`) + expect(item.participant).not.toBeDefined(); + expect(item.participant).not.toBeDefined(); +}); + +test("should signal participant on a connection that is publishing", () => { + const fakeConnection = { + transport: aliceRtcMember.getTransport(aliceRtcMember) as LivekitTransport, + } as unknown as Connection; + + fakeMemberships$.next([aliceRtcMember]); + const aliceParticipantId = getParticipantId( + aliceRtcMember.userId, + aliceRtcMember.deviceId, + ); + + const managerData: ConnectionManagerData = new ConnectionManagerData(); + managerData.add(fakeConnection, [ + mockRemoteParticipant({ identity: aliceParticipantId }), + ]); + fakeManagerData$.next(managerData); + + let items: MatrixLivekitItem[] = []; + matrixLivekitMerger.matrixLivekitItems$.pipe(take(1)).subscribe((emitted) => { + items = emitted; + }); + expect(items).toHaveLength(1); + const item = items[0]; + + // Assert the expected membership + expect(item.membership).toBe(aliceRtcMember); + expect(item.participant?.identity).toBe(aliceParticipantId); + expect(item.connection?.transport).toEqual(fakeConnection.transport); +}); + +test("should signal participant on a connection that is not publishing", () => { + const fakeConnection = { + transport: aliceRtcMember.getTransport(aliceRtcMember) as LivekitTransport, + } as unknown as Connection; + + fakeMemberships$.next([aliceRtcMember]); + + const managerData: ConnectionManagerData = new ConnectionManagerData(); + managerData.add(fakeConnection, []); + fakeManagerData$.next(managerData); + + matrixLivekitMerger.matrixLivekitItems$.pipe(take(1)).subscribe((items) => { + expect(items).toHaveLength(1); + const item = items[0]; + + // Assert the expected membership + expect(item.membership).toBe(aliceRtcMember); + expect(item.participant).not.toBeDefined(); + // We have the connection + expect(item.connection?.transport).toEqual(fakeConnection.transport); + }); +}); + +describe("Publication edge case", () => { + const connectionA = { + transport: { + type: "livekit", + livekit_service_url: "https://lk.example.org", + livekit_alias: "!alias:example.org", + }, + } as unknown as Connection; + + const connectionB = { + transport: { + type: "livekit", + livekit_service_url: "https://lk.sample.com", + livekit_alias: "!alias:sample.com", + }, + } as unknown as Connection; + + const bobMembership = { + userId: "@bob:example.org", + deviceId: "DEV000", + transports: [connectionA.transport], + } as unknown as CallMembership; + + const bobParticipantId = getParticipantId( + bobMembership.userId, + bobMembership.deviceId, + ); + + test("bob is publishing in several connections", () => { + let lastMatrixLkItems: MatrixLivekitItem[] = []; + matrixLivekitMerger.matrixLivekitItems$.subscribe((items) => { + lastMatrixLkItems = items; + }); + + vi.mocked(bobMembership).getTransport = vi + .fn() + .mockReturnValue(connectionA.transport); + + fakeMemberships$.next([bobMembership]); + + const lkMap = new ConnectionManagerData(); + lkMap.add(connectionA, [ + mockRemoteParticipant({ identity: bobParticipantId }), + ]); + lkMap.add(connectionB, [ + mockRemoteParticipant({ identity: bobParticipantId }), + ]); + + fakeManagerData$.next(lkMap); + + const items = lastMatrixLkItems; + expect(items).toHaveLength(1); + const item = items[0]; + + // Assert the expected membership + expect(item.membership.userId).toEqual(bobMembership.userId); + expect(item.membership.deviceId).toEqual(bobMembership.deviceId); + + expect(item.participant?.identity).toEqual(bobParticipantId); + + // The transport info should come from the membership transports and not only from the publishing connection + expect(item.connection?.transport?.livekit_service_url).toEqual( + bobMembership.transports[0]?.livekit_service_url, + ); + expect(item.connection?.transport?.livekit_alias).toEqual( + bobMembership.transports[0]?.livekit_alias, + ); + }); + + test("bob is publishing in the wrong connection", () => { + let lastMatrixLkItems: MatrixLivekitItem[] = []; + matrixLivekitMerger.matrixLivekitItems$.subscribe((items) => { + lastMatrixLkItems = items; + }); + + vi.mocked(bobMembership).getTransport = vi + .fn() + .mockReturnValue(connectionA.transport); + + fakeMemberships$.next([bobMembership]); + + const lkMap = new ConnectionManagerData(); + lkMap.add(connectionA, []); + lkMap.add(connectionB, [ + mockRemoteParticipant({ identity: bobParticipantId }), + ]); + + fakeManagerData$.next(lkMap); + + const items = lastMatrixLkItems; + expect(items).toHaveLength(1); + const item = items[0]; + + // Assert the expected membership + expect(item.membership.userId).toEqual(bobMembership.userId); + expect(item.membership.deviceId).toEqual(bobMembership.deviceId); + + expect(item.participant).not.toBeDefined(); + + // The transport info should come from the membership transports and not only from the publishing connection + expect(item.connection?.transport?.livekit_service_url).toEqual( + bobMembership.transports[0]?.livekit_service_url, + ); + expect(item.connection?.transport?.livekit_alias).toEqual( + bobMembership.transports[0]?.livekit_alias, + ); + }); +}); diff --git a/src/state/remoteMembers/matrixLivekitMerger.ts b/src/state/remoteMembers/matrixLivekitMerger.ts index bd6ed353..f2f106f2 100644 --- a/src/state/remoteMembers/matrixLivekitMerger.ts +++ b/src/state/remoteMembers/matrixLivekitMerger.ts @@ -13,7 +13,7 @@ import { } from "matrix-js-sdk/lib/matrixrtc"; import { combineLatest, map, startWith, type Observable } from "rxjs"; // eslint-disable-next-line rxjs/no-internal -import { type HasEventTargetAddRemove } from "rxjs/internal/observable/fromEvent"; +import { type NodeStyleEventEmitter } from "rxjs/src/internal/observable/fromEvent.ts"; import type { Room as MatrixRoom, RoomMember } from "matrix-js-sdk"; // import type { Logger } from "matrix-js-sdk/lib/logger"; @@ -71,8 +71,7 @@ export class MatrixLivekitMerger { // apparently needed to get a room member to later get the Avatar // => Extract an AvatarService instead? // Better with just `getMember` - private matrixRoom: Pick & - HasEventTargetAddRemove, + private matrixRoom: Pick & NodeStyleEventEmitter, private userId: string, private deviceId: string, // parentLogger: Logger, @@ -102,28 +101,32 @@ export class MatrixLivekitMerger { return combineLatest([ membershipsWithTransport$, - this.connectionManager.allParticipantsByMemberId$, + this.connectionManager.connectionManagerData$, displaynameMap$, ]).pipe( - map(([memberships, participantsByMemberId, displayNameMap]) => { + map(([memberships, managerData, displayNameMap]) => { const items: MatrixLivekitItem[] = memberships.map( ({ membership, transport }) => { - const participantsWithConnection = participantsByMemberId.get( - // membership.membershipID, Currently its hardcoded by the jwt service to - `${membership.userId}:${membership.deviceId}`, + // TODO! cannot use membership.membershipID yet, Currently its hardcoded by the jwt service to + const participantId = /*membership.membershipID*/ `${membership.userId}:${membership.deviceId}`; + + const participants = transport + ? managerData.getParticipantForTransport(transport) + : []; + const participant = participants.find( + (p) => p.identity == participantId, ); - const participant = - transport && - participantsWithConnection?.find((p) => - areLivekitTransportsEqual(p.connection.transport, transport), - ); const member = getRoomMemberFromRtcMember( membership, this.matrixRoom, )?.member; + const connection = transport + ? managerData.getConnectionForTransport(transport) + : undefined; return { - ...participant, + participant, membership, + connection, // This makes sense to add the the js-sdk callMembership (we only need the avatar so probably the call memberhsip just should aquire the avatar) member, displayName: displayNameMap.get(membership.membershipID) ?? "---", From a7d2a3b9dbf2367104921c213dfe0f7ef58db560 Mon Sep 17 00:00:00 2001 From: Valere Date: Mon, 3 Nov 2025 13:18:33 +0100 Subject: [PATCH 14/75] es lint fixes --- src/state/Async.ts | 1 - src/state/CallViewModel.ts | 2 +- src/state/ownMember/OwnMembership.ts | 3 ++- src/state/remoteMembers/Connection.test.ts | 2 -- src/utils/test.ts | 12 ++++++------ 5 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/state/Async.ts b/src/state/Async.ts index f2e0376b..e95eaa39 100644 --- a/src/state/Async.ts +++ b/src/state/Async.ts @@ -6,7 +6,6 @@ Please see LICENSE in the repository root for full details. */ import { catchError, from, map, type Observable, of, startWith } from "rxjs"; -import { Behavior } from "./Behavior"; /** * Data that may need to be loaded asynchronously. diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 31aa2533..15db43c4 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -134,7 +134,7 @@ import { type SpotlightLandscapeLayoutMedia, type SpotlightPortraitLayoutMedia, } from "./layout-types.ts"; -import { ElementCallError, UnknownCallError } from "../utils/errors.ts"; +import { type ElementCallError, UnknownCallError } from "../utils/errors.ts"; import { ObservableScope } from "./ObservableScope.ts"; import { memberDisplaynames$ } from "./remoteMembers/displayname.ts"; import { ConnectionManager } from "./remoteMembers/ConnectionManager.ts"; diff --git a/src/state/ownMember/OwnMembership.ts b/src/state/ownMember/OwnMembership.ts index 6ffdb5b5..3b6d1987 100644 --- a/src/state/ownMember/OwnMembership.ts +++ b/src/state/ownMember/OwnMembership.ts @@ -31,9 +31,10 @@ import { startWith, switchMap, } from "rxjs"; + import { multiSfu } from "../../settings/settings"; import { type Behavior } from "../Behavior"; -import { ConnectionManager } from "../remoteMembers/ConnectionManager"; +import { type ConnectionManager } from "../remoteMembers/ConnectionManager"; import { makeTransport } from "../../rtcSessionHelpers"; import { type ObservableScope } from "../ObservableScope"; import { async$, unwrapAsync } from "../Async"; diff --git a/src/state/remoteMembers/Connection.test.ts b/src/state/remoteMembers/Connection.test.ts index 4af64578..7e2d39f8 100644 --- a/src/state/remoteMembers/Connection.test.ts +++ b/src/state/remoteMembers/Connection.test.ts @@ -17,7 +17,6 @@ import { } from "vitest"; import { BehaviorSubject, of } from "rxjs"; import { - ConnectionState as LivekitConnectionState, type LocalParticipant, type RemoteParticipant, type Room as LivekitRoom, @@ -36,7 +35,6 @@ import { type ConnectionOpts, type ConnectionState, type PublishingParticipant, - Connection, } from "./Connection.ts"; import { ObservableScope } from "../ObservableScope.ts"; import { type OpenIDClientParts } from "../../livekit/openIDSFU.ts"; diff --git a/src/utils/test.ts b/src/utils/test.ts index db85da4a..e474cec5 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -8,23 +8,23 @@ import { map, type Observable, of, type SchedulerLike } from "rxjs"; import { type RunHelpers, TestScheduler } from "rxjs/testing"; import { expect, type MockedObject, onTestFinished, vi, vitest } from "vitest"; import { - type RoomMember, - type Room as MatrixRoom, MatrixEvent, + type Room as MatrixRoom, type Room, + type RoomMember, TypedEventEmitter, } from "matrix-js-sdk"; import { CallMembership, - type Transport, + type LivekitFocusSelection, + type LivekitTransport, + type MatrixRTCSession, MatrixRTCSessionEvent, type MatrixRTCSessionEventHandlerMap, MembershipManagerEvent, type SessionMembershipData, Status, - type LivekitFocusSelection, - type MatrixRTCSession, - type LivekitTransport, + type Transport, } from "matrix-js-sdk/lib/matrixrtc"; import { type MembershipManagerEventHandlerMap } from "matrix-js-sdk/lib/matrixrtc/IMembershipManager"; import { From 5961cb65df8dbf2ec408f9b4790016f193a5838f Mon Sep 17 00:00:00 2001 From: Valere Date: Mon, 3 Nov 2025 17:19:17 +0100 Subject: [PATCH 15/75] test with marbles --- .../remoteMembers/ConnectionManager.test.ts | 165 ++++++++---------- src/utils/test.ts | 2 +- 2 files changed, 75 insertions(+), 92 deletions(-) diff --git a/src/state/remoteMembers/ConnectionManager.test.ts b/src/state/remoteMembers/ConnectionManager.test.ts index a0203840..79881f66 100644 --- a/src/state/remoteMembers/ConnectionManager.test.ts +++ b/src/state/remoteMembers/ConnectionManager.test.ts @@ -14,12 +14,11 @@ import { type Participant as LivekitParticipant } from "livekit-client"; import { ObservableScope } from "../ObservableScope.ts"; import { ConnectionManager, - type ConnectionManagerData, } from "./ConnectionManager.ts"; import { type ConnectionFactory } from "./ConnectionFactory.ts"; import { type Connection } from "./Connection.ts"; import { areLivekitTransportsEqual } from "./matrixLivekitMerger.ts"; -import { flushPromises } from "../../utils/test.ts"; +import { flushPromises, withTestScheduler } from "../../utils/test.ts"; // Some test constants @@ -213,95 +212,79 @@ describe("connectionManagerData$ stream", () => { }); test("Should report connections with the publishing participants", async () => { - const managerDataUpdates: ConnectionManagerData[] = []; - manager.connectionManagerData$.subscribe((data) => { - managerDataUpdates.push(data); + withTestScheduler(({ expectObservable, schedule, cold, behavior }) => { + manager.registerTransports( + behavior("a", { + a: [TRANSPORT_1, TRANSPORT_2], + }), + ); + const conn1Participants$ = fakePublishingParticipantsStreams.get( + keyForTransport(TRANSPORT_1), + )!; + + schedule("-a-b", { + a: () => { + conn1Participants$.next([ + { identity: "user1A" } as LivekitParticipant, + ]); + }, + b: () => { + conn1Participants$.next([ + { identity: "user1A" } as LivekitParticipant, + { identity: "user1B" } as LivekitParticipant, + ]); + }, + }); + + const conn2Participants$ = fakePublishingParticipantsStreams.get( + keyForTransport(TRANSPORT_2), + )!; + + schedule("--a", { + a: () => { + conn2Participants$.next([ + { identity: "user2A" } as LivekitParticipant, + ]); + }, + }); + + expectObservable(manager.connectionManagerData$).toBe("abcd", { + a: expect.toSatisfy((data) => { + return ( + data.getConnections().length == 2 && + data.getParticipantForTransport(TRANSPORT_1).length == 0 && + data.getParticipantForTransport(TRANSPORT_2).length == 0 + ); + }), + b: expect.toSatisfy((data) => { + return ( + data.getConnections().length == 2 && + data.getParticipantForTransport(TRANSPORT_1).length == 1 && + data.getParticipantForTransport(TRANSPORT_2).length == 0 && + data.getParticipantForTransport(TRANSPORT_1)[0].identity == "user1A" + ); + }), + c: expect.toSatisfy((data) => { + return ( + data.getConnections().length == 2 && + data.getParticipantForTransport(TRANSPORT_1).length == 1 && + data.getParticipantForTransport(TRANSPORT_2).length == 1 && + data.getParticipantForTransport(TRANSPORT_1)[0].identity == "user1A"&& + data.getParticipantForTransport(TRANSPORT_2)[0].identity == "user2A" + ); + }), + d: expect.toSatisfy((data) => { + return ( + data.getConnections().length == 2 && + data.getParticipantForTransport(TRANSPORT_1).length == 2 && + data.getParticipantForTransport(TRANSPORT_2).length == 1 && + data.getParticipantForTransport(TRANSPORT_1)[0].identity == "user1A"&& + data.getParticipantForTransport(TRANSPORT_1)[1].identity == "user1B"&& + data.getParticipantForTransport(TRANSPORT_2)[0].identity == "user2A" + ); + }), + }); }); - - testTransportStream$.next([TRANSPORT_1, TRANSPORT_2]); - await flushPromises(); - - const conn1Participants$ = fakePublishingParticipantsStreams.get( - keyForTransport(TRANSPORT_1), - )!; - - conn1Participants$.next([{ identity: "user1A" } as LivekitParticipant]); - - const conn2Participants$ = fakePublishingParticipantsStreams.get( - keyForTransport(TRANSPORT_2), - )!; - conn2Participants$.next([{ identity: "user2A" } as LivekitParticipant]); - - conn1Participants$.next([ - { identity: "user1A" } as LivekitParticipant, - { identity: "user1B" } as LivekitParticipant, - ]); - - testTransportStream$.next([TRANSPORT_1, TRANSPORT_2, TRANSPORT_3]); - - expect(managerDataUpdates[0].getConnections().length).toEqual(0); - - { - const data = managerDataUpdates[1]; - expect(data.getConnections().length).toEqual(2); - expect(data.getParticipantForTransport(TRANSPORT_1).length).toEqual(0); - expect(data.getParticipantForTransport(TRANSPORT_1).length).toEqual(0); - } - - { - const data = managerDataUpdates[2]; - expect(data.getConnections().length).toEqual(2); - expect(data.getParticipantForTransport(TRANSPORT_1).length).toEqual(1); - expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toEqual( - "user1A", - ); - expect(data.getParticipantForTransport(TRANSPORT_2).length).toEqual(0); - } - { - const data = managerDataUpdates[3]; - expect(data.getConnections().length).toEqual(2); - expect(data.getParticipantForTransport(TRANSPORT_1).length).toEqual(1); - expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toEqual( - "user1A", - ); - expect(data.getParticipantForTransport(TRANSPORT_2).length).toEqual(1); - expect(data.getParticipantForTransport(TRANSPORT_2)[0].identity).toEqual( - "user2A", - ); - } - - { - const data = managerDataUpdates[4]; - expect(data.getConnections().length).toEqual(2); - expect(data.getParticipantForTransport(TRANSPORT_1).length).toEqual(2); - expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toEqual( - "user1A", - ); - expect(data.getParticipantForTransport(TRANSPORT_1)[1].identity).toEqual( - "user1B", - ); - expect(data.getParticipantForTransport(TRANSPORT_2).length).toEqual(1); - expect(data.getParticipantForTransport(TRANSPORT_2)[0].identity).toEqual( - "user2A", - ); - } - - { - const data = managerDataUpdates[5]; - expect(data.getConnections().length).toEqual(3); - expect(data.getParticipantForTransport(TRANSPORT_1).length).toEqual(2); - expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toEqual( - "user1A", - ); - expect(data.getParticipantForTransport(TRANSPORT_1)[1].identity).toEqual( - "user1B", - ); - expect(data.getParticipantForTransport(TRANSPORT_2).length).toEqual(1); - expect(data.getParticipantForTransport(TRANSPORT_2)[0].identity).toEqual( - "user2A", - ); - - expect(data.getParticipantForTransport(TRANSPORT_3).length).toEqual(0); - } }); + }); diff --git a/src/utils/test.ts b/src/utils/test.ts index e474cec5..b60492f6 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -106,7 +106,7 @@ export function withTestScheduler( continuation: (helpers: OurRunHelpers) => void, ): void { const scheduler = new TestScheduler((actual, expected) => { - expect(actual).deep.equals(expected); + expect(actual).toStrictEqual(expected); }); const scope = new ObservableScope(); // we set the test scheduler as a global so that you can watch it in a debugger From 79bd458dc2f1a57e38813d8e8a3d687837f3df77 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 3 Nov 2025 17:09:43 -0500 Subject: [PATCH 16/75] Remove use of npm access token now that trusted publishing is set up npm has recently limited the lifetime of all access tokens to 90 days (https://gh.io/npm-token-changes), so it would be a bit inconvenient to stick to our current access token-based method of publishing releases. Meanwhile npm has implemented a more secure publishing method based on OIDC in which you tell the registry that a particular GitHub Actions workflow should be a "trusted publisher" for a given package, and then the CLI will authenticate automatically. (https://docs.npmjs.com/trusted-publishers) I've already set trusted publishing up on the registry side, and since we're already granting the job permission to generate ID tokens for provenance, there should be no additional lines of config needed to make it work. Let's take away the access token and see how this goes next time we release. --- .github/workflows/publish-embedded-packages.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/publish-embedded-packages.yaml b/.github/workflows/publish-embedded-packages.yaml index 256e440e..434f473c 100644 --- a/.github/workflows/publish-embedded-packages.yaml +++ b/.github/workflows/publish-embedded-packages.yaml @@ -100,7 +100,7 @@ jobs: ARTIFACT_VERSION: ${{ steps.artifact_version.outputs.ARTIFACT_VERSION }} permissions: contents: read - id-token: write # required for the provenance flag on npm publish + id-token: write # Allow npm to authenticate as a trusted publisher steps: - name: Checkout uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 @@ -126,8 +126,6 @@ jobs: npm version ${{ needs.versioning.outputs.PREFIXED_VERSION }} --no-git-tag-version echo "ARTIFACT_VERSION=$(jq '.version' --raw-output package.json)" >> "$GITHUB_ENV" npm publish --provenance --access public --tag ${{ needs.versioning.outputs.TAG }} ${{ needs.versioning.outputs.DRY_RUN == 'true' && '--dry-run' || '' }} - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_RELEASE_TOKEN }} - id: artifact_version name: Output artifact version From 06734ae086ff1460652c03c74813cf7c18d73084 Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 4 Nov 2025 17:12:44 +0100 Subject: [PATCH 17/75] quick refactor, use object instead of tupple --- src/state/remoteMembers/ConnectionManager.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/state/remoteMembers/ConnectionManager.ts b/src/state/remoteMembers/ConnectionManager.ts index 1f4b3a90..239cf3c9 100644 --- a/src/state/remoteMembers/ConnectionManager.ts +++ b/src/state/remoteMembers/ConnectionManager.ts @@ -221,14 +221,14 @@ export class ConnectionManager { this.scope.behavior( this.connections$.pipe( switchMap((connections) => { - // Map the connections to list of (connection, participant[])[] tuples + // Map the connections to list of {connection, participants}[] const listOfConnectionsWithPublishingParticipants = connections.map( (connection) => { return connection.participantsWithTrack$.pipe( - map((participants): [Connection, LivekitParticipant[]] => [ + map((participants) => ({ connection, participants, - ]), + })), ); }, ); @@ -237,7 +237,7 @@ export class ConnectionManager { listOfConnectionsWithPublishingParticipants, ).pipe( map((lists) => - lists.reduce((data, [connection, participants]) => { + lists.reduce((data, { connection, participants }) => { data.add(connection, participants); return data; }, new ConnectionManagerData()), From 870b7066722bf559ed2d6fd5e2bd7bb72606822a Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 4 Nov 2025 17:13:28 +0100 Subject: [PATCH 18/75] Connection & Livekit integ test WIP --- src/state/ownMember/OwnMembership.ts | 15 +- src/state/remoteMembers/Connection.ts | 16 +- .../remoteMembers/ConnectionManager.test.ts | 34 ++-- src/state/remoteMembers/displayname.ts | 28 ++-- src/state/remoteMembers/integration.test.ts | 157 ++++++++++++++++++ .../remoteMembers/matrixLivekitMerger.ts | 1 + 6 files changed, 208 insertions(+), 43 deletions(-) create mode 100644 src/state/remoteMembers/integration.test.ts diff --git a/src/state/ownMember/OwnMembership.ts b/src/state/ownMember/OwnMembership.ts index 3b6d1987..c096c2da 100644 --- a/src/state/ownMember/OwnMembership.ts +++ b/src/state/ownMember/OwnMembership.ts @@ -5,8 +5,7 @@ SPDX-License-IdFentifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type E2EEOptions } from "livekit-client"; -import { logger } from "matrix-js-sdk/lib/logger"; +import { type E2EEOptions, type Track } from "livekit-client"; import { type LivekitTransport, type MatrixRTCSession, @@ -22,6 +21,7 @@ import { import { BehaviorSubject, combineLatest, + distinctUntilChanged, from, fromEvent, map, @@ -31,19 +31,20 @@ import { startWith, switchMap, } from "rxjs"; +import { deepCompare } from "matrix-js-sdk/lib/utils"; import { multiSfu } from "../../settings/settings"; import { type Behavior } from "../Behavior"; import { type ConnectionManager } from "../remoteMembers/ConnectionManager"; import { makeTransport } from "../../rtcSessionHelpers"; import { type ObservableScope } from "../ObservableScope"; -import { async$, unwrapAsync } from "../Async"; import { Publisher } from "./Publisher"; import { type MuteStates } from "../MuteStates"; import { type ProcessorState } from "../../livekit/TrackProcessorContext"; import { type MediaDevices } from "../../state/MediaDevices"; import { and$ } from "../../utils/observable"; import { areLivekitTransportsEqual } from "../remoteMembers/matrixLivekitMerger"; +import { type ElementCallError } from "../../utils/errors.ts"; /* * - get well known @@ -70,6 +71,10 @@ interface Props { trackerProcessorState$: Behavior; } +export type JoinedState = + | { state: "Initialized" } + | { state: "Error"; error: ElementCallError }; + /** * This class is responsible for managing the own membership in a room. * We want @@ -96,11 +101,11 @@ export const ownMembership$ = ({ trackerProcessorState$, }: Props): { // publisher: Publisher - requestJoin(): Observable; + requestJoin$(): Observable; startTracks(): Track[]; } => { // This should be used in a combineLatest with publisher$ to connect. - const shouldStartTracks$ = BehaviorSubject(false); + const shouldStartTracks$ = new BehaviorSubject(false); // to make it possible to call startTracks before the preferredTransport$ has resolved. const startTracks = () => { diff --git a/src/state/remoteMembers/Connection.ts b/src/state/remoteMembers/Connection.ts index 67b2dc8e..03a9e137 100644 --- a/src/state/remoteMembers/Connection.ts +++ b/src/state/remoteMembers/Connection.ts @@ -211,16 +211,12 @@ export class Connection { this.client = client; this.participantsWithTrack$ = scope.behavior( - connectedParticipantsObserver( - this.livekitRoom, - // VALR: added that while I think about it - { - additionalRoomEvents: [ - RoomEvent.TrackPublished, - RoomEvent.TrackUnpublished, - ], - }, - ), + connectedParticipantsObserver(this.livekitRoom, { + additionalRoomEvents: [ + RoomEvent.TrackPublished, + RoomEvent.TrackUnpublished, + ], + }), [], ); diff --git a/src/state/remoteMembers/ConnectionManager.test.ts b/src/state/remoteMembers/ConnectionManager.test.ts index 79881f66..48c897e3 100644 --- a/src/state/remoteMembers/ConnectionManager.test.ts +++ b/src/state/remoteMembers/ConnectionManager.test.ts @@ -12,9 +12,7 @@ import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { type Participant as LivekitParticipant } from "livekit-client"; import { ObservableScope } from "../ObservableScope.ts"; -import { - ConnectionManager, -} from "./ConnectionManager.ts"; +import { ConnectionManager } from "./ConnectionManager.ts"; import { type ConnectionFactory } from "./ConnectionFactory.ts"; import { type Connection } from "./Connection.ts"; import { areLivekitTransportsEqual } from "./matrixLivekitMerger.ts"; @@ -34,11 +32,11 @@ const TRANSPORT_2: LivekitTransport = { livekit_alias: "!alias:sample.com", }; -const TRANSPORT_3: LivekitTransport = { - type: "livekit", - livekit_service_url: "https://lk-other.sample.com", - livekit_alias: "!alias:sample.com", -}; +// const TRANSPORT_3: LivekitTransport = { +// type: "livekit", +// livekit_service_url: "https://lk-other.sample.com", +// livekit_alias: "!alias:sample.com", +// }; let testScope: ObservableScope; let fakeConnectionFactory: ConnectionFactory; @@ -211,8 +209,8 @@ describe("connectionManagerData$ stream", () => { ); }); - test("Should report connections with the publishing participants", async () => { - withTestScheduler(({ expectObservable, schedule, cold, behavior }) => { + test("Should report connections with the publishing participants", () => { + withTestScheduler(({ expectObservable, schedule, behavior }) => { manager.registerTransports( behavior("a", { a: [TRANSPORT_1, TRANSPORT_2], @@ -257,7 +255,7 @@ describe("connectionManagerData$ stream", () => { ); }), b: expect.toSatisfy((data) => { - return ( + return ( data.getConnections().length == 2 && data.getParticipantForTransport(TRANSPORT_1).length == 1 && data.getParticipantForTransport(TRANSPORT_2).length == 0 && @@ -265,26 +263,28 @@ describe("connectionManagerData$ stream", () => { ); }), c: expect.toSatisfy((data) => { - return ( + return ( data.getConnections().length == 2 && data.getParticipantForTransport(TRANSPORT_1).length == 1 && data.getParticipantForTransport(TRANSPORT_2).length == 1 && - data.getParticipantForTransport(TRANSPORT_1)[0].identity == "user1A"&& + data.getParticipantForTransport(TRANSPORT_1)[0].identity == + "user1A" && data.getParticipantForTransport(TRANSPORT_2)[0].identity == "user2A" ); }), d: expect.toSatisfy((data) => { - return ( + return ( data.getConnections().length == 2 && data.getParticipantForTransport(TRANSPORT_1).length == 2 && data.getParticipantForTransport(TRANSPORT_2).length == 1 && - data.getParticipantForTransport(TRANSPORT_1)[0].identity == "user1A"&& - data.getParticipantForTransport(TRANSPORT_1)[1].identity == "user1B"&& + data.getParticipantForTransport(TRANSPORT_1)[0].identity == + "user1A" && + data.getParticipantForTransport(TRANSPORT_1)[1].identity == + "user1B" && data.getParticipantForTransport(TRANSPORT_2)[0].identity == "user2A" ); }), }); }); }); - }); diff --git a/src/state/remoteMembers/displayname.ts b/src/state/remoteMembers/displayname.ts index 67e11f99..825ad5a1 100644 --- a/src/state/remoteMembers/displayname.ts +++ b/src/state/remoteMembers/displayname.ts @@ -6,7 +6,13 @@ Please see LICENSE in the repository root for full details. */ import { type RoomMember, RoomStateEvent } from "matrix-js-sdk"; -import { combineLatest, fromEvent, type Observable, startWith } from "rxjs"; +import { + combineLatest, + fromEvent, + map, + type Observable, + startWith, +} from "rxjs"; import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; import { logger } from "matrix-js-sdk/lib/logger"; import { type Room as MatrixRoom } from "matrix-js-sdk/lib/matrix"; @@ -36,15 +42,14 @@ export const memberDisplaynames$ = ( deviceId: string, ): Behavior> => scope.behavior( - combineLatest( - [ - // Handle call membership changes - memberships$, - // Additionally handle display name changes (implicitly reacting to them) - fromEvent(matrixRoom, RoomStateEvent.Members).pipe(startWith(null)), - // TODO: do we need: pauseWhen(this.pretendToBeDisconnected$), - ], - (memberships, _displaynames) => { + combineLatest([ + // Handle call membership changes + memberships$, + // Additionally handle display name changes (implicitly reacting to them) + fromEvent(matrixRoom, RoomStateEvent.Members).pipe(startWith(null)), + // TODO: do we need: pauseWhen(this.pretendToBeDisconnected$), + ]).pipe( + map((memberships, _displaynames) => { const displaynameMap = new Map([ [ `${userId}:${deviceId}`, @@ -71,8 +76,9 @@ export const memberDisplaynames$ = ( ); } return displaynameMap; - }, + }), ), + new Map(), ); export function getRoomMemberFromRtcMember( diff --git a/src/state/remoteMembers/integration.test.ts b/src/state/remoteMembers/integration.test.ts new file mode 100644 index 00000000..36594680 --- /dev/null +++ b/src/state/remoteMembers/integration.test.ts @@ -0,0 +1,157 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { test, vi, beforeEach, afterEach } from "vitest"; +import { BehaviorSubject, type Observable } from "rxjs"; +import { type Room as LivekitRoom } from "livekit-client"; +import { logger } from "matrix-js-sdk/lib/logger"; +import EventEmitter from "events"; +import fetchMock from "fetch-mock"; + +import { ConnectionManager } from "./ConnectionManager.ts"; +import { ObservableScope } from "../ObservableScope.ts"; +import { ECConnectionFactory } from "./ConnectionFactory.ts"; +import { type OpenIDClientParts } from "../../livekit/openIDSFU.ts"; +import { mockMediaDevices, withTestScheduler } from "../../utils/test"; +import { type ProcessorState } from "../../livekit/TrackProcessorContext.tsx"; +import { MatrixLivekitMerger } from "./matrixLivekitMerger.ts"; +import type { CallMembership, Transport } from "matrix-js-sdk/lib/matrixrtc"; +import { TRANSPORT_1 } from "./ConnectionManager.test.ts"; + +// Test the integration of ConnectionManager and MatrixLivekitMerger + +let testScope: ObservableScope; +let ecConnectionFactory: ECConnectionFactory; +let mockClient: OpenIDClientParts; +let lkRoomFactory: () => LivekitRoom; + +const createdMockLivekitRooms: Map = new Map(); + +// Main test input +const memberships$ = new BehaviorSubject([]); + +// under test +let connectionManager: ConnectionManager; + +function createLkMerger( + memberships$: Observable, +): MatrixLivekitMerger { + const mockRoomEmitter = new EventEmitter(); + return new MatrixLivekitMerger( + testScope, + memberships$, + connectionManager, + { + on: mockRoomEmitter.on.bind(mockRoomEmitter), + off: mockRoomEmitter.off.bind(mockRoomEmitter), + getMember: vi.fn().mockReturnValue(undefined), + }, + "@user:example.com", + "DEV000", + ); +} + +beforeEach(() => { + testScope = new ObservableScope(); + mockClient = { + getOpenIdToken: vi.fn().mockReturnValue(""), + getDeviceId: vi.fn().mockReturnValue("DEV000"), + }; + + lkRoomFactory = vi.fn().mockImplementation(() => { + const emitter = new EventEmitter(); + const base = { + on: emitter.on.bind(emitter), + off: emitter.off.bind(emitter), + emit: emitter.emit.bind(emitter), + disconnect: vi.fn(), + remoteParticipants: new Map(), + } as unknown as LivekitRoom; + + vi.mocked(base).connect = vi.fn().mockImplementation(({ url }) => { + createdMockLivekitRooms.set(url, base); + }); + return base; + }); + + ecConnectionFactory = new ECConnectionFactory( + mockClient, + mockMediaDevices({}), + new BehaviorSubject({ + supported: true, + processor: undefined, + }), + undefined, + false, + lkRoomFactory, + ); + + connectionManager = new ConnectionManager( + testScope, + ecConnectionFactory, + logger, + ); + + //TODO a bit annoying to have to do a http mock? + fetchMock.post(`**/sfu/get`, (url) => { + const domain = new URL(url).hostname; // Extract the domain from the URL + + return { + status: 200, + body: { + url: `wss://${domain}/livekit/sfu`, + jwt: "ATOKEN", + }, + }; + }); +}); + +afterEach(() => { + testScope.end(); + fetchMock.reset(); +}); + +test("example test", () => { + withTestScheduler(({ schedule, expectObservable, cold }) => { + connectionManager.connections$.subscribe((connections) => { + // console.log( + // "Connections updated:", + // connections.map((c) => c.transport), + // ); + }); + + const memberships$ = cold("-a-b-c", { + a: [mockCallmembership("@bob:example.com", "BDEV000")], + b: [ + mockCallmembership("@bob:example.com", "BDEV000"), + mockCallmembership("@carl:example.com", "CDEV000"), + ], + c: [ + mockCallmembership("@bob:example.com", "BDEV000"), + mockCallmembership("@carl:example.com", "CDEV000"), + mockCallmembership("@dave:foo.bar", "DDEV000"), + ], + }); + + // TODO IN PROGRESS + const merger = createLkMerger(memberships$); + }); +}); + +function mockCallmembership( + userId: string, + deviceId: string, + transport?: Transport, +): CallMembership { + const t = transport ?? TRANSPORT_1; + return { + userId: userId, + deviceId: deviceId, + getTransport: vi.fn().mockReturnValue(t), + transports: [t], + } as unknown as CallMembership; +} diff --git a/src/state/remoteMembers/matrixLivekitMerger.ts b/src/state/remoteMembers/matrixLivekitMerger.ts index f2f106f2..1487636c 100644 --- a/src/state/remoteMembers/matrixLivekitMerger.ts +++ b/src/state/remoteMembers/matrixLivekitMerger.ts @@ -177,6 +177,7 @@ export class MatrixLivekitMerger { }); }), ), + [], ); } } From 57bf86fc4c29d1fd99f16c377402b54bc5c490dc Mon Sep 17 00:00:00 2001 From: Timo K Date: Tue, 4 Nov 2025 20:24:15 +0100 Subject: [PATCH 19/75] finish up most of our helper classes. there are no lint issues in the new classes. The CallViewModel is not done yet however --- src/state/CallViewModel.ts | 128 ++++--- src/state/SessionBehaviors.ts | 72 ++++ src/state/localMember/LocalMembership.ts | 328 ++++++++++++++++++ src/state/localMember/LocalTransport.ts | 166 +++++++++ .../{ownMember => localMember}/Publisher.ts | 3 +- src/state/ownMember/OwnMembership.ts | 287 --------------- src/state/remoteMembers/ConnectionManager.ts | 97 ++---- src/state/remoteMembers/displayname.ts | 31 +- .../remoteMembers/matrixLivekitMerger.ts | 58 +--- 9 files changed, 669 insertions(+), 501 deletions(-) create mode 100644 src/state/SessionBehaviors.ts create mode 100644 src/state/localMember/LocalMembership.ts create mode 100644 src/state/localMember/LocalTransport.ts rename src/state/{ownMember => localMember}/Publisher.ts (99%) delete mode 100644 src/state/ownMember/OwnMembership.ts diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 15db43c4..436255eb 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -121,7 +121,7 @@ import { type MuteStates } from "./MuteStates"; import { getUrlParams } from "../UrlParams"; import { type ProcessorState } from "../livekit/TrackProcessorContext"; import { ElementWidgetActions, widget } from "../widget"; -import { PublishConnection } from "./ownMember/Publisher.ts"; +import { PublishConnection } from "./localMember/Publisher.ts"; import { type Async, async$, mapAsync, ready } from "./Async"; import { sharingScreen$, UserMedia } from "./UserMedia.ts"; import { ScreenShare } from "./ScreenShare.ts"; @@ -139,7 +139,10 @@ import { ObservableScope } from "./ObservableScope.ts"; import { memberDisplaynames$ } from "./remoteMembers/displayname.ts"; import { ConnectionManager } from "./remoteMembers/ConnectionManager.ts"; import { MatrixLivekitMerger } from "./remoteMembers/matrixLivekitMerger.ts"; -import { ownMembership$ } from "./ownMember/OwnMembership.ts"; +import { ownMembership$ } from "./localMember/LocalMembership.ts"; +import { localTransport$ as computeLocalTransport$ } from "./localMember/LocalTransport.ts"; +import { sessionBehaviors$ } from "./SessionBehaviors.ts"; +import { ECConnectionFactory } from "./remoteMembers/ConnectionFactory.ts"; //TODO // Larger rename @@ -197,6 +200,8 @@ type MediaItem = UserMedia | ScreenShare; export class CallViewModel { private readonly urlParams = getUrlParams(); + private readonly userId = this.matrixRoom.client.getUserId()!; + private readonly deviceId = this.matrixRoom.client.getDeviceId()!; private readonly livekitAlias = getLivekitAlias(this.matrixRTCSession); private readonly livekitE2EEKeyProvider = getE2eeKeyProvider( @@ -214,31 +219,52 @@ export class CallViewModel { private readonly _configError$ = new BehaviorSubject( null, ); + private sessionBehaviors = sessionBehaviors$( + this.scope, + this.matrixRTCSession, + ); + private memberships$ = this.sessionBehaviors.memberships$; - private memberships$ = this.scope.behavior( - fromEvent( - this.matrixRTCSession, - MatrixRTCSessionEvent.MembershipsChanged, - (_, memberships: CallMembership[]) => memberships, + private localTransport$ = computeLocalTransport$({ + scope: this.scope, + memberships$: this.memberships$, + client: this.matrixRoom.client, + roomId: this.matrixRoom.roomId, + useOldestMember$: multiSfu.value$, + }); + + private connectionFactory = new ECConnectionFactory( + this.matrixRoom.client, + this.mediaDevices, + this.trackProcessorState$, + this.e2eeLivekitOptions(), + getUrlParams().controlledAudioDevices, + ); + + private allTransports$ = this.scope.behavior( + combineLatest( + [this.localTransport$, this.sessionBehaviors.transports$], + (l, t) => [...(l ? [l] : []), ...t], ), ); private connectionManager = new ConnectionManager( this.scope, - this.matrixRoom.client, - this.mediaDevices, - this.trackProcessorState$, - this.e2eeLivekitOptions(), + this.connectionFactory, + this.allTransports$, + logger, ); private matrixLivekitMerger = new MatrixLivekitMerger( this.scope, - this.memberships$, + this.sessionBehaviors.membershipsWithTransport$, this.connectionManager, this.matrixRoom, + this.userId, + this.deviceId, ); - private ownMembership = ownMembership$({ + private localMembership = this.localMembership$({ scope: this.scope, muteStates: this.muteStates, multiSfu: this.multiSfu, @@ -247,6 +273,7 @@ export class CallViewModel { e2eeLivekitOptions: this.e2eeLivekitOptions, }); + private matrixLivekitItems$ = this.matrixLivekitMerger.matrixLivekitItems$; /** * If there is a configuration error with the call (e.g. misconfigured E2EE). * This is a fatal error that prevents the call from being created/joined. @@ -289,60 +316,27 @@ export class CallViewModel { ), ); - /** - * The transport that we would personally prefer to publish on (if not for the - * transport preferences of others, perhaps). - */ - // DISCUSS move to ownMembership - private readonly preferredTransport$ = this.scope.behavior( - async$(makeTransport(this.matrixRTCSession)), - ); + // /** + // * The transport that we would personally prefer to publish on (if not for the + // * transport preferences of others, perhaps). + // */ + // // DISCUSS move to ownMembership + // private readonly preferredTransport$ = this.scope.behavior( + // async$(makeTransport(this.matrixRTCSession)), + // ); - /** - * The transport over which we should be actively publishing our media. - * null when not joined. - */ - // DISCUSSION ownMembershipManager - private readonly localTransport$: Behavior | null> = - this.scope.behavior( - this.transports$.pipe( - map((transports) => transports?.local ?? null), - distinctUntilChanged | null>(deepCompare), - ), - ); - - /** - * The transport we should advertise in our MatrixRTC membership (plus whether - * it is a multi-SFU transport and whether we should use sticky events). - */ - // DISCUSSION ownMembershipManager - private readonly advertisedTransport$: Behavior<{ - multiSfu: boolean; - preferStickyEvents: boolean; - transport: LivekitTransport; - } | null> = this.scope.behavior( - this.transports$.pipe( - map((transports) => - transports?.local.state === "ready" && - transports.preferred.state === "ready" - ? { - multiSfu: transports.multiSfu, - preferStickyEvents: transports.preferStickyEvents, - // In non-multi-SFU mode we should always advertise the preferred - // SFU to minimize the number of membership updates - transport: transports.multiSfu - ? transports.local.value - : transports.preferred.value, - } - : null, - ), - distinctUntilChanged<{ - multiSfu: boolean; - preferStickyEvents: boolean; - transport: LivekitTransport; - } | null>(deepCompare), - ), - ); + // /** + // * The transport over which we should be actively publishing our media. + // * null when not joined. + // */ + // // DISCUSSION ownMembershipManager + // private readonly localTransport$: Behavior | null> = + // this.scope.behavior( + // this.transports$.pipe( + // map((transports) => transports?.local ?? null), + // distinctUntilChanged | null>(deepCompare), + // ), + // ); // // DISCUSSION move to ConnectionManager // public readonly livekitConnectionState$ = @@ -367,8 +361,6 @@ export class CallViewModel { // ), // ); - private readonly userId = this.matrixRoom.client.getUserId()!; - /** * Whether various media/event sources should pretend to be disconnected from * all network input, even if their connection still technically works. diff --git a/src/state/SessionBehaviors.ts b/src/state/SessionBehaviors.ts new file mode 100644 index 00000000..6c16ace4 --- /dev/null +++ b/src/state/SessionBehaviors.ts @@ -0,0 +1,72 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + type CallMembership, + isLivekitTransport, + type LivekitTransport, + type MatrixRTCSession, + MatrixRTCSessionEvent, +} from "matrix-js-sdk/lib/matrixrtc"; +import { fromEvent, map } from "rxjs"; + +import { type ObservableScope } from "./ObservableScope"; +import { type Behavior } from "./Behavior"; + +export const sessionBehaviors$ = ( + scope: ObservableScope, + matrixRTCSession: MatrixRTCSession, +): { + memberships$: Behavior; + membershipsWithTransport$: Behavior< + { membership: CallMembership; transport?: LivekitTransport }[] + >; + transports$: Behavior; +} => { + const memberships$ = scope.behavior( + fromEvent( + matrixRTCSession, + MatrixRTCSessionEvent.MembershipsChanged, + (_, memberships: CallMembership[]) => memberships, + ), + ); + /** + * Lists the transports used by ourselves, plus all other MatrixRTC session + * members. For completeness this also lists the preferred transport and + * whether we are in multi-SFU mode or sticky events mode (because + * advertisedTransport$ wants to read them at the same time, and bundling data + * together when it might change together is what you have to do in RxJS to + * avoid reading inconsistent state or observing too many changes.) + */ + const membershipsWithTransport$: Behavior< + { membership: CallMembership; transport?: LivekitTransport }[] + > = scope.behavior( + memberships$.pipe( + map((memberships) => { + return memberships.map((membership) => { + const oldestMembership = memberships[0] ?? membership; + const transport = membership.getTransport(oldestMembership); + return { + membership, + transport: isLivekitTransport(transport) ? transport : undefined, + }; + }); + }), + ), + ); + + const transports$ = scope.behavior( + membershipsWithTransport$.pipe( + map((mts) => mts.flatMap(({ transport: t }) => (t ? [t] : []))), + ), + ); + return { + memberships$, + membershipsWithTransport$, + transports$, + }; +}; diff --git a/src/state/localMember/LocalMembership.ts b/src/state/localMember/LocalMembership.ts new file mode 100644 index 00000000..7448c2ee --- /dev/null +++ b/src/state/localMember/LocalMembership.ts @@ -0,0 +1,328 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-IdFentifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type LocalTrack, type E2EEOptions } from "livekit-client"; +import { + type LivekitTransport, + type MatrixRTCSession, + MembershipManagerEvent, + Status, +} from "matrix-js-sdk/lib/matrixrtc"; +import { + ClientEvent, + type MatrixClient, + SyncState, + type Room as MatrixRoom, +} from "matrix-js-sdk"; +import { + BehaviorSubject, + combineLatest, + fromEvent, + map, + type Observable, + of, + startWith, + switchMap, + tap, +} from "rxjs"; +import { logger } from "matrix-js-sdk/lib/logger"; + +import { type Behavior } from "../Behavior"; +import { type ConnectionManager } from "../remoteMembers/ConnectionManager"; +import { type ObservableScope } from "../ObservableScope"; +import { Publisher } from "./Publisher"; +import { type MuteStates } from "../MuteStates"; +import { type ProcessorState } from "../../livekit/TrackProcessorContext"; +import { type MediaDevices } from "../MediaDevices"; +import { and$ } from "../../utils/observable"; +import { areLivekitTransportsEqual } from "../remoteMembers/matrixLivekitMerger"; +import { + enterRTCSession, + type EnterRTCSessionOptions, +} from "../../rtcSessionHelpers"; + +/* + * - get well known + * - get oldest membership + * - get transport to use + * - get openId + jwt token + * - wait for createTrack() call + * - create tracks + * - wait for join() call + * - Publisher.publishTracks() + * - send join state/sticky event + */ +interface Props { + scope: ObservableScope; + mediaDevices: MediaDevices; + muteStates: MuteStates; + connectionManager: ConnectionManager; + matrixRTCSession: MatrixRTCSession; + matrixRoom: MatrixRoom; + localTransport$: Behavior; + client: MatrixClient; + roomId: string; + e2eeLivekitOptions: E2EEOptions | undefined; + trackerProcessorState$: Behavior; +} +enum LivekitState { + UNINITIALIZED = "uninitialized", + CONNECTING = "connecting", + CONNECTED = "connected", + ERROR = "error", + DISCONNECTED = "disconnected", + DISCONNECTING = "disconnecting", +} +type LocalMemberLivekitState = + | { state: LivekitState.ERROR; error: string } + | { state: LivekitState.CONNECTED } + | { state: LivekitState.CONNECTING } + | { state: LivekitState.UNINITIALIZED } + | { state: LivekitState.DISCONNECTED } + | { state: LivekitState.DISCONNECTING }; + +enum MatrixState { + CONNECTED = "connected", + DISCONNECTED = "disconnected", + CONNECTING = "connecting", +} +type LocalMemberMatrixState = + | { state: MatrixState.CONNECTED } + | { state: MatrixState.CONNECTING } + | { state: MatrixState.DISCONNECTED }; + +interface LocalMemberState { + livekit$: BehaviorSubject; + matrix$: BehaviorSubject; +} +/** + * This class is responsible for managing the own membership in a room. + * We want + * - a publisher + * - + * @param param0 + * @returns + * - publisher: The handle to create tracks and publish them to the room. + * - connected$: the current connection state. Including matrix server and livekit server connection. (only the livekit server relevant for our own participation) + * - transport$: the transport object the ownMembership$ ended up using. + * + */ +export const localMembership$ = ({ + scope, + muteStates, + mediaDevices, + connectionManager, + matrixRTCSession, + localTransport$, + matrixRoom, + e2eeLivekitOptions, + trackerProcessorState$, +}: Props): { + // publisher: Publisher + requestConnect: (options: EnterRTCSessionOptions) => LocalMemberState; + startTracks: () => Behavior; + requestDisconnect: () => Observable | null; + state: LocalMemberState; // TODO this is probably superseeded by joinState$ + homeserverConnected$: Behavior; + connected$: Behavior; +} => { + const state = { + livekit$: new BehaviorSubject({ + state: LivekitState.UNINITIALIZED, + }), + matrix$: new BehaviorSubject({ + state: MatrixState.DISCONNECTED, + }), + }; + + // This should be used in a combineLatest with publisher$ to connect. + // to make it possible to call startTracks before the preferredTransport$ has resolved. + const shouldStartTracks$ = new BehaviorSubject(false); + + // This should be used in a combineLatest with publisher$ to connect. + const tracks$ = new BehaviorSubject([]); + + const connection$ = scope.behavior( + combineLatest([connectionManager.connections$, localTransport$]).pipe( + map(([connections, transport]) => + connections.find((connection) => + areLivekitTransportsEqual(connection.transport, transport), + ), + ), + ), + ); + /** + * Whether we are connected to the MatrixRTC session. + */ + const homeserverConnected$ = scope.behavior( + // To consider ourselves connected to MatrixRTC, we check the following: + and$( + // The client is connected to the sync loop + ( + fromEvent(matrixRoom.client, ClientEvent.Sync) as Observable< + [SyncState] + > + ).pipe( + startWith([matrixRoom.client.getSyncState()]), + map(([state]) => state === SyncState.Syncing), + ), + // Room state observed by session says we're connected + fromEvent(matrixRTCSession, MembershipManagerEvent.StatusChanged).pipe( + startWith(null), + map(() => matrixRTCSession.membershipStatus === Status.Connected), + ), + // Also watch out for warnings that we've likely hit a timeout and our + // delayed leave event is being sent (this condition is here because it + // provides an earlier warning than the sync loop timeout, and we wouldn't + // see the actual leave event until we reconnect to the sync loop) + fromEvent(matrixRTCSession, MembershipManagerEvent.ProbablyLeft).pipe( + startWith(null), + map(() => matrixRTCSession.probablyLeft !== true), + ), + ), + ); + + // /** + // * Whether we are "fully" connected to the call. Accounts for both the + // * connection to the MatrixRTC session and the LiveKit publish connection. + // */ + // // TODO use this in combination with the MemberState. + const connected$ = scope.behavior( + and$( + homeserverConnected$, + connection$.pipe( + switchMap((c) => + c + ? c.state$.pipe(map((state) => state.state === "ConnectedToLkRoom")) + : of(false), + ), + ), + ), + ); + + const publisher$ = scope.behavior( + connection$.pipe( + map((connection) => + connection + ? new Publisher( + scope, + connection, + mediaDevices, + muteStates, + e2eeLivekitOptions, + trackerProcessorState$, + ) + : null, + ), + ), + ); + + combineLatest( + [publisher$, shouldStartTracks$], + (publisher, shouldStartTracks) => { + if (publisher && shouldStartTracks) { + publisher + .createAndSetupTracks() + .then((tracks) => { + tracks$.next(tracks); + }) + .catch((error) => { + logger.error("Error creating tracks:", error); + }); + } + }, + ); + + // MATRIX RELATED + + // /** + // * Whether we should tell the user that we're reconnecting to the call. + // */ + // // DISCUSSION own membership manager + // const reconnecting$ = scope.behavior( + // connected$.pipe( + // // We are reconnecting if we previously had some successful initial + // // connection but are now disconnected + // scan( + // ({ connectedPreviously }, connectedNow) => ({ + // connectedPreviously: connectedPreviously || connectedNow, + // reconnecting: connectedPreviously && !connectedNow, + // }), + // { connectedPreviously: false, reconnecting: false }, + // ), + // map(({ reconnecting }) => reconnecting), + // ), + // ); + + const startTracks = (): Behavior => { + shouldStartTracks$.next(true); + return tracks$; + }; + + // const joinState$ = new BehaviorSubject({ + // state: LivekitState.UNINITIALIZED, + // }); + + const requestConnect = ( + options: EnterRTCSessionOptions, + ): LocalMemberState => { + if (state.livekit$.value === null) { + startTracks(); + state.livekit$.next({ state: LivekitState.CONNECTING }); + combineLatest([publisher$, tracks$], (publisher, tracks) => { + publisher + ?.startPublishing() + .then(() => { + state.livekit$.next({ state: LivekitState.CONNECTED }); + }) + .catch((error) => { + state.livekit$.next({ state: LivekitState.ERROR, error }); + }); + }); + } + if (state.matrix$.value.state !== MatrixState.DISCONNECTED) { + state.matrix$.next({ state: MatrixState.CONNECTING }); + localTransport$.pipe( + tap((transport) => { + enterRTCSession(matrixRTCSession, transport, options).catch( + (error) => { + logger.error(error); + }, + ); + }), + ); + } + return state; + }; + + const requestDisconnect = (): Behavior | null => { + if (state.livekit$.value.state !== LivekitState.CONNECTED) return null; + state.livekit$.next({ state: LivekitState.DISCONNECTING }); + combineLatest([publisher$, tracks$], (publisher, tracks) => { + publisher + ?.stopPublishing() + .then(() => { + tracks.forEach((track) => track.stop()); + state.livekit$.next({ state: LivekitState.DISCONNECTED }); + }) + .catch((error) => { + state.livekit$.next({ state: LivekitState.ERROR, error }); + }); + }); + + return state.livekit$; + }; + + return { + startTracks, + requestConnect, + requestDisconnect, + state, + homeserverConnected$, + connected$, + }; +}; diff --git a/src/state/localMember/LocalTransport.ts b/src/state/localMember/LocalTransport.ts new file mode 100644 index 00000000..7a5202a9 --- /dev/null +++ b/src/state/localMember/LocalTransport.ts @@ -0,0 +1,166 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-IdFentifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + type CallMembership, + isLivekitTransport, + type LivekitTransportConfig, + type LivekitTransport, + isLivekitTransportConfig, +} from "matrix-js-sdk/lib/matrixrtc"; +import { type MatrixClient } from "matrix-js-sdk"; +import { combineLatest, distinctUntilChanged, first, from, map } from "rxjs"; +import { logger } from "matrix-js-sdk/lib/logger"; +import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery"; +import { deepCompare } from "matrix-js-sdk/lib/utils"; + +import { type Behavior } from "../Behavior.ts"; +import { type ObservableScope } from "../ObservableScope.ts"; +import { Config } from "../../config/Config.ts"; +import { MatrixRTCTransportMissingError } from "../../utils/errors.ts"; +import { getSFUConfigWithOpenID } from "../../livekit/openIDSFU.ts"; + +/* + * - get well known + * - get oldest membership + * - get transport to use + * - get openId + jwt token + * - wait for createTrack() call + * - create tracks + * - wait for join() call + * - Publisher.publishTracks() + * - send join state/sticky event + */ +interface Props { + scope: ObservableScope; + memberships$: Behavior; + client: MatrixClient; + roomId: string; + useOldestMember$: Behavior; +} + +/** + * This class is responsible for managing the local transport. + * "Which transport is the local member going to use" + * + * @prop useOldestMember Whether to use the same transport as the oldest member. + * This will only update once the first oldest member appears. Will not recompute if the oldest member leaves. + */ +export const localTransport$ = ({ + scope, + memberships$, + client, + roomId, + useOldestMember$, +}: Props): Behavior => { + /** + * The transport over which we should be actively publishing our media. + * undefined when not joined. + */ + const oldestMemberTransport$ = scope.behavior( + memberships$.pipe( + map((memberships) => memberships[0].getTransport(memberships[0])), + first((t) => t != undefined && isLivekitTransport(t)), + ), + undefined, + ); + + /** + * The transport that we would personally prefer to publish on (if not for the + * transport preferences of others, perhaps). + */ + const preferredTransport$: Behavior = + scope.behavior(from(makeTransport(client, roomId)), undefined); + + /** + * The transport we should advertise in our MatrixRTC membership (plus whether + * it is a multi-SFU transport and whether we should use sticky events). + */ + const advertisedTransport$ = scope.behavior( + combineLatest( + [useOldestMember$, preferredTransport$, oldestMemberTransport$], + (useOldestMember, preferredTransport, oldestMemberTransport) => + useOldestMember ? oldestMemberTransport : preferredTransport, + ).pipe(distinctUntilChanged(deepCompare)), + undefined, + ); + return advertisedTransport$; +}; + +const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci"; + +async function makeTransportInternal( + client: MatrixClient, + roomId: string, +): Promise { + logger.log("Searching for a preferred transport"); + //TODO refactor this to use the jwt service returned alias. + const livekitAlias = roomId; + + // TODO-MULTI-SFU: Either remove this dev tool or make it more official + const urlFromStorage = + localStorage.getItem("robin-matrixrtc-auth") ?? + localStorage.getItem("timo-focus-url"); + if (urlFromStorage !== null) { + const transportFromStorage: LivekitTransport = { + type: "livekit", + livekit_service_url: urlFromStorage, + livekit_alias: livekitAlias, + }; + logger.log( + "Using LiveKit transport from local storage: ", + transportFromStorage, + ); + return transportFromStorage; + } + + // Prioritize the .well-known/matrix/client, if available, over the configured SFU + const domain = client.getDomain(); + if (domain) { + // we use AutoDiscovery instead of relying on the MatrixClient having already + // been fully configured and started + const wellKnownFoci = (await AutoDiscovery.getRawClientConfig(domain))?.[ + FOCI_WK_KEY + ]; + if (Array.isArray(wellKnownFoci)) { + const transport: LivekitTransportConfig | undefined = wellKnownFoci.find( + (f) => f && isLivekitTransportConfig(f), + ); + if (transport !== undefined) { + logger.log("Using LiveKit transport from .well-known: ", transport); + return { ...transport, livekit_alias: livekitAlias }; + } + } + } + + const urlFromConf = Config.get().livekit?.livekit_service_url; + if (urlFromConf) { + const transportFromConf: LivekitTransport = { + type: "livekit", + livekit_service_url: urlFromConf, + livekit_alias: livekitAlias, + }; + logger.log("Using LiveKit transport from config: ", transportFromConf); + return transportFromConf; + } + + throw new MatrixRTCTransportMissingError(domain ?? ""); +} + +async function makeTransport( + client: MatrixClient, + roomId: string, +): Promise { + const transport = await makeTransportInternal(client, roomId); + // this will call the jwt/sfu/get endpoint to pre create the livekit room. + await getSFUConfigWithOpenID( + client, + transport.livekit_service_url, + transport.livekit_alias, + ); + return transport; +} diff --git a/src/state/ownMember/Publisher.ts b/src/state/localMember/Publisher.ts similarity index 99% rename from src/state/ownMember/Publisher.ts rename to src/state/localMember/Publisher.ts index c37445b0..6a1079fd 100644 --- a/src/state/ownMember/Publisher.ts +++ b/src/state/localMember/Publisher.ts @@ -89,7 +89,7 @@ export class Publisher { * @throws {InsufficientCapacityError} if the LiveKit server indicates that it has insufficient capacity to accept the connection. * @throws {SFURoomCreationRestrictedError} if the LiveKit server indicates that the room does not exist and cannot be created. */ - public async createAndSetupTracks(): Promise { + public async createAndSetupTracks(): Promise { const lkRoom = this.connection.livekitRoom; // Observe mute state changes and update LiveKit microphone/camera states accordingly this.observeMuteStates(this.scope); @@ -125,6 +125,7 @@ export class Publisher { video, }); } + return this.tracks; } public async startPublishing(): Promise { diff --git a/src/state/ownMember/OwnMembership.ts b/src/state/ownMember/OwnMembership.ts deleted file mode 100644 index c096c2da..00000000 --- a/src/state/ownMember/OwnMembership.ts +++ /dev/null @@ -1,287 +0,0 @@ -/* -Copyright 2025 New Vector Ltd. - -SPDX-License-IdFentifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { type E2EEOptions, type Track } from "livekit-client"; -import { - type LivekitTransport, - type MatrixRTCSession, - MembershipManagerEvent, - Status, -} from "matrix-js-sdk/lib/matrixrtc"; -import { - ClientEvent, - type MatrixClient, - SyncState, - type Room as MatrixRoom, -} from "matrix-js-sdk"; -import { - BehaviorSubject, - combineLatest, - distinctUntilChanged, - from, - fromEvent, - map, - type Observable, - of, - scan, - startWith, - switchMap, -} from "rxjs"; -import { deepCompare } from "matrix-js-sdk/lib/utils"; - -import { multiSfu } from "../../settings/settings"; -import { type Behavior } from "../Behavior"; -import { type ConnectionManager } from "../remoteMembers/ConnectionManager"; -import { makeTransport } from "../../rtcSessionHelpers"; -import { type ObservableScope } from "../ObservableScope"; -import { Publisher } from "./Publisher"; -import { type MuteStates } from "../MuteStates"; -import { type ProcessorState } from "../../livekit/TrackProcessorContext"; -import { type MediaDevices } from "../../state/MediaDevices"; -import { and$ } from "../../utils/observable"; -import { areLivekitTransportsEqual } from "../remoteMembers/matrixLivekitMerger"; -import { type ElementCallError } from "../../utils/errors.ts"; - -/* - * - get well known - * - get oldest membership - * - get transport to use - * - get openId + jwt token - * - wait for createTrack() call - * - create tracks - * - wait for join() call - * - Publisher.publishTracks() - * - send join state/sticky event - */ -interface Props { - scope: ObservableScope; - mediaDevices: MediaDevices; - muteStates: MuteStates; - connectionManager: ConnectionManager; - matrixRTCSession: MatrixRTCSession; - matrixRoom: MatrixRoom; - client: MatrixClient; - preferStickyEvents: boolean; - roomId: string; - e2eeLivekitOptions: E2EEOptions | undefined; - trackerProcessorState$: Behavior; -} - -export type JoinedState = - | { state: "Initialized" } - | { state: "Error"; error: ElementCallError }; - -/** - * This class is responsible for managing the own membership in a room. - * We want - * - a publisher - * - - * @param param0 - * @returns - * - publisher: The handle to create tracks and publish them to the room. - * - connected$: the current connection state. Including matrix server and livekit server connection. (only the livekit server relevant for our own participation) - * - transport$: the transport object the ownMembership$ ended up using. - * - */ -export const ownMembership$ = ({ - scope, - muteStates, - mediaDevices, - preferStickyEvents, - connectionManager, - matrixRTCSession, - matrixRoom, - e2eeLivekitOptions, - client, - roomId, - trackerProcessorState$, -}: Props): { - // publisher: Publisher - requestJoin$(): Observable; - startTracks(): Track[]; -} => { - // This should be used in a combineLatest with publisher$ to connect. - const shouldStartTracks$ = new BehaviorSubject(false); - - // to make it possible to call startTracks before the preferredTransport$ has resolved. - const startTracks = () => { - shouldStartTracks$.next(true); - }; - - const userId = client.getUserId()!; - const deviceId = client.getDeviceId()!; - const multiSfu$ = multiSfu.value$; - /** - * The transport that we would personally prefer to publish on (if not for the - * transport preferences of others, perhaps). - */ - const preferredTransport$: Behavior = scope.behavior( - from(makeTransport(client, roomId)), - ); - - connectionManager.registerTransports( - scope.behavior(preferredTransport$.pipe(map((t) => (t ? [t] : [])))), - ); - - const connection$ = scope.behavior( - combineLatest([connectionManager.connections$, preferredTransport$]).pipe( - map(([connections, transport]) => - connections.find((connection) => - areLivekitTransportsEqual(connection.transport, transport), - ), - ), - ), - ); - /** - * Whether we are connected to the MatrixRTC session. - */ - // DISCUSSION own membership manager - const matrixConnected$ = scope.behavior( - // To consider ourselves connected to MatrixRTC, we check the following: - and$( - // The client is connected to the sync loop - ( - fromEvent(matrixRoom.client, ClientEvent.Sync) as Observable< - [SyncState] - > - ).pipe( - startWith([matrixRoom.client.getSyncState()]), - map(([state]) => state === SyncState.Syncing), - ), - // Room state observed by session says we're connected - fromEvent(matrixRTCSession, MembershipManagerEvent.StatusChanged).pipe( - startWith(null), - map(() => matrixRTCSession.membershipStatus === Status.Connected), - ), - // Also watch out for warnings that we've likely hit a timeout and our - // delayed leave event is being sent (this condition is here because it - // provides an earlier warning than the sync loop timeout, and we wouldn't - // see the actual leave event until we reconnect to the sync loop) - fromEvent(matrixRTCSession, MembershipManagerEvent.ProbablyLeft).pipe( - startWith(null), - map(() => matrixRTCSession.probablyLeft !== true), - ), - ), - ); - - /** - * Whether we are "fully" connected to the call. Accounts for both the - * connection to the MatrixRTC session and the LiveKit publish connection. - */ - const connected$ = scope.behavior( - and$( - matrixConnected$, - connection$.pipe( - switchMap((c) => - c - ? c.state$.pipe(map((state) => state.state === "ConnectedToLkRoom")) - : of(false), - ), - ), - ), - ); - - const publisher = scope.behavior( - connection$.pipe( - map((c) => - c - ? new Publisher( - scope, - c, - mediaDevices, - muteStates, - e2eeLivekitOptions, - trackerProcessorState$, - ) - : null, - ), - ), - ); - - // HOW IT WAS PREVIEOUSLY CREATED - // new PublishConnection( - // { - // transport, - // client: this.matrixRoom.client, - // scope, - // remoteTransports$: this.remoteTransports$, - // livekitRoomFactory: this.options.livekitRoomFactory, - // }, - // this.mediaDevices, - // this.muteStates, - // this.e2eeLivekitOptions(), - // this.scope.behavior(this.trackProcessorState$), - // ), - - /** - * The transport over which we should be actively publishing our media. - * null when not joined. - */ - // DISCUSSION ownMembershipManager - const localTransport$: Behavior = - this.scope.behavior( - this.transports$.pipe( - map((transports) => transports?.local ?? null), - distinctUntilChanged(deepCompare), - ), - ); - - /** - * The transport we should advertise in our MatrixRTC membership (plus whether - * it is a multi-SFU transport and whether we should use sticky events). - */ - // DISCUSSION ownMembershipManager - const advertisedTransport$: Behavior<{ - multiSfu: boolean; - preferStickyEvents: boolean; - transport: LivekitTransport; - } | null> = this.scope.behavior( - this.transports$.pipe( - map((transports) => - transports?.local.state === "ready" && - transports.preferred.state === "ready" - ? { - multiSfu: transports.multiSfu, - preferStickyEvents: transports.preferStickyEvents, - // In non-multi-SFU mode we should always advertise the preferred - // SFU to minimize the number of membership updates - transport: transports.multiSfu - ? transports.local.value - : transports.preferred.value, - } - : null, - ), - distinctUntilChanged<{ - multiSfu: boolean; - preferStickyEvents: boolean; - transport: LivekitTransport; - } | null>(deepCompare), - ), - ); - - // MATRIX RELATED - - /** - * Whether we should tell the user that we're reconnecting to the call. - */ - // DISCUSSION own membership manager - const reconnecting$ = scope.behavior( - connected$.pipe( - // We are reconnecting if we previously had some successful initial - // connection but are now disconnected - scan( - ({ connectedPreviously }, connectedNow) => ({ - connectedPreviously: connectedPreviously || connectedNow, - reconnecting: connectedPreviously && !connectedNow, - }), - { connectedPreviously: false, reconnecting: false }, - ), - map(({ reconnecting }) => reconnecting), - ), - ); - return { connected$, transport$: preferredTransport$, publisher }; -}; diff --git a/src/state/remoteMembers/ConnectionManager.ts b/src/state/remoteMembers/ConnectionManager.ts index 239cf3c9..e333173e 100644 --- a/src/state/remoteMembers/ConnectionManager.ts +++ b/src/state/remoteMembers/ConnectionManager.ts @@ -87,29 +87,31 @@ export class ConnectionManagerData { export class ConnectionManager { private readonly logger: Logger; + private running$ = new BehaviorSubject(true); + /** + * Crete a `ConnectionManager` + * @param scope the observable scope used by this object. + * @param connectionFactory used to create new connections. + * @param _transportsSubscriptions$ A list of Behaviors each containing a LIST of LivekitTransport. + * Each of these behaviors can be interpreted as subscribed list of transports. + * + * Using `registerTransports` independent external modules can control what connections + * are created by the ConnectionManager. + * + * The connection manager will remove all duplicate transports in each subscibed list. + * + * See `unregisterAllTransports` and `unregisterTransport` for details on how to unsubscribe. + */ public constructor( private readonly scope: ObservableScope, private readonly connectionFactory: ConnectionFactory, + private readonly inputTransports$: Behavior, logger: Logger, ) { this.logger = logger.getChild("ConnectionManager"); + scope.onEnd(() => this.running$.next(false)); } - /** - * A list of Behaviors each containing a LIST of LivekitTransport. - * Each of these behaviors can be interpreted as subscribed list of transports. - * - * Using `registerTransports` independent external modules can control what connections - * are created by the ConnectionManager. - * - * The connection manager will remove all duplicate transports in each subscibed list. - * - * See `unregisterAllTransports` and `unregisterTransport` for details on how to unsubscribe. - */ - private readonly transportsSubscriptions$ = new BehaviorSubject< - Behavior[] - >([]); - /** * All transports currently managed by the ConnectionManager. * @@ -119,15 +121,10 @@ export class ConnectionManager { * externally this is modified via `registerTransports()`. */ private readonly transports$ = this.scope.behavior( - this.transportsSubscriptions$.pipe( - switchMap((subscriptions) => - combineLatest(subscriptions).pipe( - map((transportsNested) => transportsNested.flat()), - map(removeDuplicateTransports), - ), - ), + combineLatest([this.running$, this.inputTransports$]).pipe( + map(([running, transports]) => (running ? transports : [])), + map(removeDuplicateTransports), ), - [], ); /** @@ -163,60 +160,6 @@ export class ConnectionManager { ), ); - /** - * Add an a Behavior containing a list of transports to this ConnectionManager. - * - * The intended usage is: - * - create a ConnectionManager - * - register one `transports$` behavior using registerTransports - * - add new connections to the `ConnectionManager` by updating the `transports$` behavior - * - remove a single connection by removing the transport. - * - remove this subscription by calling `unregisterTransports` and passing - * the same `transports$` behavior reference. - * @param transports$ The Behavior containing a list of transports to subscribe to. - */ - public registerTransports(transports$: Behavior): void { - if (!this.transportsSubscriptions$.value.some((t$) => t$ === transports$)) { - this.transportsSubscriptions$.next( - this.transportsSubscriptions$.value.concat(transports$), - ); - } - // // After updating the subscriptions our connection list is also updated. - // return transports$.value - // .map((transport) => { - // const isConnectionForTransport = (connection: Connection): boolean => - // areLivekitTransportsEqual(connection.transport, transport); - // return this.connections$.value.find(isConnectionForTransport); - // }) - // .filter((c) => c !== undefined); - } - - /** - * Unsubscribe from the given transports. - * @param transports$ The behavior to unsubscribe from - * @returns - */ - public unregisterTransports( - transports$: Behavior, - ): boolean { - const subscriptions = this.transportsSubscriptions$.value; - const subscriptionsUnregistered = subscriptions.filter( - (t$) => t$ !== transports$, - ); - const canUnregister = - subscriptions.length !== subscriptionsUnregistered.length; - if (canUnregister) - this.transportsSubscriptions$.next(subscriptionsUnregistered); - return canUnregister; - } - - /** - * Unsubscribe from all transports. - */ - public unregisterAllTransports(): void { - this.transportsSubscriptions$.next([]); - } - public connectionManagerData$: Behavior = this.scope.behavior( this.connections$.pipe( diff --git a/src/state/remoteMembers/displayname.ts b/src/state/remoteMembers/displayname.ts index 825ad5a1..a5d1ae3d 100644 --- a/src/state/remoteMembers/displayname.ts +++ b/src/state/remoteMembers/displayname.ts @@ -6,18 +6,12 @@ Please see LICENSE in the repository root for full details. */ import { type RoomMember, RoomStateEvent } from "matrix-js-sdk"; -import { - combineLatest, - fromEvent, - map, - type Observable, - startWith, -} from "rxjs"; +import { combineLatest, fromEvent, type Observable, startWith } from "rxjs"; import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; import { logger } from "matrix-js-sdk/lib/logger"; import { type Room as MatrixRoom } from "matrix-js-sdk/lib/matrix"; // eslint-disable-next-line rxjs/no-internal -import { type HasEventTargetAddRemove } from "rxjs/internal/observable/fromEvent"; +import { type NodeStyleEventEmitter } from "rxjs/internal/observable/fromEvent"; import { type ObservableScope } from "../ObservableScope"; import { @@ -36,20 +30,21 @@ import { type Behavior } from "../Behavior"; // don't do this work more times than we need to. This is achieved by converting to a behavior: export const memberDisplaynames$ = ( scope: ObservableScope, - matrixRoom: Pick & HasEventTargetAddRemove, + matrixRoom: Pick & NodeStyleEventEmitter, memberships$: Observable, userId: string, deviceId: string, ): Behavior> => scope.behavior( - combineLatest([ - // Handle call membership changes - memberships$, - // Additionally handle display name changes (implicitly reacting to them) - fromEvent(matrixRoom, RoomStateEvent.Members).pipe(startWith(null)), - // TODO: do we need: pauseWhen(this.pretendToBeDisconnected$), - ]).pipe( - map((memberships, _displaynames) => { + combineLatest( + [ + // Handle call membership changes + memberships$, + // Additionally handle display name changes (implicitly reacting to them) + fromEvent(matrixRoom, RoomStateEvent.Members).pipe(startWith(null)), + // TODO: do we need: pauseWhen(this.pretendToBeDisconnected$), + ], + (memberships, _displaynames) => { const displaynameMap = new Map([ [ `${userId}:${deviceId}`, @@ -76,7 +71,7 @@ export const memberDisplaynames$ = ( ); } return displaynameMap; - }), + }, ), new Map(), ); diff --git a/src/state/remoteMembers/matrixLivekitMerger.ts b/src/state/remoteMembers/matrixLivekitMerger.ts index 1487636c..39acc65b 100644 --- a/src/state/remoteMembers/matrixLivekitMerger.ts +++ b/src/state/remoteMembers/matrixLivekitMerger.ts @@ -7,13 +7,12 @@ Please see LICENSE in the repository root for full details. import { type Participant as LivekitParticipant } from "livekit-client"; import { - isLivekitTransport, type LivekitTransport, type CallMembership, } from "matrix-js-sdk/lib/matrixrtc"; import { combineLatest, map, startWith, type Observable } from "rxjs"; // eslint-disable-next-line rxjs/no-internal -import { type NodeStyleEventEmitter } from "rxjs/src/internal/observable/fromEvent.ts"; +import { type NodeStyleEventEmitter } from "rxjs/internal/observable/fromEvent"; import type { Room as MatrixRoom, RoomMember } from "matrix-js-sdk"; // import type { Logger } from "matrix-js-sdk/lib/logger"; @@ -65,7 +64,9 @@ export class MatrixLivekitMerger { public constructor( private scope: ObservableScope, - private memberships$: Observable, + private membershipsWithTransport$: Behavior< + { membership: CallMembership; transport?: LivekitTransport }[] + >, private connectionManager: ConnectionManager, // TODO this is too much information for that class, // apparently needed to get a room member to later get the Avatar @@ -90,14 +91,13 @@ export class MatrixLivekitMerger { const displaynameMap$ = memberDisplaynames$( this.scope, this.matrixRoom, - this.memberships$, + this.membershipsWithTransport$.pipe( + map((v) => v.map((v) => v.membership)), + ), this.userId, this.deviceId, ); - const membershipsWithTransport$ = - this.mapMembershipsToMembershipWithTransport$(); - - this.startFeedingConnectionManager(membershipsWithTransport$); + const membershipsWithTransport$ = this.membershipsWithTransport$; return combineLatest([ membershipsWithTransport$, @@ -138,48 +138,6 @@ export class MatrixLivekitMerger { }), ); } - - private startFeedingConnectionManager( - membershipsWithTransport$: Behavior< - { membership: CallMembership; transport?: LivekitTransport }[] - >, - ): void { - const transports$ = this.scope.behavior( - membershipsWithTransport$.pipe( - map((mts) => mts.flatMap(({ transport: t }) => (t ? [t] : []))), - ), - ); - // duplicated transports will be elimiated by the connection manager - this.connectionManager.registerTransports(transports$); - } - - /** - * Lists the transports used by ourselves, plus all other MatrixRTC session - * members. For completeness this also lists the preferred transport and - * whether we are in multi-SFU mode or sticky events mode (because - * advertisedTransport$ wants to read them at the same time, and bundling data - * together when it might change together is what you have to do in RxJS to - * avoid reading inconsistent state or observing too many changes.) - */ - private mapMembershipsToMembershipWithTransport$(): Behavior< - { membership: CallMembership; transport?: LivekitTransport }[] - > { - return this.scope.behavior( - this.memberships$.pipe( - map((memberships) => { - return memberships.map((membership) => { - const oldestMembership = memberships[0] ?? membership; - const transport = membership.getTransport(oldestMembership); - return { - membership, - transport: isLivekitTransport(transport) ? transport : undefined, - }; - }); - }), - ), - [], - ); - } } // TODO add back in the callviewmodel pauseWhen(this.pretendToBeDisconnected$) From 107ef16d945de97d7df4149e2b0195df22aa8857 Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 5 Nov 2025 12:56:58 +0100 Subject: [PATCH 20/75] Add MatrixRTCMode and refactor local membership Remove preferStickyEvents and multiSfu in favor of a MatrixRTCMode enum/setting (Legacy, Compatibil, Matrix_2_0). Move session join/leave, track pause/resume, and config error handling out of CallViewModel into the localMembership module. Update developer settings UI, i18n strings, and related RTC session helpers and wiring accordingly. --- locales/en/app.json | 19 +- src/rtcSessionHelpers.ts | 14 +- src/settings/DeveloperSettingsTab.tsx | 71 ++--- src/settings/settings.ts | 18 +- src/state/CallViewModel.ts | 262 ++++--------------- src/state/localMember/LocalMembership.ts | 218 ++++++++++----- src/state/remoteMembers/ConnectionManager.ts | 4 +- 7 files changed, 277 insertions(+), 329 deletions(-) diff --git a/locales/en/app.json b/locales/en/app.json index 11267439..104af750 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -72,12 +72,21 @@ "livekit_server_info": "LiveKit Server Info", "livekit_sfu": "LiveKit SFU: {{url}}", "matrix_id": "Matrix ID: {{id}}", - "multi_sfu": "Multi-SFU media transport", - "mute_all_audio": "Mute all audio (participants, reactions, join sounds)", - "prefer_sticky_events": { - "description": "Improves reliability of calls (requires homeserver support)", - "label": "Prefer sticky events" + "matrixRTCMode": { + "Comptibility": { + "label": "Compatibility: state events & multi SFU" + "description": "Compatible with homeservers that do not support sticky events (but all other EC clients are v0.17.0 or later)", + }, + "Legacy": { + "label": "Legacy: state events & oldest membership SFU" + "description": "Compatible with old versions of EC that do not support multi SFU", + }, + "Matrix_2_0": { + "label": "Matrix 2.0: sticky events & multi SFU" + "description": "Compatible only with homservers supporting sticky events and all EC clients v0.17.0 or later", + } }, + "mute_all_audio": "Mute all audio (participants, reactions, join sounds)", "show_connection_stats": "Show connection statistics", "url_params": "URL parameters" }, diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index 74023c22..a53418f7 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -21,6 +21,7 @@ import { ElementWidgetActions, widget } from "./widget"; import { MatrixRTCTransportMissingError } from "./utils/errors"; import { getUrlParams } from "./UrlParams"; import { getSFUConfigWithOpenID } from "./livekit/openIDSFU.ts"; +import { MatrixRTCMode } from "./settings/settings.ts"; const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci"; @@ -98,9 +99,7 @@ export async function makeTransport( export interface EnterRTCSessionOptions { encryptMedia: boolean; - /** EXPERIMENTAL: If true, will use the multi-sfu codepath where each member connects to its SFU instead of everyone connecting to an elected on. */ - useMultiSfu: boolean; - preferStickyEvents: boolean; + matrixRTCMode: MatrixRTCMode; } /** @@ -112,7 +111,7 @@ export interface EnterRTCSessionOptions { export async function enterRTCSession( rtcSession: MatrixRTCSession, transport: LivekitTransport, - { encryptMedia, useMultiSfu, preferStickyEvents }: EnterRTCSessionOptions, + { encryptMedia, matrixRTCMode }: EnterRTCSessionOptions, ): Promise { PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date()); PosthogAnalytics.instance.eventCallStarted.track(rtcSession.room.roomId); @@ -125,10 +124,11 @@ export async function enterRTCSession( const useDeviceSessionMemberEvents = features?.feature_use_device_session_member_events; const { sendNotificationType: notificationType, callIntent } = getUrlParams(); + const multiSFU = matrixRTCMode !== MatrixRTCMode.Legacy; // Multi-sfu does not need a preferred foci list. just the focus that is actually used. rtcSession.joinRoomSession( - useMultiSfu ? [] : [transport], - useMultiSfu ? transport : undefined, + multiSFU ? [] : [transport], + multiSFU ? transport : undefined, { notificationType, callIntent, @@ -147,7 +147,7 @@ export async function enterRTCSession( membershipEventExpiryMs: matrixRtcSessionConfig?.membership_event_expiry_ms, useExperimentalToDeviceTransport: true, - unstableSendStickyEvents: preferStickyEvents, + unstableSendStickyEvents: matrixRTCMode === MatrixRTCMode.Matrix_2_0, }, ); if (widget) { diff --git a/src/settings/DeveloperSettingsTab.tsx b/src/settings/DeveloperSettingsTab.tsx index 08c22557..e29e9c15 100644 --- a/src/settings/DeveloperSettingsTab.tsx +++ b/src/settings/DeveloperSettingsTab.tsx @@ -29,7 +29,8 @@ import { multiSfu as multiSfuSetting, muteAllAudio as muteAllAudioSetting, alwaysShowIphoneEarpiece as alwaysShowIphoneEarpieceSetting, - preferStickyEvents as preferStickyEventsSetting, + matrixRTCMode as matrixRTCModeSetting, + MatrixRTCMode, } from "./settings"; import type { Room as LivekitRoom } from "livekit-client"; import styles from "./DeveloperSettingsTab.module.css"; @@ -59,9 +60,7 @@ export const DeveloperSettingsTab: FC = ({ client, livekitRooms }) => { }); }, [client]); - const [preferStickyEvents, setPreferStickyEvents] = useSetting( - preferStickyEventsSetting, - ); + const [matrixRTCMode, setMatrixRTCMode] = useSetting(matrixRTCModeSetting); const [showConnectionStats, setShowConnectionStats] = useSetting( showConnectionStatsSetting, @@ -71,8 +70,6 @@ export const DeveloperSettingsTab: FC = ({ client, livekitRooms }) => { alwaysShowIphoneEarpieceSetting, ); - const [multiSfu, setMultiSfu] = useSetting(multiSfuSetting); - const [muteAllAudio, setMuteAllAudio] = useSetting(muteAllAudioSetting); const urlParams = useUrlParams(); @@ -148,17 +145,47 @@ export const DeveloperSettingsTab: FC = ({ client, livekitRooms }) => { ): void => { - setPreferStickyEvents(event.target.checked); + (event: ChangeEvent) => { + if (event.target.checked) setMatrixRTCMode(MatrixRTCMode.Legacy); }, - [setPreferStickyEvents], + [setMatrixRTCMode], + )} + /> + ) => { + if (event.target.checked) + setMatrixRTCMode(MatrixRTCMode.Compatibil); + }, + [setMatrixRTCMode], + )} + /> + ) => { + if (event.target.checked) + setMatrixRTCMode(MatrixRTCMode.Matrix_2_0); + }, + [setMatrixRTCMode], )} /> @@ -176,22 +203,6 @@ export const DeveloperSettingsTab: FC = ({ client, livekitRooms }) => { )} /> - - ): void => { - setMultiSfu(event.target.checked); - }, - [setMultiSfu], - )} - /> - ( false, ); -export const preferStickyEvents = new Setting( - "prefer-sticky-events", - false, -); - export const audioInput = new Setting( "audio-input", undefined, @@ -120,8 +115,6 @@ export const soundEffectVolume = new Setting( 0.5, ); -export const multiSfu = new Setting("multi-sfu", false); - export const muteAllAudio = new Setting("mute-all-audio", false); export const alwaysShowSelf = new Setting("always-show-self", true); @@ -130,3 +123,14 @@ export const alwaysShowIphoneEarpiece = new Setting( "always-show-iphone-earpiece", false, ); + +export enum MatrixRTCMode { + Legacy = "legacy", + Compatibil = "compatibil", + Matrix_2_0 = "matrix_2_0", +} + +export const matrixRTCMode = new Setting( + "matrix-rtc-mode", + MatrixRTCMode.Legacy, +); diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 436255eb..7396a515 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -10,22 +10,16 @@ import { ConnectionState, type E2EEOptions, ExternalE2EEKeyProvider, - type LocalParticipant, - RemoteParticipant, type Room as LivekitRoom, type RoomOptions, } from "livekit-client"; import E2EEWorker from "livekit-client/e2ee-worker?worker"; import { - ClientEvent, type EventTimelineSetHandlerMap, EventType, type Room as MatrixRoom, RoomEvent, - type RoomMember, - SyncState, } from "matrix-js-sdk"; -import { deepCompare } from "matrix-js-sdk/lib/utils"; import { BehaviorSubject, combineLatest, @@ -62,14 +56,9 @@ import { } from "rxjs"; import { logger } from "matrix-js-sdk/lib/logger"; import { - type CallMembership, - isLivekitTransport, - type LivekitTransport, type MatrixRTCSession, MatrixRTCSessionEvent, type MatrixRTCSessionEventHandlerMap, - MembershipManagerEvent, - Status, } from "matrix-js-sdk/lib/matrixrtc"; import { type IWidgetApiRequest } from "matrix-widget-api"; @@ -80,17 +69,12 @@ import { ScreenShareViewModel, type UserMediaViewModel, } from "./MediaViewModel"; -import { - accumulate, - and$, - generateKeyed$, - pauseWhen, -} from "../utils/observable"; +import { accumulate, generateKeyed$, pauseWhen } from "../utils/observable"; import { duplicateTiles, - multiSfu, + MatrixRTCMode, + matrixRTCMode, playReactionsSound, - preferStickyEvents, showReactions, } from "../settings/settings"; import { isFirefox } from "../Platform"; @@ -109,20 +93,13 @@ import { import { shallowEquals } from "../utils/array"; import { type MediaDevices } from "./MediaDevices"; import { type Behavior, constant } from "./Behavior"; -import { - enterRTCSession, - getLivekitAlias, - makeTransport, -} from "../rtcSessionHelpers"; +import { enterRTCSession } from "../rtcSessionHelpers"; import { E2eeType } from "../e2ee/e2eeType"; import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider"; -import { type Connection } from "./remoteMembers/Connection.ts"; import { type MuteStates } from "./MuteStates"; import { getUrlParams } from "../UrlParams"; import { type ProcessorState } from "../livekit/TrackProcessorContext"; import { ElementWidgetActions, widget } from "../widget"; -import { PublishConnection } from "./localMember/Publisher.ts"; -import { type Async, async$, mapAsync, ready } from "./Async"; import { sharingScreen$, UserMedia } from "./UserMedia.ts"; import { ScreenShare } from "./ScreenShare.ts"; import { @@ -134,12 +111,14 @@ import { type SpotlightLandscapeLayoutMedia, type SpotlightPortraitLayoutMedia, } from "./layout-types.ts"; -import { type ElementCallError, UnknownCallError } from "../utils/errors.ts"; +import { type ElementCallError } from "../utils/errors.ts"; import { ObservableScope } from "./ObservableScope.ts"; -import { memberDisplaynames$ } from "./remoteMembers/displayname.ts"; import { ConnectionManager } from "./remoteMembers/ConnectionManager.ts"; import { MatrixLivekitMerger } from "./remoteMembers/matrixLivekitMerger.ts"; -import { ownMembership$ } from "./localMember/LocalMembership.ts"; +import { + localMembership$, + LocalMemberState, +} from "./localMember/LocalMembership.ts"; import { localTransport$ as computeLocalTransport$ } from "./localMember/LocalTransport.ts"; import { sessionBehaviors$ } from "./SessionBehaviors.ts"; import { ECConnectionFactory } from "./remoteMembers/ConnectionFactory.ts"; @@ -202,23 +181,20 @@ export class CallViewModel { private readonly userId = this.matrixRoom.client.getUserId()!; private readonly deviceId = this.matrixRoom.client.getDeviceId()!; - private readonly livekitAlias = getLivekitAlias(this.matrixRTCSession); private readonly livekitE2EEKeyProvider = getE2eeKeyProvider( this.options.encryptionSystem, this.matrixRTCSession, ); - private readonly e2eeLivekitOptions = (): E2EEOptions | undefined => - this.livekitE2EEKeyProvider - ? { - keyProvider: this.livekitE2EEKeyProvider, - worker: new E2EEWorker(), - } - : undefined; - private readonly _configError$ = new BehaviorSubject( - null, - ); + private readonly e2eeLivekitOptions: E2EEOptions | undefined = this + .livekitE2EEKeyProvider + ? { + keyProvider: this.livekitE2EEKeyProvider, + worker: new E2EEWorker(), + } + : undefined; + private sessionBehaviors = sessionBehaviors$( this.scope, this.matrixRTCSession, @@ -230,14 +206,16 @@ export class CallViewModel { memberships$: this.memberships$, client: this.matrixRoom.client, roomId: this.matrixRoom.roomId, - useOldestMember$: multiSfu.value$, + useOldestMember$: this.scope.behavior( + matrixRTCMode.value$.pipe(map((v) => v === MatrixRTCMode.Legacy)), + ), }); private connectionFactory = new ECConnectionFactory( this.matrixRoom.client, this.mediaDevices, this.trackProcessorState$, - this.e2eeLivekitOptions(), + this.e2eeLivekitOptions, getUrlParams().controlledAudioDevices, ); @@ -252,7 +230,6 @@ export class CallViewModel { this.scope, this.connectionFactory, this.allTransports$, - logger, ); private matrixLivekitMerger = new MatrixLivekitMerger( @@ -263,31 +240,36 @@ export class CallViewModel { this.userId, this.deviceId, ); + private matrixLivekitItems$ = this.matrixLivekitMerger.matrixLivekitItems$; - private localMembership = this.localMembership$({ + private localMembership = localMembership$({ scope: this.scope, muteStates: this.muteStates, - multiSfu: this.multiSfu, mediaDevices: this.mediaDevices, - trackProcessorState$: this.trackProcessorState$, + connectionManager: this.connectionManager, + matrixRTCSession: this.matrixRTCSession, + matrixRoom: this.matrixRoom, + localTransport$: this.localTransport$, e2eeLivekitOptions: this.e2eeLivekitOptions, + trackProcessorState$: this.trackProcessorState$, + widget, }); - private matrixLivekitItems$ = this.matrixLivekitMerger.matrixLivekitItems$; /** * If there is a configuration error with the call (e.g. misconfigured E2EE). * This is a fatal error that prevents the call from being created/joined. * Should render a blocking error screen. */ public get configError$(): Behavior { - return this._configError$; + return this.localMembership.configError$; } - private readonly join$ = new Subject(); - - // DISCUSS BAD ? - public join(): void { - this.join$.next(); + public join(): LocalMemberState { + return this.localMembership.requestConnect({ + encryptMedia: this.e2eeLivekitOptions !== undefined, + // TODO. This might need to get called again on each cahnge of matrixRTCMode... + matrixRTCMode: matrixRTCMode.getValue(), + }); } // CODESMELL? @@ -304,62 +286,7 @@ export class CallViewModel { * than whether all connections are truly up and running. */ // DISCUSS ? lets think why we need joined and how to do it better - private readonly joined$ = this.scope.behavior( - this.join$.pipe( - map(() => true), - // Using takeUntil with the repeat operator is perfectly valid. - // eslint-disable-next-line rxjs/no-unsafe-takeuntil - takeUntil(this.leaveHoisted$), - endWith(false), - repeat(), - startWith(false), - ), - ); - - // /** - // * The transport that we would personally prefer to publish on (if not for the - // * transport preferences of others, perhaps). - // */ - // // DISCUSS move to ownMembership - // private readonly preferredTransport$ = this.scope.behavior( - // async$(makeTransport(this.matrixRTCSession)), - // ); - - // /** - // * The transport over which we should be actively publishing our media. - // * null when not joined. - // */ - // // DISCUSSION ownMembershipManager - // private readonly localTransport$: Behavior | null> = - // this.scope.behavior( - // this.transports$.pipe( - // map((transports) => transports?.local ?? null), - // distinctUntilChanged | null>(deepCompare), - // ), - // ); - - // // DISCUSSION move to ConnectionManager - // public readonly livekitConnectionState$ = - // // TODO: This options.connectionState$ behavior is a small hack inserted - // // here to facilitate testing. This would likely be better served by - // // breaking CallViewModel down into more naturally testable components. - // this.options.connectionState$ ?? - // this.scope.behavior( - // this.localConnection$.pipe( - // switchMap((c) => - // c?.state === "ready" - // ? // TODO mapping to ConnectionState for compatibility, but we should use the full state? - // c.value.state$.pipe( - // switchMap((s) => { - // if (s.state === "ConnectedToLkRoom") - // return s.connectionState$; - // return of(ConnectionState.Disconnected); - // }), - // ) - // : of(ConnectionState.Disconnected), - // ), - // ), - // ); + private readonly joined$ = this.localMembership.connected$; /** * Whether various media/event sources should pretend to be disconnected from @@ -370,9 +297,14 @@ export class CallViewModel { // down, for example, and we want to avoid making people worry that the app is // in a split-brained state. // DISCUSSION own membership manager ALSO this probably can be simplifis - private readonly pretendToBeDisconnected$ = this.reconnecting$; + private readonly pretendToBeDisconnected$ = + this.localMembership.reconnecting$; - public readonly audioParticipants$; // now will be created based on the connectionmanager + public readonly audioParticipants$ = this.scope.behavior( + this.matrixLivekitItems$.pipe( + map((items) => items.map((item) => item.participant)), + ), + ); public readonly handsRaised$ = this.scope.behavior( this.handsRaisedSubject$.pipe(pauseWhen(this.pretendToBeDisconnected$)), @@ -392,15 +324,6 @@ export class CallViewModel { ), ); - // Now will be added to the matricLivekitMerger - // memberDisplaynames$ = memberDisplaynames$( - // this.matrixRoom, - // this.memberships$, - // this.scope, - // this.userId, - // this.deviceId, - // ); - /** * List of MediaItems that we want to have tiles for. */ @@ -1352,6 +1275,7 @@ export class CallViewModel { /** * Whether we are sharing our screen. */ + // TODO move to LocalMembership public readonly sharingScreen$ = this.scope.behavior( from(this.localConnection$).pipe( switchMap((c) => @@ -1366,6 +1290,7 @@ export class CallViewModel { * Callback for toggling screen sharing. If null, screen sharing is not * available. */ + // TODO move to LocalMembership public readonly toggleScreenSharing = "getDisplayMedia" in (navigator.mediaDevices ?? {}) && !this.urlParams.hideScreensharing @@ -1408,101 +1333,6 @@ export class CallViewModel { >, private readonly trackProcessorState$: Behavior, ) { - // Start and stop session membership as needed - this.scope.reconcile(this.advertisedTransport$, async (advertised) => { - if (advertised !== null) { - try { - this._configError$.next(null); - await enterRTCSession(this.matrixRTCSession, advertised.transport, { - encryptMedia: this.options.encryptionSystem.kind !== E2eeType.NONE, - useMultiSfu: advertised.multiSfu, - preferStickyEvents: advertised.preferStickyEvents, - }); - } catch (e) { - logger.error("Error entering RTC session", e); - } - - // Update our member event when our mute state changes. - const intentScope = new ObservableScope(); - intentScope.reconcile( - this.muteStates.video.enabled$, - async (videoEnabled) => - this.matrixRTCSession.updateCallIntent( - videoEnabled ? "video" : "audio", - ), - ); - - return async (): Promise => { - intentScope.end(); - // Only sends Matrix leave event. The LiveKit session will disconnect - // as soon as either the stopConnection$ handler above gets to it or - // the view model is destroyed. - try { - await this.matrixRTCSession.leaveRoomSession(); - } catch (e) { - logger.error("Error leaving RTC session", e); - } - try { - await widget?.api.transport.send( - ElementWidgetActions.HangupCall, - {}, - ); - } catch (e) { - logger.error("Failed to send hangup action", e); - } - }; - } - }); - - // Pause upstream of all local media tracks when we're disconnected from - // MatrixRTC, because it can be an unpleasant surprise for the app to say - // 'reconnecting' and yet still be transmitting your media to others. - // We use matrixConnected$ rather than reconnecting$ because we want to - // pause tracks during the initial joining sequence too until we're sure - // that our own media is displayed on screen. - combineLatest([this.localConnection$, this.matrixConnected$]) - .pipe(this.scope.bind()) - .subscribe(([connection, connected]) => { - if (connection?.state !== "ready") return; - const publications = - connection.value.livekitRoom.localParticipant.trackPublications.values(); - if (connected) { - for (const p of publications) { - if (p.track?.isUpstreamPaused === true) { - const kind = p.track.kind; - logger.log( - `Resuming ${kind} track (MatrixRTC connection present)`, - ); - p.track - .resumeUpstream() - .catch((e) => - logger.error( - `Failed to resume ${kind} track after MatrixRTC reconnection`, - e, - ), - ); - } - } - } else { - for (const p of publications) { - if (p.track?.isUpstreamPaused === false) { - const kind = p.track.kind; - logger.log( - `Pausing ${kind} track (uncertain MatrixRTC connection)`, - ); - p.track - .pauseUpstream() - .catch((e) => - logger.error( - `Failed to pause ${kind} track after entering uncertain MatrixRTC connection`, - e, - ), - ); - } - } - } - }); - // Join automatically this.join(); // TODO-MULTI-SFU: Use this view model for the lobby as well, and only call this once 'join' is clicked? } diff --git a/src/state/localMember/LocalMembership.ts b/src/state/localMember/LocalMembership.ts index 7448c2ee..0bd5fbb1 100644 --- a/src/state/localMember/LocalMembership.ts +++ b/src/state/localMember/LocalMembership.ts @@ -12,12 +12,7 @@ import { MembershipManagerEvent, Status, } from "matrix-js-sdk/lib/matrixrtc"; -import { - ClientEvent, - type MatrixClient, - SyncState, - type Room as MatrixRoom, -} from "matrix-js-sdk"; +import { ClientEvent, SyncState, type Room as MatrixRoom } from "matrix-js-sdk"; import { BehaviorSubject, combineLatest, @@ -25,6 +20,7 @@ import { map, type Observable, of, + scan, startWith, switchMap, tap, @@ -33,7 +29,7 @@ import { logger } from "matrix-js-sdk/lib/logger"; import { type Behavior } from "../Behavior"; import { type ConnectionManager } from "../remoteMembers/ConnectionManager"; -import { type ObservableScope } from "../ObservableScope"; +import { ObservableScope } from "../ObservableScope"; import { Publisher } from "./Publisher"; import { type MuteStates } from "../MuteStates"; import { type ProcessorState } from "../../livekit/TrackProcessorContext"; @@ -44,31 +40,10 @@ import { enterRTCSession, type EnterRTCSessionOptions, } from "../../rtcSessionHelpers"; +import { ElementCallError } from "../../utils/errors"; +import { Widget } from "matrix-widget-api"; +import { ElementWidgetActions, WidgetHelpers } from "../../widget"; -/* - * - get well known - * - get oldest membership - * - get transport to use - * - get openId + jwt token - * - wait for createTrack() call - * - create tracks - * - wait for join() call - * - Publisher.publishTracks() - * - send join state/sticky event - */ -interface Props { - scope: ObservableScope; - mediaDevices: MediaDevices; - muteStates: MuteStates; - connectionManager: ConnectionManager; - matrixRTCSession: MatrixRTCSession; - matrixRoom: MatrixRoom; - localTransport$: Behavior; - client: MatrixClient; - roomId: string; - e2eeLivekitOptions: E2EEOptions | undefined; - trackerProcessorState$: Behavior; -} enum LivekitState { UNINITIALIZED = "uninitialized", CONNECTING = "connecting", @@ -95,10 +70,35 @@ type LocalMemberMatrixState = | { state: MatrixState.CONNECTING } | { state: MatrixState.DISCONNECTED }; -interface LocalMemberState { +export interface LocalMemberState { livekit$: BehaviorSubject; matrix$: BehaviorSubject; } + +/* + * - get well known + * - get oldest membership + * - get transport to use + * - get openId + jwt token + * - wait for createTrack() call + * - create tracks + * - wait for join() call + * - Publisher.publishTracks() + * - send join state/sticky event + */ +interface Props { + scope: ObservableScope; + mediaDevices: MediaDevices; + muteStates: MuteStates; + connectionManager: ConnectionManager; + matrixRTCSession: MatrixRTCSession; + matrixRoom: MatrixRoom; + localTransport$: Behavior; + e2eeLivekitOptions: E2EEOptions | undefined; + trackProcessorState$: Behavior; + widget: WidgetHelpers | null; +} + /** * This class is responsible for managing the own membership in a room. * We want @@ -120,7 +120,8 @@ export const localMembership$ = ({ localTransport$, matrixRoom, e2eeLivekitOptions, - trackerProcessorState$, + trackProcessorState$, + widget, }: Props): { // publisher: Publisher requestConnect: (options: EnterRTCSessionOptions) => LocalMemberState; @@ -129,6 +130,8 @@ export const localMembership$ = ({ state: LocalMemberState; // TODO this is probably superseeded by joinState$ homeserverConnected$: Behavior; connected$: Behavior; + reconnecting$: Behavior; + configError$: Behavior; } => { const state = { livekit$: new BehaviorSubject({ @@ -148,11 +151,12 @@ export const localMembership$ = ({ const connection$ = scope.behavior( combineLatest([connectionManager.connections$, localTransport$]).pipe( - map(([connections, transport]) => - connections.find((connection) => + map(([connections, transport]) => { + if (transport === undefined) return undefined; + return connections.find((connection) => areLivekitTransportsEqual(connection.transport, transport), - ), - ), + ); + }), ), ); /** @@ -214,7 +218,7 @@ export const localMembership$ = ({ mediaDevices, muteStates, e2eeLivekitOptions, - trackerProcessorState$, + trackProcessorState$, ) : null, ), @@ -242,31 +246,28 @@ export const localMembership$ = ({ // /** // * Whether we should tell the user that we're reconnecting to the call. // */ - // // DISCUSSION own membership manager - // const reconnecting$ = scope.behavior( - // connected$.pipe( - // // We are reconnecting if we previously had some successful initial - // // connection but are now disconnected - // scan( - // ({ connectedPreviously }, connectedNow) => ({ - // connectedPreviously: connectedPreviously || connectedNow, - // reconnecting: connectedPreviously && !connectedNow, - // }), - // { connectedPreviously: false, reconnecting: false }, - // ), - // map(({ reconnecting }) => reconnecting), - // ), - // ); + // DISCUSSION is there a better way to do this? + // sth that is more deriectly implied from the membership manager of the js sdk. (fromEvent(matrixRTCSession, Reconnecting)) ??? or similar + const reconnecting$ = scope.behavior( + connected$.pipe( + // We are reconnecting if we previously had some successful initial + // connection but are now disconnected + scan( + ({ connectedPreviously }, connectedNow) => ({ + connectedPreviously: connectedPreviously || connectedNow, + reconnecting: connectedPreviously && !connectedNow, + }), + { connectedPreviously: false, reconnecting: false }, + ), + map(({ reconnecting }) => reconnecting), + ), + ); const startTracks = (): Behavior => { shouldStartTracks$.next(true); return tracks$; }; - // const joinState$ = new BehaviorSubject({ - // state: LivekitState.UNINITIALIZED, - // }); - const requestConnect = ( options: EnterRTCSessionOptions, ): LocalMemberState => { @@ -288,11 +289,15 @@ export const localMembership$ = ({ state.matrix$.next({ state: MatrixState.CONNECTING }); localTransport$.pipe( tap((transport) => { - enterRTCSession(matrixRTCSession, transport, options).catch( - (error) => { - logger.error(error); - }, - ); + if (transport !== undefined) { + enterRTCSession(matrixRTCSession, transport, options).catch( + (error) => { + logger.error(error); + }, + ); + } else { + logger.info("Waiting for transport to enter rtc session"); + } }), ); } @@ -317,6 +322,93 @@ export const localMembership$ = ({ return state.livekit$; }; + // Pause upstream of all local media tracks when we're disconnected from + // MatrixRTC, because it can be an unpleasant surprise for the app to say + // 'reconnecting' and yet still be transmitting your media to others. + // We use matrixConnected$ rather than reconnecting$ because we want to + // pause tracks during the initial joining sequence too until we're sure + // that our own media is displayed on screen. + combineLatest([connection$, homeserverConnected$]) + .pipe(scope.bind()) + .subscribe(([connection, connected]) => { + if (connection?.state$.value.state !== "ConnectedToLkRoom") return; + const publications = + connection.livekitRoom.localParticipant.trackPublications.values(); + if (connected) { + for (const p of publications) { + if (p.track?.isUpstreamPaused === true) { + const kind = p.track.kind; + logger.log(`Resuming ${kind} track (MatrixRTC connection present)`); + p.track + .resumeUpstream() + .catch((e) => + logger.error( + `Failed to resume ${kind} track after MatrixRTC reconnection`, + e, + ), + ); + } + } + } else { + for (const p of publications) { + if (p.track?.isUpstreamPaused === false) { + const kind = p.track.kind; + logger.log( + `Pausing ${kind} track (uncertain MatrixRTC connection)`, + ); + p.track + .pauseUpstream() + .catch((e) => + logger.error( + `Failed to pause ${kind} track after entering uncertain MatrixRTC connection`, + e, + ), + ); + } + } + } + }); + + const configError$ = new BehaviorSubject(null); + // TODO I do not fully understand what this does. + // Is it needed? + // Is this at the right place? + // Can this be simplified? + // Start and stop session membership as needed + scope.reconcile(localTransport$, async (advertised) => { + if (advertised !== null && advertised !== undefined) { + try { + configError$.next(null); + await enterRTCSession(matrixRTCSession, advertised, options); + } catch (e) { + logger.error("Error entering RTC session", e); + } + + // Update our member event when our mute state changes. + const intentScope = new ObservableScope(); + intentScope.reconcile(muteStates.video.enabled$, async (videoEnabled) => + matrixRTCSession.updateCallIntent(videoEnabled ? "video" : "audio"), + ); + + return async (): Promise => { + intentScope.end(); + // Only sends Matrix leave event. The LiveKit session will disconnect + // as soon as either the stopConnection$ handler above gets to it or + // the view model is destroyed. + try { + await matrixRTCSession.leaveRoomSession(); + } catch (e) { + logger.error("Error leaving RTC session", e); + } + try { + await widget?.api.transport.send(ElementWidgetActions.HangupCall, {}); + } catch (e) { + logger.error("Failed to send hangup action", e); + } + }; + } + }); + return { startTracks, requestConnect, @@ -324,5 +416,7 @@ export const localMembership$ = ({ state, homeserverConnected$, connected$, + reconnecting$, + configError$, }; }; diff --git a/src/state/remoteMembers/ConnectionManager.ts b/src/state/remoteMembers/ConnectionManager.ts index e333173e..37c616f8 100644 --- a/src/state/remoteMembers/ConnectionManager.ts +++ b/src/state/remoteMembers/ConnectionManager.ts @@ -14,7 +14,7 @@ import { type ParticipantId, } from "matrix-js-sdk/lib/matrixrtc"; import { BehaviorSubject, combineLatest, map, switchMap } from "rxjs"; -import { type Logger } from "matrix-js-sdk/lib/logger"; +import { logger, type Logger } from "matrix-js-sdk/lib/logger"; import { type Participant as LivekitParticipant } from "livekit-client"; import { type Behavior } from "../Behavior"; @@ -106,8 +106,8 @@ export class ConnectionManager { private readonly scope: ObservableScope, private readonly connectionFactory: ConnectionFactory, private readonly inputTransports$: Behavior, - logger: Logger, ) { + // TODO logger: only construct one logger from the client and make it compatible via a EC specific singleton. this.logger = logger.getChild("ConnectionManager"); scope.onEnd(() => this.running$.next(false)); } From 4d0de2fb71e4320ee4919656a1e1b044a3f95c65 Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 5 Nov 2025 17:55:36 +0100 Subject: [PATCH 21/75] Refactor Matrix/LiveKit session merging - Replace MatrixLivekitItem with MatrixLivekitMember, add displayName$ and participantId, and use explicit LiveKit participant types - Make sessionBehaviors$ accept a props object and return a typed RxRtcSession - Update CallViewModel to use the new session behaviors, rebuild media items from matrixLivekitMembers, handle missing connections and use participantId-based keys - Change localMembership/localTransport to accept Behavior-based options, read options.value for enterRTCSession, and fix advertised transport selection order - Update tests and minor UI adjustments (settings modal livekitRooms stubbed) and fix JSON formatting in locales --- locales/en/app.json | 12 +- src/room/InCallView.tsx | 7 +- src/rtcSessionHelpers.test.ts | 9 +- src/state/CallViewModel.ts | 153 ++++++++++-------- src/state/SessionBehaviors.ts | 23 ++- src/state/localMember/LocalMembership.ts | 17 +- src/state/localMember/LocalTransport.ts | 7 +- src/state/remoteMembers/ConnectionManager.ts | 2 +- .../remoteMembers/MatrixLivekitMerger.test.ts | 32 ++-- .../remoteMembers/matrixLivekitMerger.ts | 40 +++-- 10 files changed, 172 insertions(+), 130 deletions(-) diff --git a/locales/en/app.json b/locales/en/app.json index 104af750..2c6801bc 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -74,16 +74,16 @@ "matrix_id": "Matrix ID: {{id}}", "matrixRTCMode": { "Comptibility": { - "label": "Compatibility: state events & multi SFU" - "description": "Compatible with homeservers that do not support sticky events (but all other EC clients are v0.17.0 or later)", + "label": "Compatibility: state events & multi SFU", + "description": "Compatible with homeservers that do not support sticky events (but all other EC clients are v0.17.0 or later)" }, "Legacy": { - "label": "Legacy: state events & oldest membership SFU" - "description": "Compatible with old versions of EC that do not support multi SFU", + "label": "Legacy: state events & oldest membership SFU", + "description": "Compatible with old versions of EC that do not support multi SFU" }, "Matrix_2_0": { - "label": "Matrix 2.0: sticky events & multi SFU" - "description": "Compatible only with homservers supporting sticky events and all EC clients v0.17.0 or later", + "label": "Matrix 2.0: sticky events & multi SFU", + "description": "Compatible only with homservers supporting sticky events and all EC clients v0.17.0 or later" } }, "mute_all_audio": "Mute all audio (participants, reactions, join sounds)", diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 6f6bae93..a6a2e897 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -138,7 +138,7 @@ export const ActiveCall: FC = (props) => { }, reactionsReader.raisedHands$, reactionsReader.reactions$, - trackProcessorState$, + scope.behavior(trackProcessorState$), ); setVm(vm); @@ -247,7 +247,7 @@ export const InCallView: FC = ({ () => void toggleRaisedHand(), ); - const allLivekitRooms = useBehavior(vm.allLivekitRooms$); + // const allLivekitRooms = useBehavior(vm.allLivekitRooms$); const audioParticipants = useBehavior(vm.audioParticipants$); const participantCount = useBehavior(vm.participantCount$); const reconnecting = useBehavior(vm.reconnecting$); @@ -841,7 +841,8 @@ export const InCallView: FC = ({ onDismiss={closeSettings} tab={settingsTab} onTabChange={setSettingsTab} - livekitRooms={allLivekitRooms} + // TODO expose correct data to setttings modal + livekitRooms={[]} /> )} diff --git a/src/rtcSessionHelpers.test.ts b/src/rtcSessionHelpers.test.ts index 8aca40f5..a2b49390 100644 --- a/src/rtcSessionHelpers.test.ts +++ b/src/rtcSessionHelpers.test.ts @@ -12,8 +12,9 @@ import EventEmitter from "events"; import { enterRTCSession } from "../src/rtcSessionHelpers"; import { mockConfig } from "./utils/test"; +import { MatrixRTCMode } from "./settings/settings"; -const USE_MUTI_SFU = false; +const MATRIX_RTC_MODE = MatrixRTCMode.Legacy; const getUrlParams = vi.hoisted(() => vi.fn(() => ({}))); vi.mock("./UrlParams", () => ({ getUrlParams })); @@ -94,8 +95,7 @@ test("It joins the correct Session", async () => { }, { encryptMedia: true, - useMultiSfu: USE_MUTI_SFU, - preferStickyEvents: false, + matrixRTCMode: MATRIX_RTC_MODE, }, ); @@ -153,8 +153,7 @@ test("It should not fail with configuration error if homeserver config has livek }, { encryptMedia: true, - useMultiSfu: USE_MUTI_SFU, - preferStickyEvents: false, + matrixRTCMode: MATRIX_RTC_MODE, }, ); }); diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 7396a515..c88348d6 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -21,7 +21,6 @@ import { RoomEvent, } from "matrix-js-sdk"; import { - BehaviorSubject, combineLatest, concat, distinctUntilChanged, @@ -38,7 +37,6 @@ import { of, pairwise, race, - repeat, scan, skip, skipWhile, @@ -93,7 +91,6 @@ import { import { shallowEquals } from "../utils/array"; import { type MediaDevices } from "./MediaDevices"; import { type Behavior, constant } from "./Behavior"; -import { enterRTCSession } from "../rtcSessionHelpers"; import { E2eeType } from "../e2ee/e2eeType"; import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider"; import { type MuteStates } from "./MuteStates"; @@ -112,12 +109,12 @@ import { type SpotlightPortraitLayoutMedia, } from "./layout-types.ts"; import { type ElementCallError } from "../utils/errors.ts"; -import { ObservableScope } from "./ObservableScope.ts"; +import { type ObservableScope } from "./ObservableScope.ts"; import { ConnectionManager } from "./remoteMembers/ConnectionManager.ts"; import { MatrixLivekitMerger } from "./remoteMembers/matrixLivekitMerger.ts"; import { localMembership$, - LocalMemberState, + type LocalMemberState, } from "./localMember/LocalMembership.ts"; import { localTransport$ as computeLocalTransport$ } from "./localMember/LocalTransport.ts"; import { sessionBehaviors$ } from "./SessionBehaviors.ts"; @@ -195,10 +192,10 @@ export class CallViewModel { } : undefined; - private sessionBehaviors = sessionBehaviors$( - this.scope, - this.matrixRTCSession, - ); + private sessionBehaviors = sessionBehaviors$({ + scope: this.scope, + matrixRTCSession: this.matrixRTCSession, + }); private memberships$ = this.sessionBehaviors.memberships$; private localTransport$ = computeLocalTransport$({ @@ -211,6 +208,8 @@ export class CallViewModel { ), }); + // ------------------------------------------------------------------------ + private connectionFactory = new ECConnectionFactory( this.matrixRoom.client, this.mediaDevices, @@ -219,10 +218,14 @@ export class CallViewModel { getUrlParams().controlledAudioDevices, ); + // Can contain duplicates. The connection manager will take care of this. private allTransports$ = this.scope.behavior( combineLatest( [this.localTransport$, this.sessionBehaviors.transports$], - (l, t) => [...(l ? [l] : []), ...t], + (localTransport, transports) => { + const localTransportAsArray = localTransport ? [localTransport] : []; + return [...localTransportAsArray, ...transports]; + }, ), ); @@ -232,6 +235,8 @@ export class CallViewModel { this.allTransports$, ); + // ------------------------------------------------------------------------ + private matrixLivekitMerger = new MatrixLivekitMerger( this.scope, this.sessionBehaviors.membershipsWithTransport$, @@ -240,7 +245,7 @@ export class CallViewModel { this.userId, this.deviceId, ); - private matrixLivekitItems$ = this.matrixLivekitMerger.matrixLivekitItems$; + private matrixLivekitMembers$ = this.matrixLivekitMerger.matrixLivekitMember$; private localMembership = localMembership$({ scope: this.scope, @@ -297,12 +302,12 @@ export class CallViewModel { // down, for example, and we want to avoid making people worry that the app is // in a split-brained state. // DISCUSSION own membership manager ALSO this probably can be simplifis - private readonly pretendToBeDisconnected$ = - this.localMembership.reconnecting$; + public reconnecting$ = this.localMembership.reconnecting$; + private readonly pretendToBeDisconnected$ = this.reconnecting$; public readonly audioParticipants$ = this.scope.behavior( - this.matrixLivekitItems$.pipe( - map((items) => items.map((item) => item.participant)), + this.matrixLivekitMembers$.pipe( + map((members) => members.map((m) => m.participant)), ), ); @@ -330,72 +335,82 @@ export class CallViewModel { // TODO KEEP THIS!! and adapt it to what our membershipManger returns private readonly mediaItems$ = this.scope.behavior( generateKeyed$< - [typeof this.participantsByRoom$.value, number], + [typeof this.matrixLivekitMembers$.value, number], MediaItem, MediaItem[] >( // Generate a collection of MediaItems from the list of expected (whether // present or missing) LiveKit participants. - combineLatest([this.participantsByRoom$, duplicateTiles.value$]), - ([participantsByRoom, duplicateTiles], createOrGet) => { + combineLatest([this.matrixLivekitMembers$, duplicateTiles.value$]), + ([matrixLivekitMembers, duplicateTiles], createOrGet) => { const items: MediaItem[] = []; - for (const { livekitRoom, participants, url } of participantsByRoom) { - for (const { id, participant, member } of participants) { - for (let i = 0; i < 1 + duplicateTiles; i++) { - const mediaId = `${id}:${i}`; - const item = createOrGet( - mediaId, - (scope) => - // We create UserMedia with or without a participant. - // This will be the initial value of a BehaviourSubject. - // Once a participant appears we will update the BehaviourSubject. (see below) - new UserMedia( - scope, - mediaId, - member, - participant, - this.options.encryptionSystem, - livekitRoom, - url, - this.mediaDevices, - this.pretendToBeDisconnected$, - this.memberDisplaynames$.pipe( - map((m) => m.get(id) ?? "[👻]"), - ), - this.handsRaised$.pipe(map((v) => v[id]?.time ?? null)), - this.reactions$.pipe(map((v) => v[id] ?? undefined)), + for (const { + connection, + participant, + member, + displayName$, + participantId, + } of matrixLivekitMembers) { + if (connection === undefined) { + logger.warn("connection is not yet initialised."); + continue; + } + for (let i = 0; i < 1 + duplicateTiles; i++) { + const mediaId = `${participantId}:${i}`; + const lkRoom = connection?.livekitRoom; + const url = connection?.transport.livekit_service_url; + const dpName$ = displayName$.pipe(map((n) => n ?? "[👻]")); + const item = createOrGet( + mediaId, + (scope) => + // We create UserMedia with or without a participant. + // This will be the initial value of a BehaviourSubject. + // Once a participant appears we will update the BehaviourSubject. (see below) + new UserMedia( + scope, + mediaId, + member, + participant, + this.options.encryptionSystem, + lkRoom, + url, + this.mediaDevices, + this.pretendToBeDisconnected$, + dpName$, + this.handsRaised$.pipe( + map((v) => v[participantId]?.time ?? null), ), - ); - items.push(item); - (item as UserMedia).updateParticipant(participant); + this.reactions$.pipe( + map((v) => v[participantId] ?? undefined), + ), + ), + ); + items.push(item); + (item as UserMedia).updateParticipant(participant); - if (participant?.isScreenShareEnabled) { - const screenShareId = `${mediaId}:screen-share`; - items.push( - createOrGet( - screenShareId, - (scope) => - new ScreenShare( - scope, - screenShareId, - member, - participant, - this.options.encryptionSystem, - livekitRoom, - url, - this.pretendToBeDisconnected$, - this.memberDisplaynames$.pipe( - map((m) => m.get(id) ?? "[👻]"), - ), - ), - ), - ); - } + if (participant?.isScreenShareEnabled) { + const screenShareId = `${mediaId}:screen-share`; + items.push( + createOrGet( + screenShareId, + (scope) => + new ScreenShare( + scope, + screenShareId, + member, + participant, + this.options.encryptionSystem, + lkRoom, + url, + this.pretendToBeDisconnected$, + dpName$, + ), + ), + ); } } } - return items; }, ), diff --git a/src/state/SessionBehaviors.ts b/src/state/SessionBehaviors.ts index 6c16ace4..250ad86a 100644 --- a/src/state/SessionBehaviors.ts +++ b/src/state/SessionBehaviors.ts @@ -17,16 +17,29 @@ import { fromEvent, map } from "rxjs"; import { type ObservableScope } from "./ObservableScope"; import { type Behavior } from "./Behavior"; -export const sessionBehaviors$ = ( - scope: ObservableScope, - matrixRTCSession: MatrixRTCSession, -): { +interface Props { + scope: ObservableScope; + matrixRTCSession: MatrixRTCSession; +} + +/** + * Wraps behaviors that we extract from an matrixRTCSession. + */ +interface RxRtcSession { + /** + * some prop + */ memberships$: Behavior; membershipsWithTransport$: Behavior< { membership: CallMembership; transport?: LivekitTransport }[] >; transports$: Behavior; -} => { +} + +export const sessionBehaviors$ = ({ + scope, + matrixRTCSession, +}: Props): RxRtcSession => { const memberships$ = scope.behavior( fromEvent( matrixRTCSession, diff --git a/src/state/localMember/LocalMembership.ts b/src/state/localMember/LocalMembership.ts index 0bd5fbb1..33a54574 100644 --- a/src/state/localMember/LocalMembership.ts +++ b/src/state/localMember/LocalMembership.ts @@ -40,9 +40,8 @@ import { enterRTCSession, type EnterRTCSessionOptions, } from "../../rtcSessionHelpers"; -import { ElementCallError } from "../../utils/errors"; -import { Widget } from "matrix-widget-api"; -import { ElementWidgetActions, WidgetHelpers } from "../../widget"; +import { type ElementCallError } from "../../utils/errors"; +import { ElementWidgetActions, type WidgetHelpers } from "../../widget"; enum LivekitState { UNINITIALIZED = "uninitialized", @@ -87,6 +86,7 @@ export interface LocalMemberState { * - send join state/sticky event */ interface Props { + options: Behavior; scope: ObservableScope; mediaDevices: MediaDevices; muteStates: MuteStates; @@ -113,6 +113,7 @@ interface Props { */ export const localMembership$ = ({ scope, + options, muteStates, mediaDevices, connectionManager, @@ -124,7 +125,7 @@ export const localMembership$ = ({ widget, }: Props): { // publisher: Publisher - requestConnect: (options: EnterRTCSessionOptions) => LocalMemberState; + requestConnect: () => LocalMemberState; startTracks: () => Behavior; requestDisconnect: () => Observable | null; state: LocalMemberState; // TODO this is probably superseeded by joinState$ @@ -268,9 +269,7 @@ export const localMembership$ = ({ return tracks$; }; - const requestConnect = ( - options: EnterRTCSessionOptions, - ): LocalMemberState => { + const requestConnect = (): LocalMemberState => { if (state.livekit$.value === null) { startTracks(); state.livekit$.next({ state: LivekitState.CONNECTING }); @@ -290,7 +289,7 @@ export const localMembership$ = ({ localTransport$.pipe( tap((transport) => { if (transport !== undefined) { - enterRTCSession(matrixRTCSession, transport, options).catch( + enterRTCSession(matrixRTCSession, transport, options.value).catch( (error) => { logger.error(error); }, @@ -379,7 +378,7 @@ export const localMembership$ = ({ if (advertised !== null && advertised !== undefined) { try { configError$.next(null); - await enterRTCSession(matrixRTCSession, advertised, options); + await enterRTCSession(matrixRTCSession, advertised, options.value); } catch (e) { logger.error("Error entering RTC session", e); } diff --git a/src/state/localMember/LocalTransport.ts b/src/state/localMember/LocalTransport.ts index 7a5202a9..9fada195 100644 --- a/src/state/localMember/LocalTransport.ts +++ b/src/state/localMember/LocalTransport.ts @@ -77,13 +77,12 @@ export const localTransport$ = ({ scope.behavior(from(makeTransport(client, roomId)), undefined); /** - * The transport we should advertise in our MatrixRTC membership (plus whether - * it is a multi-SFU transport and whether we should use sticky events). + * The transport we should advertise in our MatrixRTC membership. */ const advertisedTransport$ = scope.behavior( combineLatest( - [useOldestMember$, preferredTransport$, oldestMemberTransport$], - (useOldestMember, preferredTransport, oldestMemberTransport) => + [useOldestMember$, oldestMemberTransport$, preferredTransport$], + (useOldestMember, oldestMemberTransport, preferredTransport) => useOldestMember ? oldestMemberTransport : preferredTransport, ).pipe(distinctUntilChanged(deepCompare)), undefined, diff --git a/src/state/remoteMembers/ConnectionManager.ts b/src/state/remoteMembers/ConnectionManager.ts index 37c616f8..d0bbfe6f 100644 --- a/src/state/remoteMembers/ConnectionManager.ts +++ b/src/state/remoteMembers/ConnectionManager.ts @@ -107,7 +107,7 @@ export class ConnectionManager { private readonly connectionFactory: ConnectionFactory, private readonly inputTransports$: Behavior, ) { - // TODO logger: only construct one logger from the client and make it compatible via a EC specific singleton. + // TODO logger: only construct one logger from the client and make it compatible via a EC specific sing this.logger = logger.getChild("ConnectionManager"); scope.onEnd(() => this.running$.next(false)); } diff --git a/src/state/remoteMembers/MatrixLivekitMerger.test.ts b/src/state/remoteMembers/MatrixLivekitMerger.test.ts index e3f08405..16fc9c0c 100644 --- a/src/state/remoteMembers/MatrixLivekitMerger.test.ts +++ b/src/state/remoteMembers/MatrixLivekitMerger.test.ts @@ -23,7 +23,7 @@ import { type Room as MatrixRoom } from "matrix-js-sdk"; import { getParticipantId } from "matrix-js-sdk/lib/matrixrtc/utils"; import { - type MatrixLivekitItem, + type MatrixLivekitMember, MatrixLivekitMerger, } from "./matrixLivekitMerger"; import { ObservableScope } from "../ObservableScope"; @@ -79,10 +79,12 @@ afterEach(() => { test("should signal participant not yet connected to livekit", () => { fakeMemberships$.next([aliceRtcMember]); - let items: MatrixLivekitItem[] = []; - matrixLivekitMerger.matrixLivekitItems$.pipe(take(1)).subscribe((emitted) => { - items = emitted; - }); + let items: MatrixLivekitMember[] = []; + matrixLivekitMerger.matrixLivekitMember$ + .pipe(take(1)) + .subscribe((emitted) => { + items = emitted; + }); expect(items).toHaveLength(1); const item = items[0]; @@ -112,10 +114,12 @@ test("should signal participant on a connection that is publishing", () => { ]); fakeManagerData$.next(managerData); - let items: MatrixLivekitItem[] = []; - matrixLivekitMerger.matrixLivekitItems$.pipe(take(1)).subscribe((emitted) => { - items = emitted; - }); + let items: MatrixLivekitMember[] = []; + matrixLivekitMerger.matrixLivekitMember$ + .pipe(take(1)) + .subscribe((emitted) => { + items = emitted; + }); expect(items).toHaveLength(1); const item = items[0]; @@ -136,7 +140,7 @@ test("should signal participant on a connection that is not publishing", () => { managerData.add(fakeConnection, []); fakeManagerData$.next(managerData); - matrixLivekitMerger.matrixLivekitItems$.pipe(take(1)).subscribe((items) => { + matrixLivekitMerger.matrixLivekitMember$.pipe(take(1)).subscribe((items) => { expect(items).toHaveLength(1); const item = items[0]; @@ -177,8 +181,8 @@ describe("Publication edge case", () => { ); test("bob is publishing in several connections", () => { - let lastMatrixLkItems: MatrixLivekitItem[] = []; - matrixLivekitMerger.matrixLivekitItems$.subscribe((items) => { + let lastMatrixLkItems: MatrixLivekitMember[] = []; + matrixLivekitMerger.matrixLivekitMember$.subscribe((items) => { lastMatrixLkItems = items; }); @@ -218,8 +222,8 @@ describe("Publication edge case", () => { }); test("bob is publishing in the wrong connection", () => { - let lastMatrixLkItems: MatrixLivekitItem[] = []; - matrixLivekitMerger.matrixLivekitItems$.subscribe((items) => { + let lastMatrixLkItems: MatrixLivekitMember[] = []; + matrixLivekitMerger.matrixLivekitMember$.subscribe((items) => { lastMatrixLkItems = items; }); diff --git a/src/state/remoteMembers/matrixLivekitMerger.ts b/src/state/remoteMembers/matrixLivekitMerger.ts index 39acc65b..94e0ebd5 100644 --- a/src/state/remoteMembers/matrixLivekitMerger.ts +++ b/src/state/remoteMembers/matrixLivekitMerger.ts @@ -5,7 +5,10 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type Participant as LivekitParticipant } from "livekit-client"; +import { + type LocalParticipant as LocalLivekitParticipant, + type RemoteParticipant as RemoteLivekitParticipant, +} from "livekit-client"; import { type LivekitTransport, type CallMembership, @@ -27,22 +30,23 @@ import { type Connection } from "./Connection"; * `livekitParticipant` can be undefined if the member is not yet connected to the livekit room * or if it has no livekit transport at all. */ -export interface MatrixLivekitItem { +export interface MatrixLivekitMember { membership: CallMembership; - displayName: string; - participant?: LivekitParticipant; + displayName$: Behavior; + participant?: LocalLivekitParticipant | RemoteLivekitParticipant; connection?: Connection; /** * TODO Try to remove this! Its waaay to much information. * Just get the member's avatar * @deprecated */ - member?: RoomMember; + member: RoomMember; mxcAvatarUrl?: string; + participantId: string; } // Alternative structure idea: -// const livekitMatrixItems$ = (callMemberships$,connectionManager,scope): Observable => { +// const livekitMatrixMember$ = (callMemberships$,connectionManager,scope): Observable => { /** * Combines MatrixRtc and Livekit worlds. @@ -52,13 +56,13 @@ export interface MatrixLivekitItem { * - an observable of CallMembership[] to track the call members (The matrix side) * - a `ConnectionManager` for the lk rooms (The livekit side) * - out (via public Observable): - * - `remoteMatrixLivekitItems` an observable of MatrixLivekitItem[] to track the remote members and associated livekit data. + * - `remoteMatrixLivekitMember` an observable of MatrixLivekitMember[] to track the remote members and associated livekit data. */ export class MatrixLivekitMerger { /** * Stream of all the call members and their associated livekit data (if available). */ - public matrixLivekitItems$: Behavior; + public matrixLivekitMember$: Behavior; // private readonly logger: Logger; @@ -79,7 +83,7 @@ export class MatrixLivekitMerger { ) { // this.logger = parentLogger.getChild("MatrixLivekitMerger"); - this.matrixLivekitItems$ = this.scope.behavior( + this.matrixLivekitMember$ = this.scope.behavior( this.start$().pipe(startWith([])), ); } @@ -87,7 +91,7 @@ export class MatrixLivekitMerger { // ======================================= /// PRIVATES // ======================================= - private start$(): Observable { + private start$(): Observable { const displaynameMap$ = memberDisplaynames$( this.scope, this.matrixRoom, @@ -102,10 +106,9 @@ export class MatrixLivekitMerger { return combineLatest([ membershipsWithTransport$, this.connectionManager.connectionManagerData$, - displaynameMap$, ]).pipe( - map(([memberships, managerData, displayNameMap]) => { - const items: MatrixLivekitItem[] = memberships.map( + map(([memberships, managerData]) => { + const items: MatrixLivekitMember[] = memberships.map( ({ membership, transport }) => { // TODO! cannot use membership.membershipID yet, Currently its hardcoded by the jwt service to const participantId = /*membership.membershipID*/ `${membership.userId}:${membership.deviceId}`; @@ -123,14 +126,23 @@ export class MatrixLivekitMerger { const connection = transport ? managerData.getConnectionForTransport(transport) : undefined; + const displayName$ = this.scope.behavior( + displaynameMap$.pipe( + map( + (displayNameMap) => + displayNameMap.get(membership.membershipID) ?? "---", + ), + ), + ); return { participant, membership, connection, // This makes sense to add the the js-sdk callMembership (we only need the avatar so probably the call memberhsip just should aquire the avatar) member, - displayName: displayNameMap.get(membership.membershipID) ?? "---", + displayName$, mxcAvatarUrl: member?.getMxcAvatarUrl(), + participantId, }; }, ); From c19e2245c81df2cc06d4e4f45f6d2f28cd64a08b Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 5 Nov 2025 18:57:24 +0100 Subject: [PATCH 22/75] use createSomething$ mathods instead of classes Rename several classes/behaviors to factory-style creators and adapt call wiring and tests accordingly: - Replace ConnectionManager class with createConnectionManager$ which returns transports$, connectionManagerData$, connections$ - Convert MatrixLivekitMerger to createMatrixLivekitMembers$ (matrixLivekitMerger$) - Rename sessionBehaviors$, localMembership$, localTransport$ to createSessionMembershipsAndTransports$, createLocalMembership$, createLocalTransport$ - Adjust participant types and hook up connectOptions$; expose join via localMembership.requestConnect - Update tests to use the new factory APIs --- src/state/CallViewModel.ts | 65 ++++++------- src/state/SessionBehaviors.ts | 18 +--- src/state/localMember/LocalMembership.ts | 13 +-- src/state/localMember/LocalTransport.ts | 2 +- src/state/remoteMembers/Connection.ts | 5 +- .../remoteMembers/ConnectionManager.test.ts | 36 +++---- src/state/remoteMembers/ConnectionManager.ts | 96 +++++++++++-------- .../remoteMembers/MatrixLivekitMerger.test.ts | 6 +- src/state/remoteMembers/integration.test.ts | 6 +- .../remoteMembers/matrixLivekitMerger.ts | 83 ++++++++-------- 10 files changed, 167 insertions(+), 163 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index c88348d6..e2cd6c55 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -110,15 +110,12 @@ import { } from "./layout-types.ts"; import { type ElementCallError } from "../utils/errors.ts"; import { type ObservableScope } from "./ObservableScope.ts"; -import { ConnectionManager } from "./remoteMembers/ConnectionManager.ts"; -import { MatrixLivekitMerger } from "./remoteMembers/matrixLivekitMerger.ts"; -import { - localMembership$, - type LocalMemberState, -} from "./localMember/LocalMembership.ts"; -import { localTransport$ as computeLocalTransport$ } from "./localMember/LocalTransport.ts"; -import { sessionBehaviors$ } from "./SessionBehaviors.ts"; +import { createMatrixLivekitMembers$ } from "./remoteMembers/matrixLivekitMerger.ts"; +import { createLocalMembership$ } from "./localMember/LocalMembership.ts"; +import { createLocalTransport$ } from "./localMember/LocalTransport.ts"; +import { createSessionMembershipsAndTransports$ } from "./SessionBehaviors.ts"; import { ECConnectionFactory } from "./remoteMembers/ConnectionFactory.ts"; +import { createConnectionManager$ } from "./remoteMembers/ConnectionManager.ts"; //TODO // Larger rename @@ -192,13 +189,13 @@ export class CallViewModel { } : undefined; - private sessionBehaviors = sessionBehaviors$({ + private sessionBehaviors = createSessionMembershipsAndTransports$({ scope: this.scope, matrixRTCSession: this.matrixRTCSession, }); private memberships$ = this.sessionBehaviors.memberships$; - private localTransport$ = computeLocalTransport$({ + private localTransport$ = createLocalTransport$({ scope: this.scope, memberships$: this.memberships$, client: this.matrixRoom.client, @@ -229,25 +226,34 @@ export class CallViewModel { ), ); - private connectionManager = new ConnectionManager( - this.scope, - this.connectionFactory, - this.allTransports$, - ); + private connectionManager = createConnectionManager$({ + scope: this.scope, + connectionFactory: this.connectionFactory, + inputTransports$: this.allTransports$, + }); // ------------------------------------------------------------------------ - private matrixLivekitMerger = new MatrixLivekitMerger( - this.scope, - this.sessionBehaviors.membershipsWithTransport$, - this.connectionManager, - this.matrixRoom, - this.userId, - this.deviceId, - ); - private matrixLivekitMembers$ = this.matrixLivekitMerger.matrixLivekitMember$; + private matrixLivekitMembers$ = createMatrixLivekitMembers$({ + scope: this.scope, + membershipsWithTransport$: this.sessionBehaviors.membershipsWithTransport$, + connectionManager: this.connectionManager, + matrixRoom: this.matrixRoom, + userId: this.userId, + deviceId: this.deviceId, + }); - private localMembership = localMembership$({ + private connectOptions$ = this.scope.behavior( + matrixRTCMode.value$.pipe( + map((mode) => ({ + encryptMedia: this.e2eeLivekitOptions !== undefined, + // TODO. This might need to get called again on each cahnge of matrixRTCMode... + matrixRTCMode: mode, + })), + ), + ); + + private localMembership = createLocalMembership$({ scope: this.scope, muteStates: this.muteStates, mediaDevices: this.mediaDevices, @@ -258,6 +264,7 @@ export class CallViewModel { e2eeLivekitOptions: this.e2eeLivekitOptions, trackProcessorState$: this.trackProcessorState$, widget, + options: this.connectOptions$, }); /** @@ -269,13 +276,7 @@ export class CallViewModel { return this.localMembership.configError$; } - public join(): LocalMemberState { - return this.localMembership.requestConnect({ - encryptMedia: this.e2eeLivekitOptions !== undefined, - // TODO. This might need to get called again on each cahnge of matrixRTCMode... - matrixRTCMode: matrixRTCMode.getValue(), - }); - } + public join = this.localMembership.requestConnect; // CODESMELL? // This is functionally the same Observable as leave$, except here it's diff --git a/src/state/SessionBehaviors.ts b/src/state/SessionBehaviors.ts index 250ad86a..aad6094e 100644 --- a/src/state/SessionBehaviors.ts +++ b/src/state/SessionBehaviors.ts @@ -22,24 +22,16 @@ interface Props { matrixRTCSession: MatrixRTCSession; } -/** - * Wraps behaviors that we extract from an matrixRTCSession. - */ -interface RxRtcSession { - /** - * some prop - */ +export const createSessionMembershipsAndTransports$ = ({ + scope, + matrixRTCSession, +}: Props): { memberships$: Behavior; membershipsWithTransport$: Behavior< { membership: CallMembership; transport?: LivekitTransport }[] >; transports$: Behavior; -} - -export const sessionBehaviors$ = ({ - scope, - matrixRTCSession, -}: Props): RxRtcSession => { +} => { const memberships$ = scope.behavior( fromEvent( matrixRTCSession, diff --git a/src/state/localMember/LocalMembership.ts b/src/state/localMember/LocalMembership.ts index 33a54574..41e199d1 100644 --- a/src/state/localMember/LocalMembership.ts +++ b/src/state/localMember/LocalMembership.ts @@ -28,7 +28,7 @@ import { import { logger } from "matrix-js-sdk/lib/logger"; import { type Behavior } from "../Behavior"; -import { type ConnectionManager } from "../remoteMembers/ConnectionManager"; +import { type createConnectionManager$ } from "../remoteMembers/ConnectionManager"; import { ObservableScope } from "../ObservableScope"; import { Publisher } from "./Publisher"; import { type MuteStates } from "../MuteStates"; @@ -90,7 +90,7 @@ interface Props { scope: ObservableScope; mediaDevices: MediaDevices; muteStates: MuteStates; - connectionManager: ConnectionManager; + connectionManager: ReturnType; matrixRTCSession: MatrixRTCSession; matrixRoom: MatrixRoom; localTransport$: Behavior; @@ -111,7 +111,7 @@ interface Props { * - transport$: the transport object the ownMembership$ ended up using. * */ -export const localMembership$ = ({ +export const createLocalMembership$ = ({ scope, options, muteStates, @@ -151,13 +151,14 @@ export const localMembership$ = ({ const tracks$ = new BehaviorSubject([]); const connection$ = scope.behavior( - combineLatest([connectionManager.connections$, localTransport$]).pipe( - map(([connections, transport]) => { + combineLatest( + [connectionManager.connections$, localTransport$], + (connections, transport) => { if (transport === undefined) return undefined; return connections.find((connection) => areLivekitTransportsEqual(connection.transport, transport), ); - }), + }, ), ); /** diff --git a/src/state/localMember/LocalTransport.ts b/src/state/localMember/LocalTransport.ts index 9fada195..a1b0d329 100644 --- a/src/state/localMember/LocalTransport.ts +++ b/src/state/localMember/LocalTransport.ts @@ -50,7 +50,7 @@ interface Props { * @prop useOldestMember Whether to use the same transport as the oldest member. * This will only update once the first oldest member appears. Will not recompute if the oldest member leaves. */ -export const localTransport$ = ({ +export const createLocalTransport$ = ({ scope, memberships$, client, diff --git a/src/state/remoteMembers/Connection.ts b/src/state/remoteMembers/Connection.ts index 03a9e137..b9cfe71f 100644 --- a/src/state/remoteMembers/Connection.ts +++ b/src/state/remoteMembers/Connection.ts @@ -13,7 +13,8 @@ import { ConnectionError, type ConnectionState as LivekitConenctionState, type Room as LivekitRoom, - type Participant, + type LocalParticipant, + type RemoteParticipant, RoomEvent, } from "livekit-client"; import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; @@ -32,7 +33,7 @@ import { SFURoomCreationRestrictedError, } from "../../utils/errors.ts"; -export type PublishingParticipant = Participant; +export type PublishingParticipant = LocalParticipant | RemoteParticipant; export interface ConnectionOpts { /** The media transport to connect to. */ diff --git a/src/state/remoteMembers/ConnectionManager.test.ts b/src/state/remoteMembers/ConnectionManager.test.ts index 48c897e3..be2211f8 100644 --- a/src/state/remoteMembers/ConnectionManager.test.ts +++ b/src/state/remoteMembers/ConnectionManager.test.ts @@ -6,13 +6,12 @@ Please see LICENSE in the repository root for full details. */ import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; -import { logger } from "matrix-js-sdk/lib/logger"; import { BehaviorSubject } from "rxjs"; import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { type Participant as LivekitParticipant } from "livekit-client"; import { ObservableScope } from "../ObservableScope.ts"; -import { ConnectionManager } from "./ConnectionManager.ts"; +import { createConnectionManager$ } from "./ConnectionManager.ts"; import { type ConnectionFactory } from "./ConnectionFactory.ts"; import { type Connection } from "./Connection.ts"; import { areLivekitTransportsEqual } from "./matrixLivekitMerger.ts"; @@ -37,15 +36,15 @@ const TRANSPORT_2: LivekitTransport = { // livekit_service_url: "https://lk-other.sample.com", // livekit_alias: "!alias:sample.com", // }; - -let testScope: ObservableScope; let fakeConnectionFactory: ConnectionFactory; - +let testScope: ObservableScope; let testTransportStream$: BehaviorSubject; - -// The connection manager under test -let manager: ConnectionManager; - +let connectionManagerInputs: { + scope: ObservableScope; + connectionFactory: ConnectionFactory; + inputTransports$: BehaviorSubject; +}; +let manager: ReturnType; beforeEach(() => { testScope = new ObservableScope(); @@ -68,9 +67,12 @@ beforeEach(() => { ); testTransportStream$ = new BehaviorSubject([]); - - manager = new ConnectionManager(testScope, fakeConnectionFactory, logger); - manager.registerTransports(testTransportStream$); + connectionManagerInputs = { + scope: testScope, + connectionFactory: fakeConnectionFactory, + inputTransports$: testTransportStream$, + }; + manager = createConnectionManager$(connectionManagerInputs); }); afterEach(() => { @@ -84,7 +86,7 @@ describe("connections$ stream", () => { if (connections.length > 0) managedConnections.resolve(connections); }); - testTransportStream$.next([TRANSPORT_1, TRANSPORT_2]); + connectionManagerInputs.inputTransports$.next([TRANSPORT_1, TRANSPORT_2]); const connections = await managedConnections.promise; @@ -211,11 +213,13 @@ describe("connectionManagerData$ stream", () => { test("Should report connections with the publishing participants", () => { withTestScheduler(({ expectObservable, schedule, behavior }) => { - manager.registerTransports( - behavior("a", { + manager = createConnectionManager$({ + ...connectionManagerInputs, + inputTransports$: behavior("a", { a: [TRANSPORT_1, TRANSPORT_2], }), - ); + }); + const conn1Participants$ = fakePublishingParticipantsStreams.get( keyForTransport(TRANSPORT_1), )!; diff --git a/src/state/remoteMembers/ConnectionManager.ts b/src/state/remoteMembers/ConnectionManager.ts index d0bbfe6f..f596de2d 100644 --- a/src/state/remoteMembers/ConnectionManager.ts +++ b/src/state/remoteMembers/ConnectionManager.ts @@ -14,8 +14,8 @@ import { type ParticipantId, } from "matrix-js-sdk/lib/matrixrtc"; import { BehaviorSubject, combineLatest, map, switchMap } from "rxjs"; -import { logger, type Logger } from "matrix-js-sdk/lib/logger"; -import { type Participant as LivekitParticipant } from "livekit-client"; +import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; +import { type LocalParticipant, type RemoteParticipant } from "livekit-client"; import { type Behavior } from "../Behavior"; import { type Connection } from "./Connection"; @@ -25,12 +25,17 @@ import { areLivekitTransportsEqual } from "./matrixLivekitMerger"; import { type ConnectionFactory } from "./ConnectionFactory.ts"; export class ConnectionManagerData { - private readonly store: Map = - new Map(); + private readonly store: Map< + string, + [Connection, (LocalParticipant | RemoteParticipant)[]] + > = new Map(); public constructor() {} - public add(connection: Connection, participants: LivekitParticipant[]): void { + public add( + connection: Connection, + participants: (LocalParticipant | RemoteParticipant)[], + ): void { const key = this.getKey(connection.transport); const existing = this.store.get(key); if (!existing) { @@ -56,7 +61,7 @@ export class ConnectionManagerData { public getParticipantForTransport( transport: LivekitTransport, - ): LivekitParticipant[] { + ): (LocalParticipant | RemoteParticipant)[] { const key = transport.livekit_service_url + "|" + transport.livekit_alias; const existing = this.store.get(key); if (existing) { @@ -82,35 +87,41 @@ export class ConnectionManagerData { return connections; } } - +interface Props { + scope: ObservableScope; + connectionFactory: ConnectionFactory; + inputTransports$: Behavior; +} // TODO - write test for scopes (do we really need to bind scope) -export class ConnectionManager { - private readonly logger: Logger; - private running$ = new BehaviorSubject(true); - /** - * Crete a `ConnectionManager` - * @param scope the observable scope used by this object. - * @param connectionFactory used to create new connections. - * @param _transportsSubscriptions$ A list of Behaviors each containing a LIST of LivekitTransport. - * Each of these behaviors can be interpreted as subscribed list of transports. - * - * Using `registerTransports` independent external modules can control what connections - * are created by the ConnectionManager. - * - * The connection manager will remove all duplicate transports in each subscibed list. - * - * See `unregisterAllTransports` and `unregisterTransport` for details on how to unsubscribe. - */ - public constructor( - private readonly scope: ObservableScope, - private readonly connectionFactory: ConnectionFactory, - private readonly inputTransports$: Behavior, - ) { - // TODO logger: only construct one logger from the client and make it compatible via a EC specific sing - this.logger = logger.getChild("ConnectionManager"); - scope.onEnd(() => this.running$.next(false)); - } +/** + * Crete a `ConnectionManager` + * @param scope the observable scope used by this object. + * @param connectionFactory used to create new connections. + * @param _transportsSubscriptions$ A list of Behaviors each containing a LIST of LivekitTransport. + * Each of these behaviors can be interpreted as subscribed list of transports. + * + * Using `registerTransports` independent external modules can control what connections + * are created by the ConnectionManager. + * + * The connection manager will remove all duplicate transports in each subscibed list. + * + * See `unregisterAllTransports` and `unregisterTransport` for details on how to unsubscribe. + */ +export function createConnectionManager$({ + scope, + connectionFactory, + inputTransports$, +}: Props): { + transports$: Behavior; + connectionManagerData$: Behavior; + connections$: Behavior; +} { + const logger = rootLogger.getChild("ConnectionManager"); + + const running$ = new BehaviorSubject(true); + scope.onEnd(() => running$.next(false)); + // TODO logger: only construct one logger from the client and make it compatible via a EC specific sing /** * All transports currently managed by the ConnectionManager. @@ -120,8 +131,8 @@ export class ConnectionManager { * It is build based on the list of subscribed transports (`transportsSubscriptions$`). * externally this is modified via `registerTransports()`. */ - private readonly transports$ = this.scope.behavior( - combineLatest([this.running$, this.inputTransports$]).pipe( + const transports$ = scope.behavior( + combineLatest([running$, inputTransports$]).pipe( map(([running, transports]) => (running ? transports : [])), map(removeDuplicateTransports), ), @@ -130,19 +141,19 @@ export class ConnectionManager { /** * Connections for each transport in use by one or more session members. */ - public readonly connections$ = this.scope.behavior( + const connections$ = scope.behavior( generateKeyed$( - this.transports$, + transports$, (transports, createOrGet) => { const createConnection = ( transport: LivekitTransport, ): ((scope: ObservableScope) => Connection) => (scope) => { - const connection = this.connectionFactory.createConnection( + const connection = connectionFactory.createConnection( transport, scope, - this.logger, + logger, ); // Start the connection immediately // Use connection state to track connection progress @@ -160,9 +171,9 @@ export class ConnectionManager { ), ); - public connectionManagerData$: Behavior = - this.scope.behavior( - this.connections$.pipe( + const connectionManagerData$: Behavior = + scope.behavior( + connections$.pipe( switchMap((connections) => { // Map the connections to list of {connection, participants}[] const listOfConnectionsWithPublishingParticipants = connections.map( @@ -191,6 +202,7 @@ export class ConnectionManager { // start empty new ConnectionManagerData(), ); + return { transports$, connectionManagerData$, connections$ }; } function removeDuplicateTransports( diff --git a/src/state/remoteMembers/MatrixLivekitMerger.test.ts b/src/state/remoteMembers/MatrixLivekitMerger.test.ts index 16fc9c0c..6b0fad5f 100644 --- a/src/state/remoteMembers/MatrixLivekitMerger.test.ts +++ b/src/state/remoteMembers/MatrixLivekitMerger.test.ts @@ -24,7 +24,7 @@ import { getParticipantId } from "matrix-js-sdk/lib/matrixrtc/utils"; import { type MatrixLivekitMember, - MatrixLivekitMerger, + matrixLivekitMerger$, } from "./matrixLivekitMerger"; import { ObservableScope } from "../ObservableScope"; import { @@ -44,7 +44,7 @@ const userId = "@local:example.com"; const deviceId = "DEVICE000"; // The merger beeing tested -let matrixLivekitMerger: MatrixLivekitMerger; +let matrixLivekitMerger: matrixLivekitMerger$; beforeEach(() => { testScope = new ObservableScope(); @@ -62,7 +62,7 @@ beforeEach(() => { removeEventListener: vi.fn(), } as unknown as MatrixRoom); - matrixLivekitMerger = new MatrixLivekitMerger( + matrixLivekitMerger = new matrixLivekitMerger$( testScope, fakeMemberships$, mockConnectionManager, diff --git a/src/state/remoteMembers/integration.test.ts b/src/state/remoteMembers/integration.test.ts index 36594680..5b3cfe7c 100644 --- a/src/state/remoteMembers/integration.test.ts +++ b/src/state/remoteMembers/integration.test.ts @@ -18,7 +18,7 @@ import { ECConnectionFactory } from "./ConnectionFactory.ts"; import { type OpenIDClientParts } from "../../livekit/openIDSFU.ts"; import { mockMediaDevices, withTestScheduler } from "../../utils/test"; import { type ProcessorState } from "../../livekit/TrackProcessorContext.tsx"; -import { MatrixLivekitMerger } from "./matrixLivekitMerger.ts"; +import { matrixLivekitMerger$ } from "./matrixLivekitMerger.ts"; import type { CallMembership, Transport } from "matrix-js-sdk/lib/matrixrtc"; import { TRANSPORT_1 } from "./ConnectionManager.test.ts"; @@ -39,9 +39,9 @@ let connectionManager: ConnectionManager; function createLkMerger( memberships$: Observable, -): MatrixLivekitMerger { +): matrixLivekitMerger$ { const mockRoomEmitter = new EventEmitter(); - return new MatrixLivekitMerger( + return new matrixLivekitMerger$( testScope, memberships$, connectionManager, diff --git a/src/state/remoteMembers/matrixLivekitMerger.ts b/src/state/remoteMembers/matrixLivekitMerger.ts index 94e0ebd5..cb9f1709 100644 --- a/src/state/remoteMembers/matrixLivekitMerger.ts +++ b/src/state/remoteMembers/matrixLivekitMerger.ts @@ -16,12 +16,12 @@ import { import { combineLatest, map, startWith, type Observable } from "rxjs"; // eslint-disable-next-line rxjs/no-internal import { type NodeStyleEventEmitter } from "rxjs/internal/observable/fromEvent"; +import { type Room as MatrixRoom, type RoomMember } from "matrix-js-sdk"; -import type { Room as MatrixRoom, RoomMember } from "matrix-js-sdk"; // import type { Logger } from "matrix-js-sdk/lib/logger"; import { type Behavior } from "../Behavior"; import { type ObservableScope } from "../ObservableScope"; -import { type ConnectionManager } from "./ConnectionManager"; +import { type createConnectionManager$ } from "./ConnectionManager"; import { getRoomMemberFromRtcMember, memberDisplaynames$ } from "./displayname"; import { type Connection } from "./Connection"; @@ -45,11 +45,25 @@ export interface MatrixLivekitMember { participantId: string; } +interface Props { + scope: ObservableScope; + membershipsWithTransport$: Behavior< + { membership: CallMembership; transport?: LivekitTransport }[] + >; + connectionManager: ReturnType; + // TODO this is too much information for that class, + // apparently needed to get a room member to later get the Avatar + // => Extract an AvatarService instead? + // Better with just `getMember` + matrixRoom: Pick & NodeStyleEventEmitter; + userId: string; + deviceId: string; +} // Alternative structure idea: // const livekitMatrixMember$ = (callMemberships$,connectionManager,scope): Observable => { /** - * Combines MatrixRtc and Livekit worlds. + * Combines MatrixRTC and Livekit worlds. * * It has a small public interface: * - in (via constructor): @@ -58,54 +72,30 @@ export interface MatrixLivekitMember { * - out (via public Observable): * - `remoteMatrixLivekitMember` an observable of MatrixLivekitMember[] to track the remote members and associated livekit data. */ -export class MatrixLivekitMerger { +export function createMatrixLivekitMembers$({ + scope, + membershipsWithTransport$, + connectionManager, + matrixRoom, + userId, + deviceId, +}: Props): Behavior { /** * Stream of all the call members and their associated livekit data (if available). */ - public matrixLivekitMember$: Behavior; - // private readonly logger: Logger; - - public constructor( - private scope: ObservableScope, - private membershipsWithTransport$: Behavior< - { membership: CallMembership; transport?: LivekitTransport }[] - >, - private connectionManager: ConnectionManager, - // TODO this is too much information for that class, - // apparently needed to get a room member to later get the Avatar - // => Extract an AvatarService instead? - // Better with just `getMember` - private matrixRoom: Pick & NodeStyleEventEmitter, - private userId: string, - private deviceId: string, - // parentLogger: Logger, - ) { - // this.logger = parentLogger.getChild("MatrixLivekitMerger"); - - this.matrixLivekitMember$ = this.scope.behavior( - this.start$().pipe(startWith([])), - ); - } - - // ======================================= - /// PRIVATES - // ======================================= - private start$(): Observable { + function createMatrixLivekitMember$(): Observable { const displaynameMap$ = memberDisplaynames$( - this.scope, - this.matrixRoom, - this.membershipsWithTransport$.pipe( - map((v) => v.map((v) => v.membership)), - ), - this.userId, - this.deviceId, + scope, + matrixRoom, + membershipsWithTransport$.pipe(map((v) => v.map((v) => v.membership))), + userId, + deviceId, ); - const membershipsWithTransport$ = this.membershipsWithTransport$; return combineLatest([ membershipsWithTransport$, - this.connectionManager.connectionManagerData$, + connectionManager.connectionManagerData$, ]).pipe( map(([memberships, managerData]) => { const items: MatrixLivekitMember[] = memberships.map( @@ -121,12 +111,12 @@ export class MatrixLivekitMerger { ); const member = getRoomMemberFromRtcMember( membership, - this.matrixRoom, + matrixRoom, )?.member; const connection = transport ? managerData.getConnectionForTransport(transport) : undefined; - const displayName$ = this.scope.behavior( + const displayName$ = scope.behavior( displaynameMap$.pipe( map( (displayNameMap) => @@ -139,7 +129,8 @@ export class MatrixLivekitMerger { membership, connection, // This makes sense to add the the js-sdk callMembership (we only need the avatar so probably the call memberhsip just should aquire the avatar) - member, + // TODO Ugh this is hidign that it might be undefined!! best we remove the member entirely. + member: member as RoomMember, displayName$, mxcAvatarUrl: member?.getMxcAvatarUrl(), participantId, @@ -150,6 +141,8 @@ export class MatrixLivekitMerger { }), ); } + + return scope.behavior(createMatrixLivekitMember$().pipe(startWith([]))); } // TODO add back in the callviewmodel pauseWhen(this.pretendToBeDisconnected$) From d8e29467f609be005d0741cb5d47862246458ddc Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 5 Nov 2025 18:58:40 +0100 Subject: [PATCH 23/75] rename merger --- .../{matrixLivekitMerger.ts => MatrixLivekitMembers.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/state/remoteMembers/{matrixLivekitMerger.ts => MatrixLivekitMembers.ts} (100%) diff --git a/src/state/remoteMembers/matrixLivekitMerger.ts b/src/state/remoteMembers/MatrixLivekitMembers.ts similarity index 100% rename from src/state/remoteMembers/matrixLivekitMerger.ts rename to src/state/remoteMembers/MatrixLivekitMembers.ts From f9cc930155798c7e7e877d9942b85fd6a269a888 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 5 Nov 2025 18:49:21 +0000 Subject: [PATCH 24/75] Update Node.js to v24 --- .node-version | 2 +- package.json | 2 +- yarn.lock | 20 ++++++++++---------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.node-version b/.node-version index 2bd5a0a9..a45fd52c 100644 --- a/.node-version +++ b/.node-version @@ -1 +1 @@ -22 +24 diff --git a/package.json b/package.json index 35468c21..62ea9f4f 100644 --- a/package.json +++ b/package.json @@ -71,7 +71,7 @@ "@types/grecaptcha": "^3.0.9", "@types/jsdom": "^21.1.7", "@types/lodash-es": "^4.17.12", - "@types/node": "^22.0.0", + "@types/node": "^24.0.0", "@types/pako": "^2.0.3", "@types/qrcode": "^1.5.5", "@types/react": "^19.0.0", diff --git a/yarn.lock b/yarn.lock index e78dbbf2..9ac83900 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5247,12 +5247,12 @@ __metadata: languageName: node linkType: hard -"@types/node@npm:^22.0.0": - version: 22.17.0 - resolution: "@types/node@npm:22.17.0" +"@types/node@npm:^24.0.0": + version: 24.10.0 + resolution: "@types/node@npm:24.10.0" dependencies: - undici-types: "npm:~6.21.0" - checksum: 10c0/e1c603b660d3de3243dfc02ded5d40623ff3f36315ffbdd8cdc81bc2c5a8da172035879d437b72e9fa61ca01827f28e9c2b0c32898f411a8e9ba0a5efac0b4ca + undici-types: "npm:~7.16.0" + checksum: 10c0/f82ed7194e16f5590ef7afdc20c6d09068c76d50278b485ede8f0c5749683536e3064ffa8def8db76915196afb3724b854aa5723c64d6571b890b14492943b46 languageName: node linkType: hard @@ -7507,7 +7507,7 @@ __metadata: "@types/grecaptcha": "npm:^3.0.9" "@types/jsdom": "npm:^21.1.7" "@types/lodash-es": "npm:^4.17.12" - "@types/node": "npm:^22.0.0" + "@types/node": "npm:^24.0.0" "@types/pako": "npm:^2.0.3" "@types/qrcode": "npm:^1.5.5" "@types/react": "npm:^19.0.0" @@ -13603,10 +13603,10 @@ __metadata: languageName: node linkType: hard -"undici-types@npm:~6.21.0": - version: 6.21.0 - resolution: "undici-types@npm:6.21.0" - checksum: 10c0/c01ed51829b10aa72fc3ce64b747f8e74ae9b60eafa19a7b46ef624403508a54c526ffab06a14a26b3120d055e1104d7abe7c9017e83ced038ea5cf52f8d5e04 +"undici-types@npm:~7.16.0": + version: 7.16.0 + resolution: "undici-types@npm:7.16.0" + checksum: 10c0/3033e2f2b5c9f1504bdc5934646cb54e37ecaca0f9249c983f7b1fc2e87c6d18399ebb05dc7fd5419e02b2e915f734d872a65da2e3eeed1813951c427d33cc9a languageName: node linkType: hard From 6e1a58226505e78f7e5a0c046fb7b124afe8aa84 Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 6 Nov 2025 12:08:46 +0100 Subject: [PATCH 25/75] fix tests compilation peer session timo - wip --- src/state/CallViewModel.ts | 13 +- src/state/SessionBehaviors.ts | 61 +- src/state/localMember/LocalMembership.ts | 7 +- .../remoteMembers/ConnectionManager.test.ts | 9 +- src/state/remoteMembers/ConnectionManager.ts | 20 +- .../remoteMembers/MatrixLivekitMembers.ts | 9 +- .../remoteMembers/MatrixLivekitMerger.test.ts | 525 +++++++++++------- src/state/remoteMembers/displayname.test.ts | 299 ++++++++++ src/state/remoteMembers/displayname.ts | 33 +- src/state/remoteMembers/integration.test.ts | 174 +++--- src/utils/test.ts | 23 + 11 files changed, 861 insertions(+), 312 deletions(-) create mode 100644 src/state/remoteMembers/displayname.test.ts diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index e2cd6c55..d9d68da3 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -113,7 +113,11 @@ import { type ObservableScope } from "./ObservableScope.ts"; import { createMatrixLivekitMembers$ } from "./remoteMembers/matrixLivekitMerger.ts"; import { createLocalMembership$ } from "./localMember/LocalMembership.ts"; import { createLocalTransport$ } from "./localMember/LocalTransport.ts"; -import { createSessionMembershipsAndTransports$ } from "./SessionBehaviors.ts"; +import { + createMemberships$, + createSessionMembershipsAndTransports$, + membershipsAndTransports$, +} from "./SessionBehaviors.ts"; import { ECConnectionFactory } from "./remoteMembers/ConnectionFactory.ts"; import { createConnectionManager$ } from "./remoteMembers/ConnectionManager.ts"; @@ -189,11 +193,14 @@ export class CallViewModel { } : undefined; - private sessionBehaviors = createSessionMembershipsAndTransports$({ + private memberships$ = createMemberships$({ scope: this.scope, matrixRTCSession: this.matrixRTCSession, }); - private memberships$ = this.sessionBehaviors.memberships$; + private membershipsAndTransports = membershipsAndTransports$( + this.scope, + this.memberships$, + ); private localTransport$ = createLocalTransport$({ scope: this.scope, diff --git a/src/state/SessionBehaviors.ts b/src/state/SessionBehaviors.ts index aad6094e..ed207835 100644 --- a/src/state/SessionBehaviors.ts +++ b/src/state/SessionBehaviors.ts @@ -22,23 +22,15 @@ interface Props { matrixRTCSession: MatrixRTCSession; } -export const createSessionMembershipsAndTransports$ = ({ - scope, - matrixRTCSession, -}: Props): { - memberships$: Behavior; +export const membershipsAndTransports$ = ( + scope: ObservableScope, + memberships$: Behavior, +): { membershipsWithTransport$: Behavior< { membership: CallMembership; transport?: LivekitTransport }[] >; transports$: Behavior; } => { - const memberships$ = scope.behavior( - fromEvent( - matrixRTCSession, - MatrixRTCSessionEvent.MembershipsChanged, - (_, memberships: CallMembership[]) => memberships, - ), - ); /** * Lists the transports used by ourselves, plus all other MatrixRTC session * members. For completeness this also lists the preferred transport and @@ -47,9 +39,7 @@ export const createSessionMembershipsAndTransports$ = ({ * together when it might change together is what you have to do in RxJS to * avoid reading inconsistent state or observing too many changes.) */ - const membershipsWithTransport$: Behavior< - { membership: CallMembership; transport?: LivekitTransport }[] - > = scope.behavior( + const membershipsWithTransport$ = scope.behavior( memberships$.pipe( map((memberships) => { return memberships.map((membership) => { @@ -69,9 +59,48 @@ export const createSessionMembershipsAndTransports$ = ({ map((mts) => mts.flatMap(({ transport: t }) => (t ? [t] : []))), ), ); + return { - memberships$, membershipsWithTransport$, transports$, }; }; + +export const createMemberships$ = ({ + scope, + matrixRTCSession, +}: Props): Behavior => { + return scope.behavior( + fromEvent( + matrixRTCSession, + MatrixRTCSessionEvent.MembershipsChanged, + (_, memberships: CallMembership[]) => memberships, + ), + ); +}; + +export const createSessionMembershipsAndTransports$ = ({ + scope, + matrixRTCSession, +}: Props): { + memberships$: Behavior; + membershipsWithTransport$: Behavior< + { membership: CallMembership; transport?: LivekitTransport }[] + >; + transports$: Behavior; +} => { + const memberships$ = scope.behavior( + fromEvent( + matrixRTCSession, + MatrixRTCSessionEvent.MembershipsChanged, + (_, memberships: CallMembership[]) => memberships, + ), + ); + + const memberAndTransport = membershipsAndTransports$(scope, memberships$); + + return { + memberships$, + ...memberAndTransport, + }; +}; diff --git a/src/state/localMember/LocalMembership.ts b/src/state/localMember/LocalMembership.ts index 41e199d1..83337064 100644 --- a/src/state/localMember/LocalMembership.ts +++ b/src/state/localMember/LocalMembership.ts @@ -28,7 +28,10 @@ import { import { logger } from "matrix-js-sdk/lib/logger"; import { type Behavior } from "../Behavior"; -import { type createConnectionManager$ } from "../remoteMembers/ConnectionManager"; +import { + type ConnectionManagerReturn, + type createConnectionManager$, +} from "../remoteMembers/ConnectionManager"; import { ObservableScope } from "../ObservableScope"; import { Publisher } from "./Publisher"; import { type MuteStates } from "../MuteStates"; @@ -90,7 +93,7 @@ interface Props { scope: ObservableScope; mediaDevices: MediaDevices; muteStates: MuteStates; - connectionManager: ReturnType; + connectionManager: ConnectionManagerReturn; matrixRTCSession: MatrixRTCSession; matrixRoom: MatrixRoom; localTransport$: Behavior; diff --git a/src/state/remoteMembers/ConnectionManager.test.ts b/src/state/remoteMembers/ConnectionManager.test.ts index be2211f8..8be5bc35 100644 --- a/src/state/remoteMembers/ConnectionManager.test.ts +++ b/src/state/remoteMembers/ConnectionManager.test.ts @@ -11,11 +11,14 @@ import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { type Participant as LivekitParticipant } from "livekit-client"; import { ObservableScope } from "../ObservableScope.ts"; -import { createConnectionManager$ } from "./ConnectionManager.ts"; +import { + type ConnectionManagerReturn, + createConnectionManager$, +} from "./ConnectionManager.ts"; import { type ConnectionFactory } from "./ConnectionFactory.ts"; import { type Connection } from "./Connection.ts"; -import { areLivekitTransportsEqual } from "./matrixLivekitMerger.ts"; import { flushPromises, withTestScheduler } from "../../utils/test.ts"; +import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts"; // Some test constants @@ -44,7 +47,7 @@ let connectionManagerInputs: { connectionFactory: ConnectionFactory; inputTransports$: BehaviorSubject; }; -let manager: ReturnType; +let manager: ConnectionManagerReturn; beforeEach(() => { testScope = new ObservableScope(); diff --git a/src/state/remoteMembers/ConnectionManager.ts b/src/state/remoteMembers/ConnectionManager.ts index f596de2d..2cb6957d 100644 --- a/src/state/remoteMembers/ConnectionManager.ts +++ b/src/state/remoteMembers/ConnectionManager.ts @@ -21,7 +21,7 @@ import { type Behavior } from "../Behavior"; import { type Connection } from "./Connection"; import { type ObservableScope } from "../ObservableScope"; import { generateKeyed$ } from "../../utils/observable"; -import { areLivekitTransportsEqual } from "./matrixLivekitMerger"; +import { areLivekitTransportsEqual } from "./MatrixLivekitMembers"; import { type ConnectionFactory } from "./ConnectionFactory.ts"; export class ConnectionManagerData { @@ -94,6 +94,12 @@ interface Props { } // TODO - write test for scopes (do we really need to bind scope) +export interface ConnectionManagerReturn { + deduplicatedTransports$: Behavior; + connectionManagerData$: Behavior; + connections$: Behavior; +} + /** * Crete a `ConnectionManager` * @param scope the observable scope used by this object. @@ -112,11 +118,7 @@ export function createConnectionManager$({ scope, connectionFactory, inputTransports$, -}: Props): { - transports$: Behavior; - connectionManagerData$: Behavior; - connections$: Behavior; -} { +}: Props): ConnectionManagerReturn { const logger = rootLogger.getChild("ConnectionManager"); const running$ = new BehaviorSubject(true); @@ -131,7 +133,7 @@ export function createConnectionManager$({ * It is build based on the list of subscribed transports (`transportsSubscriptions$`). * externally this is modified via `registerTransports()`. */ - const transports$ = scope.behavior( + const deduplicatedTransports$ = scope.behavior( combineLatest([running$, inputTransports$]).pipe( map(([running, transports]) => (running ? transports : [])), map(removeDuplicateTransports), @@ -143,7 +145,7 @@ export function createConnectionManager$({ */ const connections$ = scope.behavior( generateKeyed$( - transports$, + deduplicatedTransports$, (transports, createOrGet) => { const createConnection = ( @@ -202,7 +204,7 @@ export function createConnectionManager$({ // start empty new ConnectionManagerData(), ); - return { transports$, connectionManagerData$, connections$ }; + return { deduplicatedTransports$, connectionManagerData$, connections$ }; } function removeDuplicateTransports( diff --git a/src/state/remoteMembers/MatrixLivekitMembers.ts b/src/state/remoteMembers/MatrixLivekitMembers.ts index cb9f1709..dd54d092 100644 --- a/src/state/remoteMembers/MatrixLivekitMembers.ts +++ b/src/state/remoteMembers/MatrixLivekitMembers.ts @@ -13,15 +13,14 @@ import { type LivekitTransport, type CallMembership, } from "matrix-js-sdk/lib/matrixrtc"; -import { combineLatest, map, startWith, type Observable } from "rxjs"; +import { combineLatest, map, type Observable } from "rxjs"; // eslint-disable-next-line rxjs/no-internal import { type NodeStyleEventEmitter } from "rxjs/internal/observable/fromEvent"; import { type Room as MatrixRoom, type RoomMember } from "matrix-js-sdk"; -// import type { Logger } from "matrix-js-sdk/lib/logger"; import { type Behavior } from "../Behavior"; import { type ObservableScope } from "../ObservableScope"; -import { type createConnectionManager$ } from "./ConnectionManager"; +import type * as ConnectionManager from "./ConnectionManager"; import { getRoomMemberFromRtcMember, memberDisplaynames$ } from "./displayname"; import { type Connection } from "./Connection"; @@ -50,7 +49,7 @@ interface Props { membershipsWithTransport$: Behavior< { membership: CallMembership; transport?: LivekitTransport }[] >; - connectionManager: ReturnType; + connectionManager: ConnectionManager.ConnectionManagerReturn; // TODO this is too much information for that class, // apparently needed to get a room member to later get the Avatar // => Extract an AvatarService instead? @@ -142,7 +141,7 @@ export function createMatrixLivekitMembers$({ ); } - return scope.behavior(createMatrixLivekitMember$().pipe(startWith([]))); + return scope.behavior(createMatrixLivekitMember$(), []); } // TODO add back in the callviewmodel pauseWhen(this.pretendToBeDisconnected$) diff --git a/src/state/remoteMembers/MatrixLivekitMerger.test.ts b/src/state/remoteMembers/MatrixLivekitMerger.test.ts index 6b0fad5f..71a1398c 100644 --- a/src/state/remoteMembers/MatrixLivekitMerger.test.ts +++ b/src/state/remoteMembers/MatrixLivekitMerger.test.ts @@ -5,71 +5,51 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { - describe, - test, - vi, - expect, - beforeEach, - afterEach, - type MockedObject, -} from "vitest"; -import { BehaviorSubject, take } from "rxjs"; +import { describe, test, vi, expect, beforeEach, afterEach } from "vitest"; +import { BehaviorSubject } from "rxjs"; import { type CallMembership, type LivekitTransport, } from "matrix-js-sdk/lib/matrixrtc"; -import { type Room as MatrixRoom } from "matrix-js-sdk"; +import { type Room as MatrixRoom, type RoomMember } from "matrix-js-sdk"; import { getParticipantId } from "matrix-js-sdk/lib/matrixrtc/utils"; +import { type ConnectionManagerReturn } from "./ConnectionManager.ts"; import { type MatrixLivekitMember, - matrixLivekitMerger$, -} from "./matrixLivekitMerger"; + createMatrixLivekitMembers$, + areLivekitTransportsEqual, +} from "./MatrixLivekitMembers"; import { ObservableScope } from "../ObservableScope"; +import { ConnectionManagerData } from "./ConnectionManager"; import { - type ConnectionManager, - ConnectionManagerData, -} from "./ConnectionManager"; -import { aliceRtcMember } from "../../utils/test-fixtures"; -import { mockRemoteParticipant } from "../../utils/test.ts"; + mockCallMembership, + mockRemoteParticipant, + type OurRunHelpers, + withTestScheduler, +} from "../../utils/test.ts"; import { type Connection } from "./Connection.ts"; let testScope: ObservableScope; -let fakeManagerData$: BehaviorSubject; -let fakeMemberships$: BehaviorSubject; -let mockConnectionManager: MockedObject; let mockMatrixRoom: MatrixRoom; const userId = "@local:example.com"; const deviceId = "DEVICE000"; // The merger beeing tested -let matrixLivekitMerger: matrixLivekitMerger$; beforeEach(() => { testScope = new ObservableScope(); - fakeMemberships$ = new BehaviorSubject([]); - fakeManagerData$ = new BehaviorSubject( - new ConnectionManagerData(), - ); - mockConnectionManager = vi.mocked({ - registerTransports: vi.fn(), - connectionManagerData$: fakeManagerData$, - } as unknown as ConnectionManager); mockMatrixRoom = vi.mocked({ - getMember: vi.fn().mockReturnValue(null), + getMember: vi.fn().mockImplementation((userId: string) => { + return { + userId, + rawDisplayName: userId.replace("@", "").replace(":example.org", ""), + getMxcAvatarUrl: vi.fn().mockReturnValue(null), + } as unknown as RoomMember; + }), addEventListener: vi.fn(), removeEventListener: vi.fn(), } as unknown as MatrixRoom); - - matrixLivekitMerger = new matrixLivekitMerger$( - testScope, - fakeMemberships$, - mockConnectionManager, - mockMatrixRoom, - userId, - deviceId, - ); }); afterEach(() => { @@ -77,186 +57,357 @@ afterEach(() => { }); test("should signal participant not yet connected to livekit", () => { - fakeMemberships$.next([aliceRtcMember]); + withTestScheduler(({ behavior, expectObservable }) => { + const bobMembership = { + userId: "@bob:example.org", + deviceId: "DEV000", + transports: [ + { + type: "livekit", + livekit_service_url: "https://lk.example.org", + livekit_alias: "!alias:example.org", + }, + ], + } as unknown as CallMembership; - let items: MatrixLivekitMember[] = []; - matrixLivekitMerger.matrixLivekitMember$ - .pipe(take(1)) - .subscribe((emitted) => { - items = emitted; + const matrixLivekitMember$ = createMatrixLivekitMembers$({ + scope: testScope, + membershipsWithTransport$: behavior("a", { + a: [ + { + membership: bobMembership, + }, + ], + }), + connectionManager: { + connectionManagerData$: behavior("a", { + a: new ConnectionManagerData(), + }), + transports$: behavior("a", { a: [] }), + connections$: behavior("a", { a: [] }), + }, + matrixRoom: mockMatrixRoom, + userId, + deviceId, }); - expect(items).toHaveLength(1); - const item = items[0]; - - // Assert the expected membership - expect(item.membership).toBe(aliceRtcMember); - - // Assert participant & connection are absent (not just `undefined`) - expect(item.participant).not.toBeDefined(); - expect(item.participant).not.toBeDefined(); + expectObservable(matrixLivekitMember$).toBe("a", { + a: expect.toSatisfy((data: MatrixLivekitMember[]) => { + return ( + data.length == 1 && + data[0].membership === bobMembership && + data[0].participant === undefined && + data[0].connection === undefined + ); + }), + }); + }); }); +function aConnectionManager( + data: ConnectionManagerData, + behavior: Pick, +): ConnectionManagerReturn { + return { + connectionManagerData$: behavior("a", { a: data }), + transports$: behavior("a", { + a: [data.getConnections().map((connection) => connection.transport)], + }), + connections$: behavior("a", { a: [data.getConnections()] }), + }; +} + test("should signal participant on a connection that is publishing", () => { - const fakeConnection = { - transport: aliceRtcMember.getTransport(aliceRtcMember) as LivekitTransport, - } as unknown as Connection; + withTestScheduler(({ behavior, expectObservable }) => { + const transport: LivekitTransport = { + type: "livekit", + livekit_service_url: "https://lk.example.org", + livekit_alias: "!alias:example.org", + }; - fakeMemberships$.next([aliceRtcMember]); - const aliceParticipantId = getParticipantId( - aliceRtcMember.userId, - aliceRtcMember.deviceId, - ); + const bobMembership = mockCallMembership( + "@bob:example.org", + "DEV000", + transport, + ); - const managerData: ConnectionManagerData = new ConnectionManagerData(); - managerData.add(fakeConnection, [ - mockRemoteParticipant({ identity: aliceParticipantId }), - ]); - fakeManagerData$.next(managerData); - - let items: MatrixLivekitMember[] = []; - matrixLivekitMerger.matrixLivekitMember$ - .pipe(take(1)) - .subscribe((emitted) => { - items = emitted; + const connectionWithPublisher = new ConnectionManagerData(); + const bobParticipantId = getParticipantId( + bobMembership.userId, + bobMembership.deviceId, + ); + const connection = { + transport: transport, + } as unknown as Connection; + connectionWithPublisher.add(connection, [ + mockRemoteParticipant({ identity: bobParticipantId }), + ]); + const matrixLivekitMember$ = createMatrixLivekitMembers$({ + scope: testScope, + membershipsWithTransport$: behavior("a", { + a: [ + { + membership: bobMembership, + transport, + }, + ], + }), + connectionManager: aConnectionManager(connectionWithPublisher, behavior), + matrixRoom: mockMatrixRoom, + userId, + deviceId, }); - expect(items).toHaveLength(1); - const item = items[0]; - // Assert the expected membership - expect(item.membership).toBe(aliceRtcMember); - expect(item.participant?.identity).toBe(aliceParticipantId); - expect(item.connection?.transport).toEqual(fakeConnection.transport); + expectObservable(matrixLivekitMember$).toBe("a", { + a: expect.toSatisfy((data: MatrixLivekitMember[]) => { + expect(data.length).toEqual(1); + expect(data[0].participant).toBeDefined(); + expect(data[0].connection).toBeDefined(); + expect(data[0].membership).toEqual(bobMembership); + expect( + areLivekitTransportsEqual(data[0].connection!.transport, transport), + ).toBe(true); + return true; + }), + }); + }); }); test("should signal participant on a connection that is not publishing", () => { - const fakeConnection = { - transport: aliceRtcMember.getTransport(aliceRtcMember) as LivekitTransport, - } as unknown as Connection; + withTestScheduler(({ behavior, expectObservable }) => { + const transport: LivekitTransport = { + type: "livekit", + livekit_service_url: "https://lk.example.org", + livekit_alias: "!alias:example.org", + }; - fakeMemberships$.next([aliceRtcMember]); + const bobMembership = mockCallMembership( + "@bob:example.org", + "DEV000", + transport, + ); - const managerData: ConnectionManagerData = new ConnectionManagerData(); - managerData.add(fakeConnection, []); - fakeManagerData$.next(managerData); + const connectionWithPublisher = new ConnectionManagerData(); + // const bobParticipantId = getParticipantId(bobMembership.userId, bobMembership.deviceId); + const connection = { + transport: transport, + } as unknown as Connection; + connectionWithPublisher.add(connection, []); + const matrixLivekitMember$ = createMatrixLivekitMembers$({ + scope: testScope, + membershipsWithTransport$: behavior("a", { + a: [ + { + membership: bobMembership, + transport, + }, + ], + }), + connectionManager: aConnectionManager(connectionWithPublisher, behavior), + matrixRoom: mockMatrixRoom, + userId, + deviceId, + }); - matrixLivekitMerger.matrixLivekitMember$.pipe(take(1)).subscribe((items) => { - expect(items).toHaveLength(1); - const item = items[0]; - - // Assert the expected membership - expect(item.membership).toBe(aliceRtcMember); - expect(item.participant).not.toBeDefined(); - // We have the connection - expect(item.connection?.transport).toEqual(fakeConnection.transport); + expectObservable(matrixLivekitMember$).toBe("a", { + a: expect.toSatisfy((data: MatrixLivekitMember[]) => { + expect(data.length).toEqual(1); + expect(data[0].participant).not.toBeDefined(); + expect(data[0].connection).toBeDefined(); + expect(data[0].membership).toEqual(bobMembership); + expect( + areLivekitTransportsEqual(data[0].connection!.transport, transport), + ).toBe(true); + return true; + }), + }); }); }); describe("Publication edge case", () => { - const connectionA = { - transport: { - type: "livekit", - livekit_service_url: "https://lk.example.org", - livekit_alias: "!alias:example.org", - }, - } as unknown as Connection; - - const connectionB = { - transport: { - type: "livekit", - livekit_service_url: "https://lk.sample.com", - livekit_alias: "!alias:sample.com", - }, - } as unknown as Connection; - - const bobMembership = { - userId: "@bob:example.org", - deviceId: "DEV000", - transports: [connectionA.transport], - } as unknown as CallMembership; - - const bobParticipantId = getParticipantId( - bobMembership.userId, - bobMembership.deviceId, - ); - test("bob is publishing in several connections", () => { - let lastMatrixLkItems: MatrixLivekitMember[] = []; - matrixLivekitMerger.matrixLivekitMember$.subscribe((items) => { - lastMatrixLkItems = items; + withTestScheduler(({ behavior, expectObservable }) => { + const transportA: LivekitTransport = { + type: "livekit", + livekit_service_url: "https://lk.example.org", + livekit_alias: "!alias:example.org", + }; + + const transportB: LivekitTransport = { + type: "livekit", + livekit_service_url: "https://lk.sample.com", + livekit_alias: "!alias:sample.com", + }; + + const bobMembership = mockCallMembership( + "@bob:example.org", + "DEV000", + transportA, + ); + + const connectionWithPublisher = new ConnectionManagerData(); + const bobParticipantId = getParticipantId( + bobMembership.userId, + bobMembership.deviceId, + ); + const connectionA = { + transport: transportA, + } as unknown as Connection; + const connectionB = { + transport: transportB, + } as unknown as Connection; + + connectionWithPublisher.add(connectionA, [ + mockRemoteParticipant({ identity: bobParticipantId }), + ]); + connectionWithPublisher.add(connectionB, [ + mockRemoteParticipant({ identity: bobParticipantId }), + ]); + const matrixLivekitMember$ = createMatrixLivekitMembers$({ + scope: testScope, + membershipsWithTransport$: behavior("a", { + a: [ + { + membership: bobMembership, + transport: transportA, + }, + ], + }), + connectionManager: aConnectionManager( + connectionWithPublisher, + behavior, + ), + matrixRoom: mockMatrixRoom, + userId, + deviceId, + }); + + expectObservable(matrixLivekitMember$).toBe("a", { + a: expect.toSatisfy((data: MatrixLivekitMember[]) => { + expect(data.length).toEqual(1); + expect(data[0].participant).toBeDefined(); + expect(data[0].participant!.identity).toEqual(bobParticipantId); + expect(data[0].connection).toBeDefined(); + expect(data[0].membership).toEqual(bobMembership); + expect( + areLivekitTransportsEqual( + data[0].connection!.transport, + transportA, + ), + ).toBe(true); + return true; + }), + }); }); - - vi.mocked(bobMembership).getTransport = vi - .fn() - .mockReturnValue(connectionA.transport); - - fakeMemberships$.next([bobMembership]); - - const lkMap = new ConnectionManagerData(); - lkMap.add(connectionA, [ - mockRemoteParticipant({ identity: bobParticipantId }), - ]); - lkMap.add(connectionB, [ - mockRemoteParticipant({ identity: bobParticipantId }), - ]); - - fakeManagerData$.next(lkMap); - - const items = lastMatrixLkItems; - expect(items).toHaveLength(1); - const item = items[0]; - - // Assert the expected membership - expect(item.membership.userId).toEqual(bobMembership.userId); - expect(item.membership.deviceId).toEqual(bobMembership.deviceId); - - expect(item.participant?.identity).toEqual(bobParticipantId); - - // The transport info should come from the membership transports and not only from the publishing connection - expect(item.connection?.transport?.livekit_service_url).toEqual( - bobMembership.transports[0]?.livekit_service_url, - ); - expect(item.connection?.transport?.livekit_alias).toEqual( - bobMembership.transports[0]?.livekit_alias, - ); }); test("bob is publishing in the wrong connection", () => { - let lastMatrixLkItems: MatrixLivekitMember[] = []; - matrixLivekitMerger.matrixLivekitMember$.subscribe((items) => { - lastMatrixLkItems = items; + withTestScheduler(({ behavior, expectObservable }) => { + const transportA: LivekitTransport = { + type: "livekit", + livekit_service_url: "https://lk.example.org", + livekit_alias: "!alias:example.org", + }; + + const transportB: LivekitTransport = { + type: "livekit", + livekit_service_url: "https://lk.sample.com", + livekit_alias: "!alias:sample.com", + }; + + const bobMembership = mockCallMembership( + "@bob:example.org", + "DEV000", + transportA, + ); + + const connectionWithPublisher = new ConnectionManagerData(); + const bobParticipantId = getParticipantId( + bobMembership.userId, + bobMembership.deviceId, + ); + const connectionA = { + transport: transportA, + } as unknown as Connection; + const connectionB = { + transport: transportB, + } as unknown as Connection; + + connectionWithPublisher.add(connectionA, []); + connectionWithPublisher.add(connectionB, [ + mockRemoteParticipant({ identity: bobParticipantId }), + ]); + const matrixLivekitMember$ = createMatrixLivekitMembers$({ + scope: testScope, + membershipsWithTransport$: behavior("a", { + a: [ + { + membership: bobMembership, + transport: transportA, + }, + ], + }), + connectionManager: aConnectionManager( + connectionWithPublisher, + behavior, + ), + matrixRoom: mockMatrixRoom, + userId, + deviceId, + }); + + expectObservable(matrixLivekitMember$).toBe("a", { + a: expect.toSatisfy((data: MatrixLivekitMember[]) => { + expect(data.length).toEqual(1); + expect(data[0].participant).not.toBeDefined(); + expect(data[0].connection).toBeDefined(); + expect(data[0].membership).toEqual(bobMembership); + expect( + areLivekitTransportsEqual( + data[0].connection!.transport, + transportA, + ), + ).toBe(true); + return true; + }), + }); }); - vi.mocked(bobMembership).getTransport = vi - .fn() - .mockReturnValue(connectionA.transport); + // let lastMatrixLkItems: MatrixLivekitMember[] = []; + // matrixLivekitMerger.matrixLivekitMember$.subscribe((items) => { + // lastMatrixLkItems = items; + // }); - fakeMemberships$.next([bobMembership]); + // vi.mocked(bobMembership).getTransport = vi + // .fn() + // .mockReturnValue(connectionA.transport); - const lkMap = new ConnectionManagerData(); - lkMap.add(connectionA, []); - lkMap.add(connectionB, [ - mockRemoteParticipant({ identity: bobParticipantId }), - ]); + // fakeMemberships$.next([bobMembership]); - fakeManagerData$.next(lkMap); + // const lkMap = new ConnectionManagerData(); + // lkMap.add(connectionA, []); + // lkMap.add(connectionB, [ + // mockRemoteParticipant({ identity: bobParticipantId }) + // ]); - const items = lastMatrixLkItems; - expect(items).toHaveLength(1); - const item = items[0]; + // fakeManagerData$.next(lkMap); - // Assert the expected membership - expect(item.membership.userId).toEqual(bobMembership.userId); - expect(item.membership.deviceId).toEqual(bobMembership.deviceId); + // const items = lastMatrixLkItems; + // expect(items).toHaveLength(1); + // const item = items[0]; - expect(item.participant).not.toBeDefined(); + // // Assert the expected membership + // expect(item.membership.userId).toEqual(bobMembership.userId); + // expect(item.membership.deviceId).toEqual(bobMembership.deviceId); - // The transport info should come from the membership transports and not only from the publishing connection - expect(item.connection?.transport?.livekit_service_url).toEqual( - bobMembership.transports[0]?.livekit_service_url, - ); - expect(item.connection?.transport?.livekit_alias).toEqual( - bobMembership.transports[0]?.livekit_alias, - ); + // expect(item.participant).not.toBeDefined(); + + // // The transport info should come from the membership transports and not only from the publishing connection + // expect(item.connection?.transport?.livekit_service_url).toEqual( + // bobMembership.transports[0]?.livekit_service_url + // ); + // expect(item.connection?.transport?.livekit_alias).toEqual( + // bobMembership.transports[0]?.livekit_alias + // ); }); }); diff --git a/src/state/remoteMembers/displayname.test.ts b/src/state/remoteMembers/displayname.test.ts new file mode 100644 index 00000000..dcd8cb0f --- /dev/null +++ b/src/state/remoteMembers/displayname.test.ts @@ -0,0 +1,299 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { afterEach, beforeEach, test, vi } from "vitest"; +import { + type MatrixEvent, + type RoomMember, + type RoomState, + RoomStateEvent, +} from "matrix-js-sdk"; +import EventEmitter from "events"; + +import { ObservableScope } from "../ObservableScope.ts"; +import type { Room as MatrixRoom } from "matrix-js-sdk/lib/models/room"; +import { mockCallMembership, withTestScheduler } from "../../utils/test.ts"; +import { memberDisplaynames$ } from "./displayname.ts"; + +let testScope: ObservableScope; +let mockMatrixRoom: MatrixRoom; + +/* + * To be populated in the test setup. + * Maps userId to a partial/mock RoomMember object. + */ +let fakeMembersMap: Map>; + +beforeEach(() => { + testScope = new ObservableScope(); + fakeMembersMap = new Map>(); + + const roomEmitter = new EventEmitter(); + mockMatrixRoom = { + on: roomEmitter.on.bind(roomEmitter), + off: roomEmitter.off.bind(roomEmitter), + emit: roomEmitter.emit.bind(roomEmitter), + // addListener: roomEmitter.addListener.bind(roomEmitter), + // removeListener: roomEmitter.removeListener.bind(roomEmitter), + getMember: vi.fn().mockImplementation((userId: string) => { + const member = fakeMembersMap.get(userId); + if (member) { + return member as RoomMember; + } + return null; + }), + } as unknown as MatrixRoom; +}); + +function fakeMemberWith(data: Partial): void { + const userId = data.userId || "@alice:example.com"; + const member: Partial = { + userId: userId, + rawDisplayName: data.rawDisplayName ?? userId, + ...data, + } as unknown as RoomMember; + fakeMembersMap.set(userId, member); + // return member as RoomMember; +} + +function updateDisplayName( + userId: `@${string}:${string}`, + newDisplayName: string, +): void { + const member = fakeMembersMap.get(userId); + if (member) { + member.rawDisplayName = newDisplayName; + // Emit the event to notify listeners + mockMatrixRoom.emit( + RoomStateEvent.Members, + {} as unknown as MatrixEvent, + {} as unknown as RoomState, + member as RoomMember, + ); + } else { + throw new Error(`No member found with userId: ${userId}`); + } +} + +afterEach(() => { + fakeMembersMap.clear(); +}); + +test("should always have our own user", () => { + withTestScheduler(({ cold, schedule, expectObservable }) => { + const dn$ = memberDisplaynames$( + testScope, + mockMatrixRoom, + cold("a", { + a: [], + }), + "@local:example.com", + "DEVICE000", + ); + + expectObservable(dn$).toBe("a", { + a: new Map([ + ["@local:example.com:DEVICE000", "@local:example.com"], + ]), + }); + }); +}); + +function setUpBasicRoom(): void { + fakeMemberWith({ userId: "@local:example.com", rawDisplayName: "it's a me" }); + fakeMemberWith({ userId: "@alice:example.com", rawDisplayName: "Alice" }); + fakeMemberWith({ userId: "@bob:example.com", rawDisplayName: "Bob" }); + fakeMemberWith({ userId: "@carl:example.com", rawDisplayName: "Carl" }); + fakeMemberWith({ userId: "@evil:example.com", rawDisplayName: "Carl" }); + fakeMemberWith({ userId: "@bob:foo.bar", rawDisplayName: "Bob" }); + fakeMemberWith({ userId: "@no-name:foo.bar" }); +} + +test("should get displayName for users", () => { + setUpBasicRoom(); + + withTestScheduler(({ cold, schedule, expectObservable }) => { + const dn$ = memberDisplaynames$( + testScope, + mockMatrixRoom, + cold("a", { + a: [ + mockCallMembership("@alice:example.com", "DEVICE1"), + mockCallMembership("@bob:example.com", "DEVICE1"), + ], + }), + "@local:example.com", + "DEVICE000", + ); + + expectObservable(dn$).toBe("a", { + a: new Map([ + ["@local:example.com:DEVICE000", "it's a me"], + ["@alice:example.com:DEVICE1", "Alice"], + ["@bob:example.com:DEVICE1", "Bob"], + ]), + }); + }); +}); + +test("should use userId if no display name", () => { + withTestScheduler(({ cold, schedule, expectObservable }) => { + setUpBasicRoom(); + + const dn$ = memberDisplaynames$( + testScope, + mockMatrixRoom, + cold("a", { + a: [mockCallMembership("@no-name:foo.bar", "D000")], + }), + "@local:example.com", + "DEVICE000", + ); + + expectObservable(dn$).toBe("a", { + a: new Map([ + ["@local:example.com:DEVICE000", "it's a me"], + ["@no-name:foo.bar:D000", "@no-name:foo.bar"], + ]), + }); + }); +}); + +test("should disambiguate users with same display name", () => { + withTestScheduler(({ cold, schedule, expectObservable }) => { + setUpBasicRoom(); + + const dn$ = memberDisplaynames$( + testScope, + mockMatrixRoom, + cold("a", { + a: [ + mockCallMembership("@bob:example.com", "DEVICE1"), + mockCallMembership("@bob:example.com", "DEVICE2"), + mockCallMembership("@bob:foo.bar", "BOB000"), + mockCallMembership("@carl:example.com", "C000"), + mockCallMembership("@evil:example.com", "E000"), + ], + }), + "@local:example.com", + "DEVICE000", + ); + + expectObservable(dn$).toBe("a", { + a: new Map([ + ["@local:example.com:DEVICE000", "it's a me"], + ["@bob:example.com:DEVICE1", "Bob (@bob:example.com)"], + ["@bob:example.com:DEVICE2", "Bob (@bob:example.com)"], + ["@bob:foo.bar:BOB000", "Bob (@bob:foo.bar)"], + ["@carl:example.com:C000", "Carl (@carl:example.com)"], + ["@evil:example.com:E000", "Carl (@evil:example.com)"], + ]), + }); + }); +}); + +test("should disambiguate when needed", () => { + withTestScheduler(({ cold, schedule, expectObservable }) => { + setUpBasicRoom(); + + const dn$ = memberDisplaynames$( + testScope, + mockMatrixRoom, + cold("ab", { + a: [mockCallMembership("@bob:example.com", "DEVICE1")], + b: [ + mockCallMembership("@bob:example.com", "DEVICE1"), + mockCallMembership("@bob:foo.bar", "BOB000"), + ], + }), + "@local:example.com", + "DEVICE000", + ); + + expectObservable(dn$).toBe("ab", { + a: new Map([ + ["@local:example.com:DEVICE000", "it's a me"], + ["@bob:example.com:DEVICE1", "Bob"], + ]), + b: new Map([ + ["@local:example.com:DEVICE000", "it's a me"], + ["@bob:example.com:DEVICE1", "Bob (@bob:example.com)"], + ["@bob:foo.bar:BOB000", "Bob (@bob:foo.bar)"], + ]), + }); + }); +}); + +test.skip("should keep disambiguated name when other leave", () => { + withTestScheduler(({ cold, schedule, expectObservable }) => { + setUpBasicRoom(); + + const dn$ = memberDisplaynames$( + testScope, + mockMatrixRoom, + cold("ab", { + a: [ + mockCallMembership("@bob:example.com", "DEVICE1"), + mockCallMembership("@bob:foo.bar", "BOB000"), + ], + b: [mockCallMembership("@bob:example.com", "DEVICE1")], + }), + "@local:example.com", + "DEVICE000", + ); + + expectObservable(dn$).toBe("ab", { + a: new Map([ + ["@local:example.com:DEVICE000", "it's a me"], + ["@bob:example.com:DEVICE1", "Bob (@bob:example.com)"], + ["@bob:foo.bar:BOB000", "Bob (@bob:foo.bar)"], + ]), + b: new Map([ + ["@local:example.com:DEVICE000", "it's a me"], + ["@bob:example.com:DEVICE1", "Bob (@bob:example.com)"], + ]), + }); + }); +}); + +test("should disambiguate on name change", () => { + withTestScheduler(({ cold, schedule, expectObservable }) => { + setUpBasicRoom(); + + const dn$ = memberDisplaynames$( + testScope, + mockMatrixRoom, + cold("a", { + a: [ + mockCallMembership("@bob:example.com", "B000"), + mockCallMembership("@carl:example.com", "C000"), + ], + }), + "@local:example.com", + "DEVICE000", + ); + + schedule("-a", { + a: () => { + updateDisplayName("@carl:example.com", "Bob"); + }, + }); + + expectObservable(dn$).toBe("ab", { + a: new Map([ + ["@local:example.com:DEVICE000", "it's a me"], + ["@bob:example.com:B000", "Bob"], + ["@carl:example.com:C000", "Carl"], + ]), + b: new Map([ + ["@local:example.com:DEVICE000", "it's a me"], + ["@bob:example.com:B000", "Bob (@bob:example.com)"], + ["@carl:example.com:C000", "Bob (@carl:example.com)"], + ]), + }); + }); +}); diff --git a/src/state/remoteMembers/displayname.ts b/src/state/remoteMembers/displayname.ts index a5d1ae3d..35236030 100644 --- a/src/state/remoteMembers/displayname.ts +++ b/src/state/remoteMembers/displayname.ts @@ -6,12 +6,16 @@ Please see LICENSE in the repository root for full details. */ import { type RoomMember, RoomStateEvent } from "matrix-js-sdk"; -import { combineLatest, fromEvent, type Observable, startWith } from "rxjs"; +import { + combineLatest, + fromEvent, + map, + type Observable, + startWith, +} from "rxjs"; import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; import { logger } from "matrix-js-sdk/lib/logger"; import { type Room as MatrixRoom } from "matrix-js-sdk/lib/matrix"; -// eslint-disable-next-line rxjs/no-internal -import { type NodeStyleEventEmitter } from "rxjs/internal/observable/fromEvent"; import { type ObservableScope } from "../ObservableScope"; import { @@ -19,6 +23,7 @@ import { shouldDisambiguate, } from "../../utils/displayname"; import { type Behavior } from "../Behavior"; +import type { NodeStyleEventEmitter } from "rxjs/src/internal/observable/fromEvent.ts"; /** * Displayname for each member of the call. This will disambiguate @@ -36,15 +41,14 @@ export const memberDisplaynames$ = ( deviceId: string, ): Behavior> => scope.behavior( - combineLatest( - [ - // Handle call membership changes - memberships$, - // Additionally handle display name changes (implicitly reacting to them) - fromEvent(matrixRoom, RoomStateEvent.Members).pipe(startWith(null)), - // TODO: do we need: pauseWhen(this.pretendToBeDisconnected$), - ], - (memberships, _displaynames) => { + combineLatest([ + // Handle call membership changes + memberships$, + // Additionally handle display name changes (implicitly reacting to them) + fromEvent(matrixRoom, RoomStateEvent.Members).pipe(startWith(null)), + // TODO: do we need: pauseWhen(this.pretendToBeDisconnected$), + ]).pipe( + map(([memberships, _displayNames]) => { const displaynameMap = new Map([ [ `${userId}:${deviceId}`, @@ -55,11 +59,12 @@ export const memberDisplaynames$ = ( // We only consider RTC members for disambiguation as they are the only visible members. for (const rtcMember of memberships) { + // TODO a hard-coded participant ID ? should use rtcMember.membershipID instead? const matrixIdentifier = `${rtcMember.userId}:${rtcMember.deviceId}`; const { member } = getRoomMemberFromRtcMember(rtcMember, room); if (!member) { logger.error( - "Could not find member for media id:", + "Could not find member for participant id:", matrixIdentifier, ); continue; @@ -71,7 +76,7 @@ export const memberDisplaynames$ = ( ); } return displaynameMap; - }, + }), ), new Map(), ); diff --git a/src/state/remoteMembers/integration.test.ts b/src/state/remoteMembers/integration.test.ts index 5b3cfe7c..be134306 100644 --- a/src/state/remoteMembers/integration.test.ts +++ b/src/state/remoteMembers/integration.test.ts @@ -5,22 +5,29 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { test, vi, beforeEach, afterEach } from "vitest"; -import { BehaviorSubject, type Observable } from "rxjs"; +import { test, vi, expect, beforeEach, afterEach } from "vitest"; +import { BehaviorSubject, map } from "rxjs"; import { type Room as LivekitRoom } from "livekit-client"; -import { logger } from "matrix-js-sdk/lib/logger"; import EventEmitter from "events"; import fetchMock from "fetch-mock"; +import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { type Room as MatrixRoom, type RoomMember } from "matrix-js-sdk"; -import { ConnectionManager } from "./ConnectionManager.ts"; import { ObservableScope } from "../ObservableScope.ts"; import { ECConnectionFactory } from "./ConnectionFactory.ts"; import { type OpenIDClientParts } from "../../livekit/openIDSFU.ts"; -import { mockMediaDevices, withTestScheduler } from "../../utils/test"; +import { + mockCallMembership, + mockMediaDevices, + withTestScheduler, +} from "../../utils/test"; import { type ProcessorState } from "../../livekit/TrackProcessorContext.tsx"; -import { matrixLivekitMerger$ } from "./matrixLivekitMerger.ts"; -import type { CallMembership, Transport } from "matrix-js-sdk/lib/matrixrtc"; -import { TRANSPORT_1 } from "./ConnectionManager.test.ts"; +import { + createMatrixLivekitMembers$, + type MatrixLivekitMember, +} from "./MatrixLivekitMembers.ts"; +import { createConnectionManager$ } from "./ConnectionManager.ts"; +import { membershipsAndTransports$ } from "../SessionBehaviors.ts"; // Test the integration of ConnectionManager and MatrixLivekitMerger @@ -28,33 +35,10 @@ let testScope: ObservableScope; let ecConnectionFactory: ECConnectionFactory; let mockClient: OpenIDClientParts; let lkRoomFactory: () => LivekitRoom; +let mockMatrixRoom: MatrixRoom; const createdMockLivekitRooms: Map = new Map(); -// Main test input -const memberships$ = new BehaviorSubject([]); - -// under test -let connectionManager: ConnectionManager; - -function createLkMerger( - memberships$: Observable, -): matrixLivekitMerger$ { - const mockRoomEmitter = new EventEmitter(); - return new matrixLivekitMerger$( - testScope, - memberships$, - connectionManager, - { - on: mockRoomEmitter.on.bind(mockRoomEmitter), - off: mockRoomEmitter.off.bind(mockRoomEmitter), - getMember: vi.fn().mockReturnValue(undefined), - }, - "@user:example.com", - "DEV000", - ); -} - beforeEach(() => { testScope = new ObservableScope(); mockClient = { @@ -90,16 +74,9 @@ beforeEach(() => { lkRoomFactory, ); - connectionManager = new ConnectionManager( - testScope, - ecConnectionFactory, - logger, - ); - //TODO a bit annoying to have to do a http mock? - fetchMock.post(`**/sfu/get`, (url) => { + fetchMock.post(`path:/sfu/get`, (url) => { const domain = new URL(url).hostname; // Extract the domain from the URL - return { status: 200, body: { @@ -108,6 +85,18 @@ beforeEach(() => { }, }; }); + + mockMatrixRoom = vi.mocked({ + getMember: vi.fn().mockImplementation((userId: string) => { + return { + userId, + rawDisplayName: userId.replace("@", "").replace(":example.org", ""), + getMxcAvatarUrl: vi.fn().mockReturnValue(null), + } as unknown as RoomMember; + }), + addEventListener: vi.fn(), + removeEventListener: vi.fn(), + } as unknown as MatrixRoom); }); afterEach(() => { @@ -115,43 +104,82 @@ afterEach(() => { fetchMock.reset(); }); -test("example test", () => { - withTestScheduler(({ schedule, expectObservable, cold }) => { - connectionManager.connections$.subscribe((connections) => { - // console.log( - // "Connections updated:", - // connections.map((c) => c.transport), - // ); +test("example test 2", () => { + withTestScheduler(({ schedule, expectObservable, behavior, cold }) => { + const bobMembership = mockCallMembership("@bob:example.com", "BDEV000"); + const carlMembership = mockCallMembership("@carl:example.com", "CDEV000"); + const daveMembership = mockCallMembership("@dave:foo.bar", "DDEV000"); + const memberships$ = behavior("ab---c", { + a: [bobMembership], + b: [bobMembership, carlMembership], + c: [bobMembership, carlMembership, daveMembership], }); - const memberships$ = cold("-a-b-c", { - a: [mockCallmembership("@bob:example.com", "BDEV000")], - b: [ - mockCallmembership("@bob:example.com", "BDEV000"), - mockCallmembership("@carl:example.com", "CDEV000"), - ], - c: [ - mockCallmembership("@bob:example.com", "BDEV000"), - mockCallmembership("@carl:example.com", "CDEV000"), - mockCallmembership("@dave:foo.bar", "DDEV000"), - ], + const transports$ = testScope.behavior( + memberships$.pipe( + map((memberships) => { + return memberships.map((membership) => { + return membership.getTransport(memberships[0]) as LivekitTransport; + }); + }), + ), + ); + + const connectionManager = createConnectionManager$({ + scope: testScope, + connectionFactory: ecConnectionFactory, + inputTransports$: transports$, }); - // TODO IN PROGRESS - const merger = createLkMerger(memberships$); + const marixLivekitItems$ = createMatrixLivekitMembers$({ + scope: testScope, + membershipsWithTransport$: membershipsAndTransports$( + testScope, + memberships$, + ).membershipsWithTransport$, + connectionManager, + matrixRoom: mockMatrixRoom, + userId: "local:example.org", + deviceId: "ME00", + }); + + expectObservable(marixLivekitItems$).toBe("a(bb)(cc)", { + a: expect.toSatisfy((items: MatrixLivekitMember[]) => { + expect(items.length).toBe(1); + const item = items[0]!; + expect(item.membership).toStrictEqual(bobMembership); + expect(item.participant).toBeUndefined(); + return true; + }), + b: expect.toSatisfy((items: MatrixLivekitMember[]) => { + // TODO + // expect(items.length).toBe(2); + // + // const item = items[0]!; + // expect(item.membership).toStrictEqual(bobMembership); + // expect(item.participant).toBeUndefined(); + // + // { + // const item = items[1]!; + // expect(item.membership).toStrictEqual(carlMembership); + // expect(item.participant).toBeUndefined(); + // } + return true; + }), + c: expect.toSatisfy(() => true), + }); }); }); -function mockCallmembership( - userId: string, - deviceId: string, - transport?: Transport, -): CallMembership { - const t = transport ?? TRANSPORT_1; - return { - userId: userId, - deviceId: deviceId, - getTransport: vi.fn().mockReturnValue(t), - transports: [t], - } as unknown as CallMembership; -} +// test("Tryng", () => { +// +// withTestScheduler(({ schedule, expectObservable, behavior, cold }) => { +// const one = cold("a-b-c", { a: 1, b: 2, c: 3 }); +// const a = one.pipe(map(() => 1)); +// const b = one.pipe(map(() => 2)); +// const combined = combineLatest([a,b]) +// .pipe(map(([a,b])=>`${a}${b}`)); +// expectObservable(combined).toBe("a-b-c", { a: 1, b: expect.anything(), c: 3 }); +// +// }) +// }) diff --git a/src/utils/test.ts b/src/utils/test.ts index b60492f6..96d274d1 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -187,6 +187,29 @@ export const exampleTransport: LivekitTransport = { livekit_alias: "!alias:example.org", }; +export function mockCallMembership( + userId: string, + deviceId: string, + transport?: Transport, +): CallMembership { + const t = transport ?? transportForUser(userId); + return { + userId: userId, + deviceId: deviceId, + getTransport: vi.fn().mockReturnValue(t), + transports: [t], + } as unknown as CallMembership; +} + +function transportForUser(userId: string): Transport { + const domain = userId.split(":")[1]; + return { + type: "livekit", + livekit_service_url: `https://lk.${domain}`, + livekit_alias: `!alias:${domain}`, + }; +} + export function mockRtcMembership( user: string | RoomMember, deviceId: string, From a55ce19048df756a148da9e31cdf5a1e0a13ef6b Mon Sep 17 00:00:00 2001 From: Timo K Date: Thu, 6 Nov 2025 15:26:17 +0100 Subject: [PATCH 26/75] cleanup --- src/state/CallViewModel.ts | 27 +++--- src/state/SessionBehaviors.ts | 39 +------- .../remoteMembers/ConnectionManager.test.ts | 4 +- src/state/remoteMembers/ConnectionManager.ts | 23 ++--- ...r.test.ts => MatrixLivekitMembers.test.ts} | 29 ++---- .../remoteMembers/MatrixLivekitMembers.ts | 56 +++++------ src/state/remoteMembers/displayname.ts | 9 +- src/state/remoteMembers/integration.test.ts | 92 ++++++++++++------- src/utils/test.ts | 4 +- 9 files changed, 122 insertions(+), 161 deletions(-) rename src/state/remoteMembers/{MatrixLivekitMerger.test.ts => MatrixLivekitMembers.test.ts} (94%) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index d9d68da3..420ecab4 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -52,7 +52,7 @@ import { throttleTime, timer, } from "rxjs"; -import { logger } from "matrix-js-sdk/lib/logger"; +import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { type MatrixRTCSession, MatrixRTCSessionEvent, @@ -110,17 +110,17 @@ import { } from "./layout-types.ts"; import { type ElementCallError } from "../utils/errors.ts"; import { type ObservableScope } from "./ObservableScope.ts"; -import { createMatrixLivekitMembers$ } from "./remoteMembers/matrixLivekitMerger.ts"; import { createLocalMembership$ } from "./localMember/LocalMembership.ts"; import { createLocalTransport$ } from "./localMember/LocalTransport.ts"; import { createMemberships$, - createSessionMembershipsAndTransports$, membershipsAndTransports$, } from "./SessionBehaviors.ts"; import { ECConnectionFactory } from "./remoteMembers/ConnectionFactory.ts"; import { createConnectionManager$ } from "./remoteMembers/ConnectionManager.ts"; +import { createMatrixLivekitMembers$ } from "./remoteMembers/MatrixLivekitMembers.ts"; +const logger = rootLogger.getChild("[CallViewModel]"); //TODO // Larger rename // member,membership -> rtcMember @@ -193,10 +193,8 @@ export class CallViewModel { } : undefined; - private memberships$ = createMemberships$({ - scope: this.scope, - matrixRTCSession: this.matrixRTCSession, - }); + private memberships$ = createMemberships$(this.scope, this.matrixRTCSession); + private membershipsAndTransports = membershipsAndTransports$( this.scope, this.memberships$, @@ -225,7 +223,7 @@ export class CallViewModel { // Can contain duplicates. The connection manager will take care of this. private allTransports$ = this.scope.behavior( combineLatest( - [this.localTransport$, this.sessionBehaviors.transports$], + [this.localTransport$, this.membershipsAndTransports.transports$], (localTransport, transports) => { const localTransportAsArray = localTransport ? [localTransport] : []; return [...localTransportAsArray, ...transports]; @@ -243,11 +241,10 @@ export class CallViewModel { private matrixLivekitMembers$ = createMatrixLivekitMembers$({ scope: this.scope, - membershipsWithTransport$: this.sessionBehaviors.membershipsWithTransport$, + membershipsWithTransport$: + this.membershipsAndTransports.membershipsWithTransport$, connectionManager: this.connectionManager, matrixRoom: this.matrixRoom, - userId: this.userId, - deviceId: this.deviceId, }); private connectOptions$ = this.scope.behavior( @@ -357,7 +354,7 @@ export class CallViewModel { connection, participant, member, - displayName$, + displayName, participantId, } of matrixLivekitMembers) { if (connection === undefined) { @@ -368,7 +365,7 @@ export class CallViewModel { const mediaId = `${participantId}:${i}`; const lkRoom = connection?.livekitRoom; const url = connection?.transport.livekit_service_url; - const dpName$ = displayName$.pipe(map((n) => n ?? "[👻]")); + const item = createOrGet( mediaId, (scope) => @@ -385,7 +382,7 @@ export class CallViewModel { url, this.mediaDevices, this.pretendToBeDisconnected$, - dpName$, + constant(displayName ?? "[👻]"), this.handsRaised$.pipe( map((v) => v[participantId]?.time ?? null), ), @@ -412,7 +409,7 @@ export class CallViewModel { lkRoom, url, this.pretendToBeDisconnected$, - dpName$, + constant(displayName ?? "[👻]"), ), ), ); diff --git a/src/state/SessionBehaviors.ts b/src/state/SessionBehaviors.ts index ed207835..80e9f09c 100644 --- a/src/state/SessionBehaviors.ts +++ b/src/state/SessionBehaviors.ts @@ -17,11 +17,6 @@ import { fromEvent, map } from "rxjs"; import { type ObservableScope } from "./ObservableScope"; import { type Behavior } from "./Behavior"; -interface Props { - scope: ObservableScope; - matrixRTCSession: MatrixRTCSession; -} - export const membershipsAndTransports$ = ( scope: ObservableScope, memberships$: Behavior, @@ -66,10 +61,10 @@ export const membershipsAndTransports$ = ( }; }; -export const createMemberships$ = ({ - scope, - matrixRTCSession, -}: Props): Behavior => { +export const createMemberships$ = ( + scope: ObservableScope, + matrixRTCSession: MatrixRTCSession, +): Behavior => { return scope.behavior( fromEvent( matrixRTCSession, @@ -78,29 +73,3 @@ export const createMemberships$ = ({ ), ); }; - -export const createSessionMembershipsAndTransports$ = ({ - scope, - matrixRTCSession, -}: Props): { - memberships$: Behavior; - membershipsWithTransport$: Behavior< - { membership: CallMembership; transport?: LivekitTransport }[] - >; - transports$: Behavior; -} => { - const memberships$ = scope.behavior( - fromEvent( - matrixRTCSession, - MatrixRTCSessionEvent.MembershipsChanged, - (_, memberships: CallMembership[]) => memberships, - ), - ); - - const memberAndTransport = membershipsAndTransports$(scope, memberships$); - - return { - memberships$, - ...memberAndTransport, - }; -}; diff --git a/src/state/remoteMembers/ConnectionManager.test.ts b/src/state/remoteMembers/ConnectionManager.test.ts index 8be5bc35..1b1a6ffe 100644 --- a/src/state/remoteMembers/ConnectionManager.test.ts +++ b/src/state/remoteMembers/ConnectionManager.test.ts @@ -12,7 +12,7 @@ import { type Participant as LivekitParticipant } from "livekit-client"; import { ObservableScope } from "../ObservableScope.ts"; import { - type ConnectionManagerReturn, + type IConnectionManager, createConnectionManager$, } from "./ConnectionManager.ts"; import { type ConnectionFactory } from "./ConnectionFactory.ts"; @@ -47,7 +47,7 @@ let connectionManagerInputs: { connectionFactory: ConnectionFactory; inputTransports$: BehaviorSubject; }; -let manager: ConnectionManagerReturn; +let manager: IConnectionManager; beforeEach(() => { testScope = new ObservableScope(); diff --git a/src/state/remoteMembers/ConnectionManager.ts b/src/state/remoteMembers/ConnectionManager.ts index 2cb6957d..7cee0756 100644 --- a/src/state/remoteMembers/ConnectionManager.ts +++ b/src/state/remoteMembers/ConnectionManager.ts @@ -21,7 +21,7 @@ import { type Behavior } from "../Behavior"; import { type Connection } from "./Connection"; import { type ObservableScope } from "../ObservableScope"; import { generateKeyed$ } from "../../utils/observable"; -import { areLivekitTransportsEqual } from "./MatrixLivekitMembers"; +import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts"; import { type ConnectionFactory } from "./ConnectionFactory.ts"; export class ConnectionManagerData { @@ -93,13 +93,11 @@ interface Props { inputTransports$: Behavior; } // TODO - write test for scopes (do we really need to bind scope) - -export interface ConnectionManagerReturn { - deduplicatedTransports$: Behavior; +export interface IConnectionManager { + transports$: Behavior; connectionManagerData$: Behavior; connections$: Behavior; } - /** * Crete a `ConnectionManager` * @param scope the observable scope used by this object. @@ -118,7 +116,7 @@ export function createConnectionManager$({ scope, connectionFactory, inputTransports$, -}: Props): ConnectionManagerReturn { +}: Props): IConnectionManager { const logger = rootLogger.getChild("ConnectionManager"); const running$ = new BehaviorSubject(true); @@ -133,10 +131,13 @@ export function createConnectionManager$({ * It is build based on the list of subscribed transports (`transportsSubscriptions$`). * externally this is modified via `registerTransports()`. */ - const deduplicatedTransports$ = scope.behavior( + const transports$ = scope.behavior( combineLatest([running$, inputTransports$]).pipe( - map(([running, transports]) => (running ? transports : [])), - map(removeDuplicateTransports), + map(([running, transports]) => ({ + epoch: transports.epoch, + value: running ? transports.value : [], + })), + map((transports) => removeDuplicateTransports(transports.value)), ), ); @@ -145,7 +146,7 @@ export function createConnectionManager$({ */ const connections$ = scope.behavior( generateKeyed$( - deduplicatedTransports$, + transports$, (transports, createOrGet) => { const createConnection = ( @@ -204,7 +205,7 @@ export function createConnectionManager$({ // start empty new ConnectionManagerData(), ); - return { deduplicatedTransports$, connectionManagerData$, connections$ }; + return { transports$, connectionManagerData$, connections$ }; } function removeDuplicateTransports( diff --git a/src/state/remoteMembers/MatrixLivekitMerger.test.ts b/src/state/remoteMembers/MatrixLivekitMembers.test.ts similarity index 94% rename from src/state/remoteMembers/MatrixLivekitMerger.test.ts rename to src/state/remoteMembers/MatrixLivekitMembers.test.ts index 71a1398c..75534e1f 100644 --- a/src/state/remoteMembers/MatrixLivekitMerger.test.ts +++ b/src/state/remoteMembers/MatrixLivekitMembers.test.ts @@ -6,7 +6,6 @@ Please see LICENSE in the repository root for full details. */ import { describe, test, vi, expect, beforeEach, afterEach } from "vitest"; -import { BehaviorSubject } from "rxjs"; import { type CallMembership, type LivekitTransport, @@ -14,14 +13,14 @@ import { import { type Room as MatrixRoom, type RoomMember } from "matrix-js-sdk"; import { getParticipantId } from "matrix-js-sdk/lib/matrixrtc/utils"; -import { type ConnectionManagerReturn } from "./ConnectionManager.ts"; +import { type IConnectionManager } from "./ConnectionManager.ts"; import { type MatrixLivekitMember, createMatrixLivekitMembers$, areLivekitTransportsEqual, -} from "./MatrixLivekitMembers"; -import { ObservableScope } from "../ObservableScope"; -import { ConnectionManagerData } from "./ConnectionManager"; +} from "./MatrixLivekitMembers.ts"; +import { ObservableScope } from "../ObservableScope.ts"; +import { ConnectionManagerData } from "./ConnectionManager.ts"; import { mockCallMembership, mockRemoteParticipant, @@ -32,8 +31,6 @@ import { type Connection } from "./Connection.ts"; let testScope: ObservableScope; let mockMatrixRoom: MatrixRoom; -const userId = "@local:example.com"; -const deviceId = "DEVICE000"; // The merger beeing tested @@ -87,8 +84,6 @@ test("should signal participant not yet connected to livekit", () => { connections$: behavior("a", { a: [] }), }, matrixRoom: mockMatrixRoom, - userId, - deviceId, }); expectObservable(matrixLivekitMember$).toBe("a", { @@ -106,14 +101,14 @@ test("should signal participant not yet connected to livekit", () => { function aConnectionManager( data: ConnectionManagerData, - behavior: Pick, -): ConnectionManagerReturn { + behavior: OurRunHelpers["behavior"], +): IConnectionManager { return { connectionManagerData$: behavior("a", { a: data }), transports$: behavior("a", { - a: [data.getConnections().map((connection) => connection.transport)], + a: data.getConnections().map((connection) => connection.transport), }), - connections$: behavior("a", { a: [data.getConnections()] }), + connections$: behavior("a", { a: data.getConnections() }), }; } @@ -154,8 +149,6 @@ test("should signal participant on a connection that is publishing", () => { }), connectionManager: aConnectionManager(connectionWithPublisher, behavior), matrixRoom: mockMatrixRoom, - userId, - deviceId, }); expectObservable(matrixLivekitMember$).toBe("a", { @@ -205,8 +198,6 @@ test("should signal participant on a connection that is not publishing", () => { }), connectionManager: aConnectionManager(connectionWithPublisher, behavior), matrixRoom: mockMatrixRoom, - userId, - deviceId, }); expectObservable(matrixLivekitMember$).toBe("a", { @@ -278,8 +269,6 @@ describe("Publication edge case", () => { behavior, ), matrixRoom: mockMatrixRoom, - userId, - deviceId, }); expectObservable(matrixLivekitMember$).toBe("a", { @@ -352,8 +341,6 @@ describe("Publication edge case", () => { behavior, ), matrixRoom: mockMatrixRoom, - userId, - deviceId, }); expectObservable(matrixLivekitMember$).toBe("a", { diff --git a/src/state/remoteMembers/MatrixLivekitMembers.ts b/src/state/remoteMembers/MatrixLivekitMembers.ts index dd54d092..0f5234fa 100644 --- a/src/state/remoteMembers/MatrixLivekitMembers.ts +++ b/src/state/remoteMembers/MatrixLivekitMembers.ts @@ -13,14 +13,14 @@ import { type LivekitTransport, type CallMembership, } from "matrix-js-sdk/lib/matrixrtc"; -import { combineLatest, map, type Observable } from "rxjs"; +import { combineLatest, filter, map, skipWhile, type Observable } from "rxjs"; // eslint-disable-next-line rxjs/no-internal import { type NodeStyleEventEmitter } from "rxjs/internal/observable/fromEvent"; import { type Room as MatrixRoom, type RoomMember } from "matrix-js-sdk"; import { type Behavior } from "../Behavior"; +import { type IConnectionManager } from "./ConnectionManager"; import { type ObservableScope } from "../ObservableScope"; -import type * as ConnectionManager from "./ConnectionManager"; import { getRoomMemberFromRtcMember, memberDisplaynames$ } from "./displayname"; import { type Connection } from "./Connection"; @@ -31,7 +31,7 @@ import { type Connection } from "./Connection"; */ export interface MatrixLivekitMember { membership: CallMembership; - displayName$: Behavior; + displayName?: string; participant?: LocalLivekitParticipant | RemoteLivekitParticipant; connection?: Connection; /** @@ -49,14 +49,12 @@ interface Props { membershipsWithTransport$: Behavior< { membership: CallMembership; transport?: LivekitTransport }[] >; - connectionManager: ConnectionManager.ConnectionManagerReturn; + connectionManager: IConnectionManager; // TODO this is too much information for that class, // apparently needed to get a room member to later get the Avatar // => Extract an AvatarService instead? // Better with just `getMember` matrixRoom: Pick & NodeStyleEventEmitter; - userId: string; - deviceId: string; } // Alternative structure idea: // const livekitMatrixMember$ = (callMemberships$,connectionManager,scope): Observable => { @@ -76,27 +74,30 @@ export function createMatrixLivekitMembers$({ membershipsWithTransport$, connectionManager, matrixRoom, - userId, - deviceId, }: Props): Behavior { /** * Stream of all the call members and their associated livekit data (if available). */ - function createMatrixLivekitMember$(): Observable { - const displaynameMap$ = memberDisplaynames$( - scope, - matrixRoom, - membershipsWithTransport$.pipe(map((v) => v.map((v) => v.membership))), - userId, - deviceId, - ); + const displaynameMap$ = memberDisplaynames$( + scope, + matrixRoom, + membershipsWithTransport$.pipe(map((v) => v.map((v) => v.membership))), + ); - return combineLatest([ + return scope.behavior( + combineLatest([ membershipsWithTransport$, connectionManager.connectionManagerData$, + displaynameMap$, ]).pipe( - map(([memberships, managerData]) => { + filter( + ([membershipsWithTransports, managerData, displaynames]) => + // for each change in + displaynames.size === membershipsWithTransports.length && + displaynames.size === managerData.getConnections().length, + ), + map(([memberships, managerData, displaynames]) => { const items: MatrixLivekitMember[] = memberships.map( ({ membership, transport }) => { // TODO! cannot use membership.membershipID yet, Currently its hardcoded by the jwt service to @@ -115,22 +116,15 @@ export function createMatrixLivekitMembers$({ const connection = transport ? managerData.getConnectionForTransport(transport) : undefined; - const displayName$ = scope.behavior( - displaynameMap$.pipe( - map( - (displayNameMap) => - displayNameMap.get(membership.membershipID) ?? "---", - ), - ), - ); + const displayName = displaynames.get(participantId); return { participant, membership, connection, - // This makes sense to add the the js-sdk callMembership (we only need the avatar so probably the call memberhsip just should aquire the avatar) + // This makes sense to add to the js-sdk callMembership (we only need the avatar so probably the call memberhsip just should aquire the avatar) // TODO Ugh this is hidign that it might be undefined!! best we remove the member entirely. member: member as RoomMember, - displayName$, + displayName, mxcAvatarUrl: member?.getMxcAvatarUrl(), participantId, }; @@ -138,10 +132,8 @@ export function createMatrixLivekitMembers$({ ); return items; }), - ); - } - - return scope.behavior(createMatrixLivekitMember$(), []); + ), + ); } // TODO add back in the callviewmodel pauseWhen(this.pretendToBeDisconnected$) diff --git a/src/state/remoteMembers/displayname.ts b/src/state/remoteMembers/displayname.ts index 35236030..e735147d 100644 --- a/src/state/remoteMembers/displayname.ts +++ b/src/state/remoteMembers/displayname.ts @@ -37,8 +37,6 @@ export const memberDisplaynames$ = ( scope: ObservableScope, matrixRoom: Pick & NodeStyleEventEmitter, memberships$: Observable, - userId: string, - deviceId: string, ): Behavior> => scope.behavior( combineLatest([ @@ -49,12 +47,7 @@ export const memberDisplaynames$ = ( // TODO: do we need: pauseWhen(this.pretendToBeDisconnected$), ]).pipe( map(([memberships, _displayNames]) => { - const displaynameMap = new Map([ - [ - `${userId}:${deviceId}`, - matrixRoom.getMember(userId)?.rawDisplayName ?? userId, - ], - ]); + const displaynameMap = new Map(); const room = matrixRoom; // We only consider RTC members for disambiguation as they are the only visible members. diff --git a/src/state/remoteMembers/integration.test.ts b/src/state/remoteMembers/integration.test.ts index be134306..55db4009 100644 --- a/src/state/remoteMembers/integration.test.ts +++ b/src/state/remoteMembers/integration.test.ts @@ -26,8 +26,12 @@ import { createMatrixLivekitMembers$, type MatrixLivekitMember, } from "./MatrixLivekitMembers.ts"; -import { createConnectionManager$ } from "./ConnectionManager.ts"; +import { + ConnectionManagerData, + createConnectionManager$, +} from "./ConnectionManager.ts"; import { membershipsAndTransports$ } from "../SessionBehaviors.ts"; +import { Connection } from "./Connection.ts"; // Test the integration of ConnectionManager and MatrixLivekitMerger @@ -109,61 +113,79 @@ test("example test 2", () => { const bobMembership = mockCallMembership("@bob:example.com", "BDEV000"); const carlMembership = mockCallMembership("@carl:example.com", "CDEV000"); const daveMembership = mockCallMembership("@dave:foo.bar", "DDEV000"); - const memberships$ = behavior("ab---c", { + const memberships$ = behavior("abc", { a: [bobMembership], b: [bobMembership, carlMembership], c: [bobMembership, carlMembership, daveMembership], }); - const transports$ = testScope.behavior( - memberships$.pipe( - map((memberships) => { - return memberships.map((membership) => { - return membership.getTransport(memberships[0]) as LivekitTransport; - }); - }), - ), + const membershipsAndTransports = membershipsAndTransports$( + testScope, + memberships$, ); const connectionManager = createConnectionManager$({ scope: testScope, connectionFactory: ecConnectionFactory, - inputTransports$: transports$, + inputTransports$: membershipsAndTransports.transports$, }); - const marixLivekitItems$ = createMatrixLivekitMembers$({ + const matrixLivekitItems$ = createMatrixLivekitMembers$({ scope: testScope, - membershipsWithTransport$: membershipsAndTransports$( - testScope, - memberships$, - ).membershipsWithTransport$, + membershipsWithTransport$: + membershipsAndTransports.membershipsWithTransport$, connectionManager, matrixRoom: mockMatrixRoom, - userId: "local:example.org", - deviceId: "ME00", }); - expectObservable(marixLivekitItems$).toBe("a(bb)(cc)", { + expectObservable(membershipsAndTransports.transports$).toBe("abc", { + a: expect.toSatisfy((t: LivekitTransport[]) => t.length === 1), + b: expect.toSatisfy((t: LivekitTransport[]) => t.length === 2), + c: expect.toSatisfy((t: LivekitTransport[]) => t.length === 3), + }); + + expectObservable(membershipsAndTransports.membershipsWithTransport$).toBe( + "abc", + { + a: expect.toSatisfy((t: LivekitTransport[]) => t.length === 1), + b: expect.toSatisfy((t: LivekitTransport[]) => t.length === 2), + c: expect.toSatisfy((t: LivekitTransport[]) => t.length === 3), + }, + ); + + expectObservable(connectionManager.transports$).toBe("abc", { + a: expect.toSatisfy((t: LivekitTransport[]) => t.length === 1), + b: expect.toSatisfy((t: LivekitTransport[]) => t.length === 1), + c: expect.toSatisfy((t: LivekitTransport[]) => t.length === 2), + }); + + expectObservable(connectionManager.connectionManagerData$).toBe("abc", { + a: expect.toSatisfy( + (d: ConnectionManagerData) => d.getConnections().length === 1, + ), + b: expect.toSatisfy( + (d: ConnectionManagerData) => d.getConnections().length === 1, + ), + c: expect.toSatisfy( + (d: ConnectionManagerData) => d.getConnections().length === 2, + ), + }); + + expectObservable(connectionManager.connections$).toBe("abc", { + a: expect.toSatisfy((t: Connection[]) => t.length === 1), + b: expect.toSatisfy((t: Connection[]) => t.length === 1), + c: expect.toSatisfy((t: Connection[]) => t.length === 2), + }); + + expectObservable(matrixLivekitItems$).toBe("abc", { a: expect.toSatisfy((items: MatrixLivekitMember[]) => { - expect(items.length).toBe(1); - const item = items[0]!; - expect(item.membership).toStrictEqual(bobMembership); - expect(item.participant).toBeUndefined(); - return true; - }), - b: expect.toSatisfy((items: MatrixLivekitMember[]) => { - // TODO - // expect(items.length).toBe(2); - // + // expect(items.length).toBe(1); // const item = items[0]!; // expect(item.membership).toStrictEqual(bobMembership); // expect(item.participant).toBeUndefined(); - // - // { - // const item = items[1]!; - // expect(item.membership).toStrictEqual(carlMembership); - // expect(item.participant).toBeUndefined(); - // } + return true; + }), + b: expect.toSatisfy((items: MatrixLivekitMember[]) => { return true; }), c: expect.toSatisfy(() => true), diff --git a/src/utils/test.ts b/src/utils/test.ts index 96d274d1..bb19f2b1 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -78,11 +78,11 @@ export interface OurRunHelpers extends RunHelpers { * diagram. */ schedule: (marbles: string, actions: Record void>) => void; - behavior( + behavior: ( marbles: string, values?: { [marble: string]: T }, error?: unknown, - ): Behavior; + ) => Behavior; scope: ObservableScope; } From 2e6b1767b9d905701eff449317265bc5e79763a0 Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 6 Nov 2025 16:48:20 +0100 Subject: [PATCH 27/75] Fixup base integration test --- src/state/remoteMembers/ConnectionManager.ts | 7 +- .../remoteMembers/MatrixLivekitMembers.ts | 14 +- src/state/remoteMembers/integration.test.ts | 145 ++++++++++-------- 3 files changed, 87 insertions(+), 79 deletions(-) diff --git a/src/state/remoteMembers/ConnectionManager.ts b/src/state/remoteMembers/ConnectionManager.ts index 7cee0756..245db7c1 100644 --- a/src/state/remoteMembers/ConnectionManager.ts +++ b/src/state/remoteMembers/ConnectionManager.ts @@ -133,11 +133,8 @@ export function createConnectionManager$({ */ const transports$ = scope.behavior( combineLatest([running$, inputTransports$]).pipe( - map(([running, transports]) => ({ - epoch: transports.epoch, - value: running ? transports.value : [], - })), - map((transports) => removeDuplicateTransports(transports.value)), + map(([running, transports]) => (running ? transports : [])), + map((transports) => removeDuplicateTransports(transports)), ), ); diff --git a/src/state/remoteMembers/MatrixLivekitMembers.ts b/src/state/remoteMembers/MatrixLivekitMembers.ts index 0f5234fa..28a9cca9 100644 --- a/src/state/remoteMembers/MatrixLivekitMembers.ts +++ b/src/state/remoteMembers/MatrixLivekitMembers.ts @@ -13,7 +13,7 @@ import { type LivekitTransport, type CallMembership, } from "matrix-js-sdk/lib/matrixrtc"; -import { combineLatest, filter, map, skipWhile, type Observable } from "rxjs"; +import { combineLatest, map } from "rxjs"; // eslint-disable-next-line rxjs/no-internal import { type NodeStyleEventEmitter } from "rxjs/internal/observable/fromEvent"; import { type Room as MatrixRoom, type RoomMember } from "matrix-js-sdk"; @@ -91,12 +91,12 @@ export function createMatrixLivekitMembers$({ connectionManager.connectionManagerData$, displaynameMap$, ]).pipe( - filter( - ([membershipsWithTransports, managerData, displaynames]) => - // for each change in - displaynames.size === membershipsWithTransports.length && - displaynames.size === managerData.getConnections().length, - ), + // filter( + // ([membershipsWithTransports, managerData, displaynames]) => + // // for each change in + // displaynames.size === membershipsWithTransports.length && + // displaynames.size === managerData.getConnections().length, + // ), map(([memberships, managerData, displaynames]) => { const items: MatrixLivekitMember[] = memberships.map( ({ membership, transport }) => { diff --git a/src/state/remoteMembers/integration.test.ts b/src/state/remoteMembers/integration.test.ts index 55db4009..14085568 100644 --- a/src/state/remoteMembers/integration.test.ts +++ b/src/state/remoteMembers/integration.test.ts @@ -6,12 +6,13 @@ Please see LICENSE in the repository root for full details. */ import { test, vi, expect, beforeEach, afterEach } from "vitest"; -import { BehaviorSubject, map } from "rxjs"; +import { BehaviorSubject } from "rxjs"; import { type Room as LivekitRoom } from "livekit-client"; import EventEmitter from "events"; import fetchMock from "fetch-mock"; import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { type Room as MatrixRoom, type RoomMember } from "matrix-js-sdk"; +import { logger } from "matrix-js-sdk/lib/logger"; import { ObservableScope } from "../ObservableScope.ts"; import { ECConnectionFactory } from "./ConnectionFactory.ts"; @@ -23,15 +24,12 @@ import { } from "../../utils/test"; import { type ProcessorState } from "../../livekit/TrackProcessorContext.tsx"; import { + areLivekitTransportsEqual, createMatrixLivekitMembers$, type MatrixLivekitMember, } from "./MatrixLivekitMembers.ts"; -import { - ConnectionManagerData, - createConnectionManager$, -} from "./ConnectionManager.ts"; +import { createConnectionManager$ } from "./ConnectionManager.ts"; import { membershipsAndTransports$ } from "../SessionBehaviors.ts"; -import { Connection } from "./Connection.ts"; // Test the integration of ConnectionManager and MatrixLivekitMerger @@ -108,12 +106,22 @@ afterEach(() => { fetchMock.reset(); }); -test("example test 2", () => { - withTestScheduler(({ schedule, expectObservable, behavior, cold }) => { +test("bob, carl, then bob joining no tracks yet", () => { + withTestScheduler(({ expectObservable, behavior }) => { const bobMembership = mockCallMembership("@bob:example.com", "BDEV000"); const carlMembership = mockCallMembership("@carl:example.com", "CDEV000"); const daveMembership = mockCallMembership("@dave:foo.bar", "DDEV000"); - const memberships$ = behavior("abc", { + + // We add the `---` because there is a limitation in rxjs marbles https://github.com/ReactiveX/rxjs/issues/5677 + // Because we several values emitted at the same frame, so we use the grouping format + // e.g. a(bc) to indicate that b and c are emitted at the same time. But rxjs marbles advance the + // time by the number of characters in the marble diagram, so we need to add some padding to avoid so that + // the next emission is testable + // ab---c--- + // a(bc)(de) + const eMarble = "ab----c----"; + const vMarble = "a(xxb)(xxc)"; + const memberships$ = behavior(eMarble, { a: [bobMembership], b: [bobMembership, carlMembership], c: [bobMembership, carlMembership, daveMembership], @@ -138,70 +146,73 @@ test("example test 2", () => { matrixRoom: mockMatrixRoom, }); - expectObservable(membershipsAndTransports.transports$).toBe("abc", { - a: expect.toSatisfy((t: LivekitTransport[]) => t.length === 1), - b: expect.toSatisfy((t: LivekitTransport[]) => t.length === 2), - c: expect.toSatisfy((t: LivekitTransport[]) => t.length === 3), - }); - - expectObservable(membershipsAndTransports.membershipsWithTransport$).toBe( - "abc", - { - a: expect.toSatisfy((t: LivekitTransport[]) => t.length === 1), - b: expect.toSatisfy((t: LivekitTransport[]) => t.length === 2), - c: expect.toSatisfy((t: LivekitTransport[]) => t.length === 3), - }, - ); - - expectObservable(connectionManager.transports$).toBe("abc", { - a: expect.toSatisfy((t: LivekitTransport[]) => t.length === 1), - b: expect.toSatisfy((t: LivekitTransport[]) => t.length === 1), - c: expect.toSatisfy((t: LivekitTransport[]) => t.length === 2), - }); - - expectObservable(connectionManager.connectionManagerData$).toBe("abc", { - a: expect.toSatisfy( - (d: ConnectionManagerData) => d.getConnections().length === 1, - ), - b: expect.toSatisfy( - (d: ConnectionManagerData) => d.getConnections().length === 1, - ), - c: expect.toSatisfy( - (d: ConnectionManagerData) => d.getConnections().length === 2, - ), - }); - - expectObservable(connectionManager.connections$).toBe("abc", { - a: expect.toSatisfy((t: Connection[]) => t.length === 1), - b: expect.toSatisfy((t: Connection[]) => t.length === 1), - c: expect.toSatisfy((t: Connection[]) => t.length === 2), - }); - - expectObservable(matrixLivekitItems$).toBe("abc", { + expectObservable(matrixLivekitItems$).toBe(vMarble, { a: expect.toSatisfy((items: MatrixLivekitMember[]) => { - // expect(items.length).toBe(1); - // const item = items[0]!; - // expect(item.membership).toStrictEqual(bobMembership); - // expect(item.participant).toBeUndefined(); + expect(items.length).toBe(1); + const item = items[0]!; + expect(item.membership).toStrictEqual(bobMembership); + expect( + areLivekitTransportsEqual( + item.connection!.transport, + bobMembership.transports[0]! as LivekitTransport, + ), + ).toBe(true); + expect(item.participant).toBeUndefined(); return true; }), b: expect.toSatisfy((items: MatrixLivekitMember[]) => { + expect(items.length).toBe(2); + + { + const item = items[0]!; + expect(item.membership).toStrictEqual(bobMembership); + expect(item.participant).toBeUndefined(); + } + + { + const item = items[1]!; + expect(item.membership).toStrictEqual(carlMembership); + expect(item.participantId).toStrictEqual( + `${carlMembership.userId}:${carlMembership.deviceId}`, + ); + expect( + areLivekitTransportsEqual( + item.connection!.transport, + carlMembership.transports[0]! as LivekitTransport, + ), + ).toBe(true); + expect(item.participant).toBeUndefined(); + } return true; }), - c: expect.toSatisfy(() => true), + c: expect.toSatisfy((items: MatrixLivekitMember[]) => { + logger.info(`E Items length: ${items.length}`); + expect(items.length).toBe(3); + { + expect(items[0]!.membership).toStrictEqual(bobMembership); + } + + { + expect(items[1]!.membership).toStrictEqual(carlMembership); + } + + { + const item = items[2]!; + expect(item.membership).toStrictEqual(daveMembership); + expect(item.participantId).toStrictEqual( + `${daveMembership.userId}:${daveMembership.deviceId}`, + ); + expect( + areLivekitTransportsEqual( + item.connection!.transport, + daveMembership.transports[0]! as LivekitTransport, + ), + ).toBe(true); + expect(item.participant).toBeUndefined(); + } + return true; + }), + x: expect.anything(), }); }); }); - -// test("Tryng", () => { -// -// withTestScheduler(({ schedule, expectObservable, behavior, cold }) => { -// const one = cold("a-b-c", { a: 1, b: 2, c: 3 }); -// const a = one.pipe(map(() => 1)); -// const b = one.pipe(map(() => 2)); -// const combined = combineLatest([a,b]) -// .pipe(map(([a,b])=>`${a}${b}`)); -// expectObservable(combined).toBe("a-b-c", { a: 1, b: expect.anything(), c: 3 }); -// -// }) -// }) From 7c41aef8013abb4cd9ff5321bbfbbe427ef88590 Mon Sep 17 00:00:00 2001 From: Timo K Date: Thu, 6 Nov 2025 21:54:34 +0100 Subject: [PATCH 28/75] Add `Epoch` and use it in for the `memberships$` behavior and its derivatives. --- src/state/Behavior.ts | 4 + src/state/CallViewModel.ts | 5 +- src/state/ObservableScope.test.ts | 56 +++++++++ src/state/ObservableScope.ts | 106 ++++++++++++++++++ src/state/SessionBehaviors.ts | 23 ++-- src/state/localMember/LocalMembership.ts | 12 +- src/state/localMember/LocalTransport.ts | 12 +- src/state/remoteMembers/ConnectionManager.ts | 92 ++++++++------- .../remoteMembers/MatrixLivekitMembers.ts | 95 ++++++++-------- src/state/remoteMembers/displayname.ts | 16 +-- src/state/remoteMembers/integration.test.ts | 36 +++--- 11 files changed, 322 insertions(+), 135 deletions(-) create mode 100644 src/state/ObservableScope.test.ts diff --git a/src/state/Behavior.ts b/src/state/Behavior.ts index 3c88dc00..71b18a55 100644 --- a/src/state/Behavior.ts +++ b/src/state/Behavior.ts @@ -18,6 +18,10 @@ import { BehaviorSubject } from "rxjs"; */ export type Behavior = Omit, "next" | "observers">; +export type BehaviorWithEpoch = Behavior & { + pipeEpoch(): Behavior<{ value: T; epoch: number }>; +}; + /** * Creates a Behavior which never changes in value. */ diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 420ecab4..714ca62c 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -226,7 +226,10 @@ export class CallViewModel { [this.localTransport$, this.membershipsAndTransports.transports$], (localTransport, transports) => { const localTransportAsArray = localTransport ? [localTransport] : []; - return [...localTransportAsArray, ...transports]; + return transports.mapInner((transports) => [ + ...localTransportAsArray, + ...transports, + ]); }, ), ); diff --git a/src/state/ObservableScope.test.ts b/src/state/ObservableScope.test.ts new file mode 100644 index 00000000..d53084da --- /dev/null +++ b/src/state/ObservableScope.test.ts @@ -0,0 +1,56 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { describe, expect, it } from "vitest"; + +import { Epoch, mapEpoch, trackEpoch } from "./ObservableScope"; +import { withTestScheduler } from "../utils/test"; + +describe("Epoch", () => { + it("should map the value correctly", () => { + const epoch = new Epoch(1); + const mappedEpoch = epoch.mapInner((v) => v + 1); + expect(mappedEpoch.value).toBe(2); + expect(mappedEpoch.epoch).toBe(0); + }); + + it("should be tracked from an observable", () => { + withTestScheduler(({ expectObservable, behavior }) => { + const observable$ = behavior("abc", { + a: 1, + b: 2, + c: 3, + }); + const epochObservable$ = observable$.pipe(trackEpoch()); + expectObservable(epochObservable$).toBe("abc", { + a: expect.toSatisfy((e) => e.epoch === 0 && e.value === 1), + b: expect.toSatisfy((e) => e.epoch === 1 && e.value === 2), + c: expect.toSatisfy((e) => e.epoch === 2 && e.value === 3), + }); + }); + }); + + it("can be mapped without loosing epoch information", () => { + withTestScheduler(({ expectObservable, behavior }) => { + const observable$ = behavior("abc", { + a: "A", + b: "B", + c: "C", + }); + const epochObservable$ = observable$.pipe(trackEpoch()); + const derivedEpoch$ = epochObservable$.pipe( + mapEpoch((e) => e + "-mapped"), + ); + + expectObservable(derivedEpoch$).toBe("abc", { + a: new Epoch("A-mapped", 0), + b: new Epoch("B-mapped", 1), + c: new Epoch("C-mapped", 2), + }); + }); + }); +}); diff --git a/src/state/ObservableScope.ts b/src/state/ObservableScope.ts index 879445e6..fbf92ada 100644 --- a/src/state/ObservableScope.ts +++ b/src/state/ObservableScope.ts @@ -12,7 +12,9 @@ import { EMPTY, endWith, filter, + map, type Observable, + type OperatorFunction, share, take, takeUntil, @@ -151,3 +153,107 @@ export class ObservableScope { * The global scope, a scope which never ends. */ export const globalScope = new ObservableScope(); + +/** + * `Epoch`'s can be used to create `Behavior`s and `Observable`s which derivitives can be merged + * with `combinedLatest` without duplicated emissions. + * + * This is useful in the following example: + * ``` + * const rootObs$ = of("red","green","blue"); + * const derivedObs$ = rootObs$.pipe( + * map((v)=> {red:"fire", green:"grass", blue:"water"}[v]) + * ); + * const otherDerivedObs$ = rootObs$.pipe( + * map((v)=> {red:"tomatoes", green:"leaves", blue:"sky"}[v]) + * ); + * const mergedObs$ = combineLatest([rootObs$, derivedObs$, otherDerivedObs$]).pipe( + * map(([color, a,b]) => color + " like " + a + " and " + b) + * ); + * + * ``` + * will result in 6 emissions with mismatching items like "red like fire and leaves" + * + * # Use Epoch + * ``` + * const rootObs$ = of(1,2,3).pipe(trackEpoch()); + * const derivedObs$ = rootObs$.pipe( + * mapEpoch((v)=> "this number: " + v) + * ); + * const otherDerivedObs$ = rootObs$.pipe( + * mapEpoch((v)=> "multiplied by: " + v) + * ); + * const mergedObs$ = combineLatest([derivedObs$, otherDerivedObs$]).pipe( + * filter((values) => values.every((v) => v.epoch === values[0].v)), + * map(([color, a, b]) => color + " like " + a + " and " + b) + * ); + * + * ``` + * will result in 3 emissions all matching (e.g. "blue like water and sky") + */ +export class Epoch { + public readonly epoch: number; + public readonly value: T; + + public constructor(value: T, epoch?: number) { + this.value = value; + this.epoch = epoch ?? 0; + } + /** + * Maps the value inside the epoch to a new value while keeping the epoch number. + * # usage + * ``` + * const myEpoch$ = myObservable$.pipe( + * map(trackEpoch()), + * // this is the preferred way using mapEpoch + * mapEpoch((v)=> v+1) + * // This is how inner map can be used: + * map((epoch) => epoch.innerMap((v)=> v+1)) + * // It is equivalent to: + * map((epoch) => new Epoch(epoch.value + 1, epoch.epoch)) + * ) + * ``` + * See also `Epoch` + */ + public mapInner(map: (value: T) => U): Epoch { + return new Epoch(map(this.value), this.epoch); + } +} + +/** + * A `pipe` compatible map oparator that keeps the epoch in tact but allows mapping the value. + * # usage + * ``` + * const myEpoch$ = myObservable$.pipe( + * map(trackEpoch()), + * // this is the preferred way using mapEpoch + * mapEpoch((v)=> v+1) + * // This is how inner map can be used: + * map((epoch) => epoch.innerMap((v)=> v+1)) + * // It is equivalent to: + * map((epoch) => new Epoch(epoch.value + 1, epoch.epoch)) + * ) + * ``` + * See also `Epoch` + */ +export function mapEpoch( + mapFn: (value: T) => U, +): OperatorFunction, Epoch> { + return map((e) => e.mapInner(mapFn)); +} +/** + * # usage + * ``` + * const myEpoch$ = myObservable$.pipe( + * map(trackEpoch()), + * map((epoch) => epoch.innerMap((v)=> v+1)) + * ) + * const derived = myEpoch$.pipe( + * mapEpoch((v)=>v^2) + * ) + * ``` + * See also `Epoch` + */ +export function trackEpoch(): OperatorFunction> { + return map>((value, number) => new Epoch(value, number)); +} diff --git a/src/state/SessionBehaviors.ts b/src/state/SessionBehaviors.ts index 80e9f09c..d44ad33a 100644 --- a/src/state/SessionBehaviors.ts +++ b/src/state/SessionBehaviors.ts @@ -12,19 +12,24 @@ import { type MatrixRTCSession, MatrixRTCSessionEvent, } from "matrix-js-sdk/lib/matrixrtc"; -import { fromEvent, map } from "rxjs"; +import { fromEvent } from "rxjs"; -import { type ObservableScope } from "./ObservableScope"; +import { + type Epoch, + mapEpoch, + trackEpoch, + type ObservableScope, +} from "./ObservableScope"; import { type Behavior } from "./Behavior"; export const membershipsAndTransports$ = ( scope: ObservableScope, - memberships$: Behavior, + memberships$: Behavior>, ): { membershipsWithTransport$: Behavior< - { membership: CallMembership; transport?: LivekitTransport }[] + Epoch<{ membership: CallMembership; transport?: LivekitTransport }[]> >; - transports$: Behavior; + transports$: Behavior>; } => { /** * Lists the transports used by ourselves, plus all other MatrixRTC session @@ -36,7 +41,7 @@ export const membershipsAndTransports$ = ( */ const membershipsWithTransport$ = scope.behavior( memberships$.pipe( - map((memberships) => { + mapEpoch((memberships) => { return memberships.map((membership) => { const oldestMembership = memberships[0] ?? membership; const transport = membership.getTransport(oldestMembership); @@ -51,7 +56,7 @@ export const membershipsAndTransports$ = ( const transports$ = scope.behavior( membershipsWithTransport$.pipe( - map((mts) => mts.flatMap(({ transport: t }) => (t ? [t] : []))), + mapEpoch((mts) => mts.flatMap(({ transport: t }) => (t ? [t] : []))), ), ); @@ -64,12 +69,12 @@ export const membershipsAndTransports$ = ( export const createMemberships$ = ( scope: ObservableScope, matrixRTCSession: MatrixRTCSession, -): Behavior => { +): Behavior> => { return scope.behavior( fromEvent( matrixRTCSession, MatrixRTCSessionEvent.MembershipsChanged, (_, memberships: CallMembership[]) => memberships, - ), + ).pipe(trackEpoch()), ); }; diff --git a/src/state/localMember/LocalMembership.ts b/src/state/localMember/LocalMembership.ts index 83337064..6a400c37 100644 --- a/src/state/localMember/LocalMembership.ts +++ b/src/state/localMember/LocalMembership.ts @@ -28,23 +28,20 @@ import { import { logger } from "matrix-js-sdk/lib/logger"; import { type Behavior } from "../Behavior"; -import { - type ConnectionManagerReturn, - type createConnectionManager$, -} from "../remoteMembers/ConnectionManager"; +import { type IConnectionManager } from "../remoteMembers/ConnectionManager"; import { ObservableScope } from "../ObservableScope"; import { Publisher } from "./Publisher"; import { type MuteStates } from "../MuteStates"; import { type ProcessorState } from "../../livekit/TrackProcessorContext"; import { type MediaDevices } from "../MediaDevices"; import { and$ } from "../../utils/observable"; -import { areLivekitTransportsEqual } from "../remoteMembers/matrixLivekitMerger"; import { enterRTCSession, type EnterRTCSessionOptions, } from "../../rtcSessionHelpers"; import { type ElementCallError } from "../../utils/errors"; import { ElementWidgetActions, type WidgetHelpers } from "../../widget"; +import { areLivekitTransportsEqual } from "../remoteMembers/MatrixLivekitMembers"; enum LivekitState { UNINITIALIZED = "uninitialized", @@ -93,7 +90,7 @@ interface Props { scope: ObservableScope; mediaDevices: MediaDevices; muteStates: MuteStates; - connectionManager: ConnectionManagerReturn; + connectionManager: IConnectionManager; matrixRTCSession: MatrixRTCSession; matrixRoom: MatrixRoom; localTransport$: Behavior; @@ -153,12 +150,13 @@ export const createLocalMembership$ = ({ // This should be used in a combineLatest with publisher$ to connect. const tracks$ = new BehaviorSubject([]); + // Drop Epoch data here since we will not combine this anymore const connection$ = scope.behavior( combineLatest( [connectionManager.connections$, localTransport$], (connections, transport) => { if (transport === undefined) return undefined; - return connections.find((connection) => + return connections.value.find((connection) => areLivekitTransportsEqual(connection.transport, transport), ); }, diff --git a/src/state/localMember/LocalTransport.ts b/src/state/localMember/LocalTransport.ts index a1b0d329..bdcfcffc 100644 --- a/src/state/localMember/LocalTransport.ts +++ b/src/state/localMember/LocalTransport.ts @@ -13,13 +13,17 @@ import { isLivekitTransportConfig, } from "matrix-js-sdk/lib/matrixrtc"; import { type MatrixClient } from "matrix-js-sdk"; -import { combineLatest, distinctUntilChanged, first, from, map } from "rxjs"; +import { combineLatest, distinctUntilChanged, first, from } from "rxjs"; import { logger } from "matrix-js-sdk/lib/logger"; import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery"; import { deepCompare } from "matrix-js-sdk/lib/utils"; import { type Behavior } from "../Behavior.ts"; -import { type ObservableScope } from "../ObservableScope.ts"; +import { + type Epoch, + mapEpoch, + type ObservableScope, +} from "../ObservableScope.ts"; import { Config } from "../../config/Config.ts"; import { MatrixRTCTransportMissingError } from "../../utils/errors.ts"; import { getSFUConfigWithOpenID } from "../../livekit/openIDSFU.ts"; @@ -37,7 +41,7 @@ import { getSFUConfigWithOpenID } from "../../livekit/openIDSFU.ts"; */ interface Props { scope: ObservableScope; - memberships$: Behavior; + memberships$: Behavior>; client: MatrixClient; roomId: string; useOldestMember$: Behavior; @@ -63,7 +67,7 @@ export const createLocalTransport$ = ({ */ const oldestMemberTransport$ = scope.behavior( memberships$.pipe( - map((memberships) => memberships[0].getTransport(memberships[0])), + mapEpoch((memberships) => memberships[0].getTransport(memberships[0])), first((t) => t != undefined && isLivekitTransport(t)), ), undefined, diff --git a/src/state/remoteMembers/ConnectionManager.ts b/src/state/remoteMembers/ConnectionManager.ts index 245db7c1..49ab6b71 100644 --- a/src/state/remoteMembers/ConnectionManager.ts +++ b/src/state/remoteMembers/ConnectionManager.ts @@ -19,7 +19,7 @@ import { type LocalParticipant, type RemoteParticipant } from "livekit-client"; import { type Behavior } from "../Behavior"; import { type Connection } from "./Connection"; -import { type ObservableScope } from "../ObservableScope"; +import { Epoch, type ObservableScope } from "../ObservableScope"; import { generateKeyed$ } from "../../utils/observable"; import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts"; import { type ConnectionFactory } from "./ConnectionFactory.ts"; @@ -90,13 +90,13 @@ export class ConnectionManagerData { interface Props { scope: ObservableScope; connectionFactory: ConnectionFactory; - inputTransports$: Behavior; + inputTransports$: Behavior>; } // TODO - write test for scopes (do we really need to bind scope) export interface IConnectionManager { - transports$: Behavior; - connectionManagerData$: Behavior; - connections$: Behavior; + transports$: Behavior>; + connectionManagerData$: Behavior>; + connections$: Behavior>; } /** * Crete a `ConnectionManager` @@ -133,8 +133,10 @@ export function createConnectionManager$({ */ const transports$ = scope.behavior( combineLatest([running$, inputTransports$]).pipe( - map(([running, transports]) => (running ? transports : [])), - map((transports) => removeDuplicateTransports(transports)), + map(([running, transports]) => + transports.mapInner((transport) => (running ? transport : [])), + ), + map((transports) => transports.mapInner(removeDuplicateTransports)), ), ); @@ -142,7 +144,7 @@ export function createConnectionManager$({ * Connections for each transport in use by one or more session members. */ const connections$ = scope.behavior( - generateKeyed$( + generateKeyed$, Connection, Epoch>( transports$, (transports, createOrGet) => { const createConnection = @@ -162,46 +164,50 @@ export function createConnectionManager$({ return connection; }; - return transports.map((transport) => { - const key = - transport.livekit_service_url + "|" + transport.livekit_alias; - return createOrGet(key, createConnection(transport)); + return transports.mapInner((transports) => { + return transports.map((transport) => { + const key = + transport.livekit_service_url + "|" + transport.livekit_alias; + return createOrGet(key, createConnection(transport)); + }); }); }, ), ); - const connectionManagerData$: Behavior = - scope.behavior( - connections$.pipe( - switchMap((connections) => { - // Map the connections to list of {connection, participants}[] - const listOfConnectionsWithPublishingParticipants = connections.map( - (connection) => { - return connection.participantsWithTrack$.pipe( - map((participants) => ({ - connection, - participants, - })), - ); - }, - ); - // combineLatest the several streams into a single stream with the ConnectionManagerData - return combineLatest( - listOfConnectionsWithPublishingParticipants, - ).pipe( - map((lists) => - lists.reduce((data, { connection, participants }) => { - data.add(connection, participants); - return data; - }, new ConnectionManagerData()), - ), - ); - }), - ), - // start empty - new ConnectionManagerData(), - ); + const connectionManagerData$ = scope.behavior( + connections$.pipe( + switchMap((connections) => { + const epoch = connections.epoch; + + // Map the connections to list of {connection, participants}[] + const listOfConnectionsWithPublishingParticipants = + connections.value.map((connection) => { + return connection.participantsWithTrack$.pipe( + map((participants) => ({ + connection, + participants, + })), + ); + }); + + // combineLatest the several streams into a single stream with the ConnectionManagerData + return combineLatest(listOfConnectionsWithPublishingParticipants).pipe( + map( + (lists) => + new Epoch( + lists.reduce((data, { connection, participants }) => { + data.add(connection, participants); + return data; + }, new ConnectionManagerData()), + epoch, + ), + ), + ); + }), + ), + ); + return { transports$, connectionManagerData$, connections$ }; } diff --git a/src/state/remoteMembers/MatrixLivekitMembers.ts b/src/state/remoteMembers/MatrixLivekitMembers.ts index 28a9cca9..d7937bbe 100644 --- a/src/state/remoteMembers/MatrixLivekitMembers.ts +++ b/src/state/remoteMembers/MatrixLivekitMembers.ts @@ -13,14 +13,15 @@ import { type LivekitTransport, type CallMembership, } from "matrix-js-sdk/lib/matrixrtc"; -import { combineLatest, map } from "rxjs"; +import { combineLatest, filter, map } from "rxjs"; // eslint-disable-next-line rxjs/no-internal import { type NodeStyleEventEmitter } from "rxjs/internal/observable/fromEvent"; import { type Room as MatrixRoom, type RoomMember } from "matrix-js-sdk"; +import { logger } from "matrix-js-sdk/lib/logger"; import { type Behavior } from "../Behavior"; import { type IConnectionManager } from "./ConnectionManager"; -import { type ObservableScope } from "../ObservableScope"; +import { Epoch, mapEpoch, type ObservableScope } from "../ObservableScope"; import { getRoomMemberFromRtcMember, memberDisplaynames$ } from "./displayname"; import { type Connection } from "./Connection"; @@ -47,7 +48,7 @@ export interface MatrixLivekitMember { interface Props { scope: ObservableScope; membershipsWithTransport$: Behavior< - { membership: CallMembership; transport?: LivekitTransport }[] + Epoch<{ membership: CallMembership; transport?: LivekitTransport }[]> >; connectionManager: IConnectionManager; // TODO this is too much information for that class, @@ -74,7 +75,7 @@ export function createMatrixLivekitMembers$({ membershipsWithTransport$, connectionManager, matrixRoom, -}: Props): Behavior { +}: Props): Behavior> { /** * Stream of all the call members and their associated livekit data (if available). */ @@ -82,7 +83,7 @@ export function createMatrixLivekitMembers$({ const displaynameMap$ = memberDisplaynames$( scope, matrixRoom, - membershipsWithTransport$.pipe(map((v) => v.map((v) => v.membership))), + membershipsWithTransport$.pipe(mapEpoch((v) => v.map((v) => v.membership))), ); return scope.behavior( @@ -91,48 +92,52 @@ export function createMatrixLivekitMembers$({ connectionManager.connectionManagerData$, displaynameMap$, ]).pipe( - // filter( - // ([membershipsWithTransports, managerData, displaynames]) => - // // for each change in - // displaynames.size === membershipsWithTransports.length && - // displaynames.size === managerData.getConnections().length, - // ), - map(([memberships, managerData, displaynames]) => { - const items: MatrixLivekitMember[] = memberships.map( - ({ membership, transport }) => { - // TODO! cannot use membership.membershipID yet, Currently its hardcoded by the jwt service to - const participantId = /*membership.membershipID*/ `${membership.userId}:${membership.deviceId}`; + filter((values) => + values.every((value) => value.epoch === values[0].epoch), + ), + map( + ([ + { value: membershipsWithTransports, epoch }, + { value: managerData }, + { value: displaynames }, + ]) => { + const items: MatrixLivekitMember[] = membershipsWithTransports.map( + ({ membership, transport }) => { + // TODO! cannot use membership.membershipID yet, Currently its hardcoded by the jwt service to + const participantId = /*membership.membershipID*/ `${membership.userId}:${membership.deviceId}`; - const participants = transport - ? managerData.getParticipantForTransport(transport) - : []; - const participant = participants.find( - (p) => p.identity == participantId, - ); - const member = getRoomMemberFromRtcMember( - membership, - matrixRoom, - )?.member; - const connection = transport - ? managerData.getConnectionForTransport(transport) - : undefined; - const displayName = displaynames.get(participantId); - return { - participant, - membership, - connection, - // This makes sense to add to the js-sdk callMembership (we only need the avatar so probably the call memberhsip just should aquire the avatar) - // TODO Ugh this is hidign that it might be undefined!! best we remove the member entirely. - member: member as RoomMember, - displayName, - mxcAvatarUrl: member?.getMxcAvatarUrl(), - participantId, - }; - }, - ); - return items; - }), + const participants = transport + ? managerData.getParticipantForTransport(transport) + : []; + const participant = participants.find( + (p) => p.identity == participantId, + ); + const member = getRoomMemberFromRtcMember( + membership, + matrixRoom, + )?.member; + const connection = transport + ? managerData.getConnectionForTransport(transport) + : undefined; + const displayName = displaynames.get(participantId); + return { + participant, + membership, + connection, + // This makes sense to add to the js-sdk callMembership (we only need the avatar so probably the call memberhsip just should aquire the avatar) + // TODO Ugh this is hidign that it might be undefined!! best we remove the member entirely. + member: member as RoomMember, + displayName, + mxcAvatarUrl: member?.getMxcAvatarUrl(), + participantId, + }; + }, + ); + return new Epoch(items, epoch); + }, + ), ), + // new Epoch([]), ); } diff --git a/src/state/remoteMembers/displayname.ts b/src/state/remoteMembers/displayname.ts index e735147d..8f2b3f64 100644 --- a/src/state/remoteMembers/displayname.ts +++ b/src/state/remoteMembers/displayname.ts @@ -16,14 +16,15 @@ import { import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; import { logger } from "matrix-js-sdk/lib/logger"; import { type Room as MatrixRoom } from "matrix-js-sdk/lib/matrix"; +// eslint-disable-next-line rxjs/no-internal +import { type NodeStyleEventEmitter } from "rxjs/internal/observable/fromEvent"; -import { type ObservableScope } from "../ObservableScope"; +import { Epoch, type ObservableScope } from "../ObservableScope"; import { calculateDisplayName, shouldDisambiguate, } from "../../utils/displayname"; import { type Behavior } from "../Behavior"; -import type { NodeStyleEventEmitter } from "rxjs/src/internal/observable/fromEvent.ts"; /** * Displayname for each member of the call. This will disambiguate @@ -36,8 +37,8 @@ import type { NodeStyleEventEmitter } from "rxjs/src/internal/observable/fromEve export const memberDisplaynames$ = ( scope: ObservableScope, matrixRoom: Pick & NodeStyleEventEmitter, - memberships$: Observable, -): Behavior> => + memberships$: Observable>, +): Behavior>> => scope.behavior( combineLatest([ // Handle call membership changes @@ -46,7 +47,8 @@ export const memberDisplaynames$ = ( fromEvent(matrixRoom, RoomStateEvent.Members).pipe(startWith(null)), // TODO: do we need: pauseWhen(this.pretendToBeDisconnected$), ]).pipe( - map(([memberships, _displayNames]) => { + map(([epochMemberships, _displayNames]) => { + const { epoch, value: memberships } = epochMemberships; const displaynameMap = new Map(); const room = matrixRoom; @@ -68,10 +70,10 @@ export const memberDisplaynames$ = ( calculateDisplayName(member, disambiguate), ); } - return displaynameMap; + return new Epoch(displaynameMap, epoch); }), ), - new Map(), + new Epoch(new Map()), ); export function getRoomMemberFromRtcMember( diff --git a/src/state/remoteMembers/integration.test.ts b/src/state/remoteMembers/integration.test.ts index 14085568..9ce4cf33 100644 --- a/src/state/remoteMembers/integration.test.ts +++ b/src/state/remoteMembers/integration.test.ts @@ -14,7 +14,7 @@ import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { type Room as MatrixRoom, type RoomMember } from "matrix-js-sdk"; import { logger } from "matrix-js-sdk/lib/logger"; -import { ObservableScope } from "../ObservableScope.ts"; +import { type Epoch, ObservableScope, trackEpoch } from "../ObservableScope.ts"; import { ECConnectionFactory } from "./ConnectionFactory.ts"; import { type OpenIDClientParts } from "../../livekit/openIDSFU.ts"; import { @@ -107,25 +107,20 @@ afterEach(() => { }); test("bob, carl, then bob joining no tracks yet", () => { - withTestScheduler(({ expectObservable, behavior }) => { + withTestScheduler(({ expectObservable, behavior, scope }) => { const bobMembership = mockCallMembership("@bob:example.com", "BDEV000"); const carlMembership = mockCallMembership("@carl:example.com", "CDEV000"); const daveMembership = mockCallMembership("@dave:foo.bar", "DDEV000"); - // We add the `---` because there is a limitation in rxjs marbles https://github.com/ReactiveX/rxjs/issues/5677 - // Because we several values emitted at the same frame, so we use the grouping format - // e.g. a(bc) to indicate that b and c are emitted at the same time. But rxjs marbles advance the - // time by the number of characters in the marble diagram, so we need to add some padding to avoid so that - // the next emission is testable - // ab---c--- - // a(bc)(de) - const eMarble = "ab----c----"; - const vMarble = "a(xxb)(xxc)"; - const memberships$ = behavior(eMarble, { - a: [bobMembership], - b: [bobMembership, carlMembership], - c: [bobMembership, carlMembership, daveMembership], - }); + const eMarble = "abc"; + const vMarble = "abc"; + const memberships$ = scope.behavior( + behavior(eMarble, { + a: [bobMembership], + b: [bobMembership, carlMembership], + c: [bobMembership, carlMembership, daveMembership], + }).pipe(trackEpoch()), + ); const membershipsAndTransports = membershipsAndTransports$( testScope, @@ -147,7 +142,8 @@ test("bob, carl, then bob joining no tracks yet", () => { }); expectObservable(matrixLivekitItems$).toBe(vMarble, { - a: expect.toSatisfy((items: MatrixLivekitMember[]) => { + a: expect.toSatisfy((e: Epoch) => { + const items = e.value; expect(items.length).toBe(1); const item = items[0]!; expect(item.membership).toStrictEqual(bobMembership); @@ -160,7 +156,8 @@ test("bob, carl, then bob joining no tracks yet", () => { expect(item.participant).toBeUndefined(); return true; }), - b: expect.toSatisfy((items: MatrixLivekitMember[]) => { + b: expect.toSatisfy((e: Epoch) => { + const items = e.value; expect(items.length).toBe(2); { @@ -185,7 +182,8 @@ test("bob, carl, then bob joining no tracks yet", () => { } return true; }), - c: expect.toSatisfy((items: MatrixLivekitMember[]) => { + c: expect.toSatisfy((e: Epoch) => { + const items = e.value; logger.info(`E Items length: ${items.length}`); expect(items.length).toBe(3); { From 92fdce33ea744007b5e6682809f07f26c03a281a Mon Sep 17 00:00:00 2001 From: Timo K Date: Fri, 7 Nov 2025 08:44:44 +0100 Subject: [PATCH 29/75] pull out all screen share related logic. --- src/room/InCallView.tsx | 5 +- .../CallNotificationLifecycle.ts | 210 +++++++++++++ .../{ => CallViewModel}/CallViewModel.ts | 275 ++++-------------- src/state/CallViewModel/callPickupState$.ts | 0 .../localMember/LocalMembership.ts | 179 +++++++++--- .../localMember/LocalTransport.ts | 10 +- .../localMember/Publisher.ts | 16 +- .../remoteMembers/Connection.test.ts | 12 +- .../remoteMembers/Connection.ts | 8 +- .../remoteMembers/ConnectionFactory.ts | 12 +- .../remoteMembers/ConnectionManager.test.ts | 4 +- .../remoteMembers/ConnectionManager.ts | 8 +- .../MatrixLivekitMembers.test.ts | 4 +- .../remoteMembers/MatrixLivekitMembers.ts | 4 +- .../remoteMembers/displayname.test.ts | 4 +- .../remoteMembers/displayname.ts | 6 +- .../remoteMembers/integration.test.ts | 14 +- 17 files changed, 461 insertions(+), 310 deletions(-) create mode 100644 src/state/CallViewModel/CallNotificationLifecycle.ts rename src/state/{ => CallViewModel}/CallViewModel.ts (81%) create mode 100644 src/state/CallViewModel/callPickupState$.ts rename src/state/{ => CallViewModel}/localMember/LocalMembership.ts (69%) rename src/state/{ => CallViewModel}/localMember/LocalTransport.ts (94%) rename src/state/{ => CallViewModel}/localMember/Publisher.ts (95%) rename src/state/{ => CallViewModel}/remoteMembers/Connection.test.ts (98%) rename src/state/{ => CallViewModel}/remoteMembers/Connection.ts (97%) rename src/state/{ => CallViewModel}/remoteMembers/ConnectionFactory.ts (89%) rename src/state/{ => CallViewModel}/remoteMembers/ConnectionManager.test.ts (98%) rename src/state/{ => CallViewModel}/remoteMembers/ConnectionManager.ts (96%) rename src/state/{ => CallViewModel}/remoteMembers/MatrixLivekitMembers.test.ts (99%) rename src/state/{ => CallViewModel}/remoteMembers/MatrixLivekitMembers.ts (97%) rename src/state/{ => CallViewModel}/remoteMembers/displayname.test.ts (98%) rename src/state/{ => CallViewModel}/remoteMembers/displayname.ts (95%) rename src/state/{ => CallViewModel}/remoteMembers/integration.test.ts (94%) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index a6a2e897..06c1ccb4 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -58,7 +58,10 @@ import { type MuteStates } from "../state/MuteStates"; import { type MatrixInfo } from "./VideoPreview"; import { InviteButton } from "../button/InviteButton"; import { LayoutToggle } from "./LayoutToggle"; -import { CallViewModel, type GridMode } from "../state/CallViewModel"; +import { + CallViewModel, + type GridMode, +} from "../state/CallViewModel/CallViewModel.ts"; import { Grid, type TileProps } from "../grid/Grid"; import { useInitial } from "../useInitial"; import { SpotlightTile } from "../tile/SpotlightTile"; diff --git a/src/state/CallViewModel/CallNotificationLifecycle.ts b/src/state/CallViewModel/CallNotificationLifecycle.ts new file mode 100644 index 00000000..baf2b665 --- /dev/null +++ b/src/state/CallViewModel/CallNotificationLifecycle.ts @@ -0,0 +1,210 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + type CallMembership, + type MatrixRTCSession, + MatrixRTCSessionEvent, + type MatrixRTCSessionEventHandlerMap, +} from "matrix-js-sdk/lib/matrixrtc"; +import { + combineLatest, + concat, + endWith, + filter, + fromEvent, + ignoreElements, + map, + merge, + NEVER, + type Observable, + of, + pairwise, + startWith, + switchMap, + takeUntil, + timer, +} from "rxjs"; +import { + type EventTimelineSetHandlerMap, + EventType, + type Room as MatrixRoom, + RoomEvent, +} from "matrix-js-sdk"; + +import { type Behavior } from "../Behavior"; +import { type Epoch, mapEpoch, type ObservableScope } from "../ObservableScope"; +export type AutoLeaveReason = "allOthersLeft" | "timeout" | "decline"; +export type CallPickupState = + | "unknown" + | "ringing" + | "timeout" + | "decline" + | "success" + | null; +export type CallNotificationWrapper = Parameters< + MatrixRTCSessionEventHandlerMap[MatrixRTCSessionEvent.DidSendCallNotification] +>; +export function createSentCallNotification$( + scope: ObservableScope, + matrixRTCSession: MatrixRTCSession, +): Behavior { + const sentCallNotification$ = scope.behavior( + fromEvent(matrixRTCSession, MatrixRTCSessionEvent.DidSendCallNotification), + null, + ) as Behavior; + return sentCallNotification$; +} + +export function createReceivedDecline$( + matrixRoom: MatrixRoom, +): Observable> { + return ( + fromEvent(matrixRoom, RoomEvent.Timeline) as Observable< + Parameters + > + ).pipe(filter(([event]) => event.getType() === EventType.RTCDecline)); +} + +interface Props { + scope: ObservableScope; + memberships$: Behavior>; + sentCallNotification$: Observable; + receivedDecline$: Observable< + Parameters + >; + options: { waitForCallPickup?: boolean; autoLeaveWhenOthersLeft?: boolean }; + localUser: { deviceId: string; userId: string }; +} +/** + * @returns {callPickupState$, autoLeave$} + * `callPickupState$` The current call pickup state of the call. + * - "unknown": The client has not yet sent the notification event. We don't know if it will because it first needs to send its own membership. + * Then we can conclude if we were the first one to join or not. + * This may also be set if we are disconnected. + * - "ringing": The call is ringing on other devices in this room (This client should give audiovisual feedback that this is happening). + * - "timeout": No-one picked up in the defined time this call should be ringing on others devices. + * The call failed. If desired this can be used as a trigger to exit the call. + * - "success": Someone else joined. The call is in a normal state. No audiovisual feedback. + * - null: EC is configured to never show any waiting for answer state. + * + * `autoLeave$` An observable that emits (null) when the call should be automatically left. + * - if options.autoLeaveWhenOthersLeft is set to true it emits when all others left. + * - if options.waitForCallPickup is set to true it emits if noone picked up the ring or if the ring got declined. + * - if options.autoLeaveWhenOthersLeft && options.waitForCallPickup is false it will never emit. + * + */ +export function createCallNotificationLifecycle$({ + scope, + memberships$, + sentCallNotification$, + receivedDecline$, + options, + localUser, +}: Props): { + callPickupState$: Behavior; + autoLeave$: Observable; +} { + // TODO convert all ring and all others left logic into one callLifecycleTracker$(didSendCallNotification$,matrixLivekitItem$): {autoLeave$,callPickupState$} + const allOthersLeft$ = memberships$.pipe( + pairwise(), + filter( + ([{ value: prev }, { value: current }]) => + current.every((m) => m.userId === localUser.userId) && + prev.some((m) => m.userId !== localUser.userId), + ), + map(() => {}), + ); + + /** + * Whether some Matrix user other than ourself is joined to the call. + */ + const someoneElseJoined$ = memberships$.pipe( + mapEpoch((ms) => ms.some((m) => m.userId !== localUser.userId)), + ) as Behavior>; + + /** + * Whenever the RTC session tells us that it intends to ring the remote + * participant's devices, this emits an Observable tracking the current state of + * that ringing process. + */ + // This is a behavior since we need to store the latest state for when we subscribe to this after `didSendCallNotification$` + // has already emitted but we still need the latest observable with a timeout timer that only gets created on after receiving `notificationEvent`. + // A behavior will emit the latest observable with the running timer to new subscribers. + // see also: callPickupState$ and in particular the line: `return this.ring$.pipe(mergeAll());` here we otherwise might get an EMPTY observable if + // `ring$` would not be a behavior. + const remoteRingState$: Behavior<"ringing" | "timeout" | "decline" | null> = + scope.behavior( + sentCallNotification$.pipe( + filter( + (newAndLegacyEvents) => + // only care about new events (legacy do not have decline pattern) + newAndLegacyEvents?.[0].notification_type === "ring", + ), + map((e) => e as CallNotificationWrapper), + switchMap(([notificastionEvent]) => { + const lifetimeMs = notificationEvent?.lifetime ?? 0; + return concat( + lifetimeMs === 0 + ? // If no lifetime, skip the ring state + of(null) + : // Ring until lifetime ms have passed + timer(lifetimeMs).pipe( + ignoreElements(), + startWith("ringing" as const), + ), + // The notification lifetime has timed out, meaning ringing has likely + // stopped on all receiving clients. + of("timeout" as const), + // This makes sure we will not drop into the `endWith("decline" as const)` state + NEVER, + ).pipe( + takeUntil( + receivedDecline$.pipe( + filter( + ([event]) => + event.getRelation()?.rel_type === "m.reference" && + event.getRelation()?.event_id === + notificationEvent.event_id && + event.getSender() !== localUser.userId, + ), + ), + ), + endWith("decline" as const), + ); + }), + ), + null, + ); + + const callPickupState$ = scope.behavior( + options.waitForCallPickup === true + ? combineLatest( + [someoneElseJoined$, remoteRingState$], + (someoneElseJoined, ring) => { + if (someoneElseJoined) { + return "success" as const; + } + // Show the ringing state of the most recent ringing attempt. + // as long as we have not yet sent an RTC notification event, ring will be null -> callPickupState$ = unknown. + return ring ?? ("unknown" as const); + }, + ) + : NEVER, + null, + ); + + const autoLeave$ = merge( + options.autoLeaveWhenOthersLeft === true + ? allOthersLeft$.pipe(map(() => "allOthersLeft" as const)) + : NEVER, + callPickupState$.pipe( + filter((state) => state === "timeout" || state === "decline"), + ), + ); + return { autoLeave$, callPickupState$ }; +} diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts similarity index 81% rename from src/state/CallViewModel.ts rename to src/state/CallViewModel/CallViewModel.ts index 714ca62c..d2931623 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -7,29 +7,20 @@ Please see LICENSE in the repository root for full details. import { type BaseKeyProvider, - ConnectionState, + type ConnectionState, type E2EEOptions, ExternalE2EEKeyProvider, type Room as LivekitRoom, type RoomOptions, } from "livekit-client"; import E2EEWorker from "livekit-client/e2ee-worker?worker"; -import { - type EventTimelineSetHandlerMap, - EventType, - type Room as MatrixRoom, - RoomEvent, -} from "matrix-js-sdk"; +import { type Room as MatrixRoom } from "matrix-js-sdk"; import { combineLatest, - concat, distinctUntilChanged, EMPTY, - endWith, filter, - from, fromEvent, - ignoreElements, map, merge, NEVER, @@ -46,18 +37,12 @@ import { switchMap, switchScan, take, - takeUntil, - takeWhile, tap, throttleTime, timer, } from "rxjs"; import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; -import { - type MatrixRTCSession, - MatrixRTCSessionEvent, - type MatrixRTCSessionEventHandlerMap, -} from "matrix-js-sdk/lib/matrixrtc"; +import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; import { type IWidgetApiRequest } from "matrix-widget-api"; import { @@ -66,39 +51,39 @@ import { type RemoteUserMediaViewModel, ScreenShareViewModel, type UserMediaViewModel, -} from "./MediaViewModel"; -import { accumulate, generateKeyed$, pauseWhen } from "../utils/observable"; +} from "../MediaViewModel"; +import { accumulate, generateKeyed$, pauseWhen } from "../../utils/observable"; import { duplicateTiles, MatrixRTCMode, matrixRTCMode, playReactionsSound, showReactions, -} from "../settings/settings"; -import { isFirefox } from "../Platform"; -import { setPipEnabled$ } from "../controls"; -import { TileStore } from "./TileStore"; -import { gridLikeLayout } from "./GridLikeLayout"; -import { spotlightExpandedLayout } from "./SpotlightExpandedLayout"; -import { oneOnOneLayout } from "./OneOnOneLayout"; -import { pipLayout } from "./PipLayout"; -import { type EncryptionSystem } from "../e2ee/sharedKeyManagement"; +} from "../../settings/settings"; +import { isFirefox } from "../../Platform"; +import { setPipEnabled$ } from "../../controls"; +import { TileStore } from "../TileStore"; +import { gridLikeLayout } from "../GridLikeLayout"; +import { spotlightExpandedLayout } from "../SpotlightExpandedLayout"; +import { oneOnOneLayout } from "../OneOnOneLayout"; +import { pipLayout } from "../PipLayout"; +import { type EncryptionSystem } from "../../e2ee/sharedKeyManagement"; import { type RaisedHandInfo, type ReactionInfo, type ReactionOption, -} from "../reactions"; -import { shallowEquals } from "../utils/array"; -import { type MediaDevices } from "./MediaDevices"; -import { type Behavior, constant } from "./Behavior"; -import { E2eeType } from "../e2ee/e2eeType"; -import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider"; -import { type MuteStates } from "./MuteStates"; -import { getUrlParams } from "../UrlParams"; -import { type ProcessorState } from "../livekit/TrackProcessorContext"; -import { ElementWidgetActions, widget } from "../widget"; -import { sharingScreen$, UserMedia } from "./UserMedia.ts"; -import { ScreenShare } from "./ScreenShare.ts"; +} from "../../reactions"; +import { shallowEquals } from "../../utils/array"; +import { type MediaDevices } from "../MediaDevices"; +import { type Behavior, constant } from "../Behavior"; +import { E2eeType } from "../../e2ee/e2eeType"; +import { MatrixKeyProvider } from "../../e2ee/matrixKeyProvider"; +import { type MuteStates } from "../MuteStates"; +import { getUrlParams } from "../../UrlParams"; +import { type ProcessorState } from "../../livekit/TrackProcessorContext"; +import { ElementWidgetActions, widget } from "../../widget"; +import { UserMedia } from "../UserMedia.ts"; +import { ScreenShare } from "../ScreenShare.ts"; import { type GridLayoutMedia, type Layout, @@ -107,18 +92,23 @@ import { type SpotlightExpandedLayoutMedia, type SpotlightLandscapeLayoutMedia, type SpotlightPortraitLayoutMedia, -} from "./layout-types.ts"; -import { type ElementCallError } from "../utils/errors.ts"; -import { type ObservableScope } from "./ObservableScope.ts"; +} from "../layout-types.ts"; +import { type ElementCallError } from "../../utils/errors.ts"; +import { type ObservableScope } from "../ObservableScope.ts"; import { createLocalMembership$ } from "./localMember/LocalMembership.ts"; import { createLocalTransport$ } from "./localMember/LocalTransport.ts"; import { createMemberships$, membershipsAndTransports$, -} from "./SessionBehaviors.ts"; +} from "../SessionBehaviors.ts"; import { ECConnectionFactory } from "./remoteMembers/ConnectionFactory.ts"; import { createConnectionManager$ } from "./remoteMembers/ConnectionManager.ts"; import { createMatrixLivekitMembers$ } from "./remoteMembers/MatrixLivekitMembers.ts"; +import { + createCallNotificationLifecycle$, + createReceivedDecline$, + createSentCallNotification$, +} from "./CallNotificationLifecycle.ts"; const logger = rootLogger.getChild("[CallViewModel]"); //TODO @@ -274,6 +264,23 @@ export class CallViewModel { options: this.connectOptions$, }); + // ------------------------------------------------------------------------ + // CallNotificationLifecycle + private sentCallNotification$ = createSentCallNotification$( + this.scope, + this.matrixRTCSession, + ); + private receivedDecline$ = createReceivedDecline$(this.matrixRoom); + + private callLifecycle = createCallNotificationLifecycle$({ + scope: this.scope, + memberships$: this.memberships$, + sentCallNotification$: this.sentCallNotification$, + receivedDecline$: this.receivedDecline$, + options: this.options, + localUser: { userId: this.userId, deviceId: this.deviceId }, + }); + /** * If there is a configuration error with the call (e.g. misconfigured E2EE). * This is a fatal error that prevents the call from being created/joined. @@ -315,7 +322,7 @@ export class CallViewModel { public readonly audioParticipants$ = this.scope.behavior( this.matrixLivekitMembers$.pipe( - map((members) => members.map((m) => m.participant)), + map((members) => members.value.map((m) => m.participant)), ), ); @@ -350,7 +357,7 @@ export class CallViewModel { // Generate a collection of MediaItems from the list of expected (whether // present or missing) LiveKit participants. combineLatest([this.matrixLivekitMembers$, duplicateTiles.value$]), - ([matrixLivekitMembers, duplicateTiles], createOrGet) => { + ([{ value: matrixLivekitMembers }, duplicateTiles], createOrGet) => { const items: MediaItem[] = []; for (const { @@ -455,129 +462,11 @@ export class CallViewModel { */ // TODO KEEP THIS!! and adapt it to what our membershipManger returns public readonly participantCount$ = this.scope.behavior( - this.memberships$.pipe(map((ms) => ms.length)), + this.memberships$.pipe(map((ms) => ms.value.length)), ); - // TODO convert all ring and all others left logic into one callLifecycleTracker$(didSendCallNotification$,matrixLivekitItem$): {autoLeave$,callPickupState$} - private readonly allOthersLeft$ = this.memberships$.pipe( - pairwise(), - filter( - ([prev, current]) => - current.every((m) => m.userId === this.userId) && - prev.some((m) => m.userId !== this.userId), - ), - map(() => {}), - ); - - private readonly didSendCallNotification$ = fromEvent( - this.matrixRTCSession, - MatrixRTCSessionEvent.DidSendCallNotification, - ) as Observable< - Parameters< - MatrixRTCSessionEventHandlerMap[MatrixRTCSessionEvent.DidSendCallNotification] - > - >; - - /** - * Whenever the RTC session tells us that it intends to ring the remote - * participant's devices, this emits an Observable tracking the current state of - * that ringing process. - */ - // This is a behavior since we need to store the latest state for when we subscribe to this after `didSendCallNotification$` - // has already emitted but we still need the latest observable with a timeout timer that only gets created on after receiving `notificationEvent`. - // A behavior will emit the latest observable with the running timer to new subscribers. - // see also: callPickupState$ and in particular the line: `return this.ring$.pipe(mergeAll());` here we otherwise might get an EMPTY observable if - // `ring$` would not be a behavior. - private readonly ring$: Behavior<"ringing" | "timeout" | "decline" | null> = - this.scope.behavior( - this.didSendCallNotification$.pipe( - filter( - ([notificationEvent]) => - notificationEvent.notification_type === "ring", - ), - switchMap(([notificationEvent]) => { - const lifetimeMs = notificationEvent?.lifetime ?? 0; - return concat( - lifetimeMs === 0 - ? // If no lifetime, skip the ring state - of(null) - : // Ring until lifetime ms have passed - timer(lifetimeMs).pipe( - ignoreElements(), - startWith("ringing" as const), - ), - // The notification lifetime has timed out, meaning ringing has likely - // stopped on all receiving clients. - of("timeout" as const), - // This makes sure we will not drop into the `endWith("decline" as const)` state - NEVER, - ).pipe( - takeUntil( - ( - fromEvent(this.matrixRoom, RoomEvent.Timeline) as Observable< - Parameters - > - ).pipe( - filter( - ([event]) => - event.getType() === EventType.RTCDecline && - event.getRelation()?.rel_type === "m.reference" && - event.getRelation()?.event_id === - notificationEvent.event_id && - event.getSender() !== this.userId, - ), - ), - ), - endWith("decline" as const), - ); - }), - ), - null, - ); - - /** - * Whether some Matrix user other than ourself is joined to the call. - */ - private readonly someoneElseJoined$ = this.memberships$.pipe( - map((ms) => ms.some((m) => m.userId !== this.userId)), - ) as Behavior; - - /** - * The current call pickup state of the call. - * - "unknown": The client has not yet sent the notification event. We don't know if it will because it first needs to send its own membership. - * Then we can conclude if we were the first one to join or not. - * This may also be set if we are disconnected. - * - "ringing": The call is ringing on other devices in this room (This client should give audiovisual feedback that this is happening). - * - "timeout": No-one picked up in the defined time this call should be ringing on others devices. - * The call failed. If desired this can be used as a trigger to exit the call. - * - "success": Someone else joined. The call is in a normal state. No audiovisual feedback. - * - null: EC is configured to never show any waiting for answer state. - */ - public readonly callPickupState$: Behavior< - "unknown" | "ringing" | "timeout" | "decline" | "success" | null - > = this.options.waitForCallPickup - ? this.scope.behavior< - "unknown" | "ringing" | "timeout" | "decline" | "success" - >( - combineLatest( - [this.livekitConnectionState$, this.someoneElseJoined$, this.ring$], - (livekitConnectionState, someoneElseJoined, ring) => { - if (livekitConnectionState === ConnectionState.Disconnected) { - // Do not ring until we're connected. - return "unknown" as const; - } else if (someoneElseJoined) { - return "success" as const; - } - // Show the ringing state of the most recent ringing attempt. - // as long as we have not yet sent an RTC notification event, ring will be null -> callPickupState$ = unknown. - return ring ?? ("unknown" as const); - }, - ), - ) - : constant(null); - public readonly leaveSoundEffect$ = combineLatest([ - this.callPickupState$, + this.callLifecycle.callPickupState$, this.userMedia$, ]).pipe( // Until the call is successful, do not play a leave sound. @@ -594,16 +483,6 @@ export class CallViewModel { throttleTime(THROTTLE_SOUND_EFFECT_MS), ); - // Public for testing - public readonly autoLeave$ = merge( - this.options.autoLeaveWhenOthersLeft - ? this.allOthersLeft$.pipe(map(() => "allOthersLeft" as const)) - : NEVER, - this.callPickupState$.pipe( - filter((state) => state === "timeout" || state === "decline"), - ), - ); - private readonly userHangup$ = new Subject(); public hangup(): void { this.userHangup$.next(); @@ -626,7 +505,7 @@ export class CallViewModel { public readonly leave$: Observable< "user" | "timeout" | "decline" | "allOthersLeft" > = merge( - this.autoLeave$, + this.callLifecycle.autoLeave$, merge(this.userHangup$, this.widgetHangup$).pipe( map(() => "user" as const), ), @@ -717,6 +596,7 @@ export class CallViewModel { private readonly pip$ = this.scope.behavior( combineLatest([ + // TODO This also needs epoch logic to dedupe the screenshares and mediaItems emits this.screenShares$, this.spotlightSpeaker$, this.mediaItems$, @@ -1298,47 +1178,16 @@ export class CallViewModel { /** * Whether we are sharing our screen. */ - // TODO move to LocalMembership - public readonly sharingScreen$ = this.scope.behavior( - from(this.localConnection$).pipe( - switchMap((c) => - c?.state === "ready" - ? sharingScreen$(c.value.livekitRoom.localParticipant) - : of(false), - ), - ), - ); + // reassigned here to make it publicly accessible + public readonly sharingScreen$ = this.localMembership.sharingScreen$; /** * Callback for toggling screen sharing. If null, screen sharing is not * available. */ - // TODO move to LocalMembership + // reassigned here to make it publicly accessible public readonly toggleScreenSharing = - "getDisplayMedia" in (navigator.mediaDevices ?? {}) && - !this.urlParams.hideScreensharing - ? (): void => - // Once a connection is ready... - void this.localConnection$ - .pipe( - takeWhile((c) => c !== null && c.state !== "error"), - switchMap((c) => (c.state === "ready" ? of(c.value) : NEVER)), - take(1), - this.scope.bind(), - ) - // ...toggle screen sharing. - .subscribe( - (c) => - void c.livekitRoom.localParticipant - .setScreenShareEnabled(!this.sharingScreen$.value, { - audio: true, - selfBrowserSurface: "include", - surfaceSwitching: "include", - systemAudio: "include", - }) - .catch(logger.error), - ) - : null; + this.localMembership.toggleScreenSharing; public constructor( private readonly scope: ObservableScope, diff --git a/src/state/CallViewModel/callPickupState$.ts b/src/state/CallViewModel/callPickupState$.ts new file mode 100644 index 00000000..e69de29b diff --git a/src/state/localMember/LocalMembership.ts b/src/state/CallViewModel/localMember/LocalMembership.ts similarity index 69% rename from src/state/localMember/LocalMembership.ts rename to src/state/CallViewModel/localMember/LocalMembership.ts index 6a400c37..1bbbcb7d 100644 --- a/src/state/localMember/LocalMembership.ts +++ b/src/state/CallViewModel/localMember/LocalMembership.ts @@ -18,58 +18,63 @@ import { combineLatest, fromEvent, map, + NEVER, type Observable, of, scan, startWith, switchMap, + take, + takeWhile, tap, } from "rxjs"; import { logger } from "matrix-js-sdk/lib/logger"; -import { type Behavior } from "../Behavior"; +import { sharingScreen$ as observeSharingScreen$ } from "../../UserMedia.ts"; +import { type Behavior } from "../../Behavior"; import { type IConnectionManager } from "../remoteMembers/ConnectionManager"; -import { ObservableScope } from "../ObservableScope"; +import { ObservableScope } from "../../ObservableScope"; import { Publisher } from "./Publisher"; -import { type MuteStates } from "../MuteStates"; -import { type ProcessorState } from "../../livekit/TrackProcessorContext"; -import { type MediaDevices } from "../MediaDevices"; -import { and$ } from "../../utils/observable"; +import { type MuteStates } from "../../MuteStates"; +import { type ProcessorState } from "../../../livekit/TrackProcessorContext"; +import { type MediaDevices } from "../../MediaDevices"; +import { and$ } from "../../../utils/observable"; import { enterRTCSession, type EnterRTCSessionOptions, -} from "../../rtcSessionHelpers"; -import { type ElementCallError } from "../../utils/errors"; -import { ElementWidgetActions, type WidgetHelpers } from "../../widget"; +} from "../../../rtcSessionHelpers"; +import { type ElementCallError } from "../../../utils/errors"; +import { ElementWidgetActions, type WidgetHelpers } from "../../../widget"; import { areLivekitTransportsEqual } from "../remoteMembers/MatrixLivekitMembers"; +import { getUrlParams } from "../../../UrlParams.ts"; -enum LivekitState { - UNINITIALIZED = "uninitialized", - CONNECTING = "connecting", - CONNECTED = "connected", - ERROR = "error", - DISCONNECTED = "disconnected", - DISCONNECTING = "disconnecting", +export enum LivekitState { + Uninitialized = "uninitialized", + Connecting = "connecting", + Connected = "connected", + Error = "error", + Disconnected = "disconnected", + Disconnecting = "disconnecting", } type LocalMemberLivekitState = - | { state: LivekitState.ERROR; error: string } - | { state: LivekitState.CONNECTED } - | { state: LivekitState.CONNECTING } - | { state: LivekitState.UNINITIALIZED } - | { state: LivekitState.DISCONNECTED } - | { state: LivekitState.DISCONNECTING }; + | { state: LivekitState.Error; error: string } + | { state: LivekitState.Connected } + | { state: LivekitState.Connecting } + | { state: LivekitState.Uninitialized } + | { state: LivekitState.Disconnected } + | { state: LivekitState.Disconnecting }; -enum MatrixState { - CONNECTED = "connected", - DISCONNECTED = "disconnected", - CONNECTING = "connecting", +export enum MatrixState { + Connected = "connected", + Disconnected = "disconnected", + Connecting = "connecting", } type LocalMemberMatrixState = - | { state: MatrixState.CONNECTED } - | { state: MatrixState.CONNECTING } - | { state: MatrixState.DISCONNECTED }; + | { state: MatrixState.Connected } + | { state: MatrixState.Connecting } + | { state: MatrixState.Disconnected }; -export interface LocalMemberState { +export interface LocalMemberConnectionState { livekit$: BehaviorSubject; matrix$: BehaviorSubject; } @@ -107,9 +112,10 @@ interface Props { * @param param0 * @returns * - publisher: The handle to create tracks and publish them to the room. - * - connected$: the current connection state. Including matrix server and livekit server connection. (only the livekit server relevant for our own participation) + * - connected$: the current connection state. Including matrix server and livekit server connection. (only considering the livekit server we are using for our own media publication) * - transport$: the transport object the ownMembership$ ended up using. - * + * - connectionState: the current connection state. Including matrix server and livekit server connection. + * - sharingScreen$: Whether we are sharing our screen. `undefined` if we cannot share the screen. */ export const createLocalMembership$ = ({ scope, @@ -125,21 +131,31 @@ export const createLocalMembership$ = ({ widget, }: Props): { // publisher: Publisher - requestConnect: () => LocalMemberState; + requestConnect: () => LocalMemberConnectionState; startTracks: () => Behavior; requestDisconnect: () => Observable | null; - state: LocalMemberState; // TODO this is probably superseeded by joinState$ + connectionState: LocalMemberConnectionState; + sharingScreen$: Behavior; + toggleScreenSharing: (() => void) | null; + + // deprecated fields + /** @deprecated use state instead*/ homeserverConnected$: Behavior; + /** @deprecated use state instead*/ connected$: Behavior; + // this needs to be discussed + /** @deprecated use state instead*/ reconnecting$: Behavior; + // also needs to be disccues + /** @deprecated use state instead*/ configError$: Behavior; } => { const state = { livekit$: new BehaviorSubject({ - state: LivekitState.UNINITIALIZED, + state: LivekitState.Uninitialized, }), matrix$: new BehaviorSubject({ - state: MatrixState.DISCONNECTED, + state: MatrixState.Disconnected, }), }; @@ -271,23 +287,23 @@ export const createLocalMembership$ = ({ return tracks$; }; - const requestConnect = (): LocalMemberState => { + const requestConnect = (): LocalMemberConnectionState => { if (state.livekit$.value === null) { startTracks(); - state.livekit$.next({ state: LivekitState.CONNECTING }); + state.livekit$.next({ state: LivekitState.Connecting }); combineLatest([publisher$, tracks$], (publisher, tracks) => { publisher ?.startPublishing() .then(() => { - state.livekit$.next({ state: LivekitState.CONNECTED }); + state.livekit$.next({ state: LivekitState.Connected }); }) .catch((error) => { - state.livekit$.next({ state: LivekitState.ERROR, error }); + state.livekit$.next({ state: LivekitState.Error, error }); }); }); } - if (state.matrix$.value.state !== MatrixState.DISCONNECTED) { - state.matrix$.next({ state: MatrixState.CONNECTING }); + if (state.matrix$.value.state !== MatrixState.Disconnected) { + state.matrix$.next({ state: MatrixState.Connecting }); localTransport$.pipe( tap((transport) => { if (transport !== undefined) { @@ -306,17 +322,17 @@ export const createLocalMembership$ = ({ }; const requestDisconnect = (): Behavior | null => { - if (state.livekit$.value.state !== LivekitState.CONNECTED) return null; - state.livekit$.next({ state: LivekitState.DISCONNECTING }); + if (state.livekit$.value.state !== LivekitState.Connected) return null; + state.livekit$.next({ state: LivekitState.Disconnecting }); combineLatest([publisher$, tracks$], (publisher, tracks) => { publisher ?.stopPublishing() .then(() => { tracks.forEach((track) => track.stop()); - state.livekit$.next({ state: LivekitState.DISCONNECTED }); + state.livekit$.next({ state: LivekitState.Disconnected }); }) .catch((error) => { - state.livekit$.next({ state: LivekitState.ERROR, error }); + state.livekit$.next({ state: LivekitState.Error, error }); }); }); @@ -410,14 +426,83 @@ export const createLocalMembership$ = ({ } }); + /** + * Returns undefined if scrennSharing is not yet ready. + */ + const sharingScreen$ = scope.behavior( + connection$.pipe( + switchMap((c) => { + if (!c) return of(undefined); + if (c.state$.value.state === "ConnectedToLkRoom") + return observeSharingScreen$(c.livekitRoom.localParticipant); + return of(false); + }), + ), + ); + + const toggleScreenSharing = + "getDisplayMedia" in (navigator.mediaDevices ?? {}) && + !getUrlParams().hideScreensharing + ? (): void => + // If a connection is ready... + void connection$ + .pipe( + // I dont see why we need this. isnt the check later on superseeding it? + takeWhile( + (c) => + c !== undefined && c.state$.value.state !== "FailedToStart", + ), + switchMap((c) => + c?.state$.value.state === "ConnectedToLkRoom" ? of(c) : NEVER, + ), + take(1), + scope.bind(), + ) + // ...toggle screen sharing. + .subscribe( + (c) => + void c.livekitRoom.localParticipant + .setScreenShareEnabled(!sharingScreen$.value, { + audio: true, + selfBrowserSurface: "include", + surfaceSwitching: "include", + systemAudio: "include", + }) + .catch(logger.error), + ) + : null; + + // we do not need all the auto waiting since we can just check via sharingScreen$.value !== undefined + let alternativeScreenshareToggle: (() => void) | null = null; + if ( + "getDisplayMedia" in (navigator.mediaDevices ?? {}) && + !getUrlParams().hideScreensharing + ) { + alternativeScreenshareToggle = (): void => + void connection$.value?.livekitRoom.localParticipant + .setScreenShareEnabled(!sharingScreen$.value, { + audio: true, + selfBrowserSurface: "include", + surfaceSwitching: "include", + systemAudio: "include", + }) + .catch(logger.error); + } + logger.log( + "alternativeScreenshareToggle so that it is used", + alternativeScreenshareToggle, + ); + return { startTracks, requestConnect, requestDisconnect, - state, + connectionState: state, homeserverConnected$, connected$, reconnecting$, configError$, + sharingScreen$, + toggleScreenSharing, }; }; diff --git a/src/state/localMember/LocalTransport.ts b/src/state/CallViewModel/localMember/LocalTransport.ts similarity index 94% rename from src/state/localMember/LocalTransport.ts rename to src/state/CallViewModel/localMember/LocalTransport.ts index bdcfcffc..d4474897 100644 --- a/src/state/localMember/LocalTransport.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.ts @@ -18,15 +18,15 @@ import { logger } from "matrix-js-sdk/lib/logger"; import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery"; import { deepCompare } from "matrix-js-sdk/lib/utils"; -import { type Behavior } from "../Behavior.ts"; +import { type Behavior } from "../../Behavior.ts"; import { type Epoch, mapEpoch, type ObservableScope, -} from "../ObservableScope.ts"; -import { Config } from "../../config/Config.ts"; -import { MatrixRTCTransportMissingError } from "../../utils/errors.ts"; -import { getSFUConfigWithOpenID } from "../../livekit/openIDSFU.ts"; +} from "../../ObservableScope.ts"; +import { Config } from "../../../config/Config.ts"; +import { MatrixRTCTransportMissingError } from "../../../utils/errors.ts"; +import { getSFUConfigWithOpenID } from "../../../livekit/openIDSFU.ts"; /* * - get well known diff --git a/src/state/localMember/Publisher.ts b/src/state/CallViewModel/localMember/Publisher.ts similarity index 95% rename from src/state/localMember/Publisher.ts rename to src/state/CallViewModel/localMember/Publisher.ts index 6a1079fd..f5a36e99 100644 --- a/src/state/localMember/Publisher.ts +++ b/src/state/CallViewModel/localMember/Publisher.ts @@ -22,17 +22,17 @@ import { } from "rxjs"; import { type Logger } from "matrix-js-sdk/lib/logger"; -import type { Behavior } from "../Behavior.ts"; -import type { MediaDevices, SelectedDevice } from "../MediaDevices.ts"; -import type { MuteStates } from "../MuteStates.ts"; +import type { Behavior } from "../../Behavior.ts"; +import type { MediaDevices, SelectedDevice } from "../../MediaDevices.ts"; +import type { MuteStates } from "../../MuteStates.ts"; import { type ProcessorState, trackProcessorSync, -} from "../../livekit/TrackProcessorContext.tsx"; -import { getUrlParams } from "../../UrlParams.ts"; -import { observeTrackReference$ } from "../MediaViewModel.ts"; -import { type Connection } from "../remoteMembers/Connection.ts"; -import { type ObservableScope } from "../ObservableScope.ts"; +} from "../../../livekit/TrackProcessorContext.tsx"; +import { getUrlParams } from "../../../UrlParams.ts"; +import { observeTrackReference$ } from "../../MediaViewModel.ts"; +import { type Connection } from "../CallViewModel/remoteMembers/Connection.ts"; +import { type ObservableScope } from "../../ObservableScope.ts"; /** * A wrapper for a Connection object. diff --git a/src/state/remoteMembers/Connection.test.ts b/src/state/CallViewModel/remoteMembers/Connection.test.ts similarity index 98% rename from src/state/remoteMembers/Connection.test.ts rename to src/state/CallViewModel/remoteMembers/Connection.test.ts index 7e2d39f8..7d87781f 100644 --- a/src/state/remoteMembers/Connection.test.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.test.ts @@ -36,12 +36,12 @@ import { type ConnectionState, type PublishingParticipant, } from "./Connection.ts"; -import { ObservableScope } from "../ObservableScope.ts"; -import { type OpenIDClientParts } from "../../livekit/openIDSFU.ts"; -import { FailToGetOpenIdToken } from "../../utils/errors.ts"; -import { mockMediaDevices, mockMuteStates } from "../../utils/test.ts"; -import type { ProcessorState } from "../../livekit/TrackProcessorContext.tsx"; -import { type MuteStates } from "../MuteStates.ts"; +import { ObservableScope } from "../../ObservableScope.ts"; +import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts"; +import { FailToGetOpenIdToken } from "../../../utils/errors.ts"; +import { mockMediaDevices, mockMuteStates } from "../../../utils/test.ts"; +import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx"; +import { type MuteStates } from "../../MuteStates.ts"; let testScope: ObservableScope; diff --git a/src/state/remoteMembers/Connection.ts b/src/state/CallViewModel/remoteMembers/Connection.ts similarity index 97% rename from src/state/remoteMembers/Connection.ts rename to src/state/CallViewModel/remoteMembers/Connection.ts index b9cfe71f..cae45d4a 100644 --- a/src/state/remoteMembers/Connection.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.ts @@ -25,13 +25,13 @@ import { getSFUConfigWithOpenID, type OpenIDClientParts, type SFUConfig, -} from "../../livekit/openIDSFU.ts"; -import { type Behavior } from "../Behavior.ts"; -import { type ObservableScope } from "../ObservableScope.ts"; +} from "../../../livekit/openIDSFU.ts"; +import { type Behavior } from "../../Behavior.ts"; +import { type ObservableScope } from "../../ObservableScope.ts"; import { InsufficientCapacityError, SFURoomCreationRestrictedError, -} from "../../utils/errors.ts"; +} from "../../../utils/errors.ts"; export type PublishingParticipant = LocalParticipant | RemoteParticipant; diff --git a/src/state/remoteMembers/ConnectionFactory.ts b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts similarity index 89% rename from src/state/remoteMembers/ConnectionFactory.ts rename to src/state/CallViewModel/remoteMembers/ConnectionFactory.ts index a2a02e3e..a9e2b8fb 100644 --- a/src/state/remoteMembers/ConnectionFactory.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts @@ -13,13 +13,13 @@ import { } from "livekit-client"; import { type Logger } from "matrix-js-sdk/lib/logger"; -import { type ObservableScope } from "../ObservableScope.ts"; +import { type ObservableScope } from "../../ObservableScope.ts"; import { Connection } from "./Connection.ts"; -import type { OpenIDClientParts } from "../../livekit/openIDSFU.ts"; -import type { MediaDevices } from "../MediaDevices.ts"; -import type { Behavior } from "../Behavior.ts"; -import type { ProcessorState } from "../../livekit/TrackProcessorContext.tsx"; -import { defaultLiveKitOptions } from "../../livekit/options.ts"; +import type { OpenIDClientParts } from "../../../livekit/openIDSFU.ts"; +import type { MediaDevices } from "../../MediaDevices.ts"; +import type { Behavior } from "../../Behavior.ts"; +import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx"; +import { defaultLiveKitOptions } from "../../../livekit/options.ts"; export interface ConnectionFactory { createConnection( diff --git a/src/state/remoteMembers/ConnectionManager.test.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts similarity index 98% rename from src/state/remoteMembers/ConnectionManager.test.ts rename to src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts index 1b1a6ffe..ec3c1b2f 100644 --- a/src/state/remoteMembers/ConnectionManager.test.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts @@ -10,14 +10,14 @@ import { BehaviorSubject } from "rxjs"; import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { type Participant as LivekitParticipant } from "livekit-client"; -import { ObservableScope } from "../ObservableScope.ts"; +import { ObservableScope } from "../../ObservableScope.ts"; import { type IConnectionManager, createConnectionManager$, } from "./ConnectionManager.ts"; import { type ConnectionFactory } from "./ConnectionFactory.ts"; import { type Connection } from "./Connection.ts"; -import { flushPromises, withTestScheduler } from "../../utils/test.ts"; +import { flushPromises, withTestScheduler } from "../../../utils/test.ts"; import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts"; // Some test constants diff --git a/src/state/remoteMembers/ConnectionManager.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts similarity index 96% rename from src/state/remoteMembers/ConnectionManager.ts rename to src/state/CallViewModel/remoteMembers/ConnectionManager.ts index 49ab6b71..485fae1b 100644 --- a/src/state/remoteMembers/ConnectionManager.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts @@ -17,10 +17,10 @@ import { BehaviorSubject, combineLatest, map, switchMap } from "rxjs"; import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { type LocalParticipant, type RemoteParticipant } from "livekit-client"; -import { type Behavior } from "../Behavior"; -import { type Connection } from "./Connection"; -import { Epoch, type ObservableScope } from "../ObservableScope"; -import { generateKeyed$ } from "../../utils/observable"; +import { type Behavior } from "../../Behavior.ts"; +import { type Connection } from "./Connection.ts"; +import { Epoch, type ObservableScope } from "../../ObservableScope.ts"; +import { generateKeyed$ } from "../../../utils/observable.ts"; import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts"; import { type ConnectionFactory } from "./ConnectionFactory.ts"; diff --git a/src/state/remoteMembers/MatrixLivekitMembers.test.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts similarity index 99% rename from src/state/remoteMembers/MatrixLivekitMembers.test.ts rename to src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts index 75534e1f..60b52d69 100644 --- a/src/state/remoteMembers/MatrixLivekitMembers.test.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts @@ -19,14 +19,14 @@ import { createMatrixLivekitMembers$, areLivekitTransportsEqual, } from "./MatrixLivekitMembers.ts"; -import { ObservableScope } from "../ObservableScope.ts"; +import { ObservableScope } from "../../ObservableScope.ts"; import { ConnectionManagerData } from "./ConnectionManager.ts"; import { mockCallMembership, mockRemoteParticipant, type OurRunHelpers, withTestScheduler, -} from "../../utils/test.ts"; +} from "../../../utils/test.ts"; import { type Connection } from "./Connection.ts"; let testScope: ObservableScope; diff --git a/src/state/remoteMembers/MatrixLivekitMembers.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts similarity index 97% rename from src/state/remoteMembers/MatrixLivekitMembers.ts rename to src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts index d7937bbe..5703fbd4 100644 --- a/src/state/remoteMembers/MatrixLivekitMembers.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts @@ -19,9 +19,9 @@ import { type NodeStyleEventEmitter } from "rxjs/internal/observable/fromEvent"; import { type Room as MatrixRoom, type RoomMember } from "matrix-js-sdk"; import { logger } from "matrix-js-sdk/lib/logger"; -import { type Behavior } from "../Behavior"; +import { type Behavior } from "../../Behavior"; import { type IConnectionManager } from "./ConnectionManager"; -import { Epoch, mapEpoch, type ObservableScope } from "../ObservableScope"; +import { Epoch, mapEpoch, type ObservableScope } from "../../ObservableScope"; import { getRoomMemberFromRtcMember, memberDisplaynames$ } from "./displayname"; import { type Connection } from "./Connection"; diff --git a/src/state/remoteMembers/displayname.test.ts b/src/state/CallViewModel/remoteMembers/displayname.test.ts similarity index 98% rename from src/state/remoteMembers/displayname.test.ts rename to src/state/CallViewModel/remoteMembers/displayname.test.ts index dcd8cb0f..9822e486 100644 --- a/src/state/remoteMembers/displayname.test.ts +++ b/src/state/CallViewModel/remoteMembers/displayname.test.ts @@ -14,9 +14,9 @@ import { } from "matrix-js-sdk"; import EventEmitter from "events"; -import { ObservableScope } from "../ObservableScope.ts"; +import { ObservableScope } from "../../ObservableScope.ts"; import type { Room as MatrixRoom } from "matrix-js-sdk/lib/models/room"; -import { mockCallMembership, withTestScheduler } from "../../utils/test.ts"; +import { mockCallMembership, withTestScheduler } from "../../../utils/test.ts"; import { memberDisplaynames$ } from "./displayname.ts"; let testScope: ObservableScope; diff --git a/src/state/remoteMembers/displayname.ts b/src/state/CallViewModel/remoteMembers/displayname.ts similarity index 95% rename from src/state/remoteMembers/displayname.ts rename to src/state/CallViewModel/remoteMembers/displayname.ts index 8f2b3f64..07ff3f59 100644 --- a/src/state/remoteMembers/displayname.ts +++ b/src/state/CallViewModel/remoteMembers/displayname.ts @@ -19,12 +19,12 @@ import { type Room as MatrixRoom } from "matrix-js-sdk/lib/matrix"; // eslint-disable-next-line rxjs/no-internal import { type NodeStyleEventEmitter } from "rxjs/internal/observable/fromEvent"; -import { Epoch, type ObservableScope } from "../ObservableScope"; +import { Epoch, type ObservableScope } from "../../ObservableScope"; import { calculateDisplayName, shouldDisambiguate, -} from "../../utils/displayname"; -import { type Behavior } from "../Behavior"; +} from "../../../utils/displayname"; +import { type Behavior } from "../../Behavior"; /** * Displayname for each member of the call. This will disambiguate diff --git a/src/state/remoteMembers/integration.test.ts b/src/state/CallViewModel/remoteMembers/integration.test.ts similarity index 94% rename from src/state/remoteMembers/integration.test.ts rename to src/state/CallViewModel/remoteMembers/integration.test.ts index 9ce4cf33..1d616700 100644 --- a/src/state/remoteMembers/integration.test.ts +++ b/src/state/CallViewModel/remoteMembers/integration.test.ts @@ -14,22 +14,26 @@ import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { type Room as MatrixRoom, type RoomMember } from "matrix-js-sdk"; import { logger } from "matrix-js-sdk/lib/logger"; -import { type Epoch, ObservableScope, trackEpoch } from "../ObservableScope.ts"; +import { + type Epoch, + ObservableScope, + trackEpoch, +} from "../../ObservableScope.ts"; import { ECConnectionFactory } from "./ConnectionFactory.ts"; -import { type OpenIDClientParts } from "../../livekit/openIDSFU.ts"; +import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts"; import { mockCallMembership, mockMediaDevices, withTestScheduler, -} from "../../utils/test"; -import { type ProcessorState } from "../../livekit/TrackProcessorContext.tsx"; +} from "../../../utils/test.ts"; +import { type ProcessorState } from "../../../livekit/TrackProcessorContext.tsx"; import { areLivekitTransportsEqual, createMatrixLivekitMembers$, type MatrixLivekitMember, } from "./MatrixLivekitMembers.ts"; import { createConnectionManager$ } from "./ConnectionManager.ts"; -import { membershipsAndTransports$ } from "../SessionBehaviors.ts"; +import { membershipsAndTransports$ } from "../../SessionBehaviors.ts"; // Test the integration of ConnectionManager and MatrixLivekitMerger From 28047217b85e2e6f491c887ac7099499662fa46e Mon Sep 17 00:00:00 2001 From: Timo K Date: Fri, 7 Nov 2025 12:32:29 +0100 Subject: [PATCH 30/75] Almost running - NEVER use undefined as the default for behaviors (FOOTGUN) --- .../CallNotificationLifecycle.ts | 2 +- src/state/CallViewModel/CallViewModel.ts | 13 +++++++-- src/state/CallViewModel/callPickupState$.ts | 0 .../localMember/LocalMembership.ts | 5 ++-- .../localMember/LocalTransport.ts | 16 ++++++----- .../CallViewModel/localMember/Publisher.ts | 1 + .../remoteMembers/ConnectionManager.ts | 8 +++++- .../remoteMembers/MatrixLivekitMembers.ts | 2 +- .../remoteMembers/displayname.ts | 12 ++++++++ src/state/ObservableScope.test.ts | 21 +++++++++++++- src/state/ObservableScope.ts | 28 +++++++++---------- src/state/SessionBehaviors.ts | 3 +- src/state/UserMedia.ts | 6 ++-- 13 files changed, 83 insertions(+), 34 deletions(-) delete mode 100644 src/state/CallViewModel/callPickupState$.ts diff --git a/src/state/CallViewModel/CallNotificationLifecycle.ts b/src/state/CallViewModel/CallNotificationLifecycle.ts index baf2b665..40826d07 100644 --- a/src/state/CallViewModel/CallNotificationLifecycle.ts +++ b/src/state/CallViewModel/CallNotificationLifecycle.ts @@ -146,7 +146,7 @@ export function createCallNotificationLifecycle$({ newAndLegacyEvents?.[0].notification_type === "ring", ), map((e) => e as CallNotificationWrapper), - switchMap(([notificastionEvent]) => { + switchMap(([notificationEvent]) => { const lifetimeMs = notificationEvent?.lifetime ?? 0; return concat( lifetimeMs === 0 diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index d2931623..f3c1bca9 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -109,6 +109,7 @@ import { createReceivedDecline$, createSentCallNotification$, } from "./CallNotificationLifecycle.ts"; +import { createRoomMembers$ } from "./remoteMembers/displayname.ts"; const logger = rootLogger.getChild("[CallViewModel]"); //TODO @@ -266,6 +267,7 @@ export class CallViewModel { // ------------------------------------------------------------------------ // CallNotificationLifecycle + // consider inlining these!!! private sentCallNotification$ = createSentCallNotification$( this.scope, this.matrixRTCSession, @@ -281,6 +283,9 @@ export class CallViewModel { localUser: { userId: this.userId, deviceId: this.deviceId }, }); + // ------------------------------------------------------------------------ + // ROOM MEMBER tracking TODO + private roomMembers$ = createRoomMembers$(this.scope, this.matrixRoom); /** * If there is a configuration error with the call (e.g. misconfigured E2EE). * This is a fatal error that prevents the call from being created/joined. @@ -440,6 +445,7 @@ export class CallViewModel { mediaItems.filter((m): m is UserMedia => m instanceof UserMedia), ), ), + [], ); public readonly joinSoundEffect$ = this.userMedia$.pipe( @@ -465,6 +471,9 @@ export class CallViewModel { this.memberships$.pipe(map((ms) => ms.value.length)), ); + // only public to expose to the view. + public readonly callPickupState$ = this.callLifecycle.callPickupState$; + public readonly leaveSoundEffect$ = combineLatest([ this.callLifecycle.callPickupState$, this.userMedia$, @@ -645,7 +654,6 @@ export class CallViewModel { private readonly naturalWindowMode$ = this.scope.behavior( fromEvent(window, "resize").pipe( - startWith(null), map(() => { const height = window.innerHeight; const width = window.innerWidth; @@ -658,6 +666,7 @@ export class CallViewModel { return "normal"; }), ), + "normal", ); /** @@ -687,7 +696,6 @@ export class CallViewModel { // automatically switch to spotlight mode and reset when screen sharing ends this.scope.behavior( this.gridModeUserSelection$.pipe( - startWith(null), switchMap((userSelection) => (userSelection === "spotlight" ? EMPTY @@ -706,6 +714,7 @@ export class CallViewModel { ).pipe(startWith(userSelection ?? "grid")), ), ), + "grid", ); public setGridMode(value: GridMode): void { diff --git a/src/state/CallViewModel/callPickupState$.ts b/src/state/CallViewModel/callPickupState$.ts deleted file mode 100644 index e69de29b..00000000 diff --git a/src/state/CallViewModel/localMember/LocalMembership.ts b/src/state/CallViewModel/localMember/LocalMembership.ts index 1bbbcb7d..96edd8da 100644 --- a/src/state/CallViewModel/localMember/LocalMembership.ts +++ b/src/state/CallViewModel/localMember/LocalMembership.ts @@ -288,7 +288,7 @@ export const createLocalMembership$ = ({ }; const requestConnect = (): LocalMemberConnectionState => { - if (state.livekit$.value === null) { + if (state.livekit$.value.state === LivekitState.Uninitialized) { startTracks(); state.livekit$.next({ state: LivekitState.Connecting }); combineLatest([publisher$, tracks$], (publisher, tracks) => { @@ -302,7 +302,7 @@ export const createLocalMembership$ = ({ }); }); } - if (state.matrix$.value.state !== MatrixState.Disconnected) { + if (state.matrix$.value.state === MatrixState.Disconnected) { state.matrix$.next({ state: MatrixState.Connecting }); localTransport$.pipe( tap((transport) => { @@ -438,6 +438,7 @@ export const createLocalMembership$ = ({ return of(false); }), ), + null, ); const toggleScreenSharing = diff --git a/src/state/CallViewModel/localMember/LocalTransport.ts b/src/state/CallViewModel/localMember/LocalTransport.ts index d4474897..69c9b934 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.ts @@ -67,18 +67,22 @@ export const createLocalTransport$ = ({ */ const oldestMemberTransport$ = scope.behavior( memberships$.pipe( - mapEpoch((memberships) => memberships[0].getTransport(memberships[0])), - first((t) => t != undefined && isLivekitTransport(t)), + mapEpoch( + (memberships) => memberships[0]?.getTransport(memberships[0]) ?? null, + ), + first((t) => t != null && isLivekitTransport(t)), ), - undefined, + null, ); /** * The transport that we would personally prefer to publish on (if not for the * transport preferences of others, perhaps). */ - const preferredTransport$: Behavior = - scope.behavior(from(makeTransport(client, roomId)), undefined); + const preferredTransport$: Behavior = scope.behavior( + from(makeTransport(client, roomId)), + null, + ); /** * The transport we should advertise in our MatrixRTC membership. @@ -89,7 +93,6 @@ export const createLocalTransport$ = ({ (useOldestMember, oldestMemberTransport, preferredTransport) => useOldestMember ? oldestMemberTransport : preferredTransport, ).pipe(distinctUntilChanged(deepCompare)), - undefined, ); return advertisedTransport$; }; @@ -103,7 +106,6 @@ async function makeTransportInternal( logger.log("Searching for a preferred transport"); //TODO refactor this to use the jwt service returned alias. const livekitAlias = roomId; - // TODO-MULTI-SFU: Either remove this dev tool or make it more official const urlFromStorage = localStorage.getItem("robin-matrixrtc-auth") ?? diff --git a/src/state/CallViewModel/localMember/Publisher.ts b/src/state/CallViewModel/localMember/Publisher.ts index f5a36e99..9be50bde 100644 --- a/src/state/CallViewModel/localMember/Publisher.ts +++ b/src/state/CallViewModel/localMember/Publisher.ts @@ -307,6 +307,7 @@ export class Publisher { return track instanceof LocalVideoTrack ? track : null; }), ), + null, ); trackProcessorSync(track$, trackerProcessorState$); } diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts index 485fae1b..2859e49b 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts @@ -13,7 +13,7 @@ import { type LivekitTransport, type ParticipantId, } from "matrix-js-sdk/lib/matrixrtc"; -import { BehaviorSubject, combineLatest, map, switchMap } from "rxjs"; +import { BehaviorSubject, combineLatest, map, of, switchMap } from "rxjs"; import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { type LocalParticipant, type RemoteParticipant } from "livekit-client"; @@ -191,6 +191,11 @@ export function createConnectionManager$({ ); }); + // probably not required + if (listOfConnectionsWithPublishingParticipants.length === 0) { + return of(new Epoch(new ConnectionManagerData(), epoch)); + } + // combineLatest the several streams into a single stream with the ConnectionManagerData return combineLatest(listOfConnectionsWithPublishingParticipants).pipe( map( @@ -206,6 +211,7 @@ export function createConnectionManager$({ ); }), ), + new Epoch(new ConnectionManagerData()), ); return { transports$, connectionManagerData$, connections$ }; diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts index 5703fbd4..544f5241 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts @@ -17,7 +17,6 @@ import { combineLatest, filter, map } from "rxjs"; // eslint-disable-next-line rxjs/no-internal import { type NodeStyleEventEmitter } from "rxjs/internal/observable/fromEvent"; import { type Room as MatrixRoom, type RoomMember } from "matrix-js-sdk"; -import { logger } from "matrix-js-sdk/lib/logger"; import { type Behavior } from "../../Behavior"; import { type IConnectionManager } from "./ConnectionManager"; @@ -56,6 +55,7 @@ interface Props { // => Extract an AvatarService instead? // Better with just `getMember` matrixRoom: Pick & NodeStyleEventEmitter; + roomMember$: Behavior>; } // Alternative structure idea: // const livekitMatrixMember$ = (callMemberships$,connectionManager,scope): Observable => { diff --git a/src/state/CallViewModel/remoteMembers/displayname.ts b/src/state/CallViewModel/remoteMembers/displayname.ts index 07ff3f59..c8484a9a 100644 --- a/src/state/CallViewModel/remoteMembers/displayname.ts +++ b/src/state/CallViewModel/remoteMembers/displayname.ts @@ -26,6 +26,17 @@ import { } from "../../../utils/displayname"; import { type Behavior } from "../../Behavior"; +export function createRoomMembers$( + scope: ObservableScope, + matrixRoom: MatrixRoom, +): Behavior[]> { + return scope.behavior( + fromEvent(matrixRoom, RoomStateEvent.Members).pipe( + map(() => matrixRoom.getMembers()), + ), + [], + ); +} /** * Displayname for each member of the call. This will disambiguate * any displayname that clashes with another member. Only members @@ -37,6 +48,7 @@ import { type Behavior } from "../../Behavior"; export const memberDisplaynames$ = ( scope: ObservableScope, matrixRoom: Pick & NodeStyleEventEmitter, + // roomMember$: Behavior>; memberships$: Observable>, ): Behavior>> => scope.behavior( diff --git a/src/state/ObservableScope.test.ts b/src/state/ObservableScope.test.ts index d53084da..19fea76c 100644 --- a/src/state/ObservableScope.test.ts +++ b/src/state/ObservableScope.test.ts @@ -7,8 +7,14 @@ Please see LICENSE in the repository root for full details. import { describe, expect, it } from "vitest"; -import { Epoch, mapEpoch, trackEpoch } from "./ObservableScope"; +import { + Epoch, + mapEpoch, + ObservableScope, + trackEpoch, +} from "./ObservableScope"; import { withTestScheduler } from "../utils/test"; +import { BehaviorSubject, timer } from "rxjs"; describe("Epoch", () => { it("should map the value correctly", () => { @@ -53,4 +59,17 @@ describe("Epoch", () => { }); }); }); + it("obs", () => { + const nothing = Symbol("nothing"); + const scope = new ObservableScope(); + const sb$ = new BehaviorSubject("initial"); + const su$ = new BehaviorSubject(undefined); + expect(sb$.value).toBe("initial"); + expect(su$.value).toBe(undefined); + expect(su$.value === nothing).toBe(false); + + const a$ = timer(10); + + scope.behavior(a$, undefined); + }); }); diff --git a/src/state/ObservableScope.ts b/src/state/ObservableScope.ts index fbf92ada..d1d6c297 100644 --- a/src/state/ObservableScope.ts +++ b/src/state/ObservableScope.ts @@ -24,7 +24,7 @@ import { type Behavior } from "./Behavior"; type MonoTypeOperator = (o: Observable) => Observable; -const nothing = Symbol("nothing"); +export const noInitialValue = Symbol("nothing"); /** * A scope which limits the execution lifetime of its bound Observables. @@ -59,10 +59,7 @@ export class ObservableScope { * Converts an Observable to a Behavior. If no initial value is specified, the * Observable must synchronously emit an initial value. */ - public behavior( - setValue$: Observable, - initialValue: T | typeof nothing = nothing, - ): Behavior { + public behavior(setValue$: Observable, initialValue?: T): Behavior { const subject$ = new BehaviorSubject(initialValue); // Push values from the Observable into the BehaviorSubject. // BehaviorSubjects have an undesirable feature where if you call 'complete', @@ -77,7 +74,7 @@ export class ObservableScope { subject$.error(err); }, }); - if (subject$.value === nothing) + if (subject$.value === noInitialValue) throw new Error("Behavior failed to synchronously emit an initial value"); return subject$ as Behavior; } @@ -118,27 +115,27 @@ export class ObservableScope { value$: Behavior, callback: (value: T) => Promise<(() => Promise) | void>, ): void { - let latestValue: T | typeof nothing = nothing; - let reconciledValue: T | typeof nothing = nothing; + let latestValue: T | typeof noInitialValue = noInitialValue; + let reconciledValue: T | typeof noInitialValue = noInitialValue; let cleanUp: (() => Promise) | void = undefined; value$ .pipe( catchError(() => EMPTY), // Ignore errors this.bind(), // Limit to the duration of the scope - endWith(nothing), // Clean up when the scope ends + endWith(noInitialValue), // Clean up when the scope ends ) .subscribe((value) => { void (async (): Promise => { - if (latestValue === nothing) { + if (latestValue === noInitialValue) { latestValue = value; while (latestValue !== reconciledValue) { await cleanUp?.(); // Call the previous value's clean-up handler reconciledValue = latestValue; - if (latestValue !== nothing) + if (latestValue !== noInitialValue) cleanUp = await callback(latestValue); // Sync current value } // Reset to signal that reconciliation is done for now - latestValue = nothing; + latestValue = noInitialValue; } else { // There's already an instance of the above 'while' loop running // concurrently. Just update the latest value and let it be handled. @@ -176,11 +173,11 @@ export const globalScope = new ObservableScope(); * * # Use Epoch * ``` - * const rootObs$ = of(1,2,3).pipe(trackEpoch()); - * const derivedObs$ = rootObs$.pipe( + * const ancestorObs$ = of(1,2,3).pipe(trackEpoch()); + * const derivedObs$ = ancestorObs$.pipe( * mapEpoch((v)=> "this number: " + v) * ); - * const otherDerivedObs$ = rootObs$.pipe( + * const otherDerivedObs$ = ancestorObs$.pipe( * mapEpoch((v)=> "multiplied by: " + v) * ); * const mergedObs$ = combineLatest([derivedObs$, otherDerivedObs$]).pipe( @@ -241,6 +238,7 @@ export function mapEpoch( ): OperatorFunction, Epoch> { return map((e) => e.mapInner(mapFn)); } + /** * # usage * ``` diff --git a/src/state/SessionBehaviors.ts b/src/state/SessionBehaviors.ts index d44ad33a..7d38ac3d 100644 --- a/src/state/SessionBehaviors.ts +++ b/src/state/SessionBehaviors.ts @@ -15,7 +15,7 @@ import { import { fromEvent } from "rxjs"; import { - type Epoch, + Epoch, mapEpoch, trackEpoch, type ObservableScope, @@ -76,5 +76,6 @@ export const createMemberships$ = ( MatrixRTCSessionEvent.MembershipsChanged, (_, memberships: CallMembership[]) => memberships, ).pipe(trackEpoch()), + new Epoch([]), ); }; diff --git a/src/state/UserMedia.ts b/src/state/UserMedia.ts index 65bd4e92..55de2061 100644 --- a/src/state/UserMedia.ts +++ b/src/state/UserMedia.ts @@ -111,7 +111,7 @@ export class UserMedia { private readonly presenter$ = this.scope.behavior( this.participant$.pipe( - switchMap((p) => (p === undefined ? of(false) : sharingScreen$(p))), + switchMap((p) => (p === null ? of(false) : sharingScreen$(p))), ), ); @@ -151,7 +151,7 @@ export class UserMedia { private readonly initialParticipant: | LocalParticipant | RemoteParticipant - | undefined, + | null = null, private readonly encryptionSystem: EncryptionSystem, private readonly livekitRoom: LivekitRoom, private readonly focusURL: string, @@ -163,7 +163,7 @@ export class UserMedia { ) {} public updateParticipant( - newParticipant: LocalParticipant | RemoteParticipant | undefined, + newParticipant: LocalParticipant | RemoteParticipant | null = null, ): void { if (this.participant$.value !== newParticipant) { // Update the BehaviourSubject in the UserMedia. From e741285b11212e9feedcb71a250232024e1aec1c Mon Sep 17 00:00:00 2001 From: Timo K Date: Fri, 7 Nov 2025 14:04:40 +0100 Subject: [PATCH 31/75] Fix lints, move CallViewModel.test.ts. Fix audio renderer --- src/button/ReactionToggleButton.test.tsx | 2 +- src/button/ReactionToggleButton.tsx | 2 +- src/reactions/useReactionsSender.tsx | 2 +- src/room/CallEventAudioRenderer.test.tsx | 2 +- src/room/CallEventAudioRenderer.tsx | 2 +- src/room/InCallView.tsx | 5 ++-- src/room/ReactionAudioRenderer.test.tsx | 2 +- src/room/ReactionAudioRenderer.tsx | 2 +- src/room/ReactionsOverlay.tsx | 2 +- src/settings/DeveloperSettingsTab.tsx | 1 - .../{ => CallViewModel}/CallViewModel.test.ts | 20 ++++++------- src/state/CallViewModel/CallViewModel.ts | 28 ++++++++++++++--- .../localMember/LocalMembership.ts | 5 ++-- .../CallViewModel/localMember/Publisher.ts | 6 ++-- .../remoteMembers/Connection.test.ts | 21 ++++++------- .../remoteMembers/MatrixLivekitMembers.ts | 2 +- .../remoteMembers/displayname.test.ts | 30 +++++-------------- src/tile/GridTile.test.tsx | 2 +- src/utils/test-viewmodel.ts | 2 +- 19 files changed, 71 insertions(+), 67 deletions(-) rename src/state/{ => CallViewModel}/CallViewModel.test.ts (99%) diff --git a/src/button/ReactionToggleButton.test.tsx b/src/button/ReactionToggleButton.test.tsx index c7ac5aa0..f6b7a2ea 100644 --- a/src/button/ReactionToggleButton.test.tsx +++ b/src/button/ReactionToggleButton.test.tsx @@ -13,7 +13,7 @@ import { type ReactNode } from "react"; import { ReactionToggleButton } from "./ReactionToggleButton"; import { ElementCallReactionEventType } from "../reactions"; -import { type CallViewModel } from "../state/CallViewModel"; +import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; import { getBasicCallViewModelEnvironment } from "../utils/test-viewmodel"; import { alice, local, localRtcMember } from "../utils/test-fixtures"; import { type MockRTCSession } from "../utils/test"; diff --git a/src/button/ReactionToggleButton.tsx b/src/button/ReactionToggleButton.tsx index 69673293..0c722baf 100644 --- a/src/button/ReactionToggleButton.tsx +++ b/src/button/ReactionToggleButton.tsx @@ -33,7 +33,7 @@ import { ReactionsRowSize, } from "../reactions"; import { Modal } from "../Modal"; -import { type CallViewModel } from "../state/CallViewModel"; +import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; import { useBehavior } from "../useBehavior"; interface InnerButtonProps extends ComponentPropsWithoutRef<"button"> { diff --git a/src/reactions/useReactionsSender.tsx b/src/reactions/useReactionsSender.tsx index ec29c2af..afb9b789 100644 --- a/src/reactions/useReactionsSender.tsx +++ b/src/reactions/useReactionsSender.tsx @@ -20,7 +20,7 @@ import { logger } from "matrix-js-sdk/lib/logger"; import { useMatrixRTCSessionMemberships } from "../useMatrixRTCSessionMemberships"; import { useClientState } from "../ClientContext"; import { ElementCallReactionEventType, type ReactionOption } from "."; -import { type CallViewModel } from "../state/CallViewModel"; +import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; import { useBehavior } from "../useBehavior"; interface ReactionsSenderContextType { diff --git a/src/room/CallEventAudioRenderer.test.tsx b/src/room/CallEventAudioRenderer.test.tsx index e49c7011..733346eb 100644 --- a/src/room/CallEventAudioRenderer.test.tsx +++ b/src/room/CallEventAudioRenderer.test.tsx @@ -38,7 +38,7 @@ import { local, localRtcMember, } from "../utils/test-fixtures"; -import { MAX_PARTICIPANT_COUNT_FOR_SOUND } from "../state/CallViewModel"; +import { MAX_PARTICIPANT_COUNT_FOR_SOUND } from "../state/CallViewModel/CallViewModel"; vitest.mock("livekit-client/e2ee-worker?worker"); vitest.mock("../useAudioContext"); diff --git a/src/room/CallEventAudioRenderer.tsx b/src/room/CallEventAudioRenderer.tsx index 23997c37..d33f3b84 100644 --- a/src/room/CallEventAudioRenderer.tsx +++ b/src/room/CallEventAudioRenderer.tsx @@ -7,7 +7,7 @@ Please see LICENSE in the repository root for full details. import { type ReactNode, useEffect } from "react"; -import { type CallViewModel } from "../state/CallViewModel"; +import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; import joinCallSoundMp3 from "../sound/join_call.mp3"; import joinCallSoundOgg from "../sound/join_call.ogg"; import leftCallSoundMp3 from "../sound/left_call.mp3"; diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 06c1ccb4..7f469460 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -734,7 +734,8 @@ export const InCallView: FC = ({ = ({ key={url} url={url} livekitRoom={livekitRoom} - validIdentities={participants.map((p) => p.identity)} + validIdentities={participants} muted={muteAllAudio} /> ))} diff --git a/src/room/ReactionAudioRenderer.test.tsx b/src/room/ReactionAudioRenderer.test.tsx index 83188be7..31c0a0cb 100644 --- a/src/room/ReactionAudioRenderer.test.tsx +++ b/src/room/ReactionAudioRenderer.test.tsx @@ -27,7 +27,7 @@ import { import { useAudioContext } from "../useAudioContext"; import { GenericReaction, ReactionSet } from "../reactions"; import { prefetchSounds } from "../soundUtils"; -import { type CallViewModel } from "../state/CallViewModel"; +import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; import { getBasicCallViewModelEnvironment } from "../utils/test-viewmodel"; import { alice, diff --git a/src/room/ReactionAudioRenderer.tsx b/src/room/ReactionAudioRenderer.tsx index 2b95acb9..06170d19 100644 --- a/src/room/ReactionAudioRenderer.tsx +++ b/src/room/ReactionAudioRenderer.tsx @@ -12,7 +12,7 @@ import { GenericReaction, ReactionSet } from "../reactions"; import { useAudioContext } from "../useAudioContext"; import { prefetchSounds } from "../soundUtils"; import { useLatest } from "../useLatest"; -import { type CallViewModel } from "../state/CallViewModel"; +import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; const soundMap = Object.fromEntries([ ...ReactionSet.filter((v) => v.sound !== undefined).map((v) => [ diff --git a/src/room/ReactionsOverlay.tsx b/src/room/ReactionsOverlay.tsx index f3dff848..e7c097d2 100644 --- a/src/room/ReactionsOverlay.tsx +++ b/src/room/ReactionsOverlay.tsx @@ -8,7 +8,7 @@ Please see LICENSE in the repository root for full details. import { type ReactNode } from "react"; import styles from "./ReactionsOverlay.module.css"; -import { type CallViewModel } from "../state/CallViewModel"; +import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; import { useBehavior } from "../useBehavior"; export function ReactionsOverlay({ vm }: { vm: CallViewModel }): ReactNode { diff --git a/src/settings/DeveloperSettingsTab.tsx b/src/settings/DeveloperSettingsTab.tsx index e29e9c15..a8f485b6 100644 --- a/src/settings/DeveloperSettingsTab.tsx +++ b/src/settings/DeveloperSettingsTab.tsx @@ -26,7 +26,6 @@ import { duplicateTiles as duplicateTilesSetting, debugTileLayout as debugTileLayoutSetting, showConnectionStats as showConnectionStatsSetting, - multiSfu as multiSfuSetting, muteAllAudio as muteAllAudioSetting, alwaysShowIphoneEarpiece as alwaysShowIphoneEarpieceSetting, matrixRTCMode as matrixRTCModeSetting, diff --git a/src/state/CallViewModel.test.ts b/src/state/CallViewModel/CallViewModel.test.ts similarity index 99% rename from src/state/CallViewModel.test.ts rename to src/state/CallViewModel/CallViewModel.test.ts index fec6b8cf..13693dc1 100644 --- a/src/state/CallViewModel.test.ts +++ b/src/state/CallViewModel/CallViewModel.test.ts @@ -50,7 +50,7 @@ import { deepCompare } from "matrix-js-sdk/lib/utils"; import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery"; import { CallViewModel, type CallViewModelOptions } from "./CallViewModel"; -import { type Layout } from "./layout-types"; +import { type Layout } from "../layout-types.ts"; import { mockLocalParticipant, mockMatrixRoom, @@ -65,9 +65,9 @@ import { testScope, mockLivekitRoom, exampleTransport, -} from "../utils/test"; -import { E2eeType } from "../e2ee/e2eeType"; -import type { RaisedHandInfo, ReactionInfo } from "../reactions"; +} from "../../utils/test.ts"; +import { E2eeType } from "../../e2ee/e2eeType.ts"; +import type { RaisedHandInfo, ReactionInfo } from "../../reactions/index.ts"; import { alice, aliceDoppelganger, @@ -89,15 +89,15 @@ import { localId, localRtcMember, localRtcMemberDevice2, -} from "../utils/test-fixtures"; -import { MediaDevices } from "./MediaDevices"; -import { getValue } from "../utils/observable"; -import { type Behavior, constant } from "./Behavior"; -import type { ProcessorState } from "../livekit/TrackProcessorContext.tsx"; +} from "../../utils/test-fixtures.ts"; +import { MediaDevices } from "../MediaDevices.ts"; +import { getValue } from "../../utils/observable.ts"; +import { type Behavior, constant } from "../Behavior.ts"; +import type { ProcessorState } from "../../livekit/TrackProcessorContext.tsx"; import { type ElementCallError, MatrixRTCTransportMissingError, -} from "../utils/errors.ts"; +} from "../../utils/errors.ts"; vi.mock("rxjs", async (importOriginal) => ({ ...(await importOriginal()), diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index f3c1bca9..ef4be238 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -156,7 +156,11 @@ interface LayoutScanState { } type MediaItem = UserMedia | ScreenShare; - +type AudioLivekitItem = { + livekitRoom: LivekitRoom; + participants: string[]; + url: string; +}; /** * A view model providing all the application logic needed to show the in-call * UI (may eventually be expanded to cover the lobby and feedback screens in the @@ -166,8 +170,6 @@ type MediaItem = UserMedia | ScreenShare; // state and LiveKit state. We use the common terminology of room "members", RTC // "memberships", and LiveKit "participants". export class CallViewModel { - private readonly urlParams = getUrlParams(); - private readonly userId = this.matrixRoom.client.getUserId()!; private readonly deviceId = this.matrixRoom.client.getDeviceId()!; @@ -285,6 +287,7 @@ export class CallViewModel { // ------------------------------------------------------------------------ // ROOM MEMBER tracking TODO + // eslint-disable-next-line @typescript-eslint/no-unused-vars private roomMembers$ = createRoomMembers$(this.scope, this.matrixRoom); /** * If there is a configuration error with the call (e.g. misconfigured E2EE). @@ -311,6 +314,7 @@ export class CallViewModel { * than whether all connections are truly up and running. */ // DISCUSS ? lets think why we need joined and how to do it better + // eslint-disable-next-line @typescript-eslint/no-unused-vars private readonly joined$ = this.localMembership.connected$; /** @@ -327,7 +331,23 @@ export class CallViewModel { public readonly audioParticipants$ = this.scope.behavior( this.matrixLivekitMembers$.pipe( - map((members) => members.value.map((m) => m.participant)), + map((members) => + members.value.reduce((acc, curr) => { + const url = curr.connection?.transport.livekit_service_url; + const livekitRoom = curr.connection?.livekitRoom; + const participant = curr.participant?.identity; + + if (!url || !livekitRoom || !participant) return acc; + + const existing = acc.find((item) => item.url === url); + if (existing) { + existing.participants.push(participant); + } else { + acc.push({ livekitRoom, participants: [participant], url }); + } + return acc; + }, []), + ), ), ); diff --git a/src/state/CallViewModel/localMember/LocalMembership.ts b/src/state/CallViewModel/localMember/LocalMembership.ts index 96edd8da..a0b06d46 100644 --- a/src/state/CallViewModel/localMember/LocalMembership.ts +++ b/src/state/CallViewModel/localMember/LocalMembership.ts @@ -135,7 +135,8 @@ export const createLocalMembership$ = ({ startTracks: () => Behavior; requestDisconnect: () => Observable | null; connectionState: LocalMemberConnectionState; - sharingScreen$: Behavior; + // Use null here since behavior cannot be initialised with undefined. + sharingScreen$: Behavior; toggleScreenSharing: (() => void) | null; // deprecated fields @@ -432,7 +433,7 @@ export const createLocalMembership$ = ({ const sharingScreen$ = scope.behavior( connection$.pipe( switchMap((c) => { - if (!c) return of(undefined); + if (!c) return of(null); if (c.state$.value.state === "ConnectedToLkRoom") return observeSharingScreen$(c.livekitRoom.localParticipant); return of(false); diff --git a/src/state/CallViewModel/localMember/Publisher.ts b/src/state/CallViewModel/localMember/Publisher.ts index 9be50bde..ffdd2487 100644 --- a/src/state/CallViewModel/localMember/Publisher.ts +++ b/src/state/CallViewModel/localMember/Publisher.ts @@ -31,7 +31,7 @@ import { } from "../../../livekit/TrackProcessorContext.tsx"; import { getUrlParams } from "../../../UrlParams.ts"; import { observeTrackReference$ } from "../../MediaViewModel.ts"; -import { type Connection } from "../CallViewModel/remoteMembers/Connection.ts"; +import { type Connection } from "../remoteMembers/Connection.ts"; import { type ObservableScope } from "../../ObservableScope.ts"; /** @@ -64,7 +64,7 @@ export class Publisher { const room = connection.livekitRoom; - room.setE2EEEnabled(e2eeLivekitOptions !== undefined)?.catch((e) => { + room.setE2EEEnabled(e2eeLivekitOptions !== undefined)?.catch((e: Error) => { this.logger?.error("Failed to set E2EE enabled on room", e); }); @@ -249,7 +249,7 @@ export class Publisher { ) { lkRoom .switchActiveDevice(kind, device.id) - .catch((e) => + .catch((e: Error) => this.logger?.error( `Failed to sync ${kind} device with LiveKit`, e, diff --git a/src/state/CallViewModel/remoteMembers/Connection.test.ts b/src/state/CallViewModel/remoteMembers/Connection.test.ts index 7d87781f..a3a42928 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.test.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.test.ts @@ -22,6 +22,7 @@ import { type Room as LivekitRoom, RoomEvent, type RoomOptions, + ConnectionState as LivekitConnectionState, } from "livekit-client"; import fetchMock from "fetch-mock"; import EventEmitter from "events"; @@ -32,6 +33,7 @@ import type { LivekitTransport, } from "matrix-js-sdk/lib/matrixrtc"; import { + Connection, type ConnectionOpts, type ConnectionState, type PublishingParticipant, @@ -103,7 +105,7 @@ function setupTest(): void { disconnect: vi.fn(), remoteParticipants: new Map(), localParticipant: fakeLocalParticipant, - state: ConnectionState.Disconnected, + state: LivekitConnectionState.Disconnected, on: fakeRoomEventEmiter.on.bind(fakeRoomEventEmiter), off: fakeRoomEventEmiter.off.bind(fakeRoomEventEmiter), addListener: fakeRoomEventEmiter.addListener.bind(fakeRoomEventEmiter), @@ -115,11 +117,10 @@ function setupTest(): void { } as unknown as LivekitRoom); } -function setupRemoteConnection(): RemoteConnection { +function setupRemoteConnection(): Connection { const opts: ConnectionOpts = { client: client, transport: livekitFocus, - remoteTransports$: fakeMembershipsFocusMap$, scope: testScope, livekitRoomFactory: () => fakeLivekitRoom, }; @@ -136,7 +137,7 @@ function setupRemoteConnection(): RemoteConnection { fakeLivekitRoom.connect.mockResolvedValue(undefined); - return new RemoteConnection(opts, undefined); + return new Connection(opts); } afterEach(() => { @@ -152,11 +153,10 @@ describe("Start connection states", () => { const opts: ConnectionOpts = { client: client, transport: livekitFocus, - remoteTransports$: fakeMembershipsFocusMap$, scope: testScope, livekitRoomFactory: () => fakeLivekitRoom, }; - const connection = new RemoteConnection(opts, undefined); + const connection = new Connection(opts); expect(connection.state$.getValue().state).toEqual("Initialized"); }); @@ -168,12 +168,11 @@ describe("Start connection states", () => { const opts: ConnectionOpts = { client: client, transport: livekitFocus, - remoteTransports$: fakeMembershipsFocusMap$, scope: testScope, livekitRoomFactory: () => fakeLivekitRoom, }; - const connection = new RemoteConnection(opts, undefined); + const connection = new Connection(opts, undefined); const capturedStates: ConnectionState[] = []; const s = connection.state$.subscribe((value) => { @@ -221,12 +220,11 @@ describe("Start connection states", () => { const opts: ConnectionOpts = { client: client, transport: livekitFocus, - remoteTransports$: fakeMembershipsFocusMap$, scope: testScope, livekitRoomFactory: () => fakeLivekitRoom, }; - const connection = new RemoteConnection(opts, undefined); + const connection = new Connection(opts, undefined); const capturedStates: ConnectionState[] = []; const s = connection.state$.subscribe((value) => { @@ -278,12 +276,11 @@ describe("Start connection states", () => { const opts: ConnectionOpts = { client: client, transport: livekitFocus, - remoteTransports$: fakeMembershipsFocusMap$, scope: testScope, livekitRoomFactory: () => fakeLivekitRoom, }; - const connection = new RemoteConnection(opts, undefined); + const connection = new Connection(opts, undefined); const capturedStates: ConnectionState[] = []; const s = connection.state$.subscribe((value) => { diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts index 544f5241..c9434327 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts @@ -55,7 +55,7 @@ interface Props { // => Extract an AvatarService instead? // Better with just `getMember` matrixRoom: Pick & NodeStyleEventEmitter; - roomMember$: Behavior>; + // roomMember$: Behavior>; } // Alternative structure idea: // const livekitMatrixMember$ = (callMemberships$,connectionManager,scope): Observable => { diff --git a/src/state/CallViewModel/remoteMembers/displayname.test.ts b/src/state/CallViewModel/remoteMembers/displayname.test.ts index 9822e486..dd359318 100644 --- a/src/state/CallViewModel/remoteMembers/displayname.test.ts +++ b/src/state/CallViewModel/remoteMembers/displayname.test.ts @@ -14,7 +14,7 @@ import { } from "matrix-js-sdk"; import EventEmitter from "events"; -import { ObservableScope } from "../../ObservableScope.ts"; +import { ObservableScope, trackEpoch } from "../../ObservableScope.ts"; import type { Room as MatrixRoom } from "matrix-js-sdk/lib/models/room"; import { mockCallMembership, withTestScheduler } from "../../../utils/test.ts"; import { memberDisplaynames$ } from "./displayname.ts"; @@ -90,9 +90,7 @@ test("should always have our own user", () => { mockMatrixRoom, cold("a", { a: [], - }), - "@local:example.com", - "DEVICE000", + }).pipe(trackEpoch()), ); expectObservable(dn$).toBe("a", { @@ -125,9 +123,7 @@ test("should get displayName for users", () => { mockCallMembership("@alice:example.com", "DEVICE1"), mockCallMembership("@bob:example.com", "DEVICE1"), ], - }), - "@local:example.com", - "DEVICE000", + }).pipe(trackEpoch()), ); expectObservable(dn$).toBe("a", { @@ -149,9 +145,7 @@ test("should use userId if no display name", () => { mockMatrixRoom, cold("a", { a: [mockCallMembership("@no-name:foo.bar", "D000")], - }), - "@local:example.com", - "DEVICE000", + }).pipe(trackEpoch()), ); expectObservable(dn$).toBe("a", { @@ -178,9 +172,7 @@ test("should disambiguate users with same display name", () => { mockCallMembership("@carl:example.com", "C000"), mockCallMembership("@evil:example.com", "E000"), ], - }), - "@local:example.com", - "DEVICE000", + }).pipe(trackEpoch()), ); expectObservable(dn$).toBe("a", { @@ -209,9 +201,7 @@ test("should disambiguate when needed", () => { mockCallMembership("@bob:example.com", "DEVICE1"), mockCallMembership("@bob:foo.bar", "BOB000"), ], - }), - "@local:example.com", - "DEVICE000", + }).pipe(trackEpoch()), ); expectObservable(dn$).toBe("ab", { @@ -241,9 +231,7 @@ test.skip("should keep disambiguated name when other leave", () => { mockCallMembership("@bob:foo.bar", "BOB000"), ], b: [mockCallMembership("@bob:example.com", "DEVICE1")], - }), - "@local:example.com", - "DEVICE000", + }).pipe(trackEpoch()), ); expectObservable(dn$).toBe("ab", { @@ -272,9 +260,7 @@ test("should disambiguate on name change", () => { mockCallMembership("@bob:example.com", "B000"), mockCallMembership("@carl:example.com", "C000"), ], - }), - "@local:example.com", - "DEVICE000", + }).pipe(trackEpoch()), ); schedule("-a", { diff --git a/src/tile/GridTile.test.tsx b/src/tile/GridTile.test.tsx index dd0bc9d6..e3172a22 100644 --- a/src/tile/GridTile.test.tsx +++ b/src/tile/GridTile.test.tsx @@ -15,7 +15,7 @@ import { GridTile } from "./GridTile"; import { mockRtcMembership, createRemoteMedia } from "../utils/test"; import { GridTileViewModel } from "../state/TileViewModel"; import { ReactionsSenderProvider } from "../reactions/useReactionsSender"; -import type { CallViewModel } from "../state/CallViewModel"; +import type { CallViewModel } from "../state/CallViewModel/CallViewModel"; import { constant } from "../state/Behavior"; global.IntersectionObserver = class MockIntersectionObserver { diff --git a/src/utils/test-viewmodel.ts b/src/utils/test-viewmodel.ts index 5cd64eb3..5a0d7526 100644 --- a/src/utils/test-viewmodel.ts +++ b/src/utils/test-viewmodel.ts @@ -22,7 +22,7 @@ import { E2eeType } from "../e2ee/e2eeType"; import { CallViewModel, type CallViewModelOptions, -} from "../state/CallViewModel"; +} from "../state/CallViewModel/CallViewModel"; import { mockLivekitRoom, mockLocalParticipant, From cf5c35bccd64cda3d6335f947339e8a9494ad778 Mon Sep 17 00:00:00 2001 From: Timo K Date: Fri, 7 Nov 2025 17:13:49 +0100 Subject: [PATCH 32/75] fix more runtime errors --- .../CallViewModel/localMember/LocalMembership.ts | 8 +++++--- .../CallViewModel/localMember/LocalTransport.ts | 4 +++- .../remoteMembers/MatrixLivekitMembers.ts | 15 +++++++-------- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/state/CallViewModel/localMember/LocalMembership.ts b/src/state/CallViewModel/localMember/LocalMembership.ts index a0b06d46..8f7d8b54 100644 --- a/src/state/CallViewModel/localMember/LocalMembership.ts +++ b/src/state/CallViewModel/localMember/LocalMembership.ts @@ -172,9 +172,11 @@ export const createLocalMembership$ = ({ combineLatest( [connectionManager.connections$, localTransport$], (connections, transport) => { - if (transport === undefined) return undefined; - return connections.value.find((connection) => - areLivekitTransportsEqual(connection.transport, transport), + if (transport === null) return null; + return ( + connections.value.find((connection) => + areLivekitTransportsEqual(connection.transport, transport), + ) ?? null ); }, ), diff --git a/src/state/CallViewModel/localMember/LocalTransport.ts b/src/state/CallViewModel/localMember/LocalTransport.ts index 69c9b934..a96962c9 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.ts @@ -91,7 +91,9 @@ export const createLocalTransport$ = ({ combineLatest( [useOldestMember$, oldestMemberTransport$, preferredTransport$], (useOldestMember, oldestMemberTransport, preferredTransport) => - useOldestMember ? oldestMemberTransport : preferredTransport, + useOldestMember + ? (oldestMemberTransport ?? preferredTransport) + : preferredTransport, ).pipe(distinctUntilChanged(deepCompare)), ); return advertisedTransport$; diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts index c9434327..05ffb4f9 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts @@ -145,13 +145,12 @@ export function createMatrixLivekitMembers$({ // TODO add this to the JS-SDK export function areLivekitTransportsEqual( - t1: LivekitTransport, - t2: LivekitTransport, + t1?: LivekitTransport, + t2?: LivekitTransport, ): boolean { - return ( - t1.livekit_service_url === t2.livekit_service_url && - // In case we have different lk rooms in the same SFU (depends on the livekit authorization service) - // It is only needed in case the livekit authorization service is not behaving as expected (or custom implementation) - t1.livekit_alias === t2.livekit_alias - ); + if (t1 && t2) return t1.livekit_service_url === t2.livekit_service_url; + // In case we have different lk rooms in the same SFU (depends on the livekit authorization service) + // It is only needed in case the livekit authorization service is not behaving as expected (or custom implementation) + if (!t1 && !t2) return true; + return false; } From b8635b52d882ea0935b8da8d64ad054a292c8851 Mon Sep 17 00:00:00 2001 From: Timo K Date: Fri, 7 Nov 2025 19:07:45 +0100 Subject: [PATCH 33/75] Working (no local feed) --- src/state/CallViewModel/CallViewModel.ts | 1 + .../localMember/LocalMembership.ts | 115 ++++++++++-------- .../localMember/LocalTransport.ts | 25 ++-- .../CallViewModel/localMember/Publisher.ts | 47 ++++--- .../remoteMembers/MatrixLivekitMembers.ts | 4 +- src/state/MediaViewModel.ts | 16 ++- src/state/UserMedia.ts | 4 +- src/tile/GridTile.tsx | 2 +- tsconfig.json | 3 +- 9 files changed, 121 insertions(+), 96 deletions(-) diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index ef4be238..dd1190b7 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -373,6 +373,7 @@ export class CallViewModel { * List of MediaItems that we want to have tiles for. */ // TODO KEEP THIS!! and adapt it to what our membershipManger returns + // TODO this also needs the local participant to be added. private readonly mediaItems$ = this.scope.behavior( generateKeyed$< [typeof this.matrixLivekitMembers$.value, number], diff --git a/src/state/CallViewModel/localMember/LocalMembership.ts b/src/state/CallViewModel/localMember/LocalMembership.ts index 8f7d8b54..1773eca1 100644 --- a/src/state/CallViewModel/localMember/LocalMembership.ts +++ b/src/state/CallViewModel/localMember/LocalMembership.ts @@ -98,7 +98,7 @@ interface Props { connectionManager: IConnectionManager; matrixRTCSession: MatrixRTCSession; matrixRoom: MatrixRoom; - localTransport$: Behavior; + localTransport$: Behavior; e2eeLivekitOptions: E2EEOptions | undefined; trackProcessorState$: Behavior; widget: WidgetHelpers | null; @@ -162,7 +162,11 @@ export const createLocalMembership$ = ({ // This should be used in a combineLatest with publisher$ to connect. // to make it possible to call startTracks before the preferredTransport$ has resolved. - const shouldStartTracks$ = new BehaviorSubject(false); + const trackStartRequested$ = new BehaviorSubject(false); + + // This should be used in a combineLatest with publisher$ to connect. + // to make it possible to call startTracks before the preferredTransport$ has resolved. + const connectRequested$ = new BehaviorSubject(false); // This should be used in a combineLatest with publisher$ to connect. const tracks$ = new BehaviorSubject([]); @@ -230,26 +234,24 @@ export const createLocalMembership$ = ({ ), ); - const publisher$ = scope.behavior( - connection$.pipe( - map((connection) => - connection - ? new Publisher( - scope, - connection, - mediaDevices, - muteStates, - e2eeLivekitOptions, - trackProcessorState$, - ) - : null, - ), - ), - ); + const publisher$ = new BehaviorSubject(null); + connection$.subscribe((connection) => { + if (connection !== null && publisher$.value === null) { + publisher$.next( + new Publisher( + scope, + connection, + mediaDevices, + muteStates, + e2eeLivekitOptions, + trackProcessorState$, + ), + ); + } + }); - combineLatest( - [publisher$, shouldStartTracks$], - (publisher, shouldStartTracks) => { + combineLatest([publisher$, trackStartRequested$]).subscribe( + ([publisher, shouldStartTracks]) => { if (publisher && shouldStartTracks) { publisher .createAndSetupTracks() @@ -286,41 +288,51 @@ export const createLocalMembership$ = ({ ); const startTracks = (): Behavior => { - shouldStartTracks$.next(true); + trackStartRequested$.next(true); return tracks$; }; - const requestConnect = (): LocalMemberConnectionState => { - if (state.livekit$.value.state === LivekitState.Uninitialized) { - startTracks(); - state.livekit$.next({ state: LivekitState.Connecting }); - combineLatest([publisher$, tracks$], (publisher, tracks) => { - publisher - ?.startPublishing() - .then(() => { - state.livekit$.next({ state: LivekitState.Connected }); - }) - .catch((error) => { - state.livekit$.next({ state: LivekitState.Error, error }); - }); + combineLatest([publisher$, tracks$]).subscribe(([publisher, tracks]) => { + if ( + tracks.length === 0 || + // change this to !== Publishing + state.livekit$.value.state !== LivekitState.Uninitialized + ) { + return; + } + state.livekit$.next({ state: LivekitState.Connecting }); + publisher + ?.startPublishing() + .then(() => { + state.livekit$.next({ state: LivekitState.Connected }); + }) + .catch((error) => { + state.livekit$.next({ state: LivekitState.Error, error }); }); - } - if (state.matrix$.value.state === MatrixState.Disconnected) { + }); + combineLatest([localTransport$, connectRequested$]).subscribe( + ([transport, connectRequested]) => { + if ( + transport === null || + !connectRequested || + state.matrix$.value.state !== MatrixState.Disconnected + ) { + logger.info("Waiting for transport to enter rtc session"); + return; + } state.matrix$.next({ state: MatrixState.Connecting }); - localTransport$.pipe( - tap((transport) => { - if (transport !== undefined) { - enterRTCSession(matrixRTCSession, transport, options.value).catch( - (error) => { - logger.error(error); - }, - ); - } else { - logger.info("Waiting for transport to enter rtc session"); - } - }), + enterRTCSession(matrixRTCSession, transport, options.value).catch( + (error) => { + logger.error(error); + }, ); - } + }, + ); + + const requestConnect = (): LocalMemberConnectionState => { + trackStartRequested$.next(true); + connectRequested$.next(true); + return state; }; @@ -453,8 +465,7 @@ export const createLocalMembership$ = ({ .pipe( // I dont see why we need this. isnt the check later on superseeding it? takeWhile( - (c) => - c !== undefined && c.state$.value.state !== "FailedToStart", + (c) => c !== null && c.state$.value.state !== "FailedToStart", ), switchMap((c) => c?.state$.value.state === "ConnectedToLkRoom" ? of(c) : NEVER, diff --git a/src/state/CallViewModel/localMember/LocalTransport.ts b/src/state/CallViewModel/localMember/LocalTransport.ts index a96962c9..b1fd71e9 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.ts @@ -13,20 +13,21 @@ import { isLivekitTransportConfig, } from "matrix-js-sdk/lib/matrixrtc"; import { type MatrixClient } from "matrix-js-sdk"; -import { combineLatest, distinctUntilChanged, first, from } from "rxjs"; +import { combineLatest, distinctUntilChanged, first, from, map } from "rxjs"; import { logger } from "matrix-js-sdk/lib/logger"; import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery"; import { deepCompare } from "matrix-js-sdk/lib/utils"; import { type Behavior } from "../../Behavior.ts"; import { - type Epoch, + Epoch, mapEpoch, type ObservableScope, } from "../../ObservableScope.ts"; import { Config } from "../../../config/Config.ts"; import { MatrixRTCTransportMissingError } from "../../../utils/errors.ts"; import { getSFUConfigWithOpenID } from "../../../livekit/openIDSFU.ts"; +import { areLivekitTransportsEqual } from "../remoteMembers/MatrixLivekitMembers.ts"; /* * - get well known @@ -60,15 +61,16 @@ export const createLocalTransport$ = ({ client, roomId, useOldestMember$, -}: Props): Behavior => { +}: Props): Behavior => { /** * The transport over which we should be actively publishing our media. * undefined when not joined. */ const oldestMemberTransport$ = scope.behavior( memberships$.pipe( - mapEpoch( - (memberships) => memberships[0]?.getTransport(memberships[0]) ?? null, + map( + (memberships) => + memberships.value[0]?.getTransport(memberships.value[0]) ?? null, ), first((t) => t != null && isLivekitTransport(t)), ), @@ -88,13 +90,18 @@ export const createLocalTransport$ = ({ * The transport we should advertise in our MatrixRTC membership. */ const advertisedTransport$ = scope.behavior( - combineLatest( - [useOldestMember$, oldestMemberTransport$, preferredTransport$], - (useOldestMember, oldestMemberTransport, preferredTransport) => + combineLatest([ + useOldestMember$, + oldestMemberTransport$, + preferredTransport$, + ]).pipe( + map(([useOldestMember, oldestMemberTransport, preferredTransport]) => useOldestMember ? (oldestMemberTransport ?? preferredTransport) : preferredTransport, - ).pipe(distinctUntilChanged(deepCompare)), + ), + distinctUntilChanged(areLivekitTransportsEqual), + ), ); return advertisedTransport$; }; diff --git a/src/state/CallViewModel/localMember/Publisher.ts b/src/state/CallViewModel/localMember/Publisher.ts index ffdd2487..c10201bf 100644 --- a/src/state/CallViewModel/localMember/Publisher.ts +++ b/src/state/CallViewModel/localMember/Publisher.ts @@ -99,31 +99,36 @@ export class Publisher { // instead? This optimization would only be safe for a publish connection, // because we don't want to leak the user's intent to perhaps join a call to // remote servers before they actually commit to it. - const { promise, resolve, reject } = Promise.withResolvers(); - const sub = this.connection.state$.subscribe((s) => { - if (s.state !== "FailedToStart") { - reject(new Error("Disconnected from LiveKit server")); - } else { - resolve(); - } - }); - try { - await promise; - } catch (e) { - throw e; - } finally { - sub.unsubscribe(); - } + // const { promise, resolve, reject } = Promise.withResolvers(); + // const sub = this.connection.state$.subscribe((s) => { + // if (s.state === "FailedToStart") { + // reject(new Error("Disconnected from LiveKit server")); + // } else if (s.state === "ConnectedToLkRoom") { + // resolve(); + // } + // }); + // try { + // await promise; + // } catch (e) { + // throw e; + // } finally { + // sub.unsubscribe(); + // } // TODO-MULTI-SFU: Prepublish a microphone track const audio = this.muteStates.audio.enabled$.value; const video = this.muteStates.video.enabled$.value; // createTracks throws if called with audio=false and video=false if (audio || video) { // TODO this can still throw errors? It will also prompt for permissions if not already granted - this.tracks = await lkRoom.localParticipant.createTracks({ - audio, - video, - }); + this.tracks = + (await lkRoom.localParticipant + .createTracks({ + audio, + video, + }) + .catch((error) => { + this.logger?.error("Failed to create tracks", error); + })) ?? []; } return this.tracks; } @@ -153,7 +158,9 @@ export class Publisher { for (const track of this.tracks) { // TODO: handle errors? Needs the signaling connection to be up, but it has some retries internally // with a timeout. - await lkRoom.localParticipant.publishTrack(track); + await lkRoom.localParticipant.publishTrack(track).catch((error) => { + this.logger?.error("Failed to publish track", error); + }); // TODO: check if the connection is still active? and break the loop if not? } diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts index 05ffb4f9..729ed547 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts @@ -145,8 +145,8 @@ export function createMatrixLivekitMembers$({ // TODO add this to the JS-SDK export function areLivekitTransportsEqual( - t1?: LivekitTransport, - t2?: LivekitTransport, + t1: LivekitTransport | null, + t2: LivekitTransport | null, ): boolean { if (t1 && t2) return t1.livekit_service_url === t2.livekit_service_url; // In case we have different lk rooms in the same SFU (depends on the livekit authorization service) diff --git a/src/state/MediaViewModel.ts b/src/state/MediaViewModel.ts index 0b79183e..b35f6112 100644 --- a/src/state/MediaViewModel.ts +++ b/src/state/MediaViewModel.ts @@ -220,7 +220,7 @@ abstract class BaseMediaViewModel { /** * The LiveKit video track for this media. */ - public readonly video$: Behavior; + public readonly video$: Behavior; /** * Whether there should be a warning that this media is unencrypted. */ @@ -235,12 +235,10 @@ abstract class BaseMediaViewModel { private observeTrackReference$( source: Track.Source, - ): Behavior { + ): Behavior { return this.scope.behavior( this.participant$.pipe( - switchMap((p) => - p === undefined ? of(undefined) : observeTrackReference$(p, source), - ), + switchMap((p) => (!p ? of(null) : observeTrackReference$(p, source))), ), ); } @@ -260,7 +258,7 @@ abstract class BaseMediaViewModel { // We don't necessarily have a participant if a user connects via MatrixRTC but not (yet) through // livekit. protected readonly participant$: Observable< - LocalParticipant | RemoteParticipant | undefined + LocalParticipant | RemoteParticipant | null >, encryptionSystem: EncryptionSystem, @@ -405,7 +403,7 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { scope: ObservableScope, id: string, member: RoomMember, - participant$: Observable, + participant$: Observable, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, focusUrl: string, @@ -541,7 +539,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { scope: ObservableScope, id: string, member: RoomMember, - participant$: Behavior, + participant$: Behavior, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, focusURL: string, @@ -651,7 +649,7 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel { scope: ObservableScope, id: string, member: RoomMember, - participant$: Observable, + participant$: Observable, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, focusUrl: string, diff --git a/src/state/UserMedia.ts b/src/state/UserMedia.ts index 55de2061..9eec3967 100644 --- a/src/state/UserMedia.ts +++ b/src/state/UserMedia.ts @@ -82,7 +82,7 @@ export class UserMedia { this.scope, this.id, this.member, - this.participant$ as Behavior, + this.participant$ as Behavior, this.encryptionSystem, this.livekitRoom, this.focusURL, @@ -95,7 +95,7 @@ export class UserMedia { this.scope, this.id, this.member, - this.participant$ as Observable, + this.participant$ as Behavior, this.encryptionSystem, this.livekitRoom, this.focusURL, diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index 421cefda..1925eff6 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -144,7 +144,7 @@ const UserMediaTile: FC = ({ const tile = ( Date: Sat, 8 Nov 2025 13:02:12 +0100 Subject: [PATCH 34/75] test: fixup ConnectionManager tests --- .../remoteMembers/ConnectionManager.test.ts | 361 ++++++++++-------- src/state/ObservableScope.test.ts | 2 +- 2 files changed, 196 insertions(+), 167 deletions(-) diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts index ec3c1b2f..17d127f3 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts @@ -10,15 +10,16 @@ import { BehaviorSubject } from "rxjs"; import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { type Participant as LivekitParticipant } from "livekit-client"; -import { ObservableScope } from "../../ObservableScope.ts"; +import { Epoch, ObservableScope } from "../../ObservableScope.ts"; import { - type IConnectionManager, createConnectionManager$, + type ConnectionManagerData, } from "./ConnectionManager.ts"; import { type ConnectionFactory } from "./ConnectionFactory.ts"; import { type Connection } from "./Connection.ts"; -import { flushPromises, withTestScheduler } from "../../../utils/test.ts"; +import { withTestScheduler } from "../../../utils/test.ts"; import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts"; +import { type Behavior } from "../../Behavior.ts"; // Some test constants @@ -34,23 +35,15 @@ const TRANSPORT_2: LivekitTransport = { livekit_alias: "!alias:sample.com", }; -// const TRANSPORT_3: LivekitTransport = { -// type: "livekit", -// livekit_service_url: "https://lk-other.sample.com", -// livekit_alias: "!alias:sample.com", -// }; let fakeConnectionFactory: ConnectionFactory; let testScope: ObservableScope; -let testTransportStream$: BehaviorSubject; -let connectionManagerInputs: { - scope: ObservableScope; - connectionFactory: ConnectionFactory; - inputTransports$: BehaviorSubject; -}; -let manager: IConnectionManager; + +// Can be useful to track all created connections in tests, even the disposed ones +let allCreatedConnections: Connection[]; + beforeEach(() => { testScope = new ObservableScope(); - + allCreatedConnections = []; fakeConnectionFactory = {} as unknown as ConnectionFactory; vi.mocked(fakeConnectionFactory).createConnection = vi .fn() @@ -58,6 +51,7 @@ beforeEach(() => { (transport: LivekitTransport, scope: ObservableScope) => { const mockConnection = { transport, + participantsWithTrack$: new BehaviorSubject([]), } as unknown as Connection; vi.mocked(mockConnection).start = vi.fn(); vi.mocked(mockConnection).stop = vi.fn(); @@ -65,17 +59,10 @@ beforeEach(() => { scope.onEnd(() => { void mockConnection.stop(); }); + allCreatedConnections.push(mockConnection); return mockConnection; }, ); - - testTransportStream$ = new BehaviorSubject([]); - connectionManagerInputs = { - scope: testScope, - connectionFactory: fakeConnectionFactory, - inputTransports$: testTransportStream$, - }; - manager = createConnectionManager$(connectionManagerInputs); }); afterEach(() => { @@ -83,93 +70,122 @@ afterEach(() => { }); describe("connections$ stream", () => { - test("Should create and start new connections for each transports", async () => { - const managedConnections = Promise.withResolvers(); - manager.connections$.subscribe((connections) => { - if (connections.length > 0) managedConnections.resolve(connections); + test("Should create and start new connections for each transports", () => { + withTestScheduler(({ behavior, expectObservable }) => { + const { connections$ } = createConnectionManager$({ + scope: testScope, + connectionFactory: fakeConnectionFactory, + inputTransports$: behavior("a", { + a: new Epoch([TRANSPORT_1, TRANSPORT_2], 0), + }), + }); + + expectObservable(connections$).toBe("a", { + a: expect.toSatisfy((e: Epoch) => { + const connections = e.value; + expect(connections.length).toBe(2); + + expect( + vi.mocked(fakeConnectionFactory).createConnection, + ).toHaveBeenCalledTimes(2); + + const conn1 = connections.find((c) => + areLivekitTransportsEqual(c.transport, TRANSPORT_1), + ); + expect(conn1).toBeDefined(); + expect(conn1!.start).toHaveBeenCalled(); + + const conn2 = connections.find((c) => + areLivekitTransportsEqual(c.transport, TRANSPORT_2), + ); + expect(conn2).toBeDefined(); + expect(conn2!.start).toHaveBeenCalled(); + return true; + }), + }); }); - - connectionManagerInputs.inputTransports$.next([TRANSPORT_1, TRANSPORT_2]); - - const connections = await managedConnections.promise; - - expect(connections.length).toBe(2); - - expect( - vi.mocked(fakeConnectionFactory).createConnection, - ).toHaveBeenCalledTimes(2); - - const conn1 = connections.find((c) => - areLivekitTransportsEqual(c.transport, TRANSPORT_1), - ); - expect(conn1).toBeDefined(); - expect(conn1!.start).toHaveBeenCalled(); - - const conn2 = connections.find((c) => - areLivekitTransportsEqual(c.transport, TRANSPORT_2), - ); - expect(conn2).toBeDefined(); - expect(conn2!.start).toHaveBeenCalled(); }); - test("Should start connection only once", async () => { - const observedConnections: Connection[][] = []; - manager.connections$.subscribe((connections) => { - observedConnections.push(connections); + test("Should start connection only once", () => { + withTestScheduler(({ behavior, expectObservable }) => { + const { connections$ } = createConnectionManager$({ + scope: testScope, + connectionFactory: fakeConnectionFactory, + inputTransports$: behavior("abcdef", { + a: new Epoch([TRANSPORT_1], 0), + b: new Epoch([TRANSPORT_1], 1), + c: new Epoch([TRANSPORT_1], 2), + d: new Epoch([TRANSPORT_1], 3), + e: new Epoch([TRANSPORT_1], 4), + f: new Epoch([TRANSPORT_1, TRANSPORT_2], 5), + }), + }); + + expectObservable(connections$).toBe("xxxxxa", { + x: expect.anything(), + a: expect.toSatisfy((e: Epoch) => { + const connections = e.value; + + expect(connections.length).toBe(2); + expect( + vi.mocked(fakeConnectionFactory).createConnection, + ).toHaveBeenCalledTimes(2); + + const conn2 = connections.find((c) => + areLivekitTransportsEqual(c.transport, TRANSPORT_2), + ); + expect(conn2).toBeDefined(); + + const conn1 = connections.find((c) => + areLivekitTransportsEqual(c.transport, TRANSPORT_1), + ); + expect(conn1).toBeDefined(); + expect(conn1!.start).toHaveBeenCalledOnce(); + + return true; + }), + }); }); - - testTransportStream$.next([TRANSPORT_1]); - testTransportStream$.next([TRANSPORT_1]); - testTransportStream$.next([TRANSPORT_1]); - testTransportStream$.next([TRANSPORT_1]); - testTransportStream$.next([TRANSPORT_1]); - testTransportStream$.next([TRANSPORT_1, TRANSPORT_2]); - - await flushPromises(); - const connections = observedConnections.pop()!; - - expect(connections.length).toBe(2); - expect( - vi.mocked(fakeConnectionFactory).createConnection, - ).toHaveBeenCalledTimes(2); - - const conn2 = connections.find((c) => - areLivekitTransportsEqual(c.transport, TRANSPORT_2), - ); - expect(conn2).toBeDefined(); - - const conn1 = connections.find((c) => - areLivekitTransportsEqual(c.transport, TRANSPORT_1), - ); - expect(conn1).toBeDefined(); - expect(conn1!.start).toHaveBeenCalledOnce(); }); - test("Should cleanup connections when not needed anymore", async () => { - const observedConnections: Connection[][] = []; - manager.connections$.subscribe((connections) => { - observedConnections.push(connections); + test("Should cleanup connections when not needed anymore", () => { + withTestScheduler(({ behavior, expectObservable }) => { + const { connections$ } = createConnectionManager$({ + scope: testScope, + connectionFactory: fakeConnectionFactory, + inputTransports$: behavior("abc", { + a: new Epoch([TRANSPORT_1], 0), + b: new Epoch([TRANSPORT_1, TRANSPORT_2], 1), + c: new Epoch([TRANSPORT_1], 2), + }), + }); + + expectObservable(connections$).toBe("xab", { + x: expect.anything(), + a: expect.toSatisfy((e: Epoch) => { + const connections = e.value; + expect(connections.length).toBe(2); + return true; + }), + b: expect.toSatisfy((e: Epoch) => { + const connections = e.value; + + expect(connections.length).toBe(1); + // The second connection should have been stopped has it is no longer needed. + const connection2 = allCreatedConnections.find((c) => + areLivekitTransportsEqual(c.transport, TRANSPORT_2), + ); + expect(connection2).toBeDefined(); + expect(connection2!.stop).toHaveBeenCalled(); + + // The first connection should still be active + const conn1 = connections[0]; + expect(conn1.stop).not.toHaveBeenCalledOnce(); + + return true; + }), + }); }); - - testTransportStream$.next([TRANSPORT_1]); - testTransportStream$.next([TRANSPORT_1, TRANSPORT_2]); - - await flushPromises(); - - const conn2 = observedConnections - .pop()! - .find((c) => areLivekitTransportsEqual(c.transport, TRANSPORT_2))!; - - testTransportStream$.next([TRANSPORT_1]); - - await flushPromises(); - - // The second connection should have been stopped has it is no longer needed - expect(conn2.stop).toHaveBeenCalled(); - - // The first connection should still be active - const conn1 = observedConnections.pop()![0]; - expect(conn1.stop).not.toHaveBeenCalledOnce(); }); }); @@ -177,7 +193,7 @@ describe("connectionManagerData$ stream", () => { // Used in test to control fake connections' participantsWithTrack$ streams let fakePublishingParticipantsStreams: Map< string, - BehaviorSubject + Behavior >; function keyForTransport(transport: LivekitTransport): string { @@ -186,6 +202,16 @@ describe("connectionManagerData$ stream", () => { beforeEach(() => { fakePublishingParticipantsStreams = new Map(); + + function getPublishingParticipantsFor( + transport: LivekitTransport, + ): Behavior { + return ( + fakePublishingParticipantsStreams.get(keyForTransport(transport)) ?? + new BehaviorSubject([]) + ); + } + // need a more advanced fake connection factory vi.mocked(fakeConnectionFactory).createConnection = vi .fn() @@ -196,7 +222,7 @@ describe("connectionManagerData$ stream", () => { >([]); const mockConnection = { transport, - participantsWithTrack$: fakePublishingParticipants$, + participantsWithTrack$: getPublishingParticipantsFor(transport), } as unknown as Connection; vi.mocked(mockConnection).start = vi.fn(); vi.mocked(mockConnection).stop = vi.fn(); @@ -216,80 +242,83 @@ describe("connectionManagerData$ stream", () => { test("Should report connections with the publishing participants", () => { withTestScheduler(({ expectObservable, schedule, behavior }) => { - manager = createConnectionManager$({ - ...connectionManagerInputs, - inputTransports$: behavior("a", { - a: [TRANSPORT_1, TRANSPORT_2], - }), - }); - - const conn1Participants$ = fakePublishingParticipantsStreams.get( + // Setup the fake participants streams behavior + // ============================== + fakePublishingParticipantsStreams.set( keyForTransport(TRANSPORT_1), - )!; - - schedule("-a-b", { - a: () => { - conn1Participants$.next([ - { identity: "user1A" } as LivekitParticipant, - ]); - }, - b: () => { - conn1Participants$.next([ + behavior("oa-b", { + o: [], + a: [{ identity: "user1A" } as LivekitParticipant], + b: [ { identity: "user1A" } as LivekitParticipant, { identity: "user1B" } as LivekitParticipant, - ]); - }, - }); + ], + }), + ); - const conn2Participants$ = fakePublishingParticipantsStreams.get( + fakePublishingParticipantsStreams.set( keyForTransport(TRANSPORT_2), - )!; + behavior("o-a", { + o: [], + a: [{ identity: "user2A" } as LivekitParticipant], + }), + ); + // ============================== - schedule("--a", { - a: () => { - conn2Participants$.next([ - { identity: "user2A" } as LivekitParticipant, - ]); - }, + const { connectionManagerData$ } = createConnectionManager$({ + scope: testScope, + connectionFactory: fakeConnectionFactory, + inputTransports$: behavior("a", { + a: new Epoch([TRANSPORT_1, TRANSPORT_2], 0), + }), }); - expectObservable(manager.connectionManagerData$).toBe("abcd", { - a: expect.toSatisfy((data) => { - return ( - data.getConnections().length == 2 && - data.getParticipantForTransport(TRANSPORT_1).length == 0 && - data.getParticipantForTransport(TRANSPORT_2).length == 0 - ); + expectObservable(connectionManagerData$).toBe("abcd", { + a: expect.toSatisfy((e) => { + const data: ConnectionManagerData = e.value; + expect(data.getConnections().length).toBe(2); + expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(0); + expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(0); + return true; }), - b: expect.toSatisfy((data) => { - return ( - data.getConnections().length == 2 && - data.getParticipantForTransport(TRANSPORT_1).length == 1 && - data.getParticipantForTransport(TRANSPORT_2).length == 0 && - data.getParticipantForTransport(TRANSPORT_1)[0].identity == "user1A" + b: expect.toSatisfy((e) => { + const data: ConnectionManagerData = e.value; + expect(data.getConnections().length).toBe(2); + expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(1); + expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(0); + expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toBe( + "user1A", ); + return true; }), - c: expect.toSatisfy((data) => { - return ( - data.getConnections().length == 2 && - data.getParticipantForTransport(TRANSPORT_1).length == 1 && - data.getParticipantForTransport(TRANSPORT_2).length == 1 && - data.getParticipantForTransport(TRANSPORT_1)[0].identity == - "user1A" && - data.getParticipantForTransport(TRANSPORT_2)[0].identity == "user2A" + c: expect.toSatisfy((e) => { + const data: ConnectionManagerData = e.value; + expect(data.getConnections().length).toBe(2); + expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(1); + expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(1); + expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toBe( + "user1A", ); + expect(data.getParticipantForTransport(TRANSPORT_2)[0].identity).toBe( + "user2A", + ); + return true; }), - d: expect.toSatisfy((data) => { - return ( - data.getConnections().length == 2 && - data.getParticipantForTransport(TRANSPORT_1).length == 2 && - data.getParticipantForTransport(TRANSPORT_2).length == 1 && - data.getParticipantForTransport(TRANSPORT_1)[0].identity == - "user1A" && - data.getParticipantForTransport(TRANSPORT_1)[1].identity == - "user1B" && - data.getParticipantForTransport(TRANSPORT_2)[0].identity == "user2A" + d: expect.toSatisfy((e) => { + const data: ConnectionManagerData = e.value; + expect(data.getConnections().length).toBe(2); + expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(2); + expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(1); + expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toBe( + "user1A", ); + expect(data.getParticipantForTransport(TRANSPORT_1)[1].identity).toBe( + "user1B", + ); + expect(data.getParticipantForTransport(TRANSPORT_2)[0].identity).toBe( + "user2A", + ); + return true; }), }); }); diff --git a/src/state/ObservableScope.test.ts b/src/state/ObservableScope.test.ts index 19fea76c..4b0f3b4f 100644 --- a/src/state/ObservableScope.test.ts +++ b/src/state/ObservableScope.test.ts @@ -6,6 +6,7 @@ Please see LICENSE in the repository root for full details. */ import { describe, expect, it } from "vitest"; +import { BehaviorSubject, timer } from "rxjs"; import { Epoch, @@ -14,7 +15,6 @@ import { trackEpoch, } from "./ObservableScope"; import { withTestScheduler } from "../utils/test"; -import { BehaviorSubject, timer } from "rxjs"; describe("Epoch", () => { it("should map the value correctly", () => { From 1f386a1d574249d2dc6e57c6958e4f1ef6926966 Mon Sep 17 00:00:00 2001 From: Valere Date: Sat, 8 Nov 2025 13:24:03 +0100 Subject: [PATCH 35/75] test: fix displayname tests due to Epoch change --- .../remoteMembers/displayname.test.ts | 36 ++++++++++--------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/src/state/CallViewModel/remoteMembers/displayname.test.ts b/src/state/CallViewModel/remoteMembers/displayname.test.ts index dd359318..1bb376ba 100644 --- a/src/state/CallViewModel/remoteMembers/displayname.test.ts +++ b/src/state/CallViewModel/remoteMembers/displayname.test.ts @@ -13,6 +13,7 @@ import { RoomStateEvent, } from "matrix-js-sdk"; import EventEmitter from "events"; +import { map } from "rxjs"; import { ObservableScope, trackEpoch } from "../../ObservableScope.ts"; import type { Room as MatrixRoom } from "matrix-js-sdk/lib/models/room"; @@ -83,7 +84,8 @@ afterEach(() => { fakeMembersMap.clear(); }); -test("should always have our own user", () => { +// TODO this is a regression, now there the own user is not always in the map. Ask Timo if fine +test.skip("should always have our own user", () => { withTestScheduler(({ cold, schedule, expectObservable }) => { const dn$ = memberDisplaynames$( testScope, @@ -93,7 +95,7 @@ test("should always have our own user", () => { }).pipe(trackEpoch()), ); - expectObservable(dn$).toBe("a", { + expectObservable(dn$.pipe(map((e) => e.value))).toBe("a", { a: new Map([ ["@local:example.com:DEVICE000", "@local:example.com"], ]), @@ -126,9 +128,9 @@ test("should get displayName for users", () => { }).pipe(trackEpoch()), ); - expectObservable(dn$).toBe("a", { + expectObservable(dn$.pipe(map((e) => e.value))).toBe("a", { a: new Map([ - ["@local:example.com:DEVICE000", "it's a me"], + // ["@local:example.com:DEVICE000", "it's a me"], ["@alice:example.com:DEVICE1", "Alice"], ["@bob:example.com:DEVICE1", "Bob"], ]), @@ -148,9 +150,9 @@ test("should use userId if no display name", () => { }).pipe(trackEpoch()), ); - expectObservable(dn$).toBe("a", { + expectObservable(dn$.pipe(map((e) => e.value))).toBe("a", { a: new Map([ - ["@local:example.com:DEVICE000", "it's a me"], + // ["@local:example.com:DEVICE000", "it's a me"], ["@no-name:foo.bar:D000", "@no-name:foo.bar"], ]), }); @@ -175,9 +177,9 @@ test("should disambiguate users with same display name", () => { }).pipe(trackEpoch()), ); - expectObservable(dn$).toBe("a", { + expectObservable(dn$.pipe(map((e) => e.value))).toBe("a", { a: new Map([ - ["@local:example.com:DEVICE000", "it's a me"], + // ["@local:example.com:DEVICE000", "it's a me"], ["@bob:example.com:DEVICE1", "Bob (@bob:example.com)"], ["@bob:example.com:DEVICE2", "Bob (@bob:example.com)"], ["@bob:foo.bar:BOB000", "Bob (@bob:foo.bar)"], @@ -204,13 +206,13 @@ test("should disambiguate when needed", () => { }).pipe(trackEpoch()), ); - expectObservable(dn$).toBe("ab", { + expectObservable(dn$.pipe(map((e) => e.value))).toBe("ab", { a: new Map([ - ["@local:example.com:DEVICE000", "it's a me"], + // ["@local:example.com:DEVICE000", "it's a me"], ["@bob:example.com:DEVICE1", "Bob"], ]), b: new Map([ - ["@local:example.com:DEVICE000", "it's a me"], + // ["@local:example.com:DEVICE000", "it's a me"], ["@bob:example.com:DEVICE1", "Bob (@bob:example.com)"], ["@bob:foo.bar:BOB000", "Bob (@bob:foo.bar)"], ]), @@ -234,14 +236,14 @@ test.skip("should keep disambiguated name when other leave", () => { }).pipe(trackEpoch()), ); - expectObservable(dn$).toBe("ab", { + expectObservable(dn$.pipe(map((e) => e.value))).toBe("ab", { a: new Map([ - ["@local:example.com:DEVICE000", "it's a me"], + // ["@local:example.com:DEVICE000", "it's a me"], ["@bob:example.com:DEVICE1", "Bob (@bob:example.com)"], ["@bob:foo.bar:BOB000", "Bob (@bob:foo.bar)"], ]), b: new Map([ - ["@local:example.com:DEVICE000", "it's a me"], + // ["@local:example.com:DEVICE000", "it's a me"], ["@bob:example.com:DEVICE1", "Bob (@bob:example.com)"], ]), }); @@ -269,14 +271,14 @@ test("should disambiguate on name change", () => { }, }); - expectObservable(dn$).toBe("ab", { + expectObservable(dn$.pipe(map((e) => e.value))).toBe("ab", { a: new Map([ - ["@local:example.com:DEVICE000", "it's a me"], + // ["@local:example.com:DEVICE000", "it's a me"], ["@bob:example.com:B000", "Bob"], ["@carl:example.com:C000", "Carl"], ]), b: new Map([ - ["@local:example.com:DEVICE000", "it's a me"], + // ["@local:example.com:DEVICE000", "it's a me"], ["@bob:example.com:B000", "Bob (@bob:example.com)"], ["@carl:example.com:C000", "Bob (@carl:example.com)"], ]), From b4c17ed26dfb54517e93a7b57b5ed20ee63f8eef Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 7 Nov 2025 17:36:16 -0500 Subject: [PATCH 36/75] Replace generateKeyed$ with a redesigned generateItems operator And use it to clean up a number of code smells, fix some reactivity bugs, and avoid some resource leaks. --- src/state/CallViewModel/CallViewModel.ts | 181 +++++++-------- .../localMember/LocalMembership.ts | 20 +- .../remoteMembers/ConnectionManager.ts | 56 +++-- .../remoteMembers/MatrixLivekitMembers.ts | 95 ++++---- .../remoteMembers/displayname.test.ts | 56 ++--- .../remoteMembers/displayname.ts | 25 +-- src/state/MediaViewModel.ts | 123 +++++----- src/state/ObservableScope.ts | 43 +++- src/state/ScreenShare.ts | 21 +- src/state/TileStore.ts | 8 +- src/state/UserMedia.ts | 117 +++++----- src/tile/GridTile.tsx | 15 +- src/tile/MediaView.tsx | 11 +- src/tile/SpotlightTile.tsx | 15 +- src/utils/observable.test.ts | 29 ++- src/utils/observable.ts | 210 +++++++++++++----- src/utils/test-viewmodel.ts | 4 +- src/utils/test.ts | 22 +- 18 files changed, 610 insertions(+), 441 deletions(-) diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index dd1190b7..f26c4b3b 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -52,7 +52,7 @@ import { ScreenShareViewModel, type UserMediaViewModel, } from "../MediaViewModel"; -import { accumulate, generateKeyed$, pauseWhen } from "../../utils/observable"; +import { accumulate, generateItems, pauseWhen } from "../../utils/observable"; import { duplicateTiles, MatrixRTCMode, @@ -75,7 +75,7 @@ import { } from "../../reactions"; import { shallowEquals } from "../../utils/array"; import { type MediaDevices } from "../MediaDevices"; -import { type Behavior, constant } from "../Behavior"; +import { type Behavior } from "../Behavior"; import { E2eeType } from "../../e2ee/e2eeType"; import { MatrixKeyProvider } from "../../e2ee/matrixKeyProvider"; import { type MuteStates } from "../MuteStates"; @@ -370,103 +370,101 @@ export class CallViewModel { ); /** - * List of MediaItems that we want to have tiles for. + * List of user media (camera feeds) that we want tiles for. */ - // TODO KEEP THIS!! and adapt it to what our membershipManger returns // TODO this also needs the local participant to be added. - private readonly mediaItems$ = this.scope.behavior( - generateKeyed$< - [typeof this.matrixLivekitMembers$.value, number], - MediaItem, - MediaItem[] - >( + private readonly userMedia$ = this.scope.behavior( + combineLatest([this.matrixLivekitMembers$, duplicateTiles.value$]).pipe( // Generate a collection of MediaItems from the list of expected (whether // present or missing) LiveKit participants. - combineLatest([this.matrixLivekitMembers$, duplicateTiles.value$]), - ([{ value: matrixLivekitMembers }, duplicateTiles], createOrGet) => { - const items: MediaItem[] = []; - - for (const { - connection, - participant, - member, - displayName, + generateItems( + function* ([{ value: matrixLivekitMembers }, duplicateTiles]) { + for (const { + participantId, + userId, + participant$, + connection$, + displayName$, + mxcAvatarUrl$, + } of matrixLivekitMembers) + for (let dup = 0; dup < 1 + duplicateTiles; dup++) + yield { + keys: [ + dup, + participantId, + userId, + participant$, + connection$, + displayName$, + mxcAvatarUrl$, + ], + data: undefined, + }; + }, + ( + scope, + _data$, + dup, participantId, - } of matrixLivekitMembers) { - if (connection === undefined) { - logger.warn("connection is not yet initialised."); - continue; - } - for (let i = 0; i < 1 + duplicateTiles; i++) { - const mediaId = `${participantId}:${i}`; - const lkRoom = connection?.livekitRoom; - const url = connection?.transport.livekit_service_url; + userId, + participant$, + connection$, + displayName$, + mxcAvatarUrl$, + ) => { + const livekitRoom$ = scope.behavior( + connection$.pipe(map((c) => c?.livekitRoom)), + ); + const focusUrl$ = scope.behavior( + connection$.pipe(map((c) => c?.transport.livekit_service_url)), + ); - const item = createOrGet( - mediaId, - (scope) => - // We create UserMedia with or without a participant. - // This will be the initial value of a BehaviourSubject. - // Once a participant appears we will update the BehaviourSubject. (see below) - new UserMedia( - scope, - mediaId, - member, - participant, - this.options.encryptionSystem, - lkRoom, - url, - this.mediaDevices, - this.pretendToBeDisconnected$, - constant(displayName ?? "[👻]"), - this.handsRaised$.pipe( - map((v) => v[participantId]?.time ?? null), - ), - this.reactions$.pipe( - map((v) => v[participantId] ?? undefined), - ), - ), - ); - items.push(item); - (item as UserMedia).updateParticipant(participant); - - if (participant?.isScreenShareEnabled) { - const screenShareId = `${mediaId}:screen-share`; - items.push( - createOrGet( - screenShareId, - (scope) => - new ScreenShare( - scope, - screenShareId, - member, - participant, - this.options.encryptionSystem, - lkRoom, - url, - this.pretendToBeDisconnected$, - constant(displayName ?? "[👻]"), - ), - ), - ); - } - } - } - return items; - }, + return new UserMedia( + scope, + `${participantId}:${dup}`, + userId, + participant$, + this.options.encryptionSystem, + livekitRoom$, + focusUrl$, + this.mediaDevices, + this.pretendToBeDisconnected$, + displayName$, + mxcAvatarUrl$, + this.handsRaised$.pipe(map((v) => v[participantId]?.time ?? null)), + this.reactions$.pipe(map((v) => v[participantId] ?? undefined)), + ); + }, + ), ), ); /** - * List of MediaItems that we want to display, that are of type UserMedia + * List of all media items (user media and screen share media) that we want + * tiles for. */ - private readonly userMedia$ = this.scope.behavior( - this.mediaItems$.pipe( - map((mediaItems) => - mediaItems.filter((m): m is UserMedia => m instanceof UserMedia), + private readonly mediaItems$ = this.scope.behavior( + this.userMedia$.pipe( + switchMap((userMedia) => + userMedia.length === 0 + ? of([]) + : combineLatest( + userMedia.map((m) => m.screenShares$), + (...screenShares) => [...userMedia, ...screenShares.flat(1)], + ), + ), + ), + ); + + /** + * List of MediaItems that we want to display, that are of type ScreenShare + */ + private readonly screenShares$ = this.scope.behavior( + this.mediaItems$.pipe( + map((mediaItems) => + mediaItems.filter((m): m is ScreenShare => m instanceof ScreenShare), ), ), - [], ); public readonly joinSoundEffect$ = this.userMedia$.pipe( @@ -544,17 +542,6 @@ export class CallViewModel { tap((reason) => this.leaveHoisted$.next(reason)), ); - /** - * List of MediaItems that we want to display, that are of type ScreenShare - */ - private readonly screenShares$ = this.scope.behavior( - this.mediaItems$.pipe( - map((mediaItems) => - mediaItems.filter((m): m is ScreenShare => m instanceof ScreenShare), - ), - ), - ); - private readonly spotlightSpeaker$ = this.scope.behavior( this.userMedia$.pipe( diff --git a/src/state/CallViewModel/localMember/LocalMembership.ts b/src/state/CallViewModel/localMember/LocalMembership.ts index 1773eca1..c6b8b170 100644 --- a/src/state/CallViewModel/localMember/LocalMembership.ts +++ b/src/state/CallViewModel/localMember/LocalMembership.ts @@ -5,7 +5,13 @@ SPDX-License-IdFentifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type LocalTrack, type E2EEOptions } from "livekit-client"; +import { + type LocalTrack, + type E2EEOptions, + type Participant, + ParticipantEvent, +} from "livekit-client"; +import { observeParticipantEvents } from "@livekit/components-core"; import { type LivekitTransport, type MatrixRTCSession, @@ -26,11 +32,9 @@ import { switchMap, take, takeWhile, - tap, } from "rxjs"; import { logger } from "matrix-js-sdk/lib/logger"; -import { sharingScreen$ as observeSharingScreen$ } from "../../UserMedia.ts"; import { type Behavior } from "../../Behavior"; import { type IConnectionManager } from "../remoteMembers/ConnectionManager"; import { ObservableScope } from "../../ObservableScope"; @@ -521,3 +525,13 @@ export const createLocalMembership$ = ({ toggleScreenSharing, }; }; + +export function observeSharingScreen$(p: Participant): Observable { + return observeParticipantEvents( + p, + ParticipantEvent.TrackPublished, + ParticipantEvent.TrackUnpublished, + ParticipantEvent.LocalTrackPublished, + ParticipantEvent.LocalTrackUnpublished, + ).pipe(map((p) => p.isScreenShareEnabled)); +} diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts index 2859e49b..ce984aec 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts @@ -20,7 +20,7 @@ import { type LocalParticipant, type RemoteParticipant } from "livekit-client"; import { type Behavior } from "../../Behavior.ts"; import { type Connection } from "./Connection.ts"; import { Epoch, type ObservableScope } from "../../ObservableScope.ts"; -import { generateKeyed$ } from "../../../utils/observable.ts"; +import { generateItemsWithEpoch } from "../../../utils/observable.ts"; import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts"; import { type ConnectionFactory } from "./ConnectionFactory.ts"; @@ -144,34 +144,32 @@ export function createConnectionManager$({ * Connections for each transport in use by one or more session members. */ const connections$ = scope.behavior( - generateKeyed$, Connection, Epoch>( - transports$, - (transports, createOrGet) => { - const createConnection = - ( - transport: LivekitTransport, - ): ((scope: ObservableScope) => Connection) => - (scope) => { - const connection = connectionFactory.createConnection( - transport, - scope, - logger, - ); - // Start the connection immediately - // Use connection state to track connection progress - void connection.start(); - // TODO subscribe to connection state to retry or log issues? - return connection; - }; - - return transports.mapInner((transports) => { - return transports.map((transport) => { - const key = - transport.livekit_service_url + "|" + transport.livekit_alias; - return createOrGet(key, createConnection(transport)); - }); - }); - }, + transports$.pipe( + generateItemsWithEpoch( + function* (transports) { + for (const transport of transports) + yield { + keys: [transport.livekit_service_url, transport.livekit_alias], + data: undefined, + }; + }, + (scope, _data$, serviceUrl, alias) => { + const connection = connectionFactory.createConnection( + { + type: "livekit", + livekit_service_url: serviceUrl, + livekit_alias: alias, + }, + scope, + logger, + ); + // Start the connection immediately + // Use connection state to track connection progress + void connection.start(); + // TODO subscribe to connection state to retry or log issues? + return connection; + }, + ), ), ); diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts index 729ed547..4aaaadd4 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts @@ -16,32 +16,31 @@ import { import { combineLatest, filter, map } from "rxjs"; // eslint-disable-next-line rxjs/no-internal import { type NodeStyleEventEmitter } from "rxjs/internal/observable/fromEvent"; -import { type Room as MatrixRoom, type RoomMember } from "matrix-js-sdk"; +import { type Room as MatrixRoom } from "matrix-js-sdk"; +import { logger } from "matrix-js-sdk/lib/logger"; import { type Behavior } from "../../Behavior"; import { type IConnectionManager } from "./ConnectionManager"; import { Epoch, mapEpoch, type ObservableScope } from "../../ObservableScope"; -import { getRoomMemberFromRtcMember, memberDisplaynames$ } from "./displayname"; +import { memberDisplaynames$ } from "./displayname"; import { type Connection } from "./Connection"; +import { generateItemsWithEpoch } from "../../../utils/observable"; /** - * Represent a matrix call member and his associated livekit participation. + * Represents a Matrix call member and their associated LiveKit participation. * `livekitParticipant` can be undefined if the member is not yet connected to the livekit room * or if it has no livekit transport at all. */ export interface MatrixLivekitMember { - membership: CallMembership; - displayName?: string; - participant?: LocalLivekitParticipant | RemoteLivekitParticipant; - connection?: Connection; - /** - * TODO Try to remove this! Its waaay to much information. - * Just get the member's avatar - * @deprecated - */ - member: RoomMember; - mxcAvatarUrl?: string; participantId: string; + userId: string; + membership$: Behavior; + participant$: Behavior< + LocalLivekitParticipant | RemoteLivekitParticipant | null + >; + connection$: Behavior; + displayName$: Behavior; + mxcAvatarUrl$: Behavior; } interface Props { @@ -100,44 +99,54 @@ export function createMatrixLivekitMembers$({ { value: membershipsWithTransports, epoch }, { value: managerData }, { value: displaynames }, - ]) => { - const items: MatrixLivekitMember[] = membershipsWithTransports.map( - ({ membership, transport }) => { - // TODO! cannot use membership.membershipID yet, Currently its hardcoded by the jwt service to - const participantId = /*membership.membershipID*/ `${membership.userId}:${membership.deviceId}`; + ]) => + new Epoch( + [membershipsWithTransports, managerData, displaynames] as const, + epoch, + ), + ), + generateItemsWithEpoch( + function* ([membershipsWithTransports, managerData, displaynames]) { + for (const { membership, transport } of membershipsWithTransports) { + // TODO! cannot use membership.membershipID yet, Currently its hardcoded by the jwt service to + const participantId = /*membership.membershipID*/ `${membership.userId}:${membership.deviceId}`; - const participants = transport - ? managerData.getParticipantForTransport(transport) - : []; - const participant = participants.find( - (p) => p.identity == participantId, - ); - const member = getRoomMemberFromRtcMember( + const participants = transport + ? managerData.getParticipantForTransport(transport) + : []; + const participant = + participants.find((p) => p.identity == participantId) ?? null; + // This makes sense to add to the js-sdk callMembership (we only need the avatar so probably the call memberhsip just should aquire the avatar) + const member = matrixRoom.getMember(membership.userId); + const connection = transport + ? managerData.getConnectionForTransport(transport) + : undefined; + + let displayName = displaynames.get(membership.userId); + if (displayName === undefined) { + logger.warn(`No display name for user ${membership.userId}`); + displayName = ""; + } + + yield { + keys: [participantId, membership.userId], + data: { membership, - matrixRoom, - )?.member; - const connection = transport - ? managerData.getConnectionForTransport(transport) - : undefined; - const displayName = displaynames.get(participantId); - return { participant, - membership, connection, - // This makes sense to add to the js-sdk callMembership (we only need the avatar so probably the call memberhsip just should aquire the avatar) - // TODO Ugh this is hidign that it might be undefined!! best we remove the member entirely. - member: member as RoomMember, displayName, mxcAvatarUrl: member?.getMxcAvatarUrl(), - participantId, - }; - }, - ); - return new Epoch(items, epoch); + }, + }; + } }, + (scope, data$, participantId, userId) => ({ + participantId, + userId, + ...scope.splitBehavior(data$), + }), ), ), - // new Epoch([]), ); } diff --git a/src/state/CallViewModel/remoteMembers/displayname.test.ts b/src/state/CallViewModel/remoteMembers/displayname.test.ts index 1bb376ba..efbe1235 100644 --- a/src/state/CallViewModel/remoteMembers/displayname.test.ts +++ b/src/state/CallViewModel/remoteMembers/displayname.test.ts @@ -97,7 +97,7 @@ test.skip("should always have our own user", () => { expectObservable(dn$.pipe(map((e) => e.value))).toBe("a", { a: new Map([ - ["@local:example.com:DEVICE000", "@local:example.com"], + ["@local:example.com", "@local:example.com"], ]), }); }); @@ -130,9 +130,9 @@ test("should get displayName for users", () => { expectObservable(dn$.pipe(map((e) => e.value))).toBe("a", { a: new Map([ - // ["@local:example.com:DEVICE000", "it's a me"], - ["@alice:example.com:DEVICE1", "Alice"], - ["@bob:example.com:DEVICE1", "Bob"], + // ["@local:example.com", "it's a me"], + ["@alice:example.com", "Alice"], + ["@bob:example.com", "Bob"], ]), }); }); @@ -152,8 +152,8 @@ test("should use userId if no display name", () => { expectObservable(dn$.pipe(map((e) => e.value))).toBe("a", { a: new Map([ - // ["@local:example.com:DEVICE000", "it's a me"], - ["@no-name:foo.bar:D000", "@no-name:foo.bar"], + // ["@local:example.com", "it's a me"], + ["@no-name:foo.bar", "@no-name:foo.bar"], ]), }); }); @@ -179,12 +179,12 @@ test("should disambiguate users with same display name", () => { expectObservable(dn$.pipe(map((e) => e.value))).toBe("a", { a: new Map([ - // ["@local:example.com:DEVICE000", "it's a me"], - ["@bob:example.com:DEVICE1", "Bob (@bob:example.com)"], - ["@bob:example.com:DEVICE2", "Bob (@bob:example.com)"], - ["@bob:foo.bar:BOB000", "Bob (@bob:foo.bar)"], - ["@carl:example.com:C000", "Carl (@carl:example.com)"], - ["@evil:example.com:E000", "Carl (@evil:example.com)"], + // ["@local:example.com", "it's a me"], + ["@bob:example.com", "Bob (@bob:example.com)"], + ["@bob:example.com", "Bob (@bob:example.com)"], + ["@bob:foo.bar", "Bob (@bob:foo.bar)"], + ["@carl:example.com", "Carl (@carl:example.com)"], + ["@evil:example.com", "Carl (@evil:example.com)"], ]), }); }); @@ -208,13 +208,13 @@ test("should disambiguate when needed", () => { expectObservable(dn$.pipe(map((e) => e.value))).toBe("ab", { a: new Map([ - // ["@local:example.com:DEVICE000", "it's a me"], - ["@bob:example.com:DEVICE1", "Bob"], + // ["@local:example.com", "it's a me"], + ["@bob:example.com", "Bob"], ]), b: new Map([ - // ["@local:example.com:DEVICE000", "it's a me"], - ["@bob:example.com:DEVICE1", "Bob (@bob:example.com)"], - ["@bob:foo.bar:BOB000", "Bob (@bob:foo.bar)"], + // ["@local:example.com", "it's a me"], + ["@bob:example.com", "Bob (@bob:example.com)"], + ["@bob:foo.bar", "Bob (@bob:foo.bar)"], ]), }); }); @@ -238,13 +238,13 @@ test.skip("should keep disambiguated name when other leave", () => { expectObservable(dn$.pipe(map((e) => e.value))).toBe("ab", { a: new Map([ - // ["@local:example.com:DEVICE000", "it's a me"], - ["@bob:example.com:DEVICE1", "Bob (@bob:example.com)"], - ["@bob:foo.bar:BOB000", "Bob (@bob:foo.bar)"], + // ["@local:example.com", "it's a me"], + ["@bob:example.com", "Bob (@bob:example.com)"], + ["@bob:foo.bar", "Bob (@bob:foo.bar)"], ]), b: new Map([ - // ["@local:example.com:DEVICE000", "it's a me"], - ["@bob:example.com:DEVICE1", "Bob (@bob:example.com)"], + // ["@local:example.com", "it's a me"], + ["@bob:example.com", "Bob (@bob:example.com)"], ]), }); }); @@ -273,14 +273,14 @@ test("should disambiguate on name change", () => { expectObservable(dn$.pipe(map((e) => e.value))).toBe("ab", { a: new Map([ - // ["@local:example.com:DEVICE000", "it's a me"], - ["@bob:example.com:B000", "Bob"], - ["@carl:example.com:C000", "Carl"], + // ["@local:example.com", "it's a me"], + ["@bob:example.com", "Bob"], + ["@carl:example.com", "Carl"], ]), b: new Map([ - // ["@local:example.com:DEVICE000", "it's a me"], - ["@bob:example.com:B000", "Bob (@bob:example.com)"], - ["@carl:example.com:C000", "Bob (@carl:example.com)"], + // ["@local:example.com", "it's a me"], + ["@bob:example.com", "Bob (@bob:example.com)"], + ["@carl:example.com", "Bob (@carl:example.com)"], ]), }); }); diff --git a/src/state/CallViewModel/remoteMembers/displayname.ts b/src/state/CallViewModel/remoteMembers/displayname.ts index c8484a9a..f56bc253 100644 --- a/src/state/CallViewModel/remoteMembers/displayname.ts +++ b/src/state/CallViewModel/remoteMembers/displayname.ts @@ -42,7 +42,7 @@ export function createRoomMembers$( * any displayname that clashes with another member. Only members * joined to the call are considered here. * - * @returns Map uses the rtc member idenitfier as the key. + * @returns Map uses the Matrix user ID as the key. */ // don't do this work more times than we need to. This is achieved by converting to a behavior: export const memberDisplaynames$ = ( @@ -66,19 +66,14 @@ export const memberDisplaynames$ = ( // We only consider RTC members for disambiguation as they are the only visible members. for (const rtcMember of memberships) { - // TODO a hard-coded participant ID ? should use rtcMember.membershipID instead? - const matrixIdentifier = `${rtcMember.userId}:${rtcMember.deviceId}`; - const { member } = getRoomMemberFromRtcMember(rtcMember, room); - if (!member) { - logger.error( - "Could not find member for participant id:", - matrixIdentifier, - ); + const member = room.getMember(rtcMember.userId); + if (member === null) { + logger.error(`Could not find member for user ${rtcMember.userId}`); continue; } const disambiguate = shouldDisambiguate(member, memberships, room); displaynameMap.set( - matrixIdentifier, + rtcMember.userId, calculateDisplayName(member, disambiguate), ); } @@ -87,13 +82,3 @@ export const memberDisplaynames$ = ( ), new Epoch(new Map()), ); - -export function getRoomMemberFromRtcMember( - rtcMember: CallMembership, - room: Pick, -): { id: string; member: RoomMember | undefined } { - return { - id: rtcMember.userId + ":" + rtcMember.deviceId, - member: room.getMember(rtcMember.userId) ?? undefined, - }; -} diff --git a/src/state/MediaViewModel.ts b/src/state/MediaViewModel.ts index b35f6112..74e64b93 100644 --- a/src/state/MediaViewModel.ts +++ b/src/state/MediaViewModel.ts @@ -27,7 +27,6 @@ import { RoomEvent as LivekitRoomEvent, RemoteTrack, } from "livekit-client"; -import { type RoomMember } from "matrix-js-sdk"; import { logger } from "matrix-js-sdk/lib/logger"; import { BehaviorSubject, @@ -44,6 +43,7 @@ import { startWith, switchMap, throttleTime, + distinctUntilChanged, } from "rxjs"; import { alwaysShowSelf } from "../settings/settings"; @@ -180,29 +180,35 @@ function observeRemoteTrackReceivingOkay$( } function encryptionErrorObservable$( - room: LivekitRoom, + room$: Behavior, participant: Participant, encryptionSystem: EncryptionSystem, criteria: string, ): Observable { - return roomEventSelector(room, LivekitRoomEvent.EncryptionError).pipe( - map((e) => { - const [err] = e; - if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) { - return ( - // Ideally we would pull the participant identity from the field on the error. - // However, it gets lost in the serialization process between workers. - // So, instead we do a string match - (err?.message.includes(participant.identity) && - err?.message.includes(criteria)) ?? - false - ); - } else if (encryptionSystem.kind === E2eeType.SHARED_KEY) { - return !!err?.message.includes(criteria); - } + return room$.pipe( + switchMap((room) => { + if (room === undefined) return of(false); + return roomEventSelector(room, LivekitRoomEvent.EncryptionError).pipe( + map((e) => { + const [err] = e; + if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) { + return ( + // Ideally we would pull the participant identity from the field on the error. + // However, it gets lost in the serialization process between workers. + // So, instead we do a string match + (err?.message.includes(participant.identity) && + err?.message.includes(criteria)) ?? + false + ); + } else if (encryptionSystem.kind === E2eeType.SHARED_KEY) { + return !!err?.message.includes(criteria); + } - return false; + return false; + }), + ); }), + distinctUntilChanged(), throttleTime(1000), // Throttle to avoid spamming the UI startWith(false), ); @@ -250,11 +256,9 @@ abstract class BaseMediaViewModel { */ public readonly id: string, /** - * The Matrix room member to which this media belongs. + * The Matrix user to which this media belongs. */ - // TODO: Fully separate the data layer from the UI layer by keeping the - // member object internal - public readonly member: RoomMember, + public readonly userId: string, // We don't necessarily have a participant if a user connects via MatrixRTC but not (yet) through // livekit. protected readonly participant$: Observable< @@ -264,9 +268,10 @@ abstract class BaseMediaViewModel { encryptionSystem: EncryptionSystem, audioSource: AudioSource, videoSource: VideoSource, - livekitRoom: LivekitRoom, - public readonly focusURL: string, + livekitRoom$: Behavior, + public readonly focusUrl$: Behavior, public readonly displayName$: Behavior, + public readonly mxcAvatarUrl$: Behavior, ) { const audio$ = this.observeTrackReference$(audioSource); this.video$ = this.observeTrackReference$(videoSource); @@ -294,13 +299,13 @@ abstract class BaseMediaViewModel { } else if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) { return combineLatest([ encryptionErrorObservable$( - livekitRoom, + livekitRoom$, participant, encryptionSystem, "MissingKey", ), encryptionErrorObservable$( - livekitRoom, + livekitRoom$, participant, encryptionSystem, "InvalidKey", @@ -320,7 +325,7 @@ abstract class BaseMediaViewModel { } else { return combineLatest([ encryptionErrorObservable$( - livekitRoom, + livekitRoom$, participant, encryptionSystem, "InvalidKey", @@ -402,26 +407,28 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { public constructor( scope: ObservableScope, id: string, - member: RoomMember, + userId: string, participant$: Observable, encryptionSystem: EncryptionSystem, - livekitRoom: LivekitRoom, - focusUrl: string, + livekitRoom$: Behavior, + focusUrl$: Behavior, displayName$: Behavior, + mxcAvatarUrl$: Behavior, public readonly handRaised$: Behavior, public readonly reaction$: Behavior, ) { super( scope, id, - member, + userId, participant$, encryptionSystem, Track.Source.Microphone, Track.Source.Camera, - livekitRoom, - focusUrl, + livekitRoom$, + focusUrl$, displayName$, + mxcAvatarUrl$, ); const media$ = this.scope.behavior( @@ -538,25 +545,27 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { public constructor( scope: ObservableScope, id: string, - member: RoomMember, + userId: string, participant$: Behavior, encryptionSystem: EncryptionSystem, - livekitRoom: LivekitRoom, - focusURL: string, + livekitRoom$: Behavior, + focusUrl$: Behavior, private readonly mediaDevices: MediaDevices, displayName$: Behavior, + mxcAvatarUrl$: Behavior, handRaised$: Behavior, reaction$: Behavior, ) { super( scope, id, - member, + userId, participant$, encryptionSystem, - livekitRoom, - focusURL, + livekitRoom$, + focusUrl$, displayName$, + mxcAvatarUrl$, handRaised$, reaction$, ); @@ -648,25 +657,27 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel { public constructor( scope: ObservableScope, id: string, - member: RoomMember, + userId: string, participant$: Observable, encryptionSystem: EncryptionSystem, - livekitRoom: LivekitRoom, - focusUrl: string, + livekitRoom$: Behavior, + focusUrl$: Behavior, private readonly pretendToBeDisconnected$: Behavior, - displayname$: Behavior, + displayName$: Behavior, + mxcAvatarUrl$: Behavior, handRaised$: Behavior, reaction$: Behavior, ) { super( scope, id, - member, + userId, participant$, encryptionSystem, - livekitRoom, - focusUrl, - displayname$, + livekitRoom$, + focusUrl$, + displayName$, + mxcAvatarUrl$, handRaised$, reaction$, ); @@ -747,26 +758,28 @@ export class ScreenShareViewModel extends BaseMediaViewModel { public constructor( scope: ObservableScope, id: string, - member: RoomMember, + userId: string, participant$: Observable, encryptionSystem: EncryptionSystem, - livekitRoom: LivekitRoom, - focusUrl: string, + livekitRoom$: Behavior, + focusUrl$: Behavior, private readonly pretendToBeDisconnected$: Behavior, - displayname$: Behavior, + displayName$: Behavior, + mxcAvatarUrl$: Behavior, public readonly local: boolean, ) { super( scope, id, - member, + userId, participant$, encryptionSystem, Track.Source.ScreenShareAudio, Track.Source.ScreenShare, - livekitRoom, - focusUrl, - displayname$, + livekitRoom$, + focusUrl$, + displayName$, + mxcAvatarUrl$, ); } } diff --git a/src/state/ObservableScope.ts b/src/state/ObservableScope.ts index d1d6c297..ae46a242 100644 --- a/src/state/ObservableScope.ts +++ b/src/state/ObservableScope.ts @@ -24,7 +24,11 @@ import { type Behavior } from "./Behavior"; type MonoTypeOperator = (o: Observable) => Observable; -export const noInitialValue = Symbol("nothing"); +type SplitBehavior = keyof T extends string | number + ? { [K in keyof T as `${K}$`]: Behavior } + : never; + +const nothing = Symbol("nothing"); /** * A scope which limits the execution lifetime of its bound Observables. @@ -59,7 +63,10 @@ export class ObservableScope { * Converts an Observable to a Behavior. If no initial value is specified, the * Observable must synchronously emit an initial value. */ - public behavior(setValue$: Observable, initialValue?: T): Behavior { + public behavior( + setValue$: Observable, + initialValue: T | typeof nothing = nothing, + ): Behavior { const subject$ = new BehaviorSubject(initialValue); // Push values from the Observable into the BehaviorSubject. // BehaviorSubjects have an undesirable feature where if you call 'complete', @@ -74,7 +81,7 @@ export class ObservableScope { subject$.error(err); }, }); - if (subject$.value === noInitialValue) + if (subject$.value === nothing) throw new Error("Behavior failed to synchronously emit an initial value"); return subject$ as Behavior; } @@ -115,27 +122,27 @@ export class ObservableScope { value$: Behavior, callback: (value: T) => Promise<(() => Promise) | void>, ): void { - let latestValue: T | typeof noInitialValue = noInitialValue; - let reconciledValue: T | typeof noInitialValue = noInitialValue; + let latestValue: T | typeof nothing = nothing; + let reconciledValue: T | typeof nothing = nothing; let cleanUp: (() => Promise) | void = undefined; value$ .pipe( catchError(() => EMPTY), // Ignore errors this.bind(), // Limit to the duration of the scope - endWith(noInitialValue), // Clean up when the scope ends + endWith(nothing), // Clean up when the scope ends ) .subscribe((value) => { void (async (): Promise => { - if (latestValue === noInitialValue) { + if (latestValue === nothing) { latestValue = value; while (latestValue !== reconciledValue) { await cleanUp?.(); // Call the previous value's clean-up handler reconciledValue = latestValue; - if (latestValue !== noInitialValue) + if (latestValue !== nothing) cleanUp = await callback(latestValue); // Sync current value } // Reset to signal that reconciliation is done for now - latestValue = noInitialValue; + latestValue = nothing; } else { // There's already an instance of the above 'while' loop running // concurrently. Just update the latest value and let it be handled. @@ -144,6 +151,24 @@ export class ObservableScope { })(); }); } + + /** + * Splits a Behavior of objects with static properties into an object with + * Behavior properties. + * + * For example, splitting a Behavior<{ name: string, age: number }> results in + * an object of type { name$: Behavior age$: Behavior }. + */ + public splitBehavior( + input$: Behavior, + ): SplitBehavior { + return Object.fromEntries( + Object.keys(input$.value).map((key) => [ + `${key}$`, + this.behavior(input$.pipe(map((input) => input[key as keyof T]))), + ]), + ) as SplitBehavior; + } } /** diff --git a/src/state/ScreenShare.ts b/src/state/ScreenShare.ts index 9803a5f4..0a241cdf 100644 --- a/src/state/ScreenShare.ts +++ b/src/state/ScreenShare.ts @@ -4,7 +4,7 @@ Copyright 2025 New Vector Ltd. SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { of, type Observable } from "rxjs"; +import { of } from "rxjs"; import { type LocalParticipant, type RemoteParticipant, @@ -13,7 +13,6 @@ import { import { type ObservableScope } from "./ObservableScope.ts"; import { ScreenShareViewModel } from "./MediaViewModel.ts"; -import type { RoomMember } from "matrix-js-sdk"; import type { EncryptionSystem } from "../e2ee/sharedKeyManagement.ts"; import type { Behavior } from "./Behavior.ts"; @@ -28,24 +27,26 @@ export class ScreenShare { public constructor( private readonly scope: ObservableScope, id: string, - member: RoomMember, + userId: string, participant: LocalParticipant | RemoteParticipant, encryptionSystem: EncryptionSystem, - livekitRoom: LivekitRoom, - focusUrl: string, + livekitRoom$: Behavior, + focusUrl$: Behavior, pretendToBeDisconnected$: Behavior, - displayName$: Observable, + displayName$: Behavior, + mxcAvatarUrl$: Behavior, ) { this.vm = new ScreenShareViewModel( this.scope, id, - member, + userId, of(participant), encryptionSystem, - livekitRoom, - focusUrl, + livekitRoom$, + focusUrl$, pretendToBeDisconnected$, - this.scope.behavior(displayName$), + displayName$, + mxcAvatarUrl$, participant.isLocal, ); } diff --git a/src/state/TileStore.ts b/src/state/TileStore.ts index f6db0930..7b95bd8e 100644 --- a/src/state/TileStore.ts +++ b/src/state/TileStore.ts @@ -14,7 +14,7 @@ import { fillGaps } from "../utils/iter"; import { debugTileLayout } from "../settings/settings"; function debugEntries(entries: GridTileData[]): string[] { - return entries.map((e) => e.media.member?.rawDisplayName ?? "[👻]"); + return entries.map((e) => e.media.displayName$.value); } let DEBUG_ENABLED = false; @@ -156,7 +156,7 @@ export class TileStoreBuilder { public registerSpotlight(media: MediaViewModel[], maximised: boolean): void { if (DEBUG_ENABLED) logger.debug( - `[TileStore, ${this.generation}] register spotlight: ${media.map((m) => m.member?.rawDisplayName ?? "[👻]")}`, + `[TileStore, ${this.generation}] register spotlight: ${media.map((m) => m.displayName$.value)}`, ); if (this.spotlight !== null) throw new Error("Spotlight already set"); @@ -180,7 +180,7 @@ export class TileStoreBuilder { public registerGridTile(media: UserMediaViewModel): void { if (DEBUG_ENABLED) logger.debug( - `[TileStore, ${this.generation}] register grid tile: ${media.member?.rawDisplayName ?? "[👻]"}`, + `[TileStore, ${this.generation}] register grid tile: ${media.displayName$.value}`, ); if (this.spotlight !== null) { @@ -263,7 +263,7 @@ export class TileStoreBuilder { public registerPipTile(media: UserMediaViewModel): void { if (DEBUG_ENABLED) logger.debug( - `[TileStore, ${this.generation}] register PiP tile: ${media.member?.rawDisplayName ?? "[👻]"}`, + `[TileStore, ${this.generation}] register PiP tile: ${media.displayName$.value}`, ); // If there is a single grid tile that we can reuse diff --git a/src/state/UserMedia.ts b/src/state/UserMedia.ts index 9eec3967..38f22122 100644 --- a/src/state/UserMedia.ts +++ b/src/state/UserMedia.ts @@ -5,17 +5,9 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { - BehaviorSubject, - combineLatest, - map, - type Observable, - of, - switchMap, -} from "rxjs"; +import { combineLatest, map, type Observable, of, switchMap } from "rxjs"; import { type LocalParticipant, - type Participant, ParticipantEvent, type RemoteParticipant, type Room as LivekitRoom, @@ -29,11 +21,12 @@ import { type UserMediaViewModel, } from "./MediaViewModel.ts"; import type { Behavior } from "./Behavior.ts"; -import type { RoomMember } from "matrix-js-sdk"; import type { EncryptionSystem } from "../e2ee/sharedKeyManagement.ts"; import type { MediaDevices } from "./MediaDevices.ts"; import type { ReactionOption } from "../reactions"; import { observeSpeaker$ } from "./observeSpeaker.ts"; +import { generateItems } from "../utils/observable.ts"; +import { ScreenShare } from "./ScreenShare.ts"; /** * Sorting bins defining the order in which media tiles appear in the layout. @@ -72,35 +65,35 @@ enum SortingBin { /** * A user media item to be presented in a tile. This is a thin wrapper around * UserMediaViewModel which additionally determines the media item's sorting bin - * for inclusion in the call layout. + * for inclusion in the call layout and tracks associated screen shares. */ export class UserMedia { - private readonly participant$ = new BehaviorSubject(this.initialParticipant); - public readonly vm: UserMediaViewModel = this.participant$.value?.isLocal ? new LocalUserMediaViewModel( this.scope, this.id, - this.member, + this.userId, this.participant$ as Behavior, this.encryptionSystem, - this.livekitRoom, - this.focusURL, + this.livekitRoom$, + this.focusUrl$, this.mediaDevices, - this.scope.behavior(this.displayname$), + this.displayName$, + this.mxcAvatarUrl$, this.scope.behavior(this.handRaised$), this.scope.behavior(this.reaction$), ) : new RemoteUserMediaViewModel( this.scope, this.id, - this.member, + this.userId, this.participant$ as Behavior, this.encryptionSystem, - this.livekitRoom, - this.focusURL, + this.livekitRoom$, + this.focusUrl$, this.pretendToBeDisconnected$, - this.scope.behavior(this.displayname$), + this.displayName$, + this.mxcAvatarUrl$, this.scope.behavior(this.handRaised$), this.scope.behavior(this.reaction$), ); @@ -109,12 +102,55 @@ export class UserMedia { observeSpeaker$(this.vm.speaking$), ); - private readonly presenter$ = this.scope.behavior( + /** + * All screen share media associated with this user media. + */ + public readonly screenShares$ = this.scope.behavior( this.participant$.pipe( - switchMap((p) => (p === null ? of(false) : sharingScreen$(p))), + switchMap((p) => + p === null + ? of([]) + : observeParticipantEvents( + p, + ParticipantEvent.TrackPublished, + ParticipantEvent.TrackUnpublished, + ParticipantEvent.LocalTrackPublished, + ParticipantEvent.LocalTrackUnpublished, + ).pipe( + // Technically more than one screen share might be possible... our + // MediaViewModels don't support it though since they look for a unique + // track for the given source. So generateItems here is a bit overkill. + generateItems( + function* (p) { + if (p.isScreenShareEnabled) + yield { + keys: ["screen-share"], + data: undefined, + }; + }, + (scope, _data$, key) => + new ScreenShare( + scope, + `${this.id}:${key}`, + this.userId, + p, + this.encryptionSystem, + this.livekitRoom$, + this.focusUrl$, + this.pretendToBeDisconnected$, + this.displayName$, + this.mxcAvatarUrl$, + ), + ), + ), + ), ), ); + private readonly presenter$ = this.scope.behavior( + this.screenShares$.pipe(map((screenShares) => screenShares.length > 0)), + ); + /** * Which sorting bin the media item should be placed in. */ @@ -147,37 +183,18 @@ export class UserMedia { public constructor( private readonly scope: ObservableScope, public readonly id: string, - private readonly member: RoomMember, - private readonly initialParticipant: - | LocalParticipant - | RemoteParticipant - | null = null, + private readonly userId: string, + private readonly participant$: Behavior< + LocalParticipant | RemoteParticipant | null + >, private readonly encryptionSystem: EncryptionSystem, - private readonly livekitRoom: LivekitRoom, - private readonly focusURL: string, + private readonly livekitRoom$: Behavior, + private readonly focusUrl$: Behavior, private readonly mediaDevices: MediaDevices, private readonly pretendToBeDisconnected$: Behavior, - private readonly displayname$: Observable, + private readonly displayName$: Behavior, + private readonly mxcAvatarUrl$: Behavior, private readonly handRaised$: Observable, private readonly reaction$: Observable, ) {} - - public updateParticipant( - newParticipant: LocalParticipant | RemoteParticipant | null = null, - ): void { - if (this.participant$.value !== newParticipant) { - // Update the BehaviourSubject in the UserMedia. - this.participant$.next(newParticipant); - } - } -} - -export function sharingScreen$(p: Participant): Observable { - return observeParticipantEvents( - p, - ParticipantEvent.TrackPublished, - ParticipantEvent.TrackUnpublished, - ParticipantEvent.LocalTrackPublished, - ParticipantEvent.LocalTrackUnpublished, - ).pipe(map((p) => p.isScreenShareEnabled)); } diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index 1925eff6..57409869 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -58,7 +58,9 @@ interface TileProps { style?: ComponentProps["style"]; targetWidth: number; targetHeight: number; + focusUrl: string | undefined; displayName: string; + mxcAvatarUrl: string | undefined; showSpeakingIndicators: boolean; focusable: boolean; } @@ -81,7 +83,9 @@ const UserMediaTile: FC = ({ menuStart, menuEnd, className, + focusUrl, displayName, + mxcAvatarUrl, focusable, ...props }) => { @@ -145,7 +149,7 @@ const UserMediaTile: FC = ({ = ({ /> } displayName={displayName} + mxcAvatarUrl={mxcAvatarUrl} focusable={focusable} primaryButton={ primaryButton ?? ( @@ -190,7 +195,7 @@ const UserMediaTile: FC = ({ currentReaction={reaction ?? undefined} raisedHandOnClick={raisedHandOnClick} localParticipant={vm.local} - focusUrl={vm.focusURL} + focusUrl={focusUrl} audioStreamStats={audioStreamStats} videoStreamStats={videoStreamStats} {...props} @@ -359,7 +364,9 @@ export const GridTile: FC = ({ const ourRef = useRef(null); const ref = useMergedRefs(ourRef, theirRef); const media = useBehavior(vm.media$); + const focusUrl = useBehavior(media.focusUrl$); const displayName = useBehavior(media.displayName$); + const mxcAvatarUrl = useBehavior(media.mxcAvatarUrl$); if (media instanceof LocalUserMediaViewModel) { return ( @@ -367,7 +374,9 @@ export const GridTile: FC = ({ ref={ref} vm={media} onOpenProfile={onOpenProfile} + focusUrl={focusUrl} displayName={displayName} + mxcAvatarUrl={mxcAvatarUrl} {...props} /> ); @@ -376,7 +385,9 @@ export const GridTile: FC = ({ ); diff --git a/src/tile/MediaView.tsx b/src/tile/MediaView.tsx index 149b4177..e8a30cd4 100644 --- a/src/tile/MediaView.tsx +++ b/src/tile/MediaView.tsx @@ -7,7 +7,6 @@ Please see LICENSE in the repository root for full details. import { type TrackReferenceOrPlaceholder } from "@livekit/components-core"; import { animated } from "@react-spring/web"; -import { type RoomMember } from "matrix-js-sdk"; import { type FC, type ComponentProps, type ReactNode } from "react"; import { useTranslation } from "react-i18next"; import classNames from "classnames"; @@ -32,12 +31,13 @@ interface Props extends ComponentProps { video: TrackReferenceOrPlaceholder | undefined; videoFit: "cover" | "contain"; mirror: boolean; - member: RoomMember; + userId: string; videoEnabled: boolean; unencryptedWarning: boolean; encryptionStatus: EncryptionStatus; nameTagLeadingIcon?: ReactNode; displayName: string; + mxcAvatarUrl: string | undefined; focusable: boolean; primaryButton?: ReactNode; raisedHandTime?: Date; @@ -59,11 +59,12 @@ export const MediaView: FC = ({ video, videoFit, mirror, - member, + userId, videoEnabled, unencryptedWarning, nameTagLeadingIcon, displayName, + mxcAvatarUrl, focusable, primaryButton, encryptionStatus, @@ -94,10 +95,10 @@ export const MediaView: FC = ({ > -
- - - - - - - 2 - -
-
- mocked: MatrixAudioRenderer -
-
-
-
-
+ class="scrolling" + style="--gap: 20px; --width: NaNpx; --height: NaNpx;" + />
-
+
as MatrixClient, - getMember: (userId) => roomMembers.get(userId) ?? null, + getMembers: () => Array.from(roomMembers.values()), }); const rtcSession = new MockRTCSession(room, []).withMemberships(rtcMembers$); const participantsSpy = vi diff --git a/src/state/CallViewModel/remoteMembers/Connection.test.ts b/src/state/CallViewModel/remoteMembers/Connection.test.ts index eb9ae89d..ec7906aa 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.test.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.test.ts @@ -367,21 +367,18 @@ describe("Start connection states", () => { }); }); -function fakeRemoteLivekitParticipant(id: string): RemoteParticipant { +function fakeRemoteLivekitParticipant( + id: string, + publications: number = 1, +): RemoteParticipant { return { identity: id, + getTrackPublications: () => Array(publications), } as unknown as RemoteParticipant; } -function fakeRtcMemberShip(userId: string, deviceId: string): CallMembership { - return { - userId, - deviceId, - } as unknown as CallMembership; -} - describe("Publishing participants observations", () => { - it("should emit the list of publishing participants", async () => { + it("should emit the list of publishing participants", () => { setupTest(); const connection = setupRemoteConnection(); @@ -406,107 +403,36 @@ describe("Publishing participants observations", () => { // on this connection. let participants: RemoteParticipant[] = [ - fakeRemoteLivekitParticipant("@alice:example.org:DEV000"), - fakeRemoteLivekitParticipant("@bob:example.org:DEV111"), - fakeRemoteLivekitParticipant("@carol:example.org:DEV222"), - fakeRemoteLivekitParticipant("@dan:example.org:DEV333"), + fakeRemoteLivekitParticipant("@alice:example.org:DEV000", 0), + fakeRemoteLivekitParticipant("@bob:example.org:DEV111", 0), + fakeRemoteLivekitParticipant("@carol:example.org:DEV222", 0), + fakeRemoteLivekitParticipant("@dan:example.org:DEV333", 0), ]; // Let's simulate 3 members on the livekitRoom - vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockReturnValue( - new Map(participants.map((p) => [p.identity, p])), + vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockImplementation( + () => new Map(participants.map((p) => [p.identity, p])), ); - for (const participant of participants) { - fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, participant); - } + participants.forEach((p) => + fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, p), + ); // At this point there should be no publishers expect(observedPublishers.pop()!.length).toEqual(0); - const otherFocus: LivekitTransport = { - livekit_alias: "!roomID:example.org", - livekit_service_url: "https://other-matrix-rtc.example.org/livekit/jwt", - type: "livekit", - }; - - const rtcMemberships = [ - // Say bob is on the same focus - { - membership: fakeRtcMemberShip("@bob:example.org", "DEV111"), - transport: livekitFocus, - }, - // Alice and carol is on a different focus - { - membership: fakeRtcMemberShip("@alice:example.org", "DEV000"), - transport: otherFocus, - }, - { - membership: fakeRtcMemberShip("@carol:example.org", "DEV222"), - transport: otherFocus, - }, - // NO DAVE YET + participants = [ + fakeRemoteLivekitParticipant("@alice:example.org:DEV000", 1), + fakeRemoteLivekitParticipant("@bob:example.org:DEV111", 1), + fakeRemoteLivekitParticipant("@carol:example.org:DEV222", 1), + fakeRemoteLivekitParticipant("@dan:example.org:DEV333", 2), ]; - // signal this change in rtc memberships - fakeMembershipsFocusMap$.next(rtcMemberships); - - // We should have bob has a publisher now - await bobIsAPublisher.promise; - const publishers = observedPublishers.pop(); - expect(publishers?.length).toEqual(1); - expect(publishers?.[0].identity).toEqual("@bob:example.org:DEV111"); - - // Now let's make dan join the rtc memberships - rtcMemberships.push({ - membership: fakeRtcMemberShip("@dan:example.org", "DEV333"), - transport: livekitFocus, - }); - fakeMembershipsFocusMap$.next(rtcMemberships); - - // We should have bob and dan has publishers now - await danIsAPublisher.promise; - const twoPublishers = observedPublishers.pop(); - expect(twoPublishers?.length).toEqual(2); - expect( - twoPublishers?.some((p) => p.identity === "@bob:example.org:DEV111"), - ).toBeTruthy(); - expect( - twoPublishers?.some((p) => p.identity === "@dan:example.org:DEV333"), - ).toBeTruthy(); - - // Now let's make bob leave the livekit room - participants = participants.filter( - (p) => p.identity !== "@bob:example.org:DEV111", - ); - vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockReturnValue( - new Map(participants.map((p) => [p.identity, p])), - ); - fakeRoomEventEmiter.emit( - RoomEvent.ParticipantDisconnected, - fakeRemoteLivekitParticipant("@bob:example.org:DEV111"), + participants.forEach((p) => + fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, p), ); - // TODO: evaluate this test. It looks like this is not the task of the Connection anymore. Valere? - // const updatedPublishers = observedPublishers.pop(); - // // Bob is not connected to the room but he is still in the rtc memberships declaring that - // // he is using that focus to publish, so he should still appear as a publisher - // expect(updatedPublishers?.length).toEqual(2); - // const pp = updatedPublishers?.find((p) => - // p.identity.startsWith("@bob:example.org"), - // ); - // expect(pp).toBeDefined(); - // expect(pp!).not.toBeDefined(); - // expect( - // updatedPublishers?.some( - // (p) => p.participant?.identity === "@dan:example.org:DEV333", - // ), - // ).toBeTruthy(); - // // Now if bob is not in the rtc memberships, he should disappear - // const noBob = rtcMemberships.filter( - // ({ membership }) => membership.userId !== "@bob:example.org", - // ); - // fakeMembershipsFocusMap$.next(noBob); - // expect(observedPublishers.pop()?.length).toEqual(1); + // At this point there should be no publishers + expect(observedPublishers.pop()!.length).toEqual(4); }); it("should be scoped to parent scope", (): void => { @@ -523,12 +449,12 @@ describe("Publishing participants observations", () => { onTestFinished(() => s.unsubscribe()); let participants: RemoteParticipant[] = [ - fakeRemoteLivekitParticipant("@bob:example.org:DEV111"), + fakeRemoteLivekitParticipant("@bob:example.org:DEV111", 0), ]; // Let's simulate 3 members on the livekitRoom - vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockReturnValue( - new Map(participants.map((p) => [p.identity, p])), + vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockImplementation( + () => new Map(participants.map((p) => [p.identity, p])), ); for (const participant of participants) { @@ -538,15 +464,11 @@ describe("Publishing participants observations", () => { // At this point there should be no publishers expect(observedPublishers.pop()!.length).toEqual(0); - const rtcMemberships = [ - // Say bob is on the same focus - { - membership: fakeRtcMemberShip("@bob:example.org", "DEV111"), - transport: livekitFocus, - }, - ]; - // signal this change in rtc memberships - fakeMembershipsFocusMap$.next(rtcMemberships); + participants = [fakeRemoteLivekitParticipant("@bob:example.org:DEV111", 1)]; + + for (const participant of participants) { + fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, participant); + } // We should have bob has a publisher now const publishers = observedPublishers.pop(); @@ -561,9 +483,7 @@ describe("Publishing participants observations", () => { participants = participants.filter( (p) => p.identity !== "@bob:example.org:DEV111", ); - vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockReturnValue( - new Map(participants.map((p) => [p.identity, p])), - ); + fakeRoomEventEmiter.emit( RoomEvent.ParticipantDisconnected, fakeRemoteLivekitParticipant("@bob:example.org:DEV111"), diff --git a/src/state/CallViewModel/remoteMembers/Connection.ts b/src/state/CallViewModel/remoteMembers/Connection.ts index fa66183a..454ae9fe 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.ts @@ -223,9 +223,10 @@ export class Connection { ], }).pipe( map((participants) => { - return participants.filter( + const partsFiltered = participants.filter( (participant) => participant.getTrackPublications().length > 0, ); + return partsFiltered; }), ), [], diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts index 17d127f3..97b6c2fd 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts @@ -51,7 +51,7 @@ beforeEach(() => { (transport: LivekitTransport, scope: ObservableScope) => { const mockConnection = { transport, - participantsWithTrack$: new BehaviorSubject([]), + remoteParticipantsWithTracks$: new BehaviorSubject([]), } as unknown as Connection; vi.mocked(mockConnection).start = vi.fn(); vi.mocked(mockConnection).stop = vi.fn(); @@ -190,7 +190,7 @@ describe("connections$ stream", () => { }); describe("connectionManagerData$ stream", () => { - // Used in test to control fake connections' participantsWithTrack$ streams + // Used in test to control fake connections' remoteParticipantsWithTracks$ streams let fakePublishingParticipantsStreams: Map< string, Behavior @@ -222,7 +222,8 @@ describe("connectionManagerData$ stream", () => { >([]); const mockConnection = { transport, - participantsWithTrack$: getPublishingParticipantsFor(transport), + remoteParticipantsWithTracks$: + getPublishingParticipantsFor(transport), } as unknown as Connection; vi.mocked(mockConnection).start = vi.fn(); vi.mocked(mockConnection).stop = vi.fn(); diff --git a/src/state/ObservableScope.test.ts b/src/state/ObservableScope.test.ts index b41f47c2..36f6f308 100644 --- a/src/state/ObservableScope.test.ts +++ b/src/state/ObservableScope.test.ts @@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details. */ import { describe, expect, it } from "vitest"; -import { BehaviorSubject, combineLatest, timer } from "rxjs"; +import { BehaviorSubject, combineLatest, of, Subject } from "rxjs"; import { logger } from "matrix-js-sdk/lib/logger"; import { @@ -60,19 +60,6 @@ describe("Epoch", () => { }); }); }); - it("obs", () => { - const nothing = Symbol("nothing"); - const scope = new ObservableScope(); - const sb$ = new BehaviorSubject("initial"); - const su$ = new BehaviorSubject(undefined); - expect(sb$.value).toBe("initial"); - expect(su$.value).toBe(undefined); - expect(su$.value === nothing).toBe(false); - - const a$ = timer(10); - - scope.behavior(a$, undefined); - }); it("diamonds emits in a predictable order", () => { const sb$ = new BehaviorSubject("initial"); @@ -97,4 +84,21 @@ describe("Epoch", () => { sb$.next("updated"); sb$.next("ANOTERUPDATE"); }); + + it("behavior test", () => { + const scope = new ObservableScope(); + const s$ = new Subject(); + const behavior$ = scope.behavior(s$, 0); + behavior$.subscribe((value) => { + logger.log(`Received value: ${value}`); + }); + s$.next(1); + s$.next(2); + s$.next(3); + s$.next(3); + s$.next(3); + s$.next(3); + s$.next(3); + s$.complete(); + }); }); diff --git a/src/state/ObservableScope.ts b/src/state/ObservableScope.ts index 27f501c7..5372246c 100644 --- a/src/state/ObservableScope.ts +++ b/src/state/ObservableScope.ts @@ -21,6 +21,7 @@ import { } from "rxjs"; import { type Behavior } from "./Behavior"; +import { logger } from "matrix-js-sdk/lib/logger"; type MonoTypeOperator = (o: Observable) => Observable; @@ -73,14 +74,22 @@ export class ObservableScope { // they will no longer re-emit their current value upon subscription. We want // to support Observables that complete (for example `of({})`), so we have to // take care to not propagate the completion event. - setValue$.pipe(this.bind(), distinctUntilChanged()).subscribe({ - next(value) { - subject$.next(value); - }, - error(err: unknown) { - subject$.error(err); - }, - }); + setValue$ + .pipe( + this.bind(), + distinctUntilChanged((a, b) => { + logger.log("distinctUntilChanged", a, b); + return a === b; + }), + ) + .subscribe({ + next(value) { + subject$.next(value); + }, + error(err: unknown) { + subject$.error(err); + }, + }); if (subject$.value === nothing) throw new Error("Behavior failed to synchronously emit an initial value"); return subject$ as Behavior; diff --git a/src/utils/test-viewmodel.ts b/src/utils/test-viewmodel.ts index 0e6c589a..314b6aa9 100644 --- a/src/utils/test-viewmodel.ts +++ b/src/utils/test-viewmodel.ts @@ -24,6 +24,7 @@ import { type CallViewModelOptions, } from "../state/CallViewModel/CallViewModel"; import { + mockConfig, mockLivekitRoom, mockLocalParticipant, mockMatrixRoom, @@ -36,6 +37,8 @@ import { aliceRtcMember, localRtcMember } from "./test-fixtures"; import { type RaisedHandInfo, type ReactionInfo } from "../reactions"; import { constant } from "../state/Behavior"; +mockConfig({ livekit: { livekit_service_url: "https://example.com" } }); + export function getBasicRTCSession( members: RoomMember[], initialRtcMemberships: CallMembership[] = [localRtcMember, aliceRtcMember], @@ -57,6 +60,7 @@ export function getBasicRTCSession( getUserId: () => localRtcMember.userId, getDeviceId: () => localRtcMember.deviceId, getSyncState: () => SyncState.Syncing, + getDomain: () => null, sendEvent: vitest.fn().mockResolvedValue({ event_id: "$fake:event" }), redactEvent: vitest.fn().mockResolvedValue({ event_id: "$fake:event" }), decryptEventIfNeeded: vitest.fn().mockResolvedValue(undefined), @@ -78,6 +82,7 @@ export function getBasicRTCSession( ), } as Partial as MatrixClient, getMember: (userId) => matrixRoomMembers.get(userId) ?? null, + getMembers: () => Array.from(matrixRoomMembers.values()), roomId: matrixRoomId, on: vitest .fn() From a3950869fb010dfa00c09c8d7bc77a1349928635 Mon Sep 17 00:00:00 2001 From: Ben Banfield-Zanin Date: Thu, 13 Nov 2025 15:33:59 +0000 Subject: [PATCH 49/75] Use the dedicated bug report sub-domain (#3557) * Update docs for 'rageshake.submit_url' to use the dedicated subdomain * Update Netlify preview deployment to use the dedicated subdomain for 'rageshake.submit_url' --- config/config_netlify_preview.json | 2 +- docs/url-params.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config_netlify_preview.json b/config/config_netlify_preview.json index cf0148e9..313f0d02 100644 --- a/config/config_netlify_preview.json +++ b/config/config_netlify_preview.json @@ -18,7 +18,7 @@ "api_host": "https://posthog-element-call.element.io" }, "rageshake": { - "submit_url": "https://element.io/bugreports/submit" + "submit_url": "https://rageshakes.element.io/api/submit" }, "sentry": { "environment": "netlify-pr-preview", diff --git a/docs/url-params.md b/docs/url-params.md index b2af8416..a474daed 100644 --- a/docs/url-params.md +++ b/docs/url-params.md @@ -96,6 +96,6 @@ These parameters are only supported in the [embedded](./embedded-standalone.md) | -------------------- | -------------------------------------------------------------------------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------- | | `posthogApiHost` | Posthog server URL | No | e.g. `https://posthog-element-call.element.io`. Only supported in embedded package. In full package the value from config is used. | | `posthogApiKey` | Posthog project API key | No | Only supported in embedded package. In full package the value from config is used. | -| `rageshakeSubmitUrl` | Rageshake server URL endpoint | No | e.g. `https://element.io/bugreports/submit`. In full package the value from config is used. | +| `rageshakeSubmitUrl` | Rageshake server URL endpoint | No | e.g. `https://rageshakes.element.io/api/submit`. In full package the value from config is used. | | `sentryDsn` | Sentry [DSN](https://docs.sentry.io/concepts/key-terms/dsn-explainer/) | No | In full package the value from config is used. | | `sentryEnvironment` | Sentry [environment](https://docs.sentry.io/concepts/key-terms/key-terms/) | No | In full package the value from config is used. | From fdce3ec1aacb221dd87c68be8184a1fbee395342 Mon Sep 17 00:00:00 2001 From: Timo K Date: Fri, 14 Nov 2025 10:44:16 +0100 Subject: [PATCH 50/75] Fix a couple of CallViewModel tests. --- src/state/CallViewModel/CallViewModel.test.ts | 2634 +++++++++-------- src/state/CallViewModel/CallViewModel.ts | 95 +- .../localMember/LocalMembership.ts | 21 +- .../remoteMembers/ConnectionManager.ts | 2 +- .../remoteMembers/MatrixLivekitMembers.ts | 2 +- src/state/ObservableScope.ts | 24 +- src/state/SessionBehaviors.ts | 2 +- src/utils/test-fixtures.ts | 4 +- src/utils/test.ts | 20 +- 9 files changed, 1426 insertions(+), 1378 deletions(-) diff --git a/src/state/CallViewModel/CallViewModel.test.ts b/src/state/CallViewModel/CallViewModel.test.ts index ba976094..1cc2ff89 100644 --- a/src/state/CallViewModel/CallViewModel.test.ts +++ b/src/state/CallViewModel/CallViewModel.test.ts @@ -93,6 +93,9 @@ import { MatrixRTCTransportMissingError, } from "../../utils/errors.ts"; +mockConfig({ + livekit: { livekit_service_url: "http://my-default-service-url.com" }, +}); vi.mock("rxjs", async (importOriginal) => ({ ...(await importOriginal()), // Disable interval Observables for the following tests since the test @@ -384,1375 +387,1386 @@ export function withCallViewModel( continuation(vm, rtcSession, { raisedHands$: raisedHands$ }, setSyncState); } -// TODO: Restore this test. It requires makeTransport to not be mocked, unlike -// the rest of the tests in this file… what do we do? -test.skip("test missing RTC config error", async () => { - const rtcMemberships$ = new BehaviorSubject([]); - const emitter = new EventEmitter(); - const client = vi.mocked({ - on: emitter.on.bind(emitter), - off: emitter.off.bind(emitter), - getSyncState: vi.fn().mockReturnValue(SyncState.Syncing), - getUserId: vi.fn().mockReturnValue("@user:localhost"), - getUser: vi.fn().mockReturnValue(null), - getDeviceId: vi.fn().mockReturnValue("DEVICE"), - credentials: { - userId: "@user:localhost", - }, - getCrypto: vi.fn().mockReturnValue(undefined), - getDomain: vi.fn().mockReturnValue("example.org"), - } as unknown as MatrixClient); - - const matrixRoom = mockMatrixRoom({ - roomId: "!myRoomId:example.com", - client, - getMember: vi.fn().mockReturnValue(undefined), - }); - - const fakeRtcSession = new MockRTCSession(matrixRoom).withMemberships( - rtcMemberships$, - ); - - mockConfig({}); - vi.spyOn(AutoDiscovery, "getRawClientConfig").mockResolvedValue({}); - - const callVM = new CallViewModel( - testScope(), - fakeRtcSession.asMockedSession(), - matrixRoom, - mockMediaDevices({}), - mockMuteStates(), - { - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - autoLeaveWhenOthersLeft: false, - livekitRoomFactory: (): LivekitRoom => - mockLivekitRoom({ - localParticipant, - disconnect: async () => Promise.resolve(), - setE2EEEnabled: async () => Promise.resolve(), - }), - }, - new BehaviorSubject({} as Record), - new BehaviorSubject({} as Record), - constant({ processor: undefined, supported: false }), - ); - - const failPromise = Promise.withResolvers(); - callVM.configError$.subscribe((error) => { - if (error) { - failPromise.resolve(error); - } - }); - - const error = await failPromise.promise; - expect(error).toBeInstanceOf(MatrixRTCTransportMissingError); -}); - -test("participants are retained during a focus switch", () => { - withTestScheduler(({ behavior, expectObservable }) => { - // Participants disappear on frame 2 and come back on frame 3 - const participantInputMarbles = "a-ba"; - // Start switching focus on frame 1 and reconnect on frame 3 - const connectionInputMarbles = " cs-c"; - // The visible participants should remain the same throughout the switch - const expectedLayoutMarbles = " a"; - - withCallViewModel( - { - remoteParticipants$: behavior(participantInputMarbles, { - a: [aliceParticipant, bobParticipant], - b: [], - }), - rtcMembers$: constant([localRtcMember, aliceRtcMember, bobRtcMember]), - livekitConnectionState$: behavior(connectionInputMarbles, { - c: ConnectionState.Connected, - s: ConnectionState.Connecting, - }), - }, - (vm) => { - expectObservable(summarizeLayout$(vm.layout$)).toBe( - expectedLayoutMarbles, - { - a: { - type: "grid", - spotlight: undefined, - grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], - }, - }, - ); +describe("CallViewModel", () => { + // TODO: Restore this test. It requires makeTransport to not be mocked, unlike + // the rest of the tests in this file… what do we do? + test.skip("test missing RTC config error", async () => { + const rtcMemberships$ = new BehaviorSubject([]); + const emitter = new EventEmitter(); + const client = vi.mocked({ + on: emitter.on.bind(emitter), + off: emitter.off.bind(emitter), + getSyncState: vi.fn().mockReturnValue(SyncState.Syncing), + getUserId: vi.fn().mockReturnValue("@user:localhost"), + getUser: vi.fn().mockReturnValue(null), + getDeviceId: vi.fn().mockReturnValue("DEVICE"), + credentials: { + userId: "@user:localhost", }, + getCrypto: vi.fn().mockReturnValue(undefined), + getDomain: vi.fn().mockReturnValue("example.org"), + } as unknown as MatrixClient); + + const matrixRoom = mockMatrixRoom({ + roomId: "!myRoomId:example.com", + client, + getMember: vi.fn().mockReturnValue(undefined), + }); + + const fakeRtcSession = new MockRTCSession(matrixRoom).withMemberships( + rtcMemberships$, ); - }); -}); -test("screen sharing activates spotlight layout", () => { - withTestScheduler(({ behavior, schedule, expectObservable }) => { - // Start with no screen shares, then have Alice and Bob share their screens, - // then return to no screen shares, then have just Alice share for a bit - const participantInputMarbles = " abcda-ba"; - // While there are no screen shares, switch to spotlight manually, and then - // switch back to grid at the end - const modeInputMarbles = " -----s--g"; - // We should automatically enter spotlight for the first round of screen - // sharing, then return to grid, then manually go into spotlight, and - // remain in spotlight until we manually go back to grid - const expectedLayoutMarbles = " abcdaefeg"; - const expectedShowSpeakingMarbles = "y----nyny"; - withCallViewModel( + mockConfig({}); + + vi.spyOn(AutoDiscovery, "getRawClientConfig").mockResolvedValue({}); + + const callVM = new CallViewModel( + testScope(), + fakeRtcSession.asMockedSession(), + matrixRoom, + mockMediaDevices({}), + mockMuteStates(), { - remoteParticipants$: behavior(participantInputMarbles, { - a: [aliceParticipant, bobParticipant], - b: [aliceSharingScreen, bobParticipant], - c: [aliceSharingScreen, bobSharingScreen], - d: [aliceParticipant, bobSharingScreen], - }), - rtcMembers$: constant([localRtcMember, aliceRtcMember, bobRtcMember]), - }, - (vm) => { - schedule(modeInputMarbles, { - s: () => vm.setGridMode("spotlight"), - g: () => vm.setGridMode("grid"), - }); - - expectObservable(summarizeLayout$(vm.layout$)).toBe( - expectedLayoutMarbles, - { - a: { - type: "grid", - spotlight: undefined, - grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], - }, - b: { - type: "spotlight-landscape", - spotlight: [`${aliceId}:0:screen-share`], - grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], - }, - c: { - type: "spotlight-landscape", - spotlight: [ - `${aliceId}:0:screen-share`, - `${bobId}:0:screen-share`, - ], - grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], - }, - d: { - type: "spotlight-landscape", - spotlight: [`${bobId}:0:screen-share`], - grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], - }, - e: { - type: "spotlight-landscape", - spotlight: [`${aliceId}:0`], - grid: [`${localId}:0`, `${bobId}:0`], - }, - f: { - type: "spotlight-landscape", - spotlight: [`${aliceId}:0:screen-share`], - grid: [`${localId}:0`, `${bobId}:0`, `${aliceId}:0`], - }, - g: { - type: "grid", - spotlight: undefined, - grid: [`${localId}:0`, `${bobId}:0`, `${aliceId}:0`], - }, - }, - ); - expectObservable(vm.showSpeakingIndicators$).toBe( - expectedShowSpeakingMarbles, - yesNo, - ); - }, - ); - }); -}); - -test("participants stay in the same order unless to appear/disappear", () => { - withTestScheduler(({ behavior, schedule, expectObservable }) => { - const visibilityInputMarbles = "a"; - // First Bob speaks, then Dave, then Alice - const aSpeakingInputMarbles = " n- 1998ms - 1999ms y"; - const bSpeakingInputMarbles = " ny 1998ms n 1999ms -"; - const dSpeakingInputMarbles = " n- 1998ms y 1999ms n"; - // Nothing should change when Bob speaks, because Bob is already on screen. - // When Dave speaks he should switch with Alice because she's the one who - // hasn't spoken at all. Then when Alice speaks, she should return to her - // place at the top. - const expectedLayoutMarbles = " a 1999ms b 1999ms a 57999ms c 1999ms a"; - - withCallViewModel( - { - remoteParticipants$: constant([ - aliceParticipant, - bobParticipant, - daveParticipant, - ]), - rtcMembers$: constant([ - localRtcMember, - aliceRtcMember, - bobRtcMember, - daveRtcMember, - ]), - speaking: new Map([ - [aliceParticipant, behavior(aSpeakingInputMarbles, yesNo)], - [bobParticipant, behavior(bSpeakingInputMarbles, yesNo)], - [daveParticipant, behavior(dSpeakingInputMarbles, yesNo)], - ]), - }, - (vm) => { - schedule(visibilityInputMarbles, { - a: () => { - // We imagine that only three tiles (the first three) will be visible - // on screen at a time - vm.layout$.subscribe((layout) => { - if (layout.type === "grid") layout.setVisibleTiles(3); - }); - }, - }); - - expectObservable(summarizeLayout$(vm.layout$)).toBe( - expectedLayoutMarbles, - { - a: { - type: "grid", - spotlight: undefined, - grid: [ - `${localId}:0`, - `${aliceId}:0`, - `${bobId}:0`, - `${daveId}:0`, - ], - }, - b: { - type: "grid", - spotlight: undefined, - grid: [ - `${localId}:0`, - `${daveId}:0`, - `${bobId}:0`, - `${aliceId}:0`, - ], - }, - c: { - type: "grid", - spotlight: undefined, - grid: [ - `${localId}:0`, - `${aliceId}:0`, - `${daveId}:0`, - `${bobId}:0`, - ], - }, - }, - ); - }, - ); - }); -}); - -test("participants adjust order when space becomes constrained", () => { - withTestScheduler(({ behavior, schedule, expectObservable }) => { - // Start with all tiles on screen then shrink to 3 - const visibilityInputMarbles = "a-b"; - // Bob and Dave speak - const bSpeakingInputMarbles = " ny"; - const dSpeakingInputMarbles = " ny"; - // Nothing should change when Bob or Dave initially speak, because they are - // on screen. When the screen becomes smaller Alice should move off screen - // to make way for the speakers (specifically, she should swap with Dave). - const expectedLayoutMarbles = " a-b"; - - withCallViewModel( - { - remoteParticipants$: constant([ - aliceParticipant, - bobParticipant, - daveParticipant, - ]), - rtcMembers$: constant([ - localRtcMember, - aliceRtcMember, - bobRtcMember, - daveRtcMember, - ]), - speaking: new Map([ - [bobParticipant, behavior(bSpeakingInputMarbles, yesNo)], - [daveParticipant, behavior(dSpeakingInputMarbles, yesNo)], - ]), - }, - (vm) => { - let setVisibleTiles: ((value: number) => void) | null = null; - vm.layout$.subscribe((layout) => { - if (layout.type === "grid") setVisibleTiles = layout.setVisibleTiles; - }); - schedule(visibilityInputMarbles, { - a: () => setVisibleTiles!(Infinity), - b: () => setVisibleTiles!(3), - }); - - expectObservable(summarizeLayout$(vm.layout$)).toBe( - expectedLayoutMarbles, - { - a: { - type: "grid", - spotlight: undefined, - grid: [ - `${localId}:0`, - `${aliceId}:0`, - `${bobId}:0`, - `${daveId}:0`, - ], - }, - b: { - type: "grid", - spotlight: undefined, - grid: [ - `${localId}:0`, - `${daveId}:0`, - `${bobId}:0`, - `${aliceId}:0`, - ], - }, - }, - ); - }, - ); - }); -}); - -test("spotlight speakers swap places", () => { - withTestScheduler(({ behavior, schedule, expectObservable }) => { - // Go immediately into spotlight mode for the test - const modeInputMarbles = " s"; - // First Bob speaks, then Dave, then Alice - const aSpeakingInputMarbles = "n--y"; - const bSpeakingInputMarbles = "nyn"; - const dSpeakingInputMarbles = "n-yn"; - // Alice should start in the spotlight, then Bob, then Dave, then Alice - // again. However, the positions of Dave and Bob in the grid should be - // reversed by the end because they've been swapped in and out of the - // spotlight. - const expectedLayoutMarbles = "abcd"; - - withCallViewModel( - { - remoteParticipants$: constant([ - aliceParticipant, - bobParticipant, - daveParticipant, - ]), - rtcMembers$: constant([ - localRtcMember, - aliceRtcMember, - bobRtcMember, - daveRtcMember, - ]), - speaking: new Map([ - [aliceParticipant, behavior(aSpeakingInputMarbles, yesNo)], - [bobParticipant, behavior(bSpeakingInputMarbles, yesNo)], - [daveParticipant, behavior(dSpeakingInputMarbles, yesNo)], - ]), - }, - (vm) => { - schedule(modeInputMarbles, { s: () => vm.setGridMode("spotlight") }); - - expectObservable(summarizeLayout$(vm.layout$)).toBe( - expectedLayoutMarbles, - { - a: { - type: "spotlight-landscape", - spotlight: [`${aliceId}:0`], - grid: [`${localId}:0`, `${bobId}:0`, `${daveId}:0`], - }, - b: { - type: "spotlight-landscape", - spotlight: [`${bobId}:0`], - grid: [`${localId}:0`, `${aliceId}:0`, `${daveId}:0`], - }, - c: { - type: "spotlight-landscape", - spotlight: [`${daveId}:0`], - grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], - }, - d: { - type: "spotlight-landscape", - spotlight: [`${aliceId}:0`], - grid: [`${localId}:0`, `${daveId}:0`, `${bobId}:0`], - }, - }, - ); - - // While we expect the media on tiles to change, layout$ itself should - // *never* meaningfully change. That is, we expect there to be no layout - // shifts as the spotlight speaker changes; instead, the same tiles - // should be reused for the whole duration of the test and simply have - // their media swapped out. This is meaningful for keeping the interface - // not too visually distracting during back-and-forth conversations, - // while still animating tiles to express people joining, leaving, etc. - expectObservable( - vm.layout$.pipe( - distinctUntilChanged(deepCompare), - debounceTime(0), - map(() => "x"), - ), - ).toBe("x"); // Expect just one emission - }, - ); - }); -}); - -test("layout enters picture-in-picture mode when requested", () => { - withTestScheduler(({ schedule, expectObservable }) => { - // Enable then disable picture-in-picture - const pipControlInputMarbles = "-ed"; - // Should go into picture-in-picture layout then back to grid - const expectedLayoutMarbles = " aba"; - - withCallViewModel( - { - remoteParticipants$: constant([aliceParticipant, bobParticipant]), - rtcMembers$: constant([localRtcMember, aliceRtcMember, bobRtcMember]), - }, - (vm) => { - schedule(pipControlInputMarbles, { - e: () => window.controls.enablePip(), - d: () => window.controls.disablePip(), - }); - - expectObservable(summarizeLayout$(vm.layout$)).toBe( - expectedLayoutMarbles, - { - a: { - type: "grid", - spotlight: undefined, - grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], - }, - b: { - type: "pip", - spotlight: [`${aliceId}:0`], - }, - }, - ); - }, - ); - }); -}); - -test("PiP tile in expanded spotlight layout switches speakers without layout shifts", () => { - withTestScheduler(({ behavior, schedule, expectObservable }) => { - // Switch to spotlight immediately - const modeInputMarbles = " s"; - // And expand the spotlight immediately - const expandInputMarbles = " a"; - // First Bob speaks, then Dave, then Bob again - const bSpeakingInputMarbles = "n-yn--yn"; - const dSpeakingInputMarbles = "n---yn"; - // Should show Alice (presenter) in the PiP, then Bob, then Dave, then Bob - // again - const expectedLayoutMarbles = "a-b-c-b"; - - withCallViewModel( - { - remoteParticipants$: constant([ - aliceSharingScreen, - bobParticipant, - daveParticipant, - ]), - rtcMembers$: constant([ - localRtcMember, - aliceRtcMember, - bobRtcMember, - daveRtcMember, - ]), - speaking: new Map([ - [bobParticipant, behavior(bSpeakingInputMarbles, yesNo)], - [daveParticipant, behavior(dSpeakingInputMarbles, yesNo)], - ]), - }, - (vm) => { - schedule(modeInputMarbles, { - s: () => vm.setGridMode("spotlight"), - }); - schedule(expandInputMarbles, { - a: () => vm.toggleSpotlightExpanded$.value!(), - }); - - expectObservable(summarizeLayout$(vm.layout$)).toBe( - expectedLayoutMarbles, - { - a: { - type: "spotlight-expanded", - spotlight: [`${aliceId}:0:screen-share`], - pip: `${aliceId}:0`, - }, - b: { - type: "spotlight-expanded", - spotlight: [`${aliceId}:0:screen-share`], - pip: `${bobId}:0`, - }, - c: { - type: "spotlight-expanded", - spotlight: [`${aliceId}:0:screen-share`], - pip: `${daveId}:0`, - }, - }, - ); - - // While we expect the media on the PiP tile to change, layout$ itself - // should *never* meaningfully change. That is, we expect the same PiP - // tile to exist throughout the test and just have its media swapped out - // when the speaker changes, rather than for tiles to animate in/out. - // This is meaningful for keeping the interface not too visually - // distracting during back-and-forth conversations. - expectObservable( - vm.layout$.pipe( - distinctUntilChanged(deepCompare), - debounceTime(0), - map(() => "x"), - ), - ).toBe("x"); // Expect just one emission - }, - ); - }); -}); - -test("spotlight remembers whether it's expanded", () => { - withTestScheduler(({ schedule, expectObservable }) => { - // Start in spotlight mode, then switch to grid and back to spotlight a - // couple times - const modeInputMarbles = " s-gs-gs"; - // Expand and collapse the spotlight - const expandInputMarbles = " -a--a"; - // Spotlight should stay expanded during the first mode switch, and stay - // collapsed during the second mode switch - const expectedLayoutMarbles = "abcbada"; - - withCallViewModel( - { - remoteParticipants$: constant([aliceParticipant, bobParticipant]), - rtcMembers$: constant([localRtcMember, aliceRtcMember, bobRtcMember]), - }, - (vm) => { - schedule(modeInputMarbles, { - s: () => vm.setGridMode("spotlight"), - g: () => vm.setGridMode("grid"), - }); - schedule(expandInputMarbles, { - a: () => vm.toggleSpotlightExpanded$.value!(), - }); - - expectObservable(summarizeLayout$(vm.layout$)).toBe( - expectedLayoutMarbles, - { - a: { - type: "spotlight-landscape", - spotlight: [`${aliceId}:0`], - grid: [`${localId}:0`, `${bobId}:0`], - }, - b: { - type: "spotlight-expanded", - spotlight: [`${aliceId}:0`], - pip: `${localId}:0`, - }, - c: { - type: "grid", - spotlight: undefined, - grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], - }, - d: { - type: "grid", - spotlight: undefined, - grid: [`${localId}:0`, `${bobId}:0`, `${aliceId}:0`], - }, - }, - ); - }, - ); - }); -}); - -test("participants must have a MatrixRTCSession to be visible", () => { - withTestScheduler(({ behavior, expectObservable }) => { - // iterate through a number of combinations of participants and MatrixRTC memberships - // Bob never has an MatrixRTC membership - const scenarioInputMarbles = " abcdec"; - // Bob should never be visible - const expectedLayoutMarbles = "a-bc-b"; - - withCallViewModel( - { - remoteParticipants$: behavior(scenarioInputMarbles, { - a: [], - b: [bobParticipant], - c: [aliceParticipant, bobParticipant], - d: [aliceParticipant, daveParticipant, bobParticipant], - e: [aliceParticipant, daveParticipant, bobSharingScreen], - }), - rtcMembers$: behavior(scenarioInputMarbles, { - a: [localRtcMember], - b: [localRtcMember], - c: [localRtcMember, aliceRtcMember], - d: [localRtcMember, aliceRtcMember, daveRtcMember], - e: [localRtcMember, aliceRtcMember, daveRtcMember], - }), - }, - (vm) => { - vm.setGridMode("grid"); - expectObservable(summarizeLayout$(vm.layout$)).toBe( - expectedLayoutMarbles, - { - a: { - type: "grid", - spotlight: undefined, - grid: [`${localId}:0`], - }, - b: { - type: "one-on-one", - local: `${localId}:0`, - remote: `${aliceId}:0`, - }, - c: { - type: "grid", - spotlight: undefined, - grid: [`${localId}:0`, `${aliceId}:0`, `${daveId}:0`], - }, - }, - ); - }, - ); - }); -}); - -it("should show at least one tile per MatrixRTCSession", () => { - withTestScheduler(({ behavior, expectObservable }) => { - // iterate through some combinations of MatrixRTC memberships - const scenarioInputMarbles = " abcd"; - // There should always be one tile for each MatrixRTCSession - const expectedLayoutMarbles = "abcd"; - - withCallViewModel( - { - rtcMembers$: behavior(scenarioInputMarbles, { - a: [localRtcMember], - b: [localRtcMember, aliceRtcMember], - c: [localRtcMember, aliceRtcMember, daveRtcMember], - d: [localRtcMember, daveRtcMember], - }), - }, - (vm) => { - vm.setGridMode("grid"); - expectObservable(summarizeLayout$(vm.layout$)).toBe( - expectedLayoutMarbles, - { - a: { - type: "grid", - spotlight: undefined, - grid: [`${localId}:0`], - }, - b: { - type: "one-on-one", - local: `${localId}:0`, - remote: `${aliceId}:0`, - }, - c: { - type: "grid", - spotlight: undefined, - grid: [`${localId}:0`, `${aliceId}:0`, `${daveId}:0`], - }, - d: { - type: "one-on-one", - local: `${localId}:0`, - remote: `${daveId}:0`, - }, - }, - ); - }, - ); - }); -}); - -it("should rank raised hands above video feeds and below speakers and presenters", () => { - withTestScheduler(({ schedule, expectObservable }) => { - // There should always be one tile for each MatrixRTCSession - const expectedLayoutMarbles = "ab"; - - withCallViewModel( - { - remoteParticipants$: constant([aliceParticipant, bobParticipant]), - rtcMembers$: constant([localRtcMember, aliceRtcMember, bobRtcMember]), - }, - (vm, _rtcSession, { raisedHands$ }) => { - schedule("ab", { - a: () => { - // We imagine that only two tiles (the first two) will be visible on screen at a time - vm.layout$.subscribe((layout) => { - if (layout.type === "grid") { - layout.setVisibleTiles(2); - } - }); - }, - b: () => { - raisedHands$.next({ - [`${bobRtcMember.userId}:${bobRtcMember.deviceId}`]: { - time: new Date(), - reactionEventId: "", - membershipEventId: "", - }, - }); - }, - }); - expectObservable(summarizeLayout$(vm.layout$)).toBe( - expectedLayoutMarbles, - { - a: { - type: "grid", - spotlight: undefined, - grid: [ - `${localId}:0`, - "@alice:example.org:AAAA:0", - "@bob:example.org:BBBB:0", - ], - }, - b: { - type: "grid", - spotlight: undefined, - grid: [ - `${localId}:0`, - // Bob shifts up! - "@bob:example.org:BBBB:0", - "@alice:example.org:AAAA:0", - ], - }, - }, - ); - }, - ); - }); -}); - -function nooneEverThere$( - behavior: (marbles: string, values: Record) => Behavior, -): Behavior { - return behavior("a-b-c-d", { - a: [], // Start empty - b: [], // Alice joins - c: [], // Alice still there - d: [], // Alice leaves - }); -} - -function participantJoinLeave$( - behavior: ( - marbles: string, - values: Record, - ) => Behavior, -): Behavior { - return behavior("a-b-c-d", { - a: [], // Start empty - b: [aliceParticipant], // Alice joins - c: [aliceParticipant], // Alice still there - d: [], // Alice leaves - }); -} - -function rtcMemberJoinLeave$( - behavior: ( - marbles: string, - values: Record, - ) => Behavior, -): Behavior { - return behavior("a-b-c-d", { - a: [localRtcMember], // Start empty - b: [localRtcMember, aliceRtcMember], // Alice joins - c: [localRtcMember, aliceRtcMember], // Alice still there - d: [localRtcMember], // Alice leaves - }); -} - -test("autoLeave$ emits only when autoLeaveWhenOthersLeft option is enabled", () => { - withTestScheduler(({ behavior, expectObservable }) => { - withCallViewModel( - { - remoteParticipants$: participantJoinLeave$(behavior), - rtcMembers$: rtcMemberJoinLeave$(behavior), - }, - (vm) => { - expectObservable(vm.autoLeave$).toBe("------a", { - a: "allOthersLeft", - }); - }, - { - autoLeaveWhenOthersLeft: true, encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - }, - ); - }); -}); - -test("autoLeave$ never emits autoLeaveWhenOthersLeft option is enabled but no-one is there", () => { - withTestScheduler(({ behavior, expectObservable }) => { - withCallViewModel( - { - remoteParticipants$: nooneEverThere$(behavior), - rtcMembers$: nooneEverThere$(behavior), - }, - (vm) => { - expectObservable(vm.autoLeave$).toBe("-"); - }, - { - autoLeaveWhenOthersLeft: true, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - }, - ); - }); -}); - -test("autoLeave$ doesn't emit when autoLeaveWhenOthersLeft option is disabled and all others left", () => { - withTestScheduler(({ behavior, expectObservable }) => { - withCallViewModel( - { - remoteParticipants$: participantJoinLeave$(behavior), - rtcMembers$: rtcMemberJoinLeave$(behavior), - }, - (vm) => { - expectObservable(vm.autoLeave$).toBe("-"); - }, - { autoLeaveWhenOthersLeft: false, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + livekitRoomFactory: (): LivekitRoom => + mockLivekitRoom({ + localParticipant, + disconnect: async () => Promise.resolve(), + setE2EEEnabled: async () => Promise.resolve(), + }), }, + new BehaviorSubject({} as Record), + new BehaviorSubject({} as Record), + constant({ processor: undefined, supported: false }), ); - }); -}); -test("autoLeave$ emits when autoLeaveWhenOthersLeft option is enabled and all others left", () => { - withTestScheduler(({ behavior, expectObservable }) => { - withCallViewModel( - { - remoteParticipants$: behavior("a-b-c-d", { - a: [], // Alone - b: [aliceParticipant], // Alice joins - c: [aliceParticipant], - d: [], // Local joins with a second device - }), - rtcMembers$: behavior("a-b-c-d", { - a: [localRtcMember], // Start empty - b: [localRtcMember, aliceRtcMember], // Alice joins - c: [localRtcMember, aliceRtcMember, localRtcMemberDevice2], // Alice still there - d: [localRtcMember, localRtcMemberDevice2], // The second Alice leaves - }), - }, - (vm) => { - expectObservable(vm.autoLeave$).toBe("------a", { - a: "allOthersLeft", - }); - }, - { - autoLeaveWhenOthersLeft: true, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - }, - ); - }); -}); - -describe("waitForCallPickup$", () => { - test("unknown -> ringing -> timeout when notified and nobody joins", () => { - withTestScheduler(({ schedule, expectObservable }) => { - // No one ever joins (only local user) - withCallViewModel( - { remoteParticipants$: constant([]) }, - (vm, rtcSession) => { - // Fire a call notification at 10ms with lifetime 30ms - schedule(" 10ms r", { - r: () => { - rtcSession.emit( - MatrixRTCSessionEvent.DidSendCallNotification, - mockRingEvent("$notif1", 30), - mockLegacyRingEvent, - ); - }, - }); - - expectObservable(vm.callPickupState$).toBe("a 9ms b 29ms c", { - a: "unknown", - b: "ringing", - c: "timeout", - }); - }, - { - waitForCallPickup: true, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - }, - ); + const failPromise = Promise.withResolvers(); + callVM.configError$.subscribe((error) => { + if (error) { + failPromise.resolve(error); + } }); + + const error = await failPromise.promise; + expect(error).toBeInstanceOf(MatrixRTCTransportMissingError); }); - test("regression test: does stop ringing in case livekitConnectionState$ emits after didSendCallNotification$ has already emitted", () => { - withTestScheduler(({ schedule, expectObservable, behavior }) => { + test("participants are retained during a focus switch", () => { + withTestScheduler(({ behavior, expectObservable }) => { + // Participants disappear on frame 2 and come back on frame 3 + const participantInputMarbles = "a-ba"; + // Start switching focus on frame 1 and reconnect on frame 3 + const connectionInputMarbles = " cs-c"; + // The visible participants should remain the same throughout the switch + const expectedLayoutMarbles = " a"; + withCallViewModel( { - livekitConnectionState$: behavior("d 9ms c", { - d: ConnectionState.Disconnected, + remoteParticipants$: behavior(participantInputMarbles, { + a: [aliceParticipant, bobParticipant], + b: [], + }), + rtcMembers$: constant([localRtcMember, aliceRtcMember, bobRtcMember]), + livekitConnectionState$: behavior(connectionInputMarbles, { c: ConnectionState.Connected, + s: ConnectionState.Connecting, }), }, - (vm, rtcSession) => { - // Fire a call notification IMMEDIATELY (its important for this test, that this happens before the livekitConnectionState$ emits) - schedule("n", { - n: () => { - rtcSession.emit( - MatrixRTCSessionEvent.DidSendCallNotification, - mockRingEvent("$notif1", 30), - mockLegacyRingEvent, - ); + (vm) => { + expectObservable(summarizeLayout$(vm.layout$)).toBe( + expectedLayoutMarbles, + { + a: { + type: "grid", + spotlight: undefined, + grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], + }, }, - }); - - expectObservable(vm.callPickupState$).toBe("a 9ms b 19ms c", { - a: "unknown", - b: "ringing", - c: "timeout", - }); - }, - { - waitForCallPickup: true, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + ); }, ); }); }); - test("ringing -> success if someone joins before timeout", () => { + test("screen sharing activates spotlight layout", () => { withTestScheduler(({ behavior, schedule, expectObservable }) => { - // Someone joins at 20ms (both LiveKit participant and MatrixRTC member) + // Start with no screen shares, then have Alice and Bob share their screens, + // then return to no screen shares, then have just Alice share for a bit + const participantInputMarbles = " abcda-ba"; + // While there are no screen shares, switch to spotlight manually, and then + // switch back to grid at the end + const modeInputMarbles = " -----s--g"; + // We should automatically enter spotlight for the first round of screen + // sharing, then return to grid, then manually go into spotlight, and + // remain in spotlight until we manually go back to grid + const expectedLayoutMarbles = " abcdaefeg"; + const expectedShowSpeakingMarbles = "y----nyny"; withCallViewModel( { - remoteParticipants$: behavior("a 19ms b", { - a: [], - b: [aliceParticipant], + remoteParticipants$: behavior(participantInputMarbles, { + a: [aliceParticipant, bobParticipant], + b: [aliceSharingScreen, bobParticipant], + c: [aliceSharingScreen, bobSharingScreen], + d: [aliceParticipant, bobSharingScreen], }), - rtcMembers$: behavior("a 19ms b", { + rtcMembers$: constant([localRtcMember, aliceRtcMember, bobRtcMember]), + }, + (vm) => { + schedule(modeInputMarbles, { + s: () => vm.setGridMode("spotlight"), + g: () => vm.setGridMode("grid"), + }); + + expectObservable(summarizeLayout$(vm.layout$)).toBe( + expectedLayoutMarbles, + { + a: { + type: "grid", + spotlight: undefined, + grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], + }, + b: { + type: "spotlight-landscape", + spotlight: [`${aliceId}:0:screen-share`], + grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], + }, + c: { + type: "spotlight-landscape", + spotlight: [ + `${aliceId}:0:screen-share`, + `${bobId}:0:screen-share`, + ], + grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], + }, + d: { + type: "spotlight-landscape", + spotlight: [`${bobId}:0:screen-share`], + grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], + }, + e: { + type: "spotlight-landscape", + spotlight: [`${aliceId}:0`], + grid: [`${localId}:0`, `${bobId}:0`], + }, + f: { + type: "spotlight-landscape", + spotlight: [`${aliceId}:0:screen-share`], + grid: [`${localId}:0`, `${bobId}:0`, `${aliceId}:0`], + }, + g: { + type: "grid", + spotlight: undefined, + grid: [`${localId}:0`, `${bobId}:0`, `${aliceId}:0`], + }, + }, + ); + expectObservable(vm.showSpeakingIndicators$).toBe( + expectedShowSpeakingMarbles, + yesNo, + ); + }, + ); + }); + }); + + test("participants stay in the same order unless to appear/disappear", () => { + withTestScheduler(({ behavior, schedule, expectObservable }) => { + const visibilityInputMarbles = "a"; + // First Bob speaks, then Dave, then Alice + const aSpeakingInputMarbles = " n- 1998ms - 1999ms y"; + const bSpeakingInputMarbles = " ny 1998ms n 1999ms -"; + const dSpeakingInputMarbles = " n- 1998ms y 1999ms n"; + // Nothing should change when Bob speaks, because Bob is already on screen. + // When Dave speaks he should switch with Alice because she's the one who + // hasn't spoken at all. Then when Alice speaks, she should return to her + // place at the top. + const expectedLayoutMarbles = " a 1999ms b 1999ms a 57999ms c 1999ms a"; + + withCallViewModel( + { + remoteParticipants$: constant([ + aliceParticipant, + bobParticipant, + daveParticipant, + ]), + rtcMembers$: constant([ + localRtcMember, + aliceRtcMember, + bobRtcMember, + daveRtcMember, + ]), + speaking: new Map([ + [aliceParticipant, behavior(aSpeakingInputMarbles, yesNo)], + [bobParticipant, behavior(bSpeakingInputMarbles, yesNo)], + [daveParticipant, behavior(dSpeakingInputMarbles, yesNo)], + ]), + }, + (vm) => { + schedule(visibilityInputMarbles, { + a: () => { + // We imagine that only three tiles (the first three) will be visible + // on screen at a time + vm.layout$.subscribe((layout) => { + if (layout.type === "grid") layout.setVisibleTiles(3); + }); + }, + }); + + expectObservable(summarizeLayout$(vm.layout$)).toBe( + expectedLayoutMarbles, + { + a: { + type: "grid", + spotlight: undefined, + grid: [ + `${localId}:0`, + `${aliceId}:0`, + `${bobId}:0`, + `${daveId}:0`, + ], + }, + b: { + type: "grid", + spotlight: undefined, + grid: [ + `${localId}:0`, + `${daveId}:0`, + `${bobId}:0`, + `${aliceId}:0`, + ], + }, + c: { + type: "grid", + spotlight: undefined, + grid: [ + `${localId}:0`, + `${aliceId}:0`, + `${daveId}:0`, + `${bobId}:0`, + ], + }, + }, + ); + }, + ); + }); + }); + + test("participants adjust order when space becomes constrained", () => { + withTestScheduler(({ behavior, schedule, expectObservable }) => { + // Start with all tiles on screen then shrink to 3 + const visibilityInputMarbles = "a-b"; + // Bob and Dave speak + const bSpeakingInputMarbles = " ny"; + const dSpeakingInputMarbles = " ny"; + // Nothing should change when Bob or Dave initially speak, because they are + // on screen. When the screen becomes smaller Alice should move off screen + // to make way for the speakers (specifically, she should swap with Dave). + const expectedLayoutMarbles = " a-b"; + + withCallViewModel( + { + remoteParticipants$: constant([ + aliceParticipant, + bobParticipant, + daveParticipant, + ]), + rtcMembers$: constant([ + localRtcMember, + aliceRtcMember, + bobRtcMember, + daveRtcMember, + ]), + speaking: new Map([ + [bobParticipant, behavior(bSpeakingInputMarbles, yesNo)], + [daveParticipant, behavior(dSpeakingInputMarbles, yesNo)], + ]), + }, + (vm) => { + let setVisibleTiles: ((value: number) => void) | null = null; + vm.layout$.subscribe((layout) => { + if (layout.type === "grid") + setVisibleTiles = layout.setVisibleTiles; + }); + schedule(visibilityInputMarbles, { + a: () => setVisibleTiles!(Infinity), + b: () => setVisibleTiles!(3), + }); + + expectObservable(summarizeLayout$(vm.layout$)).toBe( + expectedLayoutMarbles, + { + a: { + type: "grid", + spotlight: undefined, + grid: [ + `${localId}:0`, + `${aliceId}:0`, + `${bobId}:0`, + `${daveId}:0`, + ], + }, + b: { + type: "grid", + spotlight: undefined, + grid: [ + `${localId}:0`, + `${daveId}:0`, + `${bobId}:0`, + `${aliceId}:0`, + ], + }, + }, + ); + }, + ); + }); + }); + + test("spotlight speakers swap places", () => { + withTestScheduler(({ behavior, schedule, expectObservable }) => { + // Go immediately into spotlight mode for the test + const modeInputMarbles = " s"; + // First Bob speaks, then Dave, then Alice + const aSpeakingInputMarbles = "n--y"; + const bSpeakingInputMarbles = "nyn"; + const dSpeakingInputMarbles = "n-yn"; + // Alice should start in the spotlight, then Bob, then Dave, then Alice + // again. However, the positions of Dave and Bob in the grid should be + // reversed by the end because they've been swapped in and out of the + // spotlight. + const expectedLayoutMarbles = "abcd"; + + withCallViewModel( + { + remoteParticipants$: constant([ + aliceParticipant, + bobParticipant, + daveParticipant, + ]), + rtcMembers$: constant([ + localRtcMember, + aliceRtcMember, + bobRtcMember, + daveRtcMember, + ]), + speaking: new Map([ + [aliceParticipant, behavior(aSpeakingInputMarbles, yesNo)], + [bobParticipant, behavior(bSpeakingInputMarbles, yesNo)], + [daveParticipant, behavior(dSpeakingInputMarbles, yesNo)], + ]), + }, + (vm) => { + schedule(modeInputMarbles, { s: () => vm.setGridMode("spotlight") }); + + expectObservable(summarizeLayout$(vm.layout$)).toBe( + expectedLayoutMarbles, + { + a: { + type: "spotlight-landscape", + spotlight: [`${aliceId}:0`], + grid: [`${localId}:0`, `${bobId}:0`, `${daveId}:0`], + }, + b: { + type: "spotlight-landscape", + spotlight: [`${bobId}:0`], + grid: [`${localId}:0`, `${aliceId}:0`, `${daveId}:0`], + }, + c: { + type: "spotlight-landscape", + spotlight: [`${daveId}:0`], + grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], + }, + d: { + type: "spotlight-landscape", + spotlight: [`${aliceId}:0`], + grid: [`${localId}:0`, `${daveId}:0`, `${bobId}:0`], + }, + }, + ); + + // While we expect the media on tiles to change, layout$ itself should + // *never* meaningfully change. That is, we expect there to be no layout + // shifts as the spotlight speaker changes; instead, the same tiles + // should be reused for the whole duration of the test and simply have + // their media swapped out. This is meaningful for keeping the interface + // not too visually distracting during back-and-forth conversations, + // while still animating tiles to express people joining, leaving, etc. + expectObservable( + vm.layout$.pipe( + distinctUntilChanged(deepCompare), + debounceTime(0), + map(() => "x"), + ), + ).toBe("x"); // Expect just one emission + }, + ); + }); + }); + + test("layout enters picture-in-picture mode when requested", () => { + withTestScheduler(({ schedule, expectObservable }) => { + // Enable then disable picture-in-picture + const pipControlInputMarbles = "-ed"; + // Should go into picture-in-picture layout then back to grid + const expectedLayoutMarbles = " aba"; + + withCallViewModel( + { + remoteParticipants$: constant([aliceParticipant, bobParticipant]), + rtcMembers$: constant([localRtcMember, aliceRtcMember, bobRtcMember]), + }, + (vm) => { + schedule(pipControlInputMarbles, { + e: () => window.controls.enablePip(), + d: () => window.controls.disablePip(), + }); + + expectObservable(summarizeLayout$(vm.layout$)).toBe( + expectedLayoutMarbles, + { + a: { + type: "grid", + spotlight: undefined, + grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], + }, + b: { + type: "pip", + spotlight: [`${aliceId}:0`], + }, + }, + ); + }, + ); + }); + }); + + test("PiP tile in expanded spotlight layout switches speakers without layout shifts", () => { + withTestScheduler(({ behavior, schedule, expectObservable }) => { + // Switch to spotlight immediately + const modeInputMarbles = " s"; + // And expand the spotlight immediately + const expandInputMarbles = " a"; + // First Bob speaks, then Dave, then Bob again + const bSpeakingInputMarbles = "n-yn--yn"; + const dSpeakingInputMarbles = "n---yn"; + // Should show Alice (presenter) in the PiP, then Bob, then Dave, then Bob + // again + const expectedLayoutMarbles = "a-b-c-b"; + + withCallViewModel( + { + remoteParticipants$: constant([ + aliceSharingScreen, + bobParticipant, + daveParticipant, + ]), + rtcMembers$: constant([ + localRtcMember, + aliceRtcMember, + bobRtcMember, + daveRtcMember, + ]), + speaking: new Map([ + [bobParticipant, behavior(bSpeakingInputMarbles, yesNo)], + [daveParticipant, behavior(dSpeakingInputMarbles, yesNo)], + ]), + }, + (vm) => { + schedule(modeInputMarbles, { + s: () => vm.setGridMode("spotlight"), + }); + schedule(expandInputMarbles, { + a: () => vm.toggleSpotlightExpanded$.value!(), + }); + + expectObservable(summarizeLayout$(vm.layout$)).toBe( + expectedLayoutMarbles, + { + a: { + type: "spotlight-expanded", + spotlight: [`${aliceId}:0:screen-share`], + pip: `${aliceId}:0`, + }, + b: { + type: "spotlight-expanded", + spotlight: [`${aliceId}:0:screen-share`], + pip: `${bobId}:0`, + }, + c: { + type: "spotlight-expanded", + spotlight: [`${aliceId}:0:screen-share`], + pip: `${daveId}:0`, + }, + }, + ); + + // While we expect the media on the PiP tile to change, layout$ itself + // should *never* meaningfully change. That is, we expect the same PiP + // tile to exist throughout the test and just have its media swapped out + // when the speaker changes, rather than for tiles to animate in/out. + // This is meaningful for keeping the interface not too visually + // distracting during back-and-forth conversations. + expectObservable( + vm.layout$.pipe( + distinctUntilChanged(deepCompare), + debounceTime(0), + map(() => "x"), + ), + ).toBe("x"); // Expect just one emission + }, + ); + }); + }); + + test("spotlight remembers whether it's expanded", () => { + withTestScheduler(({ schedule, expectObservable }) => { + // Start in spotlight mode, then switch to grid and back to spotlight a + // couple times + const modeInputMarbles = " s-gs-gs"; + // Expand and collapse the spotlight + const expandInputMarbles = " -a--a"; + // Spotlight should stay expanded during the first mode switch, and stay + // collapsed during the second mode switch + const expectedLayoutMarbles = "abcbada"; + + withCallViewModel( + { + remoteParticipants$: constant([aliceParticipant, bobParticipant]), + rtcMembers$: constant([localRtcMember, aliceRtcMember, bobRtcMember]), + }, + (vm) => { + schedule(modeInputMarbles, { + s: () => vm.setGridMode("spotlight"), + g: () => vm.setGridMode("grid"), + }); + schedule(expandInputMarbles, { + a: () => vm.toggleSpotlightExpanded$.value!(), + }); + + expectObservable(summarizeLayout$(vm.layout$)).toBe( + expectedLayoutMarbles, + { + a: { + type: "spotlight-landscape", + spotlight: [`${aliceId}:0`], + grid: [`${localId}:0`, `${bobId}:0`], + }, + b: { + type: "spotlight-expanded", + spotlight: [`${aliceId}:0`], + pip: `${localId}:0`, + }, + c: { + type: "grid", + spotlight: undefined, + grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], + }, + d: { + type: "grid", + spotlight: undefined, + grid: [`${localId}:0`, `${bobId}:0`, `${aliceId}:0`], + }, + }, + ); + }, + ); + }); + }); + + test("participants must have a MatrixRTCSession to be visible", () => { + withTestScheduler(({ behavior, expectObservable }) => { + // iterate through a number of combinations of participants and MatrixRTC memberships + // Bob never has an MatrixRTC membership + const scenarioInputMarbles = " abcdec"; + // Bob should never be visible + const expectedLayoutMarbles = "a-bc-b"; + + withCallViewModel( + { + remoteParticipants$: behavior(scenarioInputMarbles, { + a: [], + b: [bobParticipant], + c: [aliceParticipant, bobParticipant], + d: [aliceParticipant, daveParticipant, bobParticipant], + e: [aliceParticipant, daveParticipant, bobSharingScreen], + }), + rtcMembers$: behavior(scenarioInputMarbles, { + a: [localRtcMember], + b: [localRtcMember], + c: [localRtcMember, aliceRtcMember], + d: [localRtcMember, aliceRtcMember, daveRtcMember], + e: [localRtcMember, aliceRtcMember, daveRtcMember], + }), + }, + (vm) => { + vm.setGridMode("grid"); + expectObservable(summarizeLayout$(vm.layout$)).toBe( + expectedLayoutMarbles, + { + a: { + type: "grid", + spotlight: undefined, + grid: [`${localId}:0`], + }, + b: { + type: "one-on-one", + local: `${localId}:0`, + remote: `${aliceId}:0`, + }, + c: { + type: "grid", + spotlight: undefined, + grid: [`${localId}:0`, `${aliceId}:0`, `${daveId}:0`], + }, + }, + ); + }, + ); + }); + }); + + it("should show at least one tile per MatrixRTCSession", () => { + withTestScheduler(({ behavior, expectObservable }) => { + // iterate through some combinations of MatrixRTC memberships + const scenarioInputMarbles = " abcd"; + // There should always be one tile for each MatrixRTCSession + const expectedLayoutMarbles = "abcd"; + + withCallViewModel( + { + rtcMembers$: behavior(scenarioInputMarbles, { a: [localRtcMember], b: [localRtcMember, aliceRtcMember], + c: [localRtcMember, aliceRtcMember, daveRtcMember], + d: [localRtcMember, daveRtcMember], }), }, - (vm, rtcSession) => { - // Notify at 5ms so we enter ringing, then success at 20ms - schedule(" 5ms r", { - r: () => { - rtcSession.emit( - MatrixRTCSessionEvent.DidSendCallNotification, - mockRingEvent("$notif2", 100), - mockLegacyRingEvent, - ); + (vm) => { + vm.setGridMode("grid"); + expectObservable(summarizeLayout$(vm.layout$)).toBe( + expectedLayoutMarbles, + { + a: { + type: "grid", + spotlight: undefined, + grid: [`${localId}:0`], + }, + b: { + type: "one-on-one", + local: `${localId}:0`, + remote: `${aliceId}:0`, + }, + c: { + type: "grid", + spotlight: undefined, + grid: [`${localId}:0`, `${aliceId}:0`, `${daveId}:0`], + }, + d: { + type: "one-on-one", + local: `${localId}:0`, + remote: `${daveId}:0`, + }, }, - }); - - expectObservable(vm.callPickupState$).toBe("a 4ms b 14ms c", { - a: "unknown", - b: "ringing", - c: "success", - }); - }, - { - waitForCallPickup: true, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + ); }, ); }); }); - test("ringing -> unknown if we get disconnected", () => { - withTestScheduler(({ behavior, schedule, expectObservable }) => { - const connectionState$ = new BehaviorSubject(ConnectionState.Connected); - // Someone joins at 20ms (both LiveKit participant and MatrixRTC member) - withCallViewModel( - { - remoteParticipants$: behavior("a 19ms b", { - a: [], - b: [aliceParticipant], - }), - rtcMembers$: behavior("a 19ms b", { - a: [localRtcMember], - b: [localRtcMember, aliceRtcMember], - }), - livekitConnectionState$: connectionState$, - }, - (vm, rtcSession) => { - // Notify at 5ms so we enter ringing, then get disconnected 5ms later - schedule(" 5ms r 5ms d", { - r: () => { - rtcSession.emit( - MatrixRTCSessionEvent.DidSendCallNotification, - mockRingEvent("$notif2", 100), - mockLegacyRingEvent, - ); - }, - d: () => { - connectionState$.next(ConnectionState.Disconnected); - }, - }); - - expectObservable(vm.callPickupState$).toBe("a 4ms b 5ms c", { - a: "unknown", - b: "ringing", - c: "unknown", - }); - }, - { - waitForCallPickup: true, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - }, - ); - }); - }); - - test("success when someone joins before we notify", () => { - withTestScheduler(({ behavior, schedule, expectObservable }) => { - // Join at 10ms, notify later at 20ms (state should stay success) - withCallViewModel( - { - remoteParticipants$: behavior("a 9ms b", { - a: [], - b: [aliceParticipant], - }), - rtcMembers$: behavior("a 9ms b", { - a: [localRtcMember], - b: [localRtcMember, aliceRtcMember], - }), - }, - (vm, rtcSession) => { - schedule(" 20ms r", { - r: () => { - rtcSession.emit( - MatrixRTCSessionEvent.DidSendCallNotification, - mockRingEvent("$notif3", 50), - mockLegacyRingEvent, - ); - }, - }); - expectObservable(vm.callPickupState$).toBe("a 9ms b", { - a: "unknown", - b: "success", - }); - }, - { - waitForCallPickup: true, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - }, - ); - }); - }); - - test("notify without lifetime -> immediate timeout", () => { + it("should rank raised hands above video feeds and below speakers and presenters", () => { withTestScheduler(({ schedule, expectObservable }) => { + // There should always be one tile for each MatrixRTCSession + const expectedLayoutMarbles = "ab"; + withCallViewModel( - {}, - (vm, rtcSession) => { - schedule(" 10ms r", { - r: () => { - rtcSession.emit( - MatrixRTCSessionEvent.DidSendCallNotification, - mockRingEvent("$notif4", undefined), - mockLegacyRingEvent, - ); + { + remoteParticipants$: constant([aliceParticipant, bobParticipant]), + rtcMembers$: constant([localRtcMember, aliceRtcMember, bobRtcMember]), + }, + (vm, _rtcSession, { raisedHands$ }) => { + schedule("ab", { + a: () => { + // We imagine that only two tiles (the first two) will be visible on screen at a time + vm.layout$.subscribe((layout) => { + if (layout.type === "grid") { + layout.setVisibleTiles(2); + } + }); + }, + b: () => { + raisedHands$.next({ + [`${bobRtcMember.userId}:${bobRtcMember.deviceId}`]: { + time: new Date(), + reactionEventId: "", + membershipEventId: "", + }, + }); }, }); - expectObservable(vm.callPickupState$).toBe("a 9ms b", { - a: "unknown", - b: "timeout", - }); - }, - { - waitForCallPickup: true, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + expectObservable(summarizeLayout$(vm.layout$)).toBe( + expectedLayoutMarbles, + { + a: { + type: "grid", + spotlight: undefined, + grid: [ + `${localId}:0`, + "@alice:example.org:AAAA:0", + "@bob:example.org:BBBB:0", + ], + }, + b: { + type: "grid", + spotlight: undefined, + grid: [ + `${localId}:0`, + // Bob shifts up! + "@bob:example.org:BBBB:0", + "@alice:example.org:AAAA:0", + ], + }, + }, + ); }, ); }); }); - test("stays null when waitForCallPickup=false", () => { - withTestScheduler(({ behavior, schedule, expectObservable }) => { - withCallViewModel( - { - remoteParticipants$: behavior("a--b", { - a: [], - b: [aliceParticipant], - }), - rtcMembers$: behavior("a--b", { - a: [localRtcMember], - b: [localRtcMember, aliceRtcMember], - }), - }, - (vm, rtcSession) => { - schedule(" 5ms r", { - r: () => { - rtcSession.emit( - MatrixRTCSessionEvent.DidSendCallNotification, - mockRingEvent("$notif5", 30), - mockLegacyRingEvent, - ); - }, - }); - expectObservable(vm.callPickupState$).toBe("(n)", { - n: null, - }); - }, - { - waitForCallPickup: false, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - }, - ); - }); - }); - - test("decline before timeout window ends -> decline", () => { - withTestScheduler(({ schedule, expectObservable }) => { - withCallViewModel( - {}, - (vm, rtcSession) => { - // Notify at 10ms with 50ms lifetime, decline at 40ms with matching id - schedule(" 10ms r 29ms d", { - r: () => { - rtcSession.emit( - MatrixRTCSessionEvent.DidSendCallNotification, - mockRingEvent("$decl1", 50), - mockLegacyRingEvent, - ); - }, - d: () => { - // Emit decline timeline event with id matching the notification - rtcSession.room.emit( - MatrixRoomEvent.Timeline, - new MatrixEvent({ - type: EventType.RTCDecline, - content: { - "m.relates_to": { - rel_type: "m.reference", - event_id: "$decl1", - }, - }, - }), - rtcSession.room, - undefined, - false, - {} as IRoomTimelineData, - ); - }, - }); - expectObservable(vm.callPickupState$).toBe("a 9ms b 29ms e", { - a: "unknown", - b: "ringing", - e: "decline", - }); - }, - { - waitForCallPickup: true, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - }, - ); - }); - }); - - test("decline after timeout window ends -> stays timeout", () => { - withTestScheduler(({ schedule, expectObservable }) => { - withCallViewModel( - {}, - (vm, rtcSession) => { - // Notify at 10ms with 20ms lifetime (timeout at 30ms), decline at 40ms - schedule(" 10ms r 20ms t 10ms d", { - r: () => { - rtcSession.emit( - MatrixRTCSessionEvent.DidSendCallNotification, - mockRingEvent("$decl2", 20), - mockLegacyRingEvent, - ); - }, - t: () => {}, - d: () => { - rtcSession.room.emit( - MatrixRoomEvent.Timeline, - new MatrixEvent({ event_id: "$decl2", type: "m.rtc.decline" }), - rtcSession.room, - undefined, - false, - {} as IRoomTimelineData, - ); - }, - }); - expectObservable(vm.callPickupState$).toBe("a 9ms b 19ms c", { - a: "unknown", - b: "ringing", - c: "timeout", - }); - }, - { - waitForCallPickup: true, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - }, - ); - }); - }); - - function testStaysRinging(declineEvent: Partial): void { - withTestScheduler(({ schedule, expectObservable }) => { - withCallViewModel( - {}, - (vm, rtcSession) => { - // Notify at 10ms with id A, decline arrives at 20ms with id B - schedule(" 10ms r 10ms d", { - r: () => { - rtcSession.emit( - MatrixRTCSessionEvent.DidSendCallNotification, - mockRingEvent("$right", 50), - mockLegacyRingEvent, - ); - }, - d: () => { - rtcSession.room.emit( - MatrixRoomEvent.Timeline, - new MatrixEvent(declineEvent), - rtcSession.room, - undefined, - false, - {} as IRoomTimelineData, - ); - }, - }); - // We assert up to 21ms to see the ringing at 10ms and no change at 20ms - expectObservable(vm.callPickupState$, "21ms !").toBe("a 9ms b", { - a: "unknown", - b: "ringing", - }); - }, - { - waitForCallPickup: true, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - }, - ); + function nooneEverThere$( + behavior: (marbles: string, values: Record) => Behavior, + ): Behavior { + return behavior("a-b-c-d", { + a: [], // Start empty + b: [], // Alice joins + c: [], // Alice still there + d: [], // Alice leaves }); } - test("decline with wrong id is ignored (stays ringing)", () => { - testStaysRinging({ - event_id: "$wrong", - type: "m.rtc.decline", - sender: local.userId, + function participantJoinLeave$( + behavior: ( + marbles: string, + values: Record, + ) => Behavior, + ): Behavior { + return behavior("a-b-c-d", { + a: [], // Start empty + b: [aliceParticipant], // Alice joins + c: [aliceParticipant], // Alice still there + d: [], // Alice leaves }); - }); + } - test("decline with sender being the local user is ignored (stays ringing)", () => { - testStaysRinging({ - event_id: "$right", - type: "m.rtc.decline", - sender: alice.userId, + function rtcMemberJoinLeave$( + behavior: ( + marbles: string, + values: Record, + ) => Behavior, + ): Behavior { + return behavior("a-b-c-d", { + a: [localRtcMember], // Start empty + b: [localRtcMember, aliceRtcMember], // Alice joins + c: [localRtcMember, aliceRtcMember], // Alice still there + d: [localRtcMember], // Alice leaves }); - }); -}); + } -test("audio output changes when toggling earpiece mode", () => { - withTestScheduler(({ schedule, expectObservable }) => { - getUrlParams.mockReturnValue({ controlledAudioDevices: true }); - vi.mocked(ComponentsCore.createMediaDeviceObserver).mockReturnValue(of([])); - - const devices = new MediaDevices(testScope()); - - window.controls.setAvailableAudioDevices([ - { id: "speaker", name: "Speaker", isSpeaker: true }, - { id: "earpiece", name: "Handset", isEarpiece: true }, - { id: "headphones", name: "Headphones" }, - ]); - window.controls.setAudioDevice("headphones"); - - const toggleInputMarbles = " -aaa"; - const expectedEarpieceModeMarbles = "n-yn"; - const expectedTargetStateMarbles = " sese"; - - withCallViewModel({ mediaDevices: devices }, (vm) => { - schedule(toggleInputMarbles, { - a: () => getValue(vm.audioOutputSwitcher$)?.switch(), - }); - expectObservable(vm.earpieceMode$).toBe( - expectedEarpieceModeMarbles, - yesNo, - ); - expectObservable( - vm.audioOutputSwitcher$.pipe(map((switcher) => switcher?.targetOutput)), - ).toBe(expectedTargetStateMarbles, { s: "speaker", e: "earpiece" }); - }); - }); -}); - -test("media tracks are paused while reconnecting to MatrixRTC", () => { - withTestScheduler(({ schedule, expectObservable }) => { - const trackRunning$ = new BehaviorSubject(true); - const originalPublications = localParticipant.trackPublications; - localParticipant.trackPublications = new Map([ - [ - "video", + test("autoLeave$ emits only when autoLeaveWhenOthersLeft option is enabled", () => { + withTestScheduler(({ behavior, expectObservable }) => { + withCallViewModel( { - track: new (class { - public get isUpstreamPaused(): boolean { - return !trackRunning$.value; - } - public async pauseUpstream(): Promise { - trackRunning$.next(false); - return Promise.resolve(); - } - public async resumeUpstream(): Promise { - trackRunning$.next(true); - return Promise.resolve(); - } - })(), - } as unknown as LocalTrackPublication, - ], - ]); - onTestFinished(() => { - localParticipant.trackPublications = originalPublications; + remoteParticipants$: participantJoinLeave$(behavior), + rtcMembers$: rtcMemberJoinLeave$(behavior), + }, + (vm) => { + expectObservable(vm.autoLeave$).toBe("------a", { + a: "allOthersLeft", + }); + }, + { + autoLeaveWhenOthersLeft: true, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); + }); + + test("autoLeave$ never emits autoLeaveWhenOthersLeft option is enabled but no-one is there", () => { + withTestScheduler(({ behavior, expectObservable }) => { + withCallViewModel( + { + remoteParticipants$: nooneEverThere$(behavior), + rtcMembers$: nooneEverThere$(behavior), + }, + (vm) => { + expectObservable(vm.autoLeave$).toBe("-"); + }, + { + autoLeaveWhenOthersLeft: true, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); + }); + + test("autoLeave$ doesn't emit when autoLeaveWhenOthersLeft option is disabled and all others left", () => { + withTestScheduler(({ behavior, expectObservable }) => { + withCallViewModel( + { + remoteParticipants$: participantJoinLeave$(behavior), + rtcMembers$: rtcMemberJoinLeave$(behavior), + }, + (vm) => { + expectObservable(vm.autoLeave$).toBe("-"); + }, + { + autoLeaveWhenOthersLeft: false, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); + }); + + test("autoLeave$ emits when autoLeaveWhenOthersLeft option is enabled and all others left", () => { + withTestScheduler(({ behavior, expectObservable }) => { + withCallViewModel( + { + remoteParticipants$: behavior("a-b-c-d", { + a: [], // Alone + b: [aliceParticipant], // Alice joins + c: [aliceParticipant], + d: [], // Local joins with a second device + }), + rtcMembers$: behavior("a-b-c-d", { + a: [localRtcMember], // Start empty + b: [localRtcMember, aliceRtcMember], // Alice joins + c: [localRtcMember, aliceRtcMember, localRtcMemberDevice2], // Alice still there + d: [localRtcMember, localRtcMemberDevice2], // The second Alice leaves + }), + }, + (vm) => { + expectObservable(vm.autoLeave$).toBe("------a", { + a: "allOthersLeft", + }); + }, + { + autoLeaveWhenOthersLeft: true, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); + }); + + describe("waitForCallPickup$", () => { + test("unknown -> ringing -> timeout when notified and nobody joins", () => { + withTestScheduler(({ schedule, expectObservable }) => { + // No one ever joins (only local user) + withCallViewModel( + { remoteParticipants$: constant([]) }, + (vm, rtcSession) => { + // Fire a call notification at 10ms with lifetime 30ms + schedule(" 10ms r", { + r: () => { + rtcSession.emit( + MatrixRTCSessionEvent.DidSendCallNotification, + mockRingEvent("$notif1", 30), + mockLegacyRingEvent, + ); + }, + }); + + expectObservable(vm.callPickupState$).toBe("a 9ms b 29ms c", { + a: "unknown", + b: "ringing", + c: "timeout", + }); + }, + { + waitForCallPickup: true, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); }); - // There are three indicators that the client might be disconnected from - // MatrixRTC: whether the sync loop is connected, whether the membership is - // present in local room state, and whether the membership manager thinks - // we've hit the timeout for the delayed leave event. Let's test all - // combinations of these conditions. - const syncingMarbles = " nyny----n--y"; - const membershipStatusMarbles = " y---ny-n-yn-y"; - const probablyLeftMarbles = " n-----y-ny---n"; - const expectedReconnectingMarbles = "n-ynyny------n"; - const expectedTrackRunningMarbles = "nynynyn------y"; + test("regression test: does stop ringing in case livekitConnectionState$ emits after didSendCallNotification$ has already emitted", () => { + withTestScheduler(({ schedule, expectObservable, behavior }) => { + withCallViewModel( + { + livekitConnectionState$: behavior("d 9ms c", { + d: ConnectionState.Disconnected, + c: ConnectionState.Connected, + }), + }, + (vm, rtcSession) => { + // Fire a call notification IMMEDIATELY (its important for this test, that this happens before the livekitConnectionState$ emits) + schedule("n", { + n: () => { + rtcSession.emit( + MatrixRTCSessionEvent.DidSendCallNotification, + mockRingEvent("$notif1", 30), + mockLegacyRingEvent, + ); + }, + }); - withCallViewModel( - { initialSyncState: SyncState.Reconnecting }, - (vm, rtcSession, _subjects, setSyncState) => { - schedule(syncingMarbles, { - y: () => setSyncState(SyncState.Syncing), - n: () => setSyncState(SyncState.Reconnecting), + expectObservable(vm.callPickupState$).toBe("a 9ms b 19ms c", { + a: "unknown", + b: "ringing", + c: "timeout", + }); + }, + { + waitForCallPickup: true, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); + }); + + test("ringing -> success if someone joins before timeout", () => { + withTestScheduler(({ behavior, schedule, expectObservable }) => { + // Someone joins at 20ms (both LiveKit participant and MatrixRTC member) + withCallViewModel( + { + remoteParticipants$: behavior("a 19ms b", { + a: [], + b: [aliceParticipant], + }), + rtcMembers$: behavior("a 19ms b", { + a: [localRtcMember], + b: [localRtcMember, aliceRtcMember], + }), + }, + (vm, rtcSession) => { + // Notify at 5ms so we enter ringing, then success at 20ms + schedule(" 5ms r", { + r: () => { + rtcSession.emit( + MatrixRTCSessionEvent.DidSendCallNotification, + mockRingEvent("$notif2", 100), + mockLegacyRingEvent, + ); + }, + }); + + expectObservable(vm.callPickupState$).toBe("a 4ms b 14ms c", { + a: "unknown", + b: "ringing", + c: "success", + }); + }, + { + waitForCallPickup: true, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); + }); + + test("ringing -> unknown if we get disconnected", () => { + withTestScheduler(({ behavior, schedule, expectObservable }) => { + const connectionState$ = new BehaviorSubject(ConnectionState.Connected); + // Someone joins at 20ms (both LiveKit participant and MatrixRTC member) + withCallViewModel( + { + remoteParticipants$: behavior("a 19ms b", { + a: [], + b: [aliceParticipant], + }), + rtcMembers$: behavior("a 19ms b", { + a: [localRtcMember], + b: [localRtcMember, aliceRtcMember], + }), + livekitConnectionState$: connectionState$, + }, + (vm, rtcSession) => { + // Notify at 5ms so we enter ringing, then get disconnected 5ms later + schedule(" 5ms r 5ms d", { + r: () => { + rtcSession.emit( + MatrixRTCSessionEvent.DidSendCallNotification, + mockRingEvent("$notif2", 100), + mockLegacyRingEvent, + ); + }, + d: () => { + connectionState$.next(ConnectionState.Disconnected); + }, + }); + + expectObservable(vm.callPickupState$).toBe("a 4ms b 5ms c", { + a: "unknown", + b: "ringing", + c: "unknown", + }); + }, + { + waitForCallPickup: true, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); + }); + + test("success when someone joins before we notify", () => { + withTestScheduler(({ behavior, schedule, expectObservable }) => { + // Join at 10ms, notify later at 20ms (state should stay success) + withCallViewModel( + { + remoteParticipants$: behavior("a 9ms b", { + a: [], + b: [aliceParticipant], + }), + rtcMembers$: behavior("a 9ms b", { + a: [localRtcMember], + b: [localRtcMember, aliceRtcMember], + }), + }, + (vm, rtcSession) => { + schedule(" 20ms r", { + r: () => { + rtcSession.emit( + MatrixRTCSessionEvent.DidSendCallNotification, + mockRingEvent("$notif3", 50), + mockLegacyRingEvent, + ); + }, + }); + expectObservable(vm.callPickupState$).toBe("a 9ms b", { + a: "unknown", + b: "success", + }); + }, + { + waitForCallPickup: true, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); + }); + + test("notify without lifetime -> immediate timeout", () => { + withTestScheduler(({ schedule, expectObservable }) => { + withCallViewModel( + {}, + (vm, rtcSession) => { + schedule(" 10ms r", { + r: () => { + rtcSession.emit( + MatrixRTCSessionEvent.DidSendCallNotification, + mockRingEvent("$notif4", undefined), + mockLegacyRingEvent, + ); + }, + }); + expectObservable(vm.callPickupState$).toBe("a 9ms b", { + a: "unknown", + b: "timeout", + }); + }, + { + waitForCallPickup: true, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); + }); + + test("stays null when waitForCallPickup=false", () => { + withTestScheduler(({ behavior, schedule, expectObservable }) => { + withCallViewModel( + { + remoteParticipants$: behavior("a--b", { + a: [], + b: [aliceParticipant], + }), + rtcMembers$: behavior("a--b", { + a: [localRtcMember], + b: [localRtcMember, aliceRtcMember], + }), + }, + (vm, rtcSession) => { + schedule(" 5ms r", { + r: () => { + rtcSession.emit( + MatrixRTCSessionEvent.DidSendCallNotification, + mockRingEvent("$notif5", 30), + mockLegacyRingEvent, + ); + }, + }); + expectObservable(vm.callPickupState$).toBe("(n)", { + n: null, + }); + }, + { + waitForCallPickup: false, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); + }); + + test("decline before timeout window ends -> decline", () => { + withTestScheduler(({ schedule, expectObservable }) => { + withCallViewModel( + {}, + (vm, rtcSession) => { + // Notify at 10ms with 50ms lifetime, decline at 40ms with matching id + schedule(" 10ms r 29ms d", { + r: () => { + rtcSession.emit( + MatrixRTCSessionEvent.DidSendCallNotification, + mockRingEvent("$decl1", 50), + mockLegacyRingEvent, + ); + }, + d: () => { + // Emit decline timeline event with id matching the notification + rtcSession.room.emit( + MatrixRoomEvent.Timeline, + new MatrixEvent({ + type: EventType.RTCDecline, + content: { + "m.relates_to": { + rel_type: "m.reference", + event_id: "$decl1", + }, + }, + }), + rtcSession.room, + undefined, + false, + {} as IRoomTimelineData, + ); + }, + }); + expectObservable(vm.callPickupState$).toBe("a 9ms b 29ms e", { + a: "unknown", + b: "ringing", + e: "decline", + }); + }, + { + waitForCallPickup: true, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); + }); + + test("decline after timeout window ends -> stays timeout", () => { + withTestScheduler(({ schedule, expectObservable }) => { + withCallViewModel( + {}, + (vm, rtcSession) => { + // Notify at 10ms with 20ms lifetime (timeout at 30ms), decline at 40ms + schedule(" 10ms r 20ms t 10ms d", { + r: () => { + rtcSession.emit( + MatrixRTCSessionEvent.DidSendCallNotification, + mockRingEvent("$decl2", 20), + mockLegacyRingEvent, + ); + }, + t: () => {}, + d: () => { + rtcSession.room.emit( + MatrixRoomEvent.Timeline, + new MatrixEvent({ + event_id: "$decl2", + type: "m.rtc.decline", + }), + rtcSession.room, + undefined, + false, + {} as IRoomTimelineData, + ); + }, + }); + expectObservable(vm.callPickupState$).toBe("a 9ms b 19ms c", { + a: "unknown", + b: "ringing", + c: "timeout", + }); + }, + { + waitForCallPickup: true, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); + }); + + function testStaysRinging(declineEvent: Partial): void { + withTestScheduler(({ schedule, expectObservable }) => { + withCallViewModel( + {}, + (vm, rtcSession) => { + // Notify at 10ms with id A, decline arrives at 20ms with id B + schedule(" 10ms r 10ms d", { + r: () => { + rtcSession.emit( + MatrixRTCSessionEvent.DidSendCallNotification, + mockRingEvent("$right", 50), + mockLegacyRingEvent, + ); + }, + d: () => { + rtcSession.room.emit( + MatrixRoomEvent.Timeline, + new MatrixEvent(declineEvent), + rtcSession.room, + undefined, + false, + {} as IRoomTimelineData, + ); + }, + }); + // We assert up to 21ms to see the ringing at 10ms and no change at 20ms + expectObservable(vm.callPickupState$, "21ms !").toBe("a 9ms b", { + a: "unknown", + b: "ringing", + }); + }, + { + waitForCallPickup: true, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); + } + + test("decline with wrong id is ignored (stays ringing)", () => { + testStaysRinging({ + event_id: "$wrong", + type: "m.rtc.decline", + sender: local.userId, + }); + }); + + test("decline with sender being the local user is ignored (stays ringing)", () => { + testStaysRinging({ + event_id: "$right", + type: "m.rtc.decline", + sender: alice.userId, + }); + }); + }); + + test("audio output changes when toggling earpiece mode", () => { + withTestScheduler(({ schedule, expectObservable }) => { + getUrlParams.mockReturnValue({ controlledAudioDevices: true }); + vi.mocked(ComponentsCore.createMediaDeviceObserver).mockReturnValue( + of([]), + ); + + const devices = new MediaDevices(testScope()); + + window.controls.setAvailableAudioDevices([ + { id: "speaker", name: "Speaker", isSpeaker: true }, + { id: "earpiece", name: "Handset", isEarpiece: true }, + { id: "headphones", name: "Headphones" }, + ]); + window.controls.setAudioDevice("headphones"); + + const toggleInputMarbles = " -aaa"; + const expectedEarpieceModeMarbles = "n-yn"; + const expectedTargetStateMarbles = " sese"; + + withCallViewModel({ mediaDevices: devices }, (vm) => { + schedule(toggleInputMarbles, { + a: () => getValue(vm.audioOutputSwitcher$)?.switch(), }); - schedule(membershipStatusMarbles, { - y: () => { - rtcSession.membershipStatus = Status.Connected; - }, - n: () => { - rtcSession.membershipStatus = Status.Reconnecting; - }, - }); - schedule(probablyLeftMarbles, { - y: () => { - rtcSession.probablyLeft = true; - }, - n: () => { - rtcSession.probablyLeft = false; - }, - }); - expectObservable(vm.reconnecting$).toBe( - expectedReconnectingMarbles, + expectObservable(vm.earpieceMode$).toBe( + expectedEarpieceModeMarbles, yesNo, ); - expectObservable(trackRunning$).toBe( - expectedTrackRunningMarbles, - yesNo, - ); - }, - ); + expectObservable( + vm.audioOutputSwitcher$.pipe( + map((switcher) => switcher?.targetOutput), + ), + ).toBe(expectedTargetStateMarbles, { s: "speaker", e: "earpiece" }); + }); + }); + }); + + test("media tracks are paused while reconnecting to MatrixRTC", () => { + withTestScheduler(({ schedule, expectObservable }) => { + const trackRunning$ = new BehaviorSubject(true); + const originalPublications = localParticipant.trackPublications; + localParticipant.trackPublications = new Map([ + [ + "video", + { + track: new (class { + public get isUpstreamPaused(): boolean { + return !trackRunning$.value; + } + public async pauseUpstream(): Promise { + trackRunning$.next(false); + return Promise.resolve(); + } + public async resumeUpstream(): Promise { + trackRunning$.next(true); + return Promise.resolve(); + } + })(), + } as unknown as LocalTrackPublication, + ], + ]); + onTestFinished(() => { + localParticipant.trackPublications = originalPublications; + }); + + // There are three indicators that the client might be disconnected from + // MatrixRTC: whether the sync loop is connected, whether the membership is + // present in local room state, and whether the membership manager thinks + // we've hit the timeout for the delayed leave event. Let's test all + // combinations of these conditions. + const syncingMarbles = " nyny----n--y"; + const membershipStatusMarbles = " y---ny-n-yn-y"; + const probablyLeftMarbles = " n-----y-ny---n"; + const expectedReconnectingMarbles = "n-ynyny------n"; + const expectedTrackRunningMarbles = "nynynyn------y"; + + withCallViewModel( + { initialSyncState: SyncState.Reconnecting }, + (vm, rtcSession, _subjects, setSyncState) => { + schedule(syncingMarbles, { + y: () => setSyncState(SyncState.Syncing), + n: () => setSyncState(SyncState.Reconnecting), + }); + schedule(membershipStatusMarbles, { + y: () => { + rtcSession.membershipStatus = Status.Connected; + }, + n: () => { + rtcSession.membershipStatus = Status.Reconnecting; + }, + }); + schedule(probablyLeftMarbles, { + y: () => { + rtcSession.probablyLeft = true; + }, + n: () => { + rtcSession.probablyLeft = false; + }, + }); + expectObservable(vm.reconnecting$).toBe( + expectedReconnectingMarbles, + yesNo, + ); + expectObservable(trackRunning$).toBe( + expectedTrackRunningMarbles, + yesNo, + ); + }, + ); + }); }); }); diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index aa7f32be..669820ab 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -182,6 +182,21 @@ export class CallViewModel { this.matrixRTCSession, ); + // Each hbar seperates a block of input variables required for the CallViewModel to function. + // The outputs of this block is written under the hbar. + // + // For mocking purposes it is recommended to only mock the functions creating those outputs. + // All other fields are just temp computations for the mentioned output. + // The class does not need anything except the values underneath the bar. + // The creation of the values under the bar are all tested independently and testing the callViewModel Should + // not test their cretation. Call view model only needs: + // - memberships$ via createMemberships$ + // - localMembership via createLocalMembership$ + // - callLifecycle via createCallNotificationLifecycle$ + // - matrixMemberMetadataStore via createMatrixMemberMetadata$ + + // ------------------------------------------------------------------------ + // memberships$ private memberships$ = createMemberships$(this.scope, this.matrixRTCSession); private membershipsAndTransports = membershipsAndTransports$( @@ -189,6 +204,9 @@ export class CallViewModel { this.memberships$, ); + // ------------------------------------------------------------------------ + // matrixLivekitMembers$ AND localMembership + private localTransport$ = createLocalTransport$({ scope: this.scope, memberships$: this.memberships$, @@ -199,38 +217,32 @@ export class CallViewModel { ), }); - // ------------------------------------------------------------------------ - private connectionFactory = new ECConnectionFactory( this.matrixRoom.client, this.mediaDevices, this.trackProcessorState$, this.livekitKeyProvider, getUrlParams().controlledAudioDevices, - ); - - // Can contain duplicates. The connection manager will take care of this. - private allTransports$ = this.scope.behavior( - combineLatest( - [this.localTransport$, this.membershipsAndTransports.transports$], - (localTransport, transports) => { - const localTransportAsArray = localTransport ? [localTransport] : []; - return transports.mapInner((transports) => [ - ...localTransportAsArray, - ...transports, - ]); - }, - ), + this.options.livekitRoomFactory, ); private connectionManager = createConnectionManager$({ scope: this.scope, connectionFactory: this.connectionFactory, - inputTransports$: this.allTransports$, + inputTransports$: this.scope.behavior( + combineLatest( + [this.localTransport$, this.membershipsAndTransports.transports$], + (localTransport, transports) => { + const localTransportAsArray = localTransport ? [localTransport] : []; + return transports.mapInner((transports) => [ + ...localTransportAsArray, + ...transports, + ]); + }, + ), + ), }); - // ------------------------------------------------------------------------ - private matrixLivekitMembers$ = createMatrixLivekitMembers$({ scope: this.scope, membershipsWithTransport$: @@ -273,6 +285,7 @@ export class CallViewModel { ), ), ); + private localMatrixLivekitMemberUninitialized = { membership$: this.localRtcMembership$, participant$: this.localMembership.participant$, @@ -294,6 +307,7 @@ export class CallViewModel { ); // ------------------------------------------------------------------------ + // callLifecycle private callLifecycle = createCallNotificationLifecycle$({ scope: this.scope, @@ -309,6 +323,13 @@ export class CallViewModel { public autoLeave$ = this.callLifecycle.autoLeave$; // ------------------------------------------------------------------------ + // matrixMemberMetadataStore + + private matrixMemberMetadataStore = createMatrixMemberMetadata$( + this.scope, + this.scope.behavior(this.memberships$.pipe(map((mems) => mems.value))), + createRoomMembers$(this.scope, this.matrixRoom), + ); /** * If there is a configuration error with the call (e.g. misconfigured E2EE). @@ -402,12 +423,6 @@ export class CallViewModel { ), ); - private matrixMemberMetadataStore = createMatrixMemberMetadata$( - this.scope, - this.scope.behavior(this.memberships$.pipe(map((mems) => mems.value))), - createRoomMembers$(this.scope, this.matrixRoom), - ); - /** * List of user media (camera feeds) that we want tiles for. */ @@ -426,20 +441,23 @@ export class CallViewModel { { value: matrixLivekitMembers }, duplicateTiles, ]) { + let localParticipantId = undefined; // add local member if available if (localMatrixLivekitMember) { - const { - userId, - participant$, - connection$, - // membership$, - } = localMatrixLivekitMember; - const participantId = participant$.value?.identity; // should be membership$.value.membershipID which is not optional + const { userId, participant$, connection$, membership$ } = + localMatrixLivekitMember; + localParticipantId = `${userId}:${membership$.value.deviceId}`; // should be membership$.value.membershipID which is not optional // const participantId = membership$.value.membershipID; - if (participantId) { + if (localParticipantId) { for (let dup = 0; dup < 1 + duplicateTiles; dup++) { yield { - keys: [dup, participantId, userId, participant$, connection$], + keys: [ + dup, + localParticipantId, + userId, + participant$, + connection$, + ], data: undefined, }; } @@ -450,11 +468,11 @@ export class CallViewModel { userId, participant$, connection$, - // membership$ + membership$, } of matrixLivekitMembers) { - const participantId = participant$.value?.identity; + const participantId = `${userId}:${membership$.value.deviceId}`; + if (participantId === localParticipantId) continue; // const participantId = membership$.value?.identity; - if (!participantId) continue; for (let dup = 0; dup < 1 + duplicateTiles; dup++) { yield { keys: [dup, participantId, userId, participant$, connection$], @@ -550,9 +568,8 @@ export class CallViewModel { * - There can be multiple participants for one Matrix user if they join from * multiple devices. */ - // TODO KEEP THIS!! and adapt it to what our membershipManger returns public readonly participantCount$ = this.scope.behavior( - this.memberships$.pipe(map((ms) => ms.value.length)), + this.matrixLivekitMembers$.pipe(map((ms) => ms.value.length)), ); // only public to expose to the view. diff --git a/src/state/CallViewModel/localMember/LocalMembership.ts b/src/state/CallViewModel/localMember/LocalMembership.ts index 1d517643..5e60562a 100644 --- a/src/state/CallViewModel/localMember/LocalMembership.ts +++ b/src/state/CallViewModel/localMember/LocalMembership.ts @@ -30,7 +30,7 @@ import { startWith, switchMap, } from "rxjs"; -import { logger } from "matrix-js-sdk/lib/logger"; +import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { type Behavior } from "../../Behavior"; import { type IConnectionManager } from "../remoteMembers/ConnectionManager"; @@ -52,7 +52,7 @@ import { PosthogAnalytics } from "../../../analytics/PosthogAnalytics.ts"; import { MatrixRTCMode } from "../../../settings/settings.ts"; import { Config } from "../../../config/Config.ts"; import { type Connection } from "../remoteMembers/Connection.ts"; - +const logger = rootLogger.getChild("[LocalMembership]"); export enum LivekitState { Uninitialized = "uninitialized", Connecting = "connecting", @@ -323,10 +323,19 @@ export const createLocalMembership$ = ({ !connectRequested || state.matrix$.value.state !== MatrixState.Disconnected ) { - logger.info("Waiting for transport to enter rtc session"); + logger.info( + "Not yet connecting because: ", + "transport === null:", + transport === null, + "!connectRequested:", + !connectRequested, + "state.matrix$.value.state !== MatrixState.Disconnected:", + state.matrix$.value.state !== MatrixState.Disconnected, + ); return; } state.matrix$.next({ state: MatrixState.Connecting }); + logger.info("Matrix State connecting"); enterRTCSession(matrixRTCSession, transport, options.value).catch( (error) => { logger.error(error); @@ -376,7 +385,9 @@ export const createLocalMembership$ = ({ for (const p of publications) { if (p.track?.isUpstreamPaused === true) { const kind = p.track.kind; - logger.log(`Resuming ${kind} track (MatrixRTC connection present)`); + logger.info( + `Resuming ${kind} track (MatrixRTC connection present)`, + ); p.track .resumeUpstream() .catch((e) => @@ -391,7 +402,7 @@ export const createLocalMembership$ = ({ for (const p of publications) { if (p.track?.isUpstreamPaused === false) { const kind = p.track.kind; - logger.log( + logger.info( `Pausing ${kind} track (uncertain MatrixRTC connection)`, ); p.track diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts index 32d42d75..b28e4f75 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts @@ -117,7 +117,7 @@ export function createConnectionManager$({ connectionFactory, inputTransports$, }: Props): IConnectionManager { - const logger = rootLogger.getChild("ConnectionManager"); + const logger = rootLogger.getChild("[ConnectionManager]"); const running$ = new BehaviorSubject(true); scope.onEnd(() => running$.next(false)); diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts index 3b31cd33..2f152630 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts @@ -22,7 +22,7 @@ import { Epoch, type ObservableScope } from "../../ObservableScope"; import { type Connection } from "./Connection"; import { generateItemsWithEpoch } from "../../../utils/observable"; -const logger = rootLogger.getChild("MatrixLivekitMembers"); +const logger = rootLogger.getChild("[MatrixLivekitMembers]"); /** * Represents a Matrix call member and their associated LiveKit participation. diff --git a/src/state/ObservableScope.ts b/src/state/ObservableScope.ts index 5372246c..282eb77f 100644 --- a/src/state/ObservableScope.ts +++ b/src/state/ObservableScope.ts @@ -74,22 +74,14 @@ export class ObservableScope { // they will no longer re-emit their current value upon subscription. We want // to support Observables that complete (for example `of({})`), so we have to // take care to not propagate the completion event. - setValue$ - .pipe( - this.bind(), - distinctUntilChanged((a, b) => { - logger.log("distinctUntilChanged", a, b); - return a === b; - }), - ) - .subscribe({ - next(value) { - subject$.next(value); - }, - error(err: unknown) { - subject$.error(err); - }, - }); + setValue$.pipe(this.bind(), distinctUntilChanged()).subscribe({ + next(value) { + subject$.next(value); + }, + error(err: unknown) { + subject$.error(err); + }, + }); if (subject$.value === nothing) throw new Error("Behavior failed to synchronously emit an initial value"); return subject$ as Behavior; diff --git a/src/state/SessionBehaviors.ts b/src/state/SessionBehaviors.ts index 7d38ac3d..153ad164 100644 --- a/src/state/SessionBehaviors.ts +++ b/src/state/SessionBehaviors.ts @@ -76,6 +76,6 @@ export const createMemberships$ = ( MatrixRTCSessionEvent.MembershipsChanged, (_, memberships: CallMembership[]) => memberships, ).pipe(trackEpoch()), - new Epoch([]), + new Epoch(matrixRTCSession.memberships), ); }; diff --git a/src/utils/test-fixtures.ts b/src/utils/test-fixtures.ts index e3824ac9..4cf330b7 100644 --- a/src/utils/test-fixtures.ts +++ b/src/utils/test-fixtures.ts @@ -11,9 +11,9 @@ import { mockRemoteParticipant, } from "./test"; -export const localRtcMember = mockRtcMembership("@carol:example.org", "1111"); +export const localRtcMember = mockRtcMembership("@local:example.org", "1111"); export const localRtcMemberDevice2 = mockRtcMembership( - "@carol:example.org", + "@local:example.org", "2222", ); export const local = mockMatrixRoomMember(localRtcMember); diff --git a/src/utils/test.ts b/src/utils/test.ts index d0f09576..4fec433c 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -6,7 +6,14 @@ Please see LICENSE in the repository root for full details. */ import { map, type Observable, of, type SchedulerLike } from "rxjs"; import { type RunHelpers, TestScheduler } from "rxjs/testing"; -import { expect, type MockedObject, onTestFinished, vi, vitest } from "vitest"; +import { + expect, + type MockedObject, + type MockInstance, + onTestFinished, + vi, + vitest, +} from "vitest"; import { MatrixEvent, type Room as MatrixRoom, @@ -269,6 +276,7 @@ export function mockLivekitRoom( }: { remoteParticipants$?: Observable } = {}, ): LivekitRoom { const livekitRoom = { + options: {}, ...mockEmitter(), ...room, } as Partial as LivekitRoom; @@ -291,6 +299,7 @@ export function mockLocalParticipant( return { isLocal: true, trackPublications: new Map(), + unpublishTracks: async () => Promise.resolve(), getTrackPublication: () => ({}) as Partial as LocalTrackPublication, ...mockEmitter(), @@ -331,6 +340,8 @@ export function mockRemoteParticipant( setVolume() {}, getTrackPublication: () => ({}) as Partial as RemoteTrackPublication, + // this will only get used for `getTrackPublications().length` + getTrackPublications: () => [0], ...mockEmitter(), ...participant, } as RemoteParticipant; @@ -363,13 +374,16 @@ export function createRemoteMedia( ); } -export function mockConfig(config: Partial = {}): void { - vi.spyOn(Config, "get").mockReturnValue({ +export function mockConfig( + config: Partial = {}, +): MockInstance<() => ResolvedConfigOptions> { + const spy = vi.spyOn(Config, "get").mockReturnValue({ ...DEFAULT_CONFIG, ...config, }); // simulate loading the config vi.spyOn(Config, "init").mockResolvedValue(void 0); + return spy; } export class MockRTCSession extends TypedEventEmitter< From f6d0d7323e141b6379261eab0f0f89f20343bace Mon Sep 17 00:00:00 2001 From: Timo K Date: Fri, 14 Nov 2025 16:01:43 +0100 Subject: [PATCH 51/75] fix call lifecycle and add tests. --- .../CallNotificationLifecycle.test.ts | 356 ++++++++++++++++++ .../CallNotificationLifecycle.ts | 5 +- src/state/CallViewModel/CallViewModel.test.ts | 135 ------- 3 files changed, 358 insertions(+), 138 deletions(-) create mode 100644 src/state/CallViewModel/CallNotificationLifecycle.test.ts diff --git a/src/state/CallViewModel/CallNotificationLifecycle.test.ts b/src/state/CallViewModel/CallNotificationLifecycle.test.ts new file mode 100644 index 00000000..94112cf0 --- /dev/null +++ b/src/state/CallViewModel/CallNotificationLifecycle.test.ts @@ -0,0 +1,356 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { ConnectionState } from "livekit-client"; +import { + EventType, + MatrixEvent, + type IEvent, + type IRoomTimelineData, +} from "matrix-js-sdk"; +import { + type ICallNotifyContent, + type IRTCNotificationContent, + MatrixRTCSessionEvent, +} from "matrix-js-sdk/lib/matrixrtc"; +import { BehaviorSubject } from "rxjs"; +import { describe, it } from "vitest"; + +import { E2eeType } from "../../e2ee/e2eeType"; +import { withTestScheduler } from "../../utils/test"; +import { + alice, + aliceParticipant, + aliceRtcMember, + local, + localRtcMember, +} from "../../utils/test-fixtures"; +import { + createCallNotificationLifecycle$, + type Props as CallNotificationLifecycleProps, +} from "./CallNotificationLifecycle"; +import { withCallViewModel } from "./CallViewModel.test"; +import { trackEpoch } from "../ObservableScope"; + +const mockLegacyRingEvent = {} as { event_id: string } & ICallNotifyContent; +function mockRingEvent( + eventId: string, + lifetimeMs: number | undefined, + sender = local.userId, +): { event_id: string } & IRTCNotificationContent { + return { + event_id: eventId, + ...(lifetimeMs === undefined ? {} : { lifetime: lifetimeMs }), + notification_type: "ring", + sender, + } as unknown as { event_id: string } & IRTCNotificationContent; +} + +describe("waitForCallPickup$", () => { + it("unknown -> ringing -> timeout when notified and nobody joins", () => { + withTestScheduler(({ scope, expectObservable, behavior, hot }) => { + // No one ever joins (only local user) + const props: CallNotificationLifecycleProps = { + scope, + memberships$: scope.behavior( + behavior("a", { a: [] }).pipe(trackEpoch()), + ), + sentCallNotification$: hot("10ms a", { + a: [mockRingEvent("$notif1", 30), mockLegacyRingEvent], + }), + receivedDecline$: hot(""), + options: { + waitForCallPickup: true, + autoLeaveWhenOthersLeft: false, + }, + localUser: localRtcMember, + }; + + const lifecycle = createCallNotificationLifecycle$(props); + + expectObservable(lifecycle.callPickupState$).toBe("a 9ms b 29ms c", { + a: "unknown", + b: "ringing", + c: "timeout", + }); + }); + }); + + it("ringing -> success if someone joins before timeout is reached", () => { + withTestScheduler(({ scope, hot, behavior, expectObservable }) => { + // Someone joins at 20ms (both LiveKit participant and MatrixRTC member) + const props: CallNotificationLifecycleProps = { + scope, + + memberships$: scope.behavior( + behavior("a 19ms b", { + a: [localRtcMember], + b: [localRtcMember, aliceRtcMember], + }).pipe(trackEpoch()), + ), + sentCallNotification$: hot("5ms a", { + a: [mockRingEvent("$notif2", 100), mockLegacyRingEvent], + }), + receivedDecline$: hot(""), + options: { + waitForCallPickup: true, + autoLeaveWhenOthersLeft: false, + }, + localUser: localRtcMember, + }; + const lifecycle = createCallNotificationLifecycle$(props); + + expectObservable(lifecycle.callPickupState$).toBe("a 4ms b 14ms c", { + a: "unknown", + b: "ringing", + c: "success", + }); + }); + }); + + it("success when someone joins before we notify", () => { + withTestScheduler(({ scope, hot, behavior, expectObservable }) => { + // Someone joins at 20ms (both LiveKit participant and MatrixRTC member) + const props: CallNotificationLifecycleProps = { + scope, + + memberships$: scope.behavior( + behavior("a 9ms b", { + a: [localRtcMember], + b: [localRtcMember, aliceRtcMember], + }).pipe(trackEpoch()), + ), + sentCallNotification$: hot("20ms a", { + a: [mockRingEvent("$notif2", 50), mockLegacyRingEvent], + }), + receivedDecline$: hot(""), + options: { + waitForCallPickup: true, + autoLeaveWhenOthersLeft: false, + }, + localUser: localRtcMember, + }; + const lifecycle = createCallNotificationLifecycle$(props); + + expectObservable(lifecycle.callPickupState$).toBe("a 9ms b", { + a: "unknown", + b: "success", + }); + }); + }); + + it("notify without lifetime -> immediate timeout", () => { + withTestScheduler(({ scope, hot, behavior, expectObservable }) => { + // Someone joins at 20ms (both LiveKit participant and MatrixRTC member) + const props: CallNotificationLifecycleProps = { + scope, + + memberships$: scope.behavior( + behavior("a", { + a: [localRtcMember], + }).pipe(trackEpoch()), + ), + sentCallNotification$: hot("10ms a", { + a: [mockRingEvent("$notif2", undefined), mockLegacyRingEvent], + }), + receivedDecline$: hot(""), + options: { + waitForCallPickup: true, + autoLeaveWhenOthersLeft: false, + }, + localUser: localRtcMember, + }; + const lifecycle = createCallNotificationLifecycle$(props); + + expectObservable(lifecycle.callPickupState$).toBe("a 9ms b", { + a: "unknown", + b: "timeout", + }); + }); + }); + + it("stays null when waitForCallPickup=false", () => { + withTestScheduler(({ behavior, schedule, expectObservable }) => { + withCallViewModel( + { + remoteParticipants$: behavior("a--b", { + a: [], + b: [aliceParticipant], + }), + rtcMembers$: behavior("a--b", { + a: [localRtcMember], + b: [localRtcMember, aliceRtcMember], + }), + }, + (vm, rtcSession) => { + schedule(" 5ms r", { + r: () => { + rtcSession.emit( + MatrixRTCSessionEvent.DidSendCallNotification, + mockRingEvent("$notif5", 30), + mockLegacyRingEvent, + ); + }, + }); + expectObservable(vm.callPickupState$).toBe("(n)", { + n: null, + }); + }, + { + waitForCallPickup: false, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); + }); + + it("decline before timeout window ends -> decline", () => { + withTestScheduler(({ schedule, expectObservable }) => { + withCallViewModel( + {}, + (vm, rtcSession) => { + // Notify at 10ms with 50ms lifetime, decline at 40ms with matching id + schedule(" 10ms r 29ms d", { + r: () => { + rtcSession.emit( + MatrixRTCSessionEvent.DidSendCallNotification, + mockRingEvent("$decl1", 50), + mockLegacyRingEvent, + ); + }, + d: () => { + // Emit decline timeline event with id matching the notification + rtcSession.room.emit( + MatrixRoomEvent.Timeline, + new MatrixEvent({ + type: EventType.RTCDecline, + content: { + "m.relates_to": { + rel_type: "m.reference", + event_id: "$decl1", + }, + }, + }), + rtcSession.room, + undefined, + false, + {} as IRoomTimelineData, + ); + }, + }); + expectObservable(vm.callPickupState$).toBe("a 9ms b 29ms e", { + a: "unknown", + b: "ringing", + e: "decline", + }); + }, + { + waitForCallPickup: true, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); + }); + + it("decline after timeout window ends -> stays timeout", () => { + withTestScheduler(({ schedule, expectObservable }) => { + withCallViewModel( + {}, + (vm, rtcSession) => { + // Notify at 10ms with 20ms lifetime (timeout at 30ms), decline at 40ms + schedule(" 10ms r 20ms t 10ms d", { + r: () => { + rtcSession.emit( + MatrixRTCSessionEvent.DidSendCallNotification, + mockRingEvent("$decl2", 20), + mockLegacyRingEvent, + ); + }, + t: () => {}, + d: () => { + rtcSession.room.emit( + MatrixRoomEvent.Timeline, + new MatrixEvent({ + event_id: "$decl2", + type: "m.rtc.decline", + }), + rtcSession.room, + undefined, + false, + {} as IRoomTimelineData, + ); + }, + }); + expectObservable(vm.callPickupState$).toBe("a 9ms b 19ms c", { + a: "unknown", + b: "ringing", + c: "timeout", + }); + }, + { + waitForCallPickup: true, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); + }); + + function testStaysRinging(declineEvent: Partial): void { + withTestScheduler(({ schedule, expectObservable }) => { + withCallViewModel( + {}, + (vm, rtcSession) => { + // Notify at 10ms with id A, decline arrives at 20ms with id B + schedule(" 10ms r 10ms d", { + r: () => { + rtcSession.emit( + MatrixRTCSessionEvent.DidSendCallNotification, + mockRingEvent("$right", 50), + mockLegacyRingEvent, + ); + }, + d: () => { + rtcSession.room.emit( + MatrixRoomEvent.Timeline, + new MatrixEvent(declineEvent), + rtcSession.room, + undefined, + false, + {} as IRoomTimelineData, + ); + }, + }); + // We assert up to 21ms to see the ringing at 10ms and no change at 20ms + expectObservable(vm.callPickupState$, "21ms !").toBe("a 9ms b", { + a: "unknown", + b: "ringing", + }); + }, + { + waitForCallPickup: true, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); + } + + it("decline with wrong id is ignored (stays ringing)", () => { + testStaysRinging({ + event_id: "$wrong", + type: "m.rtc.decline", + sender: local.userId, + }); + }); + + it("decline with sender being the local user is ignored (stays ringing)", () => { + testStaysRinging({ + event_id: "$right", + type: "m.rtc.decline", + sender: alice.userId, + }); + }); +}); diff --git a/src/state/CallViewModel/CallNotificationLifecycle.ts b/src/state/CallViewModel/CallNotificationLifecycle.ts index 40826d07..ba15a981 100644 --- a/src/state/CallViewModel/CallNotificationLifecycle.ts +++ b/src/state/CallViewModel/CallNotificationLifecycle.ts @@ -70,7 +70,7 @@ export function createReceivedDecline$( ).pipe(filter(([event]) => event.getType() === EventType.RTCDecline)); } -interface Props { +export interface Props { scope: ObservableScope; memberships$: Behavior>; sentCallNotification$: Observable; @@ -109,7 +109,6 @@ export function createCallNotificationLifecycle$({ callPickupState$: Behavior; autoLeave$: Observable; } { - // TODO convert all ring and all others left logic into one callLifecycleTracker$(didSendCallNotification$,matrixLivekitItem$): {autoLeave$,callPickupState$} const allOthersLeft$ = memberships$.pipe( pairwise(), filter( @@ -186,7 +185,7 @@ export function createCallNotificationLifecycle$({ ? combineLatest( [someoneElseJoined$, remoteRingState$], (someoneElseJoined, ring) => { - if (someoneElseJoined) { + if (someoneElseJoined.value === true) { return "success" as const; } // Show the ringing state of the most recent ringing attempt. diff --git a/src/state/CallViewModel/CallViewModel.test.ts b/src/state/CallViewModel/CallViewModel.test.ts index 1cc2ff89..4c5e9cc5 100644 --- a/src/state/CallViewModel/CallViewModel.test.ts +++ b/src/state/CallViewModel/CallViewModel.test.ts @@ -1261,37 +1261,6 @@ describe("CallViewModel", () => { }); describe("waitForCallPickup$", () => { - test("unknown -> ringing -> timeout when notified and nobody joins", () => { - withTestScheduler(({ schedule, expectObservable }) => { - // No one ever joins (only local user) - withCallViewModel( - { remoteParticipants$: constant([]) }, - (vm, rtcSession) => { - // Fire a call notification at 10ms with lifetime 30ms - schedule(" 10ms r", { - r: () => { - rtcSession.emit( - MatrixRTCSessionEvent.DidSendCallNotification, - mockRingEvent("$notif1", 30), - mockLegacyRingEvent, - ); - }, - }); - - expectObservable(vm.callPickupState$).toBe("a 9ms b 29ms c", { - a: "unknown", - b: "ringing", - c: "timeout", - }); - }, - { - waitForCallPickup: true, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - }, - ); - }); - }); - test("regression test: does stop ringing in case livekitConnectionState$ emits after didSendCallNotification$ has already emitted", () => { withTestScheduler(({ schedule, expectObservable, behavior }) => { withCallViewModel( @@ -1327,46 +1296,6 @@ describe("CallViewModel", () => { }); }); - test("ringing -> success if someone joins before timeout", () => { - withTestScheduler(({ behavior, schedule, expectObservable }) => { - // Someone joins at 20ms (both LiveKit participant and MatrixRTC member) - withCallViewModel( - { - remoteParticipants$: behavior("a 19ms b", { - a: [], - b: [aliceParticipant], - }), - rtcMembers$: behavior("a 19ms b", { - a: [localRtcMember], - b: [localRtcMember, aliceRtcMember], - }), - }, - (vm, rtcSession) => { - // Notify at 5ms so we enter ringing, then success at 20ms - schedule(" 5ms r", { - r: () => { - rtcSession.emit( - MatrixRTCSessionEvent.DidSendCallNotification, - mockRingEvent("$notif2", 100), - mockLegacyRingEvent, - ); - }, - }); - - expectObservable(vm.callPickupState$).toBe("a 4ms b 14ms c", { - a: "unknown", - b: "ringing", - c: "success", - }); - }, - { - waitForCallPickup: true, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - }, - ); - }); - }); - test("ringing -> unknown if we get disconnected", () => { withTestScheduler(({ behavior, schedule, expectObservable }) => { const connectionState$ = new BehaviorSubject(ConnectionState.Connected); @@ -1412,70 +1341,6 @@ describe("CallViewModel", () => { }); }); - test("success when someone joins before we notify", () => { - withTestScheduler(({ behavior, schedule, expectObservable }) => { - // Join at 10ms, notify later at 20ms (state should stay success) - withCallViewModel( - { - remoteParticipants$: behavior("a 9ms b", { - a: [], - b: [aliceParticipant], - }), - rtcMembers$: behavior("a 9ms b", { - a: [localRtcMember], - b: [localRtcMember, aliceRtcMember], - }), - }, - (vm, rtcSession) => { - schedule(" 20ms r", { - r: () => { - rtcSession.emit( - MatrixRTCSessionEvent.DidSendCallNotification, - mockRingEvent("$notif3", 50), - mockLegacyRingEvent, - ); - }, - }); - expectObservable(vm.callPickupState$).toBe("a 9ms b", { - a: "unknown", - b: "success", - }); - }, - { - waitForCallPickup: true, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - }, - ); - }); - }); - - test("notify without lifetime -> immediate timeout", () => { - withTestScheduler(({ schedule, expectObservable }) => { - withCallViewModel( - {}, - (vm, rtcSession) => { - schedule(" 10ms r", { - r: () => { - rtcSession.emit( - MatrixRTCSessionEvent.DidSendCallNotification, - mockRingEvent("$notif4", undefined), - mockLegacyRingEvent, - ); - }, - }); - expectObservable(vm.callPickupState$).toBe("a 9ms b", { - a: "unknown", - b: "timeout", - }); - }, - { - waitForCallPickup: true, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - }, - ); - }); - }); - test("stays null when waitForCallPickup=false", () => { withTestScheduler(({ behavior, schedule, expectObservable }) => { withCallViewModel( From 8ce806795605b02d5da94d5aa14c0b5890497fc5 Mon Sep 17 00:00:00 2001 From: Valere Fedronic Date: Fri, 14 Nov 2025 16:18:31 +0100 Subject: [PATCH 52/75] Adding more logs (#3563) * Adding more logs * post merge fix * fixup merge error * review --------- Co-authored-by: Timo K --- src/state/CallViewModel/CallViewModel.ts | 2 + .../localMember/LocalMembership.ts | 58 ++++++++++++------- .../remoteMembers/Connection.test.ts | 27 ++++----- .../CallViewModel/remoteMembers/Connection.ts | 22 +++++-- .../remoteMembers/ConnectionManager.test.ts | 5 ++ .../remoteMembers/ConnectionManager.ts | 14 ++++- .../remoteMembers/integration.test.ts | 2 + src/state/ObservableScope.test.ts | 2 +- src/state/ObservableScope.ts | 1 - 9 files changed, 85 insertions(+), 48 deletions(-) diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 669820ab..72f10034 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -241,6 +241,7 @@ export class CallViewModel { }, ), ), + logger: logger, }); private matrixLivekitMembers$ = createMatrixLivekitMembers$({ @@ -271,6 +272,7 @@ export class CallViewModel { trackProcessorState$: this.trackProcessorState$, widget, options: this.connectOptions$, + logger: logger.getChild(`[${Date.now()}]`), }); private localRtcMembership$ = this.scope.behavior( diff --git a/src/state/CallViewModel/localMember/LocalMembership.ts b/src/state/CallViewModel/localMember/LocalMembership.ts index 5e60562a..1a789ee8 100644 --- a/src/state/CallViewModel/localMember/LocalMembership.ts +++ b/src/state/CallViewModel/localMember/LocalMembership.ts @@ -22,6 +22,7 @@ import { ClientEvent, SyncState, type Room as MatrixRoom } from "matrix-js-sdk"; import { BehaviorSubject, combineLatest, + distinctUntilChanged, fromEvent, map, type Observable, @@ -29,8 +30,9 @@ import { scan, startWith, switchMap, + tap, } from "rxjs"; -import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; +import { type Logger } from "matrix-js-sdk/lib/logger"; import { type Behavior } from "../../Behavior"; import { type IConnectionManager } from "../remoteMembers/ConnectionManager"; @@ -52,7 +54,7 @@ import { PosthogAnalytics } from "../../../analytics/PosthogAnalytics.ts"; import { MatrixRTCMode } from "../../../settings/settings.ts"; import { Config } from "../../../config/Config.ts"; import { type Connection } from "../remoteMembers/Connection.ts"; -const logger = rootLogger.getChild("[LocalMembership]"); + export enum LivekitState { Uninitialized = "uninitialized", Connecting = "connecting", @@ -61,6 +63,7 @@ export enum LivekitState { Disconnected = "disconnected", Disconnecting = "disconnecting", } + type LocalMemberLivekitState = | { state: LivekitState.Error; error: string } | { state: LivekitState.Connected } @@ -74,6 +77,7 @@ export enum MatrixState { Disconnected = "disconnected", Connecting = "connecting", } + type LocalMemberMatrixState = | { state: MatrixState.Connected } | { state: MatrixState.Connecting } @@ -106,6 +110,7 @@ interface Props { localTransport$: Behavior; trackProcessorState$: Behavior; widget: WidgetHelpers | null; + logger: Logger; } /** @@ -132,6 +137,7 @@ export const createLocalMembership$ = ({ matrixRoom, trackProcessorState$, widget, + logger: parentLogger, }: Props): { // publisher: Publisher requestConnect: () => LocalMemberConnectionState; @@ -157,6 +163,8 @@ export const createLocalMembership$ = ({ /** @deprecated use state instead*/ configError$: Behavior; } => { + const logger = parentLogger.getChild("[LocalMembership]"); + logger.debug(`Creating local membership..`); const state = { livekit$: new BehaviorSubject({ state: LivekitState.Uninitialized, @@ -178,17 +186,23 @@ export const createLocalMembership$ = ({ const tracks$ = new BehaviorSubject([]); // Drop Epoch data here since we will not combine this anymore - const connection$ = scope.behavior( - combineLatest( - [connectionManager.connections$, localTransport$], - (connections, transport) => { - if (transport === null) return null; + const localConnection$ = scope.behavior( + combineLatest([connectionManager.connections$, localTransport$]).pipe( + map(([connections, localTransport]) => { + if (localTransport === null) { + return null; + } return ( connections.value.find((connection) => - areLivekitTransportsEqual(connection.transport, transport), + areLivekitTransportsEqual(connection.transport, localTransport), ) ?? null ); - }, + }), + tap((connection) => { + logger.info( + `Local connection updated: ${connection?.transport?.livekit_service_url}`, + ); + }), ), ); /** @@ -219,6 +233,10 @@ export const createLocalMembership$ = ({ startWith(null), map(() => matrixRTCSession.probablyLeft !== true), ), + ).pipe( + tap((connected) => { + logger.info(`Homeserver connected update: ${connected}`); + }), ), ); @@ -230,7 +248,7 @@ export const createLocalMembership$ = ({ const connected$ = scope.behavior( and$( homeserverConnected$, - connection$.pipe( + localConnection$.pipe( switchMap((c) => c ? c.state$.pipe(map((state) => state.state === "ConnectedToLkRoom")) @@ -241,8 +259,9 @@ export const createLocalMembership$ = ({ ); const publisher$ = new BehaviorSubject(null); - connection$.subscribe((connection) => { + localConnection$.subscribe((connection) => { if (connection !== null && publisher$.value === null) { + // TODO looks strange to not change publisher if connection changes. publisher$.next( new Publisher( scope, @@ -375,7 +394,7 @@ export const createLocalMembership$ = ({ // We use matrixConnected$ rather than reconnecting$ because we want to // pause tracks during the initial joining sequence too until we're sure // that our own media is displayed on screen. - combineLatest([connection$, homeserverConnected$]) + combineLatest([localConnection$, homeserverConnected$]) .pipe(scope.bind()) .subscribe(([connection, connected]) => { if (connection?.state$.value.state !== "ConnectedToLkRoom") return; @@ -462,7 +481,7 @@ export const createLocalMembership$ = ({ * Whether the user is currently sharing their screen. */ const sharingScreen$ = scope.behavior( - connection$.pipe( + localConnection$.pipe( switchMap((c) => c === null ? of(false) @@ -483,7 +502,7 @@ export const createLocalMembership$ = ({ // We also allow screen sharing to be toggled even if the connection // is still initializing or publishing tracks, because there's no // technical reason to disallow this. LiveKit will publish if it can. - void connection$.value?.livekitRoom.localParticipant + void localConnection$.value?.livekitRoom.localParticipant .setScreenShareEnabled(!sharingScreen$.value, { audio: true, selfBrowserSurface: "include", @@ -494,7 +513,7 @@ export const createLocalMembership$ = ({ : null; const participant$ = scope.behavior( - connection$.pipe(map((c) => c?.livekitRoom.localParticipant ?? null)), + localConnection$.pipe(map((c) => c?.livekitRoom.localParticipant ?? null)), ); return { startTracks, @@ -508,7 +527,7 @@ export const createLocalMembership$ = ({ sharingScreen$, toggleScreenSharing, participant$, - connection$, + connection$: localConnection$, }; }; @@ -532,6 +551,7 @@ interface EnterRTCSessionOptions { * @param rtcSession * @param transport * @param options + * @throws If the widget could not send ElementWidgetActions.JoinCall action. */ async function enterRTCSession( rtcSession: MatrixRTCSession, @@ -576,10 +596,6 @@ async function enterRTCSession( }, ); if (widget) { - try { - await widget.api.transport.send(ElementWidgetActions.JoinCall, {}); - } catch (e) { - logger.error("Failed to send join action", e); - } + await widget.api.transport.send(ElementWidgetActions.JoinCall, {}); } } diff --git a/src/state/CallViewModel/remoteMembers/Connection.test.ts b/src/state/CallViewModel/remoteMembers/Connection.test.ts index ec7906aa..9d48e893 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.test.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.test.ts @@ -14,7 +14,6 @@ import { onTestFinished, vi, } from "vitest"; -import { BehaviorSubject } from "rxjs"; import { type LocalParticipant, type RemoteParticipant, @@ -25,11 +24,9 @@ import { import fetchMock from "fetch-mock"; import EventEmitter from "events"; import { type IOpenIDToken } from "matrix-js-sdk"; +import { logger } from "matrix-js-sdk/lib/logger"; -import type { - CallMembership, - LivekitTransport, -} from "matrix-js-sdk/lib/matrixrtc"; +import type { LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { Connection, type ConnectionOpts, @@ -39,6 +36,7 @@ import { import { ObservableScope } from "../../ObservableScope.ts"; import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts"; import { FailToGetOpenIdToken } from "../../../utils/errors.ts"; + let testScope: ObservableScope; let client: MockedObject; @@ -49,9 +47,9 @@ let localParticipantEventEmiter: EventEmitter; let fakeLocalParticipant: MockedObject; let fakeRoomEventEmiter: EventEmitter; -let fakeMembershipsFocusMap$: BehaviorSubject< - { membership: CallMembership; transport: LivekitTransport }[] ->; +// let fakeMembershipsFocusMap$: BehaviorSubject< +// { membership: CallMembership; transport: LivekitTransport }[] +// >; const livekitFocus: LivekitTransport = { livekit_alias: "!roomID:example.org", @@ -70,9 +68,6 @@ function setupTest(): void { }), getDeviceId: vi.fn().mockReturnValue("ABCDEF"), } as unknown as OpenIDClientParts); - fakeMembershipsFocusMap$ = new BehaviorSubject< - { membership: CallMembership; transport: LivekitTransport }[] - >([]); localParticipantEventEmiter = new EventEmitter(); @@ -131,7 +126,7 @@ function setupRemoteConnection(): Connection { fakeLivekitRoom.connect.mockResolvedValue(undefined); - return new Connection(opts); + return new Connection(opts, logger); } afterEach(() => { @@ -150,7 +145,7 @@ describe("Start connection states", () => { scope: testScope, livekitRoomFactory: () => fakeLivekitRoom, }; - const connection = new Connection(opts); + const connection = new Connection(opts, logger); expect(connection.state$.getValue().state).toEqual("Initialized"); }); @@ -166,7 +161,7 @@ describe("Start connection states", () => { livekitRoomFactory: () => fakeLivekitRoom, }; - const connection = new Connection(opts, undefined); + const connection = new Connection(opts, logger); const capturedStates: ConnectionState[] = []; const s = connection.state$.subscribe((value) => { @@ -218,7 +213,7 @@ describe("Start connection states", () => { livekitRoomFactory: () => fakeLivekitRoom, }; - const connection = new Connection(opts, undefined); + const connection = new Connection(opts, logger); const capturedStates: ConnectionState[] = []; const s = connection.state$.subscribe((value) => { @@ -274,7 +269,7 @@ describe("Start connection states", () => { livekitRoomFactory: () => fakeLivekitRoom, }; - const connection = new Connection(opts, undefined); + const connection = new Connection(opts, logger); const capturedStates: ConnectionState[] = []; const s = connection.state$.subscribe((value) => { diff --git a/src/state/CallViewModel/remoteMembers/Connection.ts b/src/state/CallViewModel/remoteMembers/Connection.ts index 454ae9fe..44ce4972 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.ts @@ -98,6 +98,7 @@ export class Connection { // TODO dont make this throw and instead store a connection error state in this class? // TODO consider an autostart pattern... public async start(): Promise { + this.logger.debug("Starting Connection"); this.stopped = false; try { this._state$.next({ @@ -145,6 +146,7 @@ export class Connection { livekitConnectionState$: connectionStateObserver(this.livekitRoom), }); } catch (error) { + this.logger.debug(`Failed to connect to LiveKit room: ${error}`); this._state$.next({ state: "FailedToStart", error: error instanceof Error ? error : new Error(`${error}`), @@ -169,6 +171,9 @@ export class Connection { * If the connection is already stopped, this is a no-op. */ public async stop(): Promise { + this.logger.debug( + `Stopping connection to ${this.transport.livekit_service_url}`, + ); if (this.stopped) return; await this.livekitRoom.disconnect(); this._state$.next({ @@ -195,15 +200,18 @@ export class Connection { private readonly client: OpenIDClientParts; public readonly livekitRoom: LivekitRoom; + private readonly logger: Logger; + /** * Creates a new connection to a matrix RTC LiveKit backend. * - * @param livekitRoom - LiveKit room instance to use. * @param opts - Connection options {@link ConnectionOpts}. * + * @param logger */ - public constructor(opts: ConnectionOpts, logger?: Logger) { - logger?.info( + public constructor(opts: ConnectionOpts, logger: Logger) { + this.logger = logger.getChild("[Connection]"); + this.logger.info( `[Connection] Creating new connection to ${opts.transport.livekit_service_url} ${opts.transport.livekit_alias}`, ); const { transport, client, scope } = opts; @@ -223,15 +231,17 @@ export class Connection { ], }).pipe( map((participants) => { - const partsFiltered = participants.filter( + return participants.filter( (participant) => participant.getTrackPublications().length > 0, ); - return partsFiltered; }), ), [], ); - scope.onEnd(() => void this.stop()); + scope.onEnd(() => { + this.logger.info(`Connection scope ended, stopping connection`); + void this.stop(); + }); } } diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts index 97b6c2fd..5887442c 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts @@ -9,6 +9,7 @@ import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; import { BehaviorSubject } from "rxjs"; import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { type Participant as LivekitParticipant } from "livekit-client"; +import { logger } from "matrix-js-sdk/lib/logger"; import { Epoch, ObservableScope } from "../../ObservableScope.ts"; import { @@ -78,6 +79,7 @@ describe("connections$ stream", () => { inputTransports$: behavior("a", { a: new Epoch([TRANSPORT_1, TRANSPORT_2], 0), }), + logger: logger, }); expectObservable(connections$).toBe("a", { @@ -119,6 +121,7 @@ describe("connections$ stream", () => { e: new Epoch([TRANSPORT_1], 4), f: new Epoch([TRANSPORT_1, TRANSPORT_2], 5), }), + logger: logger, }); expectObservable(connections$).toBe("xxxxxa", { @@ -158,6 +161,7 @@ describe("connections$ stream", () => { b: new Epoch([TRANSPORT_1, TRANSPORT_2], 1), c: new Epoch([TRANSPORT_1], 2), }), + logger: logger, }); expectObservable(connections$).toBe("xab", { @@ -272,6 +276,7 @@ describe("connectionManagerData$ stream", () => { inputTransports$: behavior("a", { a: new Epoch([TRANSPORT_1, TRANSPORT_2], 0), }), + logger, }); expectObservable(connectionManagerData$).toBe("abcd", { diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts index b28e4f75..e446571c 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts @@ -13,8 +13,8 @@ import { type LivekitTransport, type ParticipantId, } from "matrix-js-sdk/lib/matrixrtc"; -import { BehaviorSubject, combineLatest, map, of, switchMap } from "rxjs"; -import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; +import { BehaviorSubject, combineLatest, map, of, switchMap, tap } from "rxjs"; +import { type Logger } from "matrix-js-sdk/lib/logger"; import { type LocalParticipant, type RemoteParticipant } from "livekit-client"; import { type Behavior } from "../../Behavior.ts"; @@ -91,6 +91,7 @@ interface Props { scope: ObservableScope; connectionFactory: ConnectionFactory; inputTransports$: Behavior>; + logger: Logger; } // TODO - write test for scopes (do we really need to bind scope) export interface IConnectionManager { @@ -116,8 +117,9 @@ export function createConnectionManager$({ scope, connectionFactory, inputTransports$, + logger: parentLogger, }: Props): IConnectionManager { - const logger = rootLogger.getChild("[ConnectionManager]"); + const logger = parentLogger.getChild("[ConnectionManager]"); const running$ = new BehaviorSubject(true); scope.onEnd(() => running$.next(false)); @@ -137,6 +139,11 @@ export function createConnectionManager$({ transports.mapInner((transport) => (running ? transport : [])), ), map((transports) => transports.mapInner(removeDuplicateTransports)), + tap(({ value: transports }) => { + logger.trace( + `Managing transports: ${transports.map((t) => t.livekit_service_url).join(", ")}`, + ); + }), ), ); @@ -154,6 +161,7 @@ export function createConnectionManager$({ }; }, (scope, _data$, serviceUrl, alias) => { + logger.debug(`Creating connection to ${serviceUrl} (${alias})`); const connection = connectionFactory.createConnection( { type: "livekit", diff --git a/src/state/CallViewModel/remoteMembers/integration.test.ts b/src/state/CallViewModel/remoteMembers/integration.test.ts index 6115694d..e3aa6be8 100644 --- a/src/state/CallViewModel/remoteMembers/integration.test.ts +++ b/src/state/CallViewModel/remoteMembers/integration.test.ts @@ -11,6 +11,7 @@ import { type Room as LivekitRoom } from "livekit-client"; import EventEmitter from "events"; import fetchMock from "fetch-mock"; import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { logger } from "matrix-js-sdk/lib/logger"; import { type Epoch, @@ -120,6 +121,7 @@ test("bob, carl, then bob joining no tracks yet", () => { scope: testScope, connectionFactory: ecConnectionFactory, inputTransports$: membershipsAndTransports.transports$, + logger: logger, }); const matrixLivekitItems$ = createMatrixLivekitMembers$({ diff --git a/src/state/ObservableScope.test.ts b/src/state/ObservableScope.test.ts index 36f6f308..e6c1249d 100644 --- a/src/state/ObservableScope.test.ts +++ b/src/state/ObservableScope.test.ts @@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details. */ import { describe, expect, it } from "vitest"; -import { BehaviorSubject, combineLatest, of, Subject } from "rxjs"; +import { BehaviorSubject, combineLatest, Subject } from "rxjs"; import { logger } from "matrix-js-sdk/lib/logger"; import { diff --git a/src/state/ObservableScope.ts b/src/state/ObservableScope.ts index 282eb77f..27f501c7 100644 --- a/src/state/ObservableScope.ts +++ b/src/state/ObservableScope.ts @@ -21,7 +21,6 @@ import { } from "rxjs"; import { type Behavior } from "./Behavior"; -import { logger } from "matrix-js-sdk/lib/logger"; type MonoTypeOperator = (o: Observable) => Observable; From da85e745966327732a054d00158eabf4605009cb Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 14 Nov 2025 10:48:24 -0500 Subject: [PATCH 53/75] Restore LocalMembership tests --- .../localMember/LocalMembership.test.ts | 28 ++++--------------- .../localMember/LocalMembership.ts | 4 +-- 2 files changed, 7 insertions(+), 25 deletions(-) diff --git a/src/state/CallViewModel/localMember/LocalMembership.test.ts b/src/state/CallViewModel/localMember/LocalMembership.test.ts index 763946d4..f16119e4 100644 --- a/src/state/CallViewModel/localMember/LocalMembership.test.ts +++ b/src/state/CallViewModel/localMember/LocalMembership.test.ts @@ -5,39 +5,21 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { - type LivekitTransport, - type MatrixRTCSession, -} from "matrix-js-sdk/lib/matrixrtc"; +import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; import { expect, test, vi } from "vitest"; import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery"; import EventEmitter from "events"; import { MatrixRTCMode } from "../../../settings/settings"; import { mockConfig } from "../../../utils/test"; -import * as LocalMembership from "./LocalMembership"; - -// Read private function so we do not have to make it public -const enterRTCSession = ( - LocalMembership as unknown as { - enterRTCSession: ( - rtcSession: MatrixRTCSession, - transport: LivekitTransport, - { - encryptMedia, - matrixRTCMode, - }: { encryptMedia: boolean; matrixRTCMode: MatrixRTCMode }, - ) => Promise; - } -).enterRTCSession; +import { enterRTCSession } from "./LocalMembership"; const MATRIX_RTC_MODE = MatrixRTCMode.Legacy; const getUrlParams = vi.hoisted(() => vi.fn(() => ({}))); -vi.mock("./UrlParams", () => ({ getUrlParams })); +vi.mock("../../../UrlParams", () => ({ getUrlParams })); -const actualWidget = await vi.hoisted(async () => vi.importActual("./widget")); -vi.mock("./widget", () => ({ - ...actualWidget, +vi.mock("../../../widget", async (importOriginal) => ({ + ...(await importOriginal()), widget: { api: { setAlwaysOnScreen: (): void => {}, diff --git a/src/state/CallViewModel/localMember/LocalMembership.ts b/src/state/CallViewModel/localMember/LocalMembership.ts index 1a789ee8..ae6c1c7c 100644 --- a/src/state/CallViewModel/localMember/LocalMembership.ts +++ b/src/state/CallViewModel/localMember/LocalMembership.ts @@ -22,7 +22,6 @@ import { ClientEvent, SyncState, type Room as MatrixRoom } from "matrix-js-sdk"; import { BehaviorSubject, combineLatest, - distinctUntilChanged, fromEvent, map, type Observable, @@ -553,7 +552,8 @@ interface EnterRTCSessionOptions { * @param options * @throws If the widget could not send ElementWidgetActions.JoinCall action. */ -async function enterRTCSession( +// Exported for unit testing +export async function enterRTCSession( rtcSession: MatrixRTCSession, transport: LivekitTransport, { encryptMedia, matrixRTCMode }: EnterRTCSessionOptions, From b731c2e9fd44732e8bf1be894d63374ef710d289 Mon Sep 17 00:00:00 2001 From: Timo K Date: Fri, 14 Nov 2025 16:53:43 +0100 Subject: [PATCH 54/75] dont import withCallViewModel in Lifecycle test --- .../CallNotificationLifecycle.test.ts | 308 ++++++++---------- 1 file changed, 142 insertions(+), 166 deletions(-) diff --git a/src/state/CallViewModel/CallNotificationLifecycle.test.ts b/src/state/CallViewModel/CallNotificationLifecycle.test.ts index 94112cf0..1fd04d7a 100644 --- a/src/state/CallViewModel/CallNotificationLifecycle.test.ts +++ b/src/state/CallViewModel/CallNotificationLifecycle.test.ts @@ -5,25 +5,16 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { ConnectionState } from "livekit-client"; -import { - EventType, - MatrixEvent, - type IEvent, - type IRoomTimelineData, -} from "matrix-js-sdk"; import { type ICallNotifyContent, type IRTCNotificationContent, MatrixRTCSessionEvent, } from "matrix-js-sdk/lib/matrixrtc"; -import { BehaviorSubject } from "rxjs"; import { describe, it } from "vitest"; import { E2eeType } from "../../e2ee/e2eeType"; import { withTestScheduler } from "../../utils/test"; import { - alice, aliceParticipant, aliceRtcMember, local, @@ -33,7 +24,7 @@ import { createCallNotificationLifecycle$, type Props as CallNotificationLifecycleProps, } from "./CallNotificationLifecycle"; -import { withCallViewModel } from "./CallViewModel.test"; +// import { withCallViewModel } from "./CallViewModel.test"; import { trackEpoch } from "../ObservableScope"; const mockLegacyRingEvent = {} as { event_id: string } & ICallNotifyContent; @@ -69,9 +60,7 @@ describe("waitForCallPickup$", () => { }, localUser: localRtcMember, }; - const lifecycle = createCallNotificationLifecycle$(props); - expectObservable(lifecycle.callPickupState$).toBe("a 9ms b 29ms c", { a: "unknown", b: "ringing", @@ -79,13 +68,11 @@ describe("waitForCallPickup$", () => { }); }); }); - it("ringing -> success if someone joins before timeout is reached", () => { withTestScheduler(({ scope, hot, behavior, expectObservable }) => { // Someone joins at 20ms (both LiveKit participant and MatrixRTC member) const props: CallNotificationLifecycleProps = { scope, - memberships$: scope.behavior( behavior("a 19ms b", { a: [localRtcMember], @@ -103,7 +90,6 @@ describe("waitForCallPickup$", () => { localUser: localRtcMember, }; const lifecycle = createCallNotificationLifecycle$(props); - expectObservable(lifecycle.callPickupState$).toBe("a 4ms b 14ms c", { a: "unknown", b: "ringing", @@ -111,13 +97,11 @@ describe("waitForCallPickup$", () => { }); }); }); - it("success when someone joins before we notify", () => { withTestScheduler(({ scope, hot, behavior, expectObservable }) => { // Someone joins at 20ms (both LiveKit participant and MatrixRTC member) const props: CallNotificationLifecycleProps = { scope, - memberships$: scope.behavior( behavior("a 9ms b", { a: [localRtcMember], @@ -135,20 +119,17 @@ describe("waitForCallPickup$", () => { localUser: localRtcMember, }; const lifecycle = createCallNotificationLifecycle$(props); - expectObservable(lifecycle.callPickupState$).toBe("a 9ms b", { a: "unknown", b: "success", }); }); }); - it("notify without lifetime -> immediate timeout", () => { withTestScheduler(({ scope, hot, behavior, expectObservable }) => { // Someone joins at 20ms (both LiveKit participant and MatrixRTC member) const props: CallNotificationLifecycleProps = { scope, - memberships$: scope.behavior( behavior("a", { a: [localRtcMember], @@ -165,7 +146,6 @@ describe("waitForCallPickup$", () => { localUser: localRtcMember, }; const lifecycle = createCallNotificationLifecycle$(props); - expectObservable(lifecycle.callPickupState$).toBe("a 9ms b", { a: "unknown", b: "timeout", @@ -208,149 +188,145 @@ describe("waitForCallPickup$", () => { }); }); - it("decline before timeout window ends -> decline", () => { - withTestScheduler(({ schedule, expectObservable }) => { - withCallViewModel( - {}, - (vm, rtcSession) => { - // Notify at 10ms with 50ms lifetime, decline at 40ms with matching id - schedule(" 10ms r 29ms d", { - r: () => { - rtcSession.emit( - MatrixRTCSessionEvent.DidSendCallNotification, - mockRingEvent("$decl1", 50), - mockLegacyRingEvent, - ); - }, - d: () => { - // Emit decline timeline event with id matching the notification - rtcSession.room.emit( - MatrixRoomEvent.Timeline, - new MatrixEvent({ - type: EventType.RTCDecline, - content: { - "m.relates_to": { - rel_type: "m.reference", - event_id: "$decl1", - }, - }, - }), - rtcSession.room, - undefined, - false, - {} as IRoomTimelineData, - ); - }, - }); - expectObservable(vm.callPickupState$).toBe("a 9ms b 29ms e", { - a: "unknown", - b: "ringing", - e: "decline", - }); - }, - { - waitForCallPickup: true, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - }, - ); - }); - }); - - it("decline after timeout window ends -> stays timeout", () => { - withTestScheduler(({ schedule, expectObservable }) => { - withCallViewModel( - {}, - (vm, rtcSession) => { - // Notify at 10ms with 20ms lifetime (timeout at 30ms), decline at 40ms - schedule(" 10ms r 20ms t 10ms d", { - r: () => { - rtcSession.emit( - MatrixRTCSessionEvent.DidSendCallNotification, - mockRingEvent("$decl2", 20), - mockLegacyRingEvent, - ); - }, - t: () => {}, - d: () => { - rtcSession.room.emit( - MatrixRoomEvent.Timeline, - new MatrixEvent({ - event_id: "$decl2", - type: "m.rtc.decline", - }), - rtcSession.room, - undefined, - false, - {} as IRoomTimelineData, - ); - }, - }); - expectObservable(vm.callPickupState$).toBe("a 9ms b 19ms c", { - a: "unknown", - b: "ringing", - c: "timeout", - }); - }, - { - waitForCallPickup: true, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - }, - ); - }); - }); - - function testStaysRinging(declineEvent: Partial): void { - withTestScheduler(({ schedule, expectObservable }) => { - withCallViewModel( - {}, - (vm, rtcSession) => { - // Notify at 10ms with id A, decline arrives at 20ms with id B - schedule(" 10ms r 10ms d", { - r: () => { - rtcSession.emit( - MatrixRTCSessionEvent.DidSendCallNotification, - mockRingEvent("$right", 50), - mockLegacyRingEvent, - ); - }, - d: () => { - rtcSession.room.emit( - MatrixRoomEvent.Timeline, - new MatrixEvent(declineEvent), - rtcSession.room, - undefined, - false, - {} as IRoomTimelineData, - ); - }, - }); - // We assert up to 21ms to see the ringing at 10ms and no change at 20ms - expectObservable(vm.callPickupState$, "21ms !").toBe("a 9ms b", { - a: "unknown", - b: "ringing", - }); - }, - { - waitForCallPickup: true, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - }, - ); - }); - } - - it("decline with wrong id is ignored (stays ringing)", () => { - testStaysRinging({ - event_id: "$wrong", - type: "m.rtc.decline", - sender: local.userId, - }); - }); - - it("decline with sender being the local user is ignored (stays ringing)", () => { - testStaysRinging({ - event_id: "$right", - type: "m.rtc.decline", - sender: alice.userId, - }); - }); + // it("decline before timeout window ends -> decline", () => { + // withTestScheduler(({ schedule, expectObservable }) => { + // withCallViewModel( + // {}, + // (vm, rtcSession) => { + // // Notify at 10ms with 50ms lifetime, decline at 40ms with matching id + // schedule(" 10ms r 29ms d", { + // r: () => { + // rtcSession.emit( + // MatrixRTCSessionEvent.DidSendCallNotification, + // mockRingEvent("$decl1", 50), + // mockLegacyRingEvent, + // ); + // }, + // d: () => { + // // Emit decline timeline event with id matching the notification + // rtcSession.room.emit( + // MatrixRoomEvent.Timeline, + // new MatrixEvent({ + // type: EventType.RTCDecline, + // content: { + // "m.relates_to": { + // rel_type: "m.reference", + // event_id: "$decl1", + // }, + // }, + // }), + // rtcSession.room, + // undefined, + // false, + // {} as IRoomTimelineData, + // ); + // }, + // }); + // expectObservable(vm.callPickupState$).toBe("a 9ms b 29ms e", { + // a: "unknown", + // b: "ringing", + // e: "decline", + // }); + // }, + // { + // waitForCallPickup: true, + // encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + // }, + // ); + // }); + // }); + // it("decline after timeout window ends -> stays timeout", () => { + // withTestScheduler(({ schedule, expectObservable }) => { + // withCallViewModel( + // {}, + // (vm, rtcSession) => { + // // Notify at 10ms with 20ms lifetime (timeout at 30ms), decline at 40ms + // schedule(" 10ms r 20ms t 10ms d", { + // r: () => { + // rtcSession.emit( + // MatrixRTCSessionEvent.DidSendCallNotification, + // mockRingEvent("$decl2", 20), + // mockLegacyRingEvent, + // ); + // }, + // t: () => {}, + // d: () => { + // rtcSession.room.emit( + // MatrixRoomEvent.Timeline, + // new MatrixEvent({ + // event_id: "$decl2", + // type: "m.rtc.decline", + // }), + // rtcSession.room, + // undefined, + // false, + // {} as IRoomTimelineData, + // ); + // }, + // }); + // expectObservable(vm.callPickupState$).toBe("a 9ms b 19ms c", { + // a: "unknown", + // b: "ringing", + // c: "timeout", + // }); + // }, + // { + // waitForCallPickup: true, + // encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + // }, + // ); + // }); + // }); + // function testStaysRinging(declineEvent: Partial): void { + // withTestScheduler(({ schedule, expectObservable }) => { + // withCallViewModel( + // {}, + // (vm, rtcSession) => { + // // Notify at 10ms with id A, decline arrives at 20ms with id B + // schedule(" 10ms r 10ms d", { + // r: () => { + // rtcSession.emit( + // MatrixRTCSessionEvent.DidSendCallNotification, + // mockRingEvent("$right", 50), + // mockLegacyRingEvent, + // ); + // }, + // d: () => { + // rtcSession.room.emit( + // MatrixRoomEvent.Timeline, + // new MatrixEvent(declineEvent), + // rtcSession.room, + // undefined, + // false, + // {} as IRoomTimelineData, + // ); + // }, + // }); + // // We assert up to 21ms to see the ringing at 10ms and no change at 20ms + // expectObservable(vm.callPickupState$, "21ms !").toBe("a 9ms b", { + // a: "unknown", + // b: "ringing", + // }); + // }, + // { + // waitForCallPickup: true, + // encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + // }, + // ); + // }); + // } + // it("decline with wrong id is ignored (stays ringing)", () => { + // testStaysRinging({ + // event_id: "$wrong", + // type: "m.rtc.decline", + // sender: local.userId, + // }); + // }); + // it("decline with sender being the local user is ignored (stays ringing)", () => { + // testStaysRinging({ + // event_id: "$right", + // type: "m.rtc.decline", + // sender: alice.userId, + // }); + // }); }); From 6d8d2d2384b16a4bf2372022eaa12c872dbdb458 Mon Sep 17 00:00:00 2001 From: Valere Date: Fri, 14 Nov 2025 17:30:02 +0100 Subject: [PATCH 55/75] refact: extract withCallViewModel in test utils file --- .../CallNotificationLifecycle.test.ts | 2 +- src/state/CallViewModel/CallViewModel.test.ts | 138 +------------ .../CallViewModelTestUtils.test.ts | 183 ++++++++++++++++++ src/state/CallViewModelWidget.test.ts | 2 +- 4 files changed, 188 insertions(+), 137 deletions(-) create mode 100644 src/state/CallViewModel/CallViewModelTestUtils.test.ts diff --git a/src/state/CallViewModel/CallNotificationLifecycle.test.ts b/src/state/CallViewModel/CallNotificationLifecycle.test.ts index 1fd04d7a..8bdbe059 100644 --- a/src/state/CallViewModel/CallNotificationLifecycle.test.ts +++ b/src/state/CallViewModel/CallNotificationLifecycle.test.ts @@ -24,8 +24,8 @@ import { createCallNotificationLifecycle$, type Props as CallNotificationLifecycleProps, } from "./CallNotificationLifecycle"; -// import { withCallViewModel } from "./CallViewModel.test"; import { trackEpoch } from "../ObservableScope"; +import { withCallViewModel } from "./CallViewModelTestUtils.test"; const mockLegacyRingEvent = {} as { event_id: string } & ICallNotifyContent; function mockRingEvent( diff --git a/src/state/CallViewModel/CallViewModel.test.ts b/src/state/CallViewModel/CallViewModel.test.ts index 4c5e9cc5..863ba16f 100644 --- a/src/state/CallViewModel/CallViewModel.test.ts +++ b/src/state/CallViewModel/CallViewModel.test.ts @@ -19,7 +19,6 @@ import { switchMap, } from "rxjs"; import { - ClientEvent, SyncState, type MatrixClient, RoomEvent as MatrixRoomEvent, @@ -30,10 +29,7 @@ import { } from "matrix-js-sdk"; import { ConnectionState, - type LocalParticipant, type LocalTrackPublication, - type Participant, - ParticipantEvent, type RemoteParticipant, type Room as LivekitRoom, } from "livekit-client"; @@ -49,7 +45,7 @@ import { import { deepCompare } from "matrix-js-sdk/lib/utils"; import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery"; -import { CallViewModel, type CallViewModelOptions } from "./CallViewModel"; +import { CallViewModel } from "./CallViewModel"; import { type Layout } from "../layout-types.ts"; import { mockLocalParticipant, @@ -70,15 +66,11 @@ import { E2eeType } from "../../e2ee/e2eeType.ts"; import type { RaisedHandInfo, ReactionInfo } from "../../reactions/index.ts"; import { alice, - aliceDoppelganger, aliceId, aliceParticipant, aliceRtcMember, - bob, bobId, bobRtcMember, - bobZeroWidthSpace, - daveRTL, local, localId, localRtcMember, @@ -87,11 +79,11 @@ import { import { MediaDevices } from "../MediaDevices.ts"; import { getValue } from "../../utils/observable.ts"; import { type Behavior, constant } from "../Behavior.ts"; -import type { ProcessorState } from "../../livekit/TrackProcessorContext.tsx"; import { type ElementCallError, MatrixRTCTransportMissingError, } from "../../utils/errors.ts"; +import { withCallViewModel } from "./CallViewModelTestUtils.test"; mockConfig({ livekit: { livekit_service_url: "http://my-default-service-url.com" }, @@ -124,7 +116,7 @@ const yesNo = { const daveRtcMember = mockRtcMembership("@dave:example.org", "DDDD"); -const carol = local; +// const carol = local; const dave = mockMatrixRoomMember(daveRtcMember, { rawDisplayName: "Dave" }); @@ -142,12 +134,6 @@ const bobSharingScreen = mockRemoteParticipant({ }); const daveParticipant = mockRemoteParticipant({ identity: daveId }); -const roomMembers = new Map( - [alice, aliceDoppelganger, bob, bobZeroWidthSpace, carol, dave, daveRTL].map( - (p) => [p.userId, p], - ), -); - export interface GridLayoutSummary { type: "grid"; spotlight?: string[]; @@ -269,124 +255,6 @@ function mockRingEvent( // need a value to fill in for them when emitting notifications const mockLegacyRingEvent = {} as { event_id: string } & ICallNotifyContent; -interface CallViewModelInputs { - remoteParticipants$: Behavior; - rtcMembers$: Behavior[]>; - livekitConnectionState$: Behavior; - speaking: Map>; - mediaDevices: MediaDevices; - initialSyncState: SyncState; -} - -export function withCallViewModel( - { - remoteParticipants$ = constant([]), - rtcMembers$ = constant([localRtcMember]), - livekitConnectionState$: connectionState$ = constant( - ConnectionState.Connected, - ), - speaking = new Map(), - mediaDevices = mockMediaDevices({}), - initialSyncState = SyncState.Syncing, - }: Partial, - continuation: ( - vm: CallViewModel, - rtcSession: MockRTCSession, - subjects: { raisedHands$: BehaviorSubject> }, - setSyncState: (value: SyncState) => void, - ) => void, - options: CallViewModelOptions = { - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - autoLeaveWhenOthersLeft: false, - }, -): void { - let syncState = initialSyncState; - const setSyncState = (value: SyncState): void => { - const prev = syncState; - syncState = value; - room.client.emit(ClientEvent.Sync, value, prev); - }; - const room = mockMatrixRoom({ - client: new (class extends EventEmitter { - public getUserId(): string | undefined { - return localRtcMember.userId; - } - public getDeviceId(): string { - return localRtcMember.deviceId; - } - public getDomain(): string { - return "example.com"; - } - public getSyncState(): SyncState { - return syncState; - } - })() as Partial as MatrixClient, - getMembers: () => Array.from(roomMembers.values()), - }); - const rtcSession = new MockRTCSession(room, []).withMemberships(rtcMembers$); - const participantsSpy = vi - .spyOn(ComponentsCore, "connectedParticipantsObserver") - .mockReturnValue(remoteParticipants$); - const mediaSpy = vi - .spyOn(ComponentsCore, "observeParticipantMedia") - .mockImplementation((p) => - of({ participant: p } as Partial< - ComponentsCore.ParticipantMedia - > as ComponentsCore.ParticipantMedia), - ); - const eventsSpy = vi - .spyOn(ComponentsCore, "observeParticipantEvents") - .mockImplementation((p, ...eventTypes) => { - if (eventTypes.includes(ParticipantEvent.IsSpeakingChanged)) { - return (speaking.get(p) ?? of(false)).pipe( - map((s) => ({ ...p, isSpeaking: s }) as Participant), - ); - } else { - return of(p); - } - }); - - const roomEventSelectorSpy = vi - .spyOn(ComponentsCore, "roomEventSelector") - .mockImplementation((_room, _eventType) => of()); - const muteStates = mockMuteStates(); - const raisedHands$ = new BehaviorSubject>({}); - const reactions$ = new BehaviorSubject>({}); - - const vm = new CallViewModel( - testScope(), - rtcSession.asMockedSession(), - room, - mediaDevices, - muteStates, - { - ...options, - livekitRoomFactory: (): LivekitRoom => - mockLivekitRoom({ - localParticipant, - disconnect: async () => Promise.resolve(), - setE2EEEnabled: async () => Promise.resolve(), - }), - connectionState$, - }, - raisedHands$, - reactions$, - new BehaviorSubject({ - processor: undefined, - supported: undefined, - }), - ); - - onTestFinished(() => { - participantsSpy.mockRestore(); - mediaSpy.mockRestore(); - eventsSpy.mockRestore(); - roomEventSelectorSpy.mockRestore(); - }); - - continuation(vm, rtcSession, { raisedHands$: raisedHands$ }, setSyncState); -} - describe("CallViewModel", () => { // TODO: Restore this test. It requires makeTransport to not be mocked, unlike // the rest of the tests in this file… what do we do? diff --git a/src/state/CallViewModel/CallViewModelTestUtils.test.ts b/src/state/CallViewModel/CallViewModelTestUtils.test.ts new file mode 100644 index 00000000..6cae1bfa --- /dev/null +++ b/src/state/CallViewModel/CallViewModelTestUtils.test.ts @@ -0,0 +1,183 @@ +/* +Copyright 2025 Element Corp. +Copyright 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + ConnectionState, + type LocalParticipant, + type Participant, + ParticipantEvent, + type RemoteParticipant, + type Room as LivekitRoom, +} from "livekit-client"; +import { SyncState } from "matrix-js-sdk/lib/sync"; +import { BehaviorSubject, type Observable, map, of } from "rxjs"; +import { onTestFinished, vi } from "vitest"; +import { ClientEvent, type MatrixClient } from "matrix-js-sdk"; +import EventEmitter from "events"; +import * as ComponentsCore from "@livekit/components-core"; + +import type { CallMembership } from "matrix-js-sdk/lib/matrixrtc"; +import { E2eeType } from "../../e2ee/e2eeType"; +import { type RaisedHandInfo, type ReactionInfo } from "../../reactions"; +import { CallViewModel, type CallViewModelOptions } from "./CallViewModel"; +import { + mockLivekitRoom, + mockLocalParticipant, + mockMatrixRoom, + mockMatrixRoomMember, + mockMediaDevices, + mockMuteStates, + MockRTCSession, + testScope, +} from "../../utils/test"; +import { + alice, + aliceDoppelganger, + bob, + bobZeroWidthSpace, + daveRTL, + daveRTLRtcMember, + local, + localRtcMember, +} from "../../utils/test-fixtures"; +import { type Behavior, constant } from "../Behavior"; +import { type ProcessorState } from "../../livekit/TrackProcessorContext"; +import { type MediaDevices } from "../MediaDevices"; + +const carol = local; + +const dave = mockMatrixRoomMember(daveRTLRtcMember, { rawDisplayName: "Dave" }); + +const roomMembers = new Map( + [alice, aliceDoppelganger, bob, bobZeroWidthSpace, carol, dave, daveRTL].map( + (p) => [p.userId, p], + ), +); + +export interface CallViewModelInputs { + remoteParticipants$: Behavior; + rtcMembers$: Behavior[]>; + livekitConnectionState$: Behavior; + speaking: Map>; + mediaDevices: MediaDevices; + initialSyncState: SyncState; +} + +const localParticipant = mockLocalParticipant({ identity: "" }); + +export function withCallViewModel( + { + remoteParticipants$ = constant([]), + rtcMembers$ = constant([localRtcMember]), + livekitConnectionState$: connectionState$ = constant( + ConnectionState.Connected, + ), + speaking = new Map(), + mediaDevices = mockMediaDevices({}), + initialSyncState = SyncState.Syncing, + }: Partial = {}, + continuation: ( + vm: CallViewModel, + rtcSession: MockRTCSession, + subjects: { raisedHands$: BehaviorSubject> }, + setSyncState: (value: SyncState) => void, + ) => void, + options: CallViewModelOptions = { + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + autoLeaveWhenOthersLeft: false, + }, +): void { + let syncState = initialSyncState; + const setSyncState = (value: SyncState): void => { + const prev = syncState; + syncState = value; + room.client.emit(ClientEvent.Sync, value, prev); + }; + const room = mockMatrixRoom({ + client: new (class extends EventEmitter { + public getUserId(): string | undefined { + return localRtcMember.userId; + } + + public getDeviceId(): string { + return localRtcMember.deviceId; + } + + public getDomain(): string { + return "example.com"; + } + + public getSyncState(): SyncState { + return syncState; + } + })() as Partial as MatrixClient, + getMembers: () => Array.from(roomMembers.values()), + }); + const rtcSession = new MockRTCSession(room, []).withMemberships(rtcMembers$); + const participantsSpy = vi + .spyOn(ComponentsCore, "connectedParticipantsObserver") + .mockReturnValue(remoteParticipants$); + const mediaSpy = vi + .spyOn(ComponentsCore, "observeParticipantMedia") + .mockImplementation((p) => + of({ participant: p } as Partial< + ComponentsCore.ParticipantMedia + > as ComponentsCore.ParticipantMedia), + ); + const eventsSpy = vi + .spyOn(ComponentsCore, "observeParticipantEvents") + .mockImplementation((p, ...eventTypes) => { + if (eventTypes.includes(ParticipantEvent.IsSpeakingChanged)) { + return (speaking.get(p) ?? of(false)).pipe( + map((s): Participant => ({ ...p, isSpeaking: s }) as Participant), + ); + } else { + return of(p); + } + }); + + const roomEventSelectorSpy = vi + .spyOn(ComponentsCore, "roomEventSelector") + .mockImplementation((_room, _eventType) => of()); + const muteStates = mockMuteStates(); + const raisedHands$ = new BehaviorSubject>({}); + const reactions$ = new BehaviorSubject>({}); + + const vm = new CallViewModel( + testScope(), + rtcSession.asMockedSession(), + room, + mediaDevices, + muteStates, + { + ...options, + livekitRoomFactory: (): LivekitRoom => + mockLivekitRoom({ + localParticipant, + disconnect: async () => Promise.resolve(), + setE2EEEnabled: async () => Promise.resolve(), + }), + connectionState$, + }, + raisedHands$, + reactions$, + new BehaviorSubject({ + processor: undefined, + supported: undefined, + }), + ); + + onTestFinished(() => { + participantsSpy.mockRestore(); + mediaSpy.mockRestore(); + eventsSpy.mockRestore(); + roomEventSelectorSpy.mockRestore(); + }); + + continuation(vm, rtcSession, { raisedHands$: raisedHands$ }, setSyncState); +} diff --git a/src/state/CallViewModelWidget.test.ts b/src/state/CallViewModelWidget.test.ts index 35a451f9..7947cc1a 100644 --- a/src/state/CallViewModelWidget.test.ts +++ b/src/state/CallViewModelWidget.test.ts @@ -12,8 +12,8 @@ import { constant } from "./Behavior.ts"; import { aliceParticipant, localRtcMember } from "../utils/test-fixtures.ts"; import { ElementWidgetActions, widget } from "../widget.ts"; import { E2eeType } from "../e2ee/e2eeType.ts"; -import { withCallViewModel } from "./CallViewModel/CallViewModel.test.ts"; import { type CallViewModel } from "./CallViewModel/CallViewModel.ts"; +import { withCallViewModel } from "./CallViewModel/CallViewModelTestUtils.test"; vi.mock("../widget", () => ({ ElementWidgetActions: { From b9ad83bf3b226a268c8d6fe2cb31f97c8755e16a Mon Sep 17 00:00:00 2001 From: Timo K Date: Fri, 14 Nov 2025 18:45:07 +0100 Subject: [PATCH 56/75] fix a couple of more tests. --- src/room/ReactionAudioRenderer.test.tsx | 222 +++++------ .../__snapshots__/InCallView.test.tsx.snap | 49 ++- .../CallNotificationLifecycle.test.ts | 364 ++++++++++-------- .../CallNotificationLifecycle.ts | 6 +- src/state/CallViewModel/CallViewModel.test.ts | 209 +--------- src/state/CallViewModel/CallViewModel.ts | 8 +- ...tils.test.ts => CallViewModelTestUtils.ts} | 5 + src/state/CallViewModelWidget.test.ts | 11 +- 8 files changed, 378 insertions(+), 496 deletions(-) rename src/state/CallViewModel/{CallViewModelTestUtils.test.ts => CallViewModelTestUtils.ts} (98%) diff --git a/src/room/ReactionAudioRenderer.test.tsx b/src/room/ReactionAudioRenderer.test.tsx index 31c0a0cb..988d43a6 100644 --- a/src/room/ReactionAudioRenderer.test.tsx +++ b/src/room/ReactionAudioRenderer.test.tsx @@ -10,8 +10,9 @@ import { afterAll, afterEach, beforeEach, + describe, expect, - test, + it, vitest, type MockedFunction, type Mock, @@ -49,122 +50,125 @@ vitest.mock("livekit-client/e2ee-worker?worker"); vitest.mock("../useAudioContext"); vitest.mock("../soundUtils"); -afterEach(() => { - vitest.resetAllMocks(); - playReactionsSoundSetting.setValue(playReactionsSoundSetting.defaultValue); - soundEffectVolumeSetting.setValue(soundEffectVolumeSetting.defaultValue); -}); - -afterAll(() => { - vitest.restoreAllMocks(); -}); - let playSound: Mock< NonNullable>["playSound"] >; -beforeEach(() => { - (prefetchSounds as MockedFunction).mockResolvedValue({ - sound: new ArrayBuffer(0), +describe("ReactionAudioRenderer", () => { + afterEach(() => { + playReactionsSoundSetting.setValue(playReactionsSoundSetting.defaultValue); + soundEffectVolumeSetting.setValue(soundEffectVolumeSetting.defaultValue); }); - playSound = vitest.fn(); - (useAudioContext as MockedFunction).mockReturnValue({ - playSound, - playSoundLooping: vitest.fn(), - soundDuration: {}, - }); -}); - -test("preloads all audio elements", () => { - const { vm } = getBasicCallViewModelEnvironment([local, alice]); - playReactionsSoundSetting.setValue(true); - render(); - expect(prefetchSounds).toHaveBeenCalledOnce(); -}); - -test("will play an audio sound when there is a reaction", () => { - const { vm, reactionsSubject$ } = getBasicCallViewModelEnvironment([ - local, - alice, - ]); - playReactionsSoundSetting.setValue(true); - render(); - - // Find the first reaction with a sound effect - const chosenReaction = ReactionSet.find((r) => !!r.sound); - if (!chosenReaction) { - throw Error( - "No reactions have sounds configured, this test cannot succeed", + beforeEach(() => { + (prefetchSounds as MockedFunction).mockResolvedValue( + { + sound: new ArrayBuffer(0), + }, ); - } - act(() => { - reactionsSubject$.next({ - [aliceRtcMember.deviceId]: { - reactionOption: chosenReaction, - expireAfter: new Date(0), + playSound = vitest.fn(); + (useAudioContext as MockedFunction).mockReturnValue( + { + playSound, + playSoundLooping: vitest.fn(), + soundDuration: {}, }, - }); - }); - expect(playSound).toHaveBeenCalledWith(chosenReaction.name); -}); - -test("will play the generic audio sound when there is soundless reaction", () => { - const { vm, reactionsSubject$ } = getBasicCallViewModelEnvironment([ - local, - alice, - ]); - playReactionsSoundSetting.setValue(true); - render(); - - // Find the first reaction with a sound effect - const chosenReaction = ReactionSet.find((r) => !r.sound); - if (!chosenReaction) { - throw Error( - "No reactions have sounds configured, this test cannot succeed", ); - } - act(() => { - reactionsSubject$.next({ - [aliceRtcMember.deviceId]: { - reactionOption: chosenReaction, - expireAfter: new Date(0), - }, - }); }); - expect(playSound).toHaveBeenCalledWith(GenericReaction.name); -}); - -test("will play multiple audio sounds when there are multiple different reactions", () => { - const { vm, reactionsSubject$ } = getBasicCallViewModelEnvironment([ - local, - alice, - ]); - playReactionsSoundSetting.setValue(true); - render(); - - // Find the first reaction with a sound effect - const [reaction1, reaction2] = ReactionSet.filter((r) => !!r.sound); - if (!reaction1 || !reaction2) { - throw Error( - "No reactions have sounds configured, this test cannot succeed", - ); - } - act(() => { - reactionsSubject$.next({ - [aliceRtcMember.deviceId]: { - reactionOption: reaction1, - expireAfter: new Date(0), - }, - [bobRtcMember.deviceId]: { - reactionOption: reaction2, - expireAfter: new Date(0), - }, - [localRtcMember.deviceId]: { - reactionOption: reaction1, - expireAfter: new Date(0), - }, - }); + afterAll(() => { + vitest.restoreAllMocks(); + }); + + it("preloads all audio elements", () => { + const { vm } = getBasicCallViewModelEnvironment([local, alice]); + playReactionsSoundSetting.setValue(true); + render(); + expect(prefetchSounds).toHaveBeenCalledOnce(); + }); + + it("will play an audio sound when there is a reaction", () => { + const { vm, reactionsSubject$ } = getBasicCallViewModelEnvironment([ + local, + alice, + ]); + playReactionsSoundSetting.setValue(true); + render(); + + // Find the first reaction with a sound effect + const chosenReaction = ReactionSet.find((r) => !!r.sound); + if (!chosenReaction) { + throw Error( + "No reactions have sounds configured, this test cannot succeed", + ); + } + act(() => { + reactionsSubject$.next({ + [aliceRtcMember.deviceId]: { + reactionOption: chosenReaction, + expireAfter: new Date(0), + }, + }); + }); + expect(playSound).toHaveBeenCalledWith(chosenReaction.name); + }); + + it("will play the generic audio sound when there is soundless reaction", () => { + const { vm, reactionsSubject$ } = getBasicCallViewModelEnvironment([ + local, + alice, + ]); + playReactionsSoundSetting.setValue(true); + render(); + + // Find the first reaction with a sound effect + const chosenReaction = ReactionSet.find((r) => !r.sound); + if (!chosenReaction) { + throw Error( + "No reactions have sounds configured, this test cannot succeed", + ); + } + act(() => { + reactionsSubject$.next({ + [aliceRtcMember.deviceId]: { + reactionOption: chosenReaction, + expireAfter: new Date(0), + }, + }); + }); + expect(playSound).toHaveBeenCalledWith(GenericReaction.name); + }); + + it("will play multiple audio sounds when there are multiple different reactions", () => { + const { vm, reactionsSubject$ } = getBasicCallViewModelEnvironment([ + local, + alice, + ]); + playReactionsSoundSetting.setValue(true); + render(); + + // Find the first reaction with a sound effect + const [reaction1, reaction2] = ReactionSet.filter((r) => !!r.sound); + if (!reaction1 || !reaction2) { + throw Error( + "No reactions have sounds configured, this test cannot succeed", + ); + } + act(() => { + reactionsSubject$.next({ + [aliceRtcMember.deviceId]: { + reactionOption: reaction1, + expireAfter: new Date(0), + }, + [bobRtcMember.deviceId]: { + reactionOption: reaction2, + expireAfter: new Date(0), + }, + [localRtcMember.deviceId]: { + reactionOption: reaction1, + expireAfter: new Date(0), + }, + }); + }); + expect(playSound).toHaveBeenCalledWith(reaction1.name); + expect(playSound).toHaveBeenCalledWith(reaction2.name); }); - expect(playSound).toHaveBeenCalledWith(reaction1.name); - expect(playSound).toHaveBeenCalledWith(reaction2.name); }); diff --git a/src/room/__snapshots__/InCallView.test.tsx.snap b/src/room/__snapshots__/InCallView.test.tsx.snap index a1b0d226..c610b1fb 100644 --- a/src/room/__snapshots__/InCallView.test.tsx.snap +++ b/src/room/__snapshots__/InCallView.test.tsx.snap @@ -49,6 +49,34 @@ exports[`InCallView > rendering > renders 1`] = `
+
+ + + + + + + 2 + +
rendering > renders 1`] = ` class="scrollingGrid grid" >
+ class="layer" + > +
+
+
+
-
+
{ }, localUser: localRtcMember, }; + const lifecycle = createCallNotificationLifecycle$(props); + expectObservable(lifecycle.callPickupState$).toBe("a 9ms b 29ms c", { a: "unknown", b: "ringing", @@ -68,6 +73,7 @@ describe("waitForCallPickup$", () => { }); }); }); + it("ringing -> success if someone joins before timeout is reached", () => { withTestScheduler(({ scope, hot, behavior, expectObservable }) => { // Someone joins at 20ms (both LiveKit participant and MatrixRTC member) @@ -154,179 +160,195 @@ describe("waitForCallPickup$", () => { }); it("stays null when waitForCallPickup=false", () => { - withTestScheduler(({ behavior, schedule, expectObservable }) => { - withCallViewModel( - { - remoteParticipants$: behavior("a--b", { - a: [], - b: [aliceParticipant], - }), - rtcMembers$: behavior("a--b", { + withTestScheduler(({ scope, hot, behavior, expectObservable }) => { + // Someone joins at 20ms (both LiveKit participant and MatrixRTC member) + const validProps: CallNotificationLifecycleProps = { + scope, + memberships$: scope.behavior( + behavior("a--b", { a: [localRtcMember], b: [localRtcMember, aliceRtcMember], - }), + }).pipe(trackEpoch()), + ), + sentCallNotification$: hot("10ms a", { + a: [mockRingEvent("$notif5", 30), mockLegacyRingEvent], + }), + receivedDecline$: hot(""), + options: { + waitForCallPickup: true, + autoLeaveWhenOthersLeft: false, }, - (vm, rtcSession) => { - schedule(" 5ms r", { - r: () => { - rtcSession.emit( - MatrixRTCSessionEvent.DidSendCallNotification, - mockRingEvent("$notif5", 30), - mockLegacyRingEvent, - ); - }, - }); - expectObservable(vm.callPickupState$).toBe("(n)", { - n: null, - }); - }, - { + localUser: localRtcMember, + }; + const propsDeactivated = { + ...validProps, + options: { + ...validProps.options, waitForCallPickup: false, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + }; + const lifecycle = createCallNotificationLifecycle$(propsDeactivated); + expectObservable(lifecycle.callPickupState$).toBe("n", { + n: null, + }); + const lifecycleReference = createCallNotificationLifecycle$(validProps); + expectObservable(lifecycleReference.callPickupState$).toBe("u--s", { + u: "unknown", + s: "success", + }); + }); + }); + + it("decline before timeout window ends -> decline", () => { + withTestScheduler(({ scope, hot, behavior, expectObservable }) => { + // Someone joins at 20ms (both LiveKit participant and MatrixRTC member) + const props: CallNotificationLifecycleProps = { + scope, + memberships$: scope.behavior( + behavior("a", { + a: [localRtcMember], + }).pipe(trackEpoch()), + ), + sentCallNotification$: hot("10ms a", { + a: [mockRingEvent("$decl1", 50), mockLegacyRingEvent], + }), + receivedDecline$: hot("40ms d", { + d: [ + new MatrixEvent({ + type: EventType.RTCDecline, + content: { + "m.relates_to": { + rel_type: "m.reference", + event_id: "$decl1", + }, + }, + }), + {} as Room, + undefined, + false, + {} as IRoomTimelineData, + ], + }), + options: { + waitForCallPickup: true, + autoLeaveWhenOthersLeft: false, + }, + localUser: localRtcMember, + }; + const lifecycle = createCallNotificationLifecycle$(props); + expectObservable(lifecycle.callPickupState$).toBe("a 9ms b 29ms e", { + a: "unknown", + b: "ringing", + e: "decline", + }); + }); + }); + it("decline after timeout window ends -> stays timeout", () => { + withTestScheduler(({ scope, hot, behavior, expectObservable }) => { + // Someone joins at 20ms (both LiveKit participant and MatrixRTC member) + const props: CallNotificationLifecycleProps = { + scope, + memberships$: scope.behavior( + behavior("a", { + a: [localRtcMember], + }).pipe(trackEpoch()), + ), + sentCallNotification$: hot("10ms a", { + a: [mockRingEvent("$decl", 20), mockLegacyRingEvent], + }), + receivedDecline$: hot("40ms d", { + d: [ + new MatrixEvent({ + type: EventType.RTCDecline, + content: { + "m.relates_to": { + rel_type: "m.reference", + event_id: "$decl", + }, + }, + }), + {} as Room, + undefined, + false, + {} as IRoomTimelineData, + ], + }), + options: { + waitForCallPickup: true, + autoLeaveWhenOthersLeft: false, + }, + localUser: localRtcMember, + }; + const lifecycle = createCallNotificationLifecycle$(props); + expectObservable(lifecycle.callPickupState$, "50ms !").toBe( + "a 9ms b 19ms e", + { + a: "unknown", + b: "ringing", + e: "timeout", }, ); }); }); - - // it("decline before timeout window ends -> decline", () => { - // withTestScheduler(({ schedule, expectObservable }) => { - // withCallViewModel( - // {}, - // (vm, rtcSession) => { - // // Notify at 10ms with 50ms lifetime, decline at 40ms with matching id - // schedule(" 10ms r 29ms d", { - // r: () => { - // rtcSession.emit( - // MatrixRTCSessionEvent.DidSendCallNotification, - // mockRingEvent("$decl1", 50), - // mockLegacyRingEvent, - // ); - // }, - // d: () => { - // // Emit decline timeline event with id matching the notification - // rtcSession.room.emit( - // MatrixRoomEvent.Timeline, - // new MatrixEvent({ - // type: EventType.RTCDecline, - // content: { - // "m.relates_to": { - // rel_type: "m.reference", - // event_id: "$decl1", - // }, - // }, - // }), - // rtcSession.room, - // undefined, - // false, - // {} as IRoomTimelineData, - // ); - // }, - // }); - // expectObservable(vm.callPickupState$).toBe("a 9ms b 29ms e", { - // a: "unknown", - // b: "ringing", - // e: "decline", - // }); - // }, - // { - // waitForCallPickup: true, - // encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - // }, - // ); - // }); - // }); - // it("decline after timeout window ends -> stays timeout", () => { - // withTestScheduler(({ schedule, expectObservable }) => { - // withCallViewModel( - // {}, - // (vm, rtcSession) => { - // // Notify at 10ms with 20ms lifetime (timeout at 30ms), decline at 40ms - // schedule(" 10ms r 20ms t 10ms d", { - // r: () => { - // rtcSession.emit( - // MatrixRTCSessionEvent.DidSendCallNotification, - // mockRingEvent("$decl2", 20), - // mockLegacyRingEvent, - // ); - // }, - // t: () => {}, - // d: () => { - // rtcSession.room.emit( - // MatrixRoomEvent.Timeline, - // new MatrixEvent({ - // event_id: "$decl2", - // type: "m.rtc.decline", - // }), - // rtcSession.room, - // undefined, - // false, - // {} as IRoomTimelineData, - // ); - // }, - // }); - // expectObservable(vm.callPickupState$).toBe("a 9ms b 19ms c", { - // a: "unknown", - // b: "ringing", - // c: "timeout", - // }); - // }, - // { - // waitForCallPickup: true, - // encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - // }, - // ); - // }); - // }); - // function testStaysRinging(declineEvent: Partial): void { - // withTestScheduler(({ schedule, expectObservable }) => { - // withCallViewModel( - // {}, - // (vm, rtcSession) => { - // // Notify at 10ms with id A, decline arrives at 20ms with id B - // schedule(" 10ms r 10ms d", { - // r: () => { - // rtcSession.emit( - // MatrixRTCSessionEvent.DidSendCallNotification, - // mockRingEvent("$right", 50), - // mockLegacyRingEvent, - // ); - // }, - // d: () => { - // rtcSession.room.emit( - // MatrixRoomEvent.Timeline, - // new MatrixEvent(declineEvent), - // rtcSession.room, - // undefined, - // false, - // {} as IRoomTimelineData, - // ); - // }, - // }); - // // We assert up to 21ms to see the ringing at 10ms and no change at 20ms - // expectObservable(vm.callPickupState$, "21ms !").toBe("a 9ms b", { - // a: "unknown", - // b: "ringing", - // }); - // }, - // { - // waitForCallPickup: true, - // encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - // }, - // ); - // }); - // } - // it("decline with wrong id is ignored (stays ringing)", () => { - // testStaysRinging({ - // event_id: "$wrong", - // type: "m.rtc.decline", - // sender: local.userId, - // }); - // }); - // it("decline with sender being the local user is ignored (stays ringing)", () => { - // testStaysRinging({ - // event_id: "$right", - // type: "m.rtc.decline", - // sender: alice.userId, - // }); - // }); + // + function testStaysRinging( + declineEvent: Partial, + expectDecline: boolean, + ): void { + withTestScheduler(({ scope, hot, behavior, expectObservable }) => { + // Someone joins at 20ms (both LiveKit participant and MatrixRTC member) + const props: CallNotificationLifecycleProps = { + scope, + memberships$: scope.behavior( + behavior("a", { + a: [localRtcMember], + }).pipe(trackEpoch()), + ), + sentCallNotification$: hot("10ms a", { + a: [mockRingEvent("$right", 50), mockLegacyRingEvent], + }), + receivedDecline$: hot("20ms d", { + d: [ + new MatrixEvent(declineEvent), + {} as Room, + undefined, + false, + {} as IRoomTimelineData, + ], + }), + options: { + waitForCallPickup: true, + autoLeaveWhenOthersLeft: false, + }, + localUser: localRtcMember, + }; + const lifecycle = createCallNotificationLifecycle$(props); + const marbles = expectDecline ? "a 9ms b 9ms d" : "a 9ms b"; + expectObservable(lifecycle.callPickupState$, "21ms !").toBe(marbles, { + a: "unknown", + b: "ringing", + d: "decline", + }); + }); + } + const reference = (refId?: string, sender?: string): Partial => ({ + event_id: "$decline", + type: EventType.RTCDecline, + sender: sender ?? "@other:example.org", + content: { + "m.relates_to": { + rel_type: "m.reference", + event_id: refId ?? "$right", + }, + }, + }); + it("decline reference works", () => { + testStaysRinging(reference(), true); + }); + it("decline with wrong id is ignored (stays ringing)", () => { + testStaysRinging(reference("$wrong"), false); + }); + it("decline with wrong id is ignored (stays ringing)", () => { + testStaysRinging(reference(undefined, local.userId), false); + }); }); diff --git a/src/state/CallViewModel/CallNotificationLifecycle.ts b/src/state/CallViewModel/CallNotificationLifecycle.ts index ba15a981..d1257b04 100644 --- a/src/state/CallViewModel/CallNotificationLifecycle.ts +++ b/src/state/CallViewModel/CallNotificationLifecycle.ts @@ -169,7 +169,8 @@ export function createCallNotificationLifecycle$({ event.getRelation()?.rel_type === "m.reference" && event.getRelation()?.event_id === notificationEvent.event_id && - event.getSender() !== localUser.userId, + event.getSender() !== localUser.userId && + callPickupState$.value !== "timeout", ), ), ), @@ -189,7 +190,8 @@ export function createCallNotificationLifecycle$({ return "success" as const; } // Show the ringing state of the most recent ringing attempt. - // as long as we have not yet sent an RTC notification event, ring will be null -> callPickupState$ = unknown. + // as long as we have not yet sent an RTC notification event or noone else joined, + // ring will be null -> callPickupState$ = unknown. return ring ?? ("unknown" as const); }, ) diff --git a/src/state/CallViewModel/CallViewModel.test.ts b/src/state/CallViewModel/CallViewModel.test.ts index 863ba16f..eb8d438d 100644 --- a/src/state/CallViewModel/CallViewModel.test.ts +++ b/src/state/CallViewModel/CallViewModel.test.ts @@ -18,15 +18,7 @@ import { of, switchMap, } from "rxjs"; -import { - SyncState, - type MatrixClient, - RoomEvent as MatrixRoomEvent, - MatrixEvent, - type IRoomTimelineData, - EventType, - type IEvent, -} from "matrix-js-sdk"; +import { SyncState, type MatrixClient } from "matrix-js-sdk"; import { ConnectionState, type LocalTrackPublication, @@ -65,7 +57,6 @@ import { import { E2eeType } from "../../e2ee/e2eeType.ts"; import type { RaisedHandInfo, ReactionInfo } from "../../reactions/index.ts"; import { - alice, aliceId, aliceParticipant, aliceRtcMember, @@ -83,11 +74,8 @@ import { type ElementCallError, MatrixRTCTransportMissingError, } from "../../utils/errors.ts"; -import { withCallViewModel } from "./CallViewModelTestUtils.test"; +import { withCallViewModel } from "./CallViewModelTestUtils.ts"; -mockConfig({ - livekit: { livekit_service_url: "http://my-default-service-url.com" }, -}); vi.mock("rxjs", async (importOriginal) => ({ ...(await importOriginal()), // Disable interval Observables for the following tests since the test @@ -258,7 +246,7 @@ const mockLegacyRingEvent = {} as { event_id: string } & ICallNotifyContent; describe("CallViewModel", () => { // TODO: Restore this test. It requires makeTransport to not be mocked, unlike // the rest of the tests in this file… what do we do? - test.skip("test missing RTC config error", async () => { + it.skip("test missing RTC config error", async () => { const rtcMemberships$ = new BehaviorSubject([]); const emitter = new EventEmitter(); const client = vi.mocked({ @@ -358,7 +346,7 @@ describe("CallViewModel", () => { }); }); - test("screen sharing activates spotlight layout", () => { + it.skip("screen sharing activates spotlight layout", () => { withTestScheduler(({ behavior, schedule, expectObservable }) => { // Start with no screen shares, then have Alice and Bob share their screens, // then return to no screen shares, then have just Alice share for a bit @@ -1129,7 +1117,7 @@ describe("CallViewModel", () => { }); describe("waitForCallPickup$", () => { - test("regression test: does stop ringing in case livekitConnectionState$ emits after didSendCallNotification$ has already emitted", () => { + it.skip("regression test: does stop ringing in case livekitConnectionState$ emits after didSendCallNotification$ has already emitted", () => { withTestScheduler(({ schedule, expectObservable, behavior }) => { withCallViewModel( { @@ -1164,7 +1152,7 @@ describe("CallViewModel", () => { }); }); - test("ringing -> unknown if we get disconnected", () => { + it.skip("ringing -> unknown if we get disconnected", () => { withTestScheduler(({ behavior, schedule, expectObservable }) => { const connectionState$ = new BehaviorSubject(ConnectionState.Connected); // Someone joins at 20ms (both LiveKit participant and MatrixRTC member) @@ -1208,190 +1196,9 @@ describe("CallViewModel", () => { ); }); }); - - test("stays null when waitForCallPickup=false", () => { - withTestScheduler(({ behavior, schedule, expectObservable }) => { - withCallViewModel( - { - remoteParticipants$: behavior("a--b", { - a: [], - b: [aliceParticipant], - }), - rtcMembers$: behavior("a--b", { - a: [localRtcMember], - b: [localRtcMember, aliceRtcMember], - }), - }, - (vm, rtcSession) => { - schedule(" 5ms r", { - r: () => { - rtcSession.emit( - MatrixRTCSessionEvent.DidSendCallNotification, - mockRingEvent("$notif5", 30), - mockLegacyRingEvent, - ); - }, - }); - expectObservable(vm.callPickupState$).toBe("(n)", { - n: null, - }); - }, - { - waitForCallPickup: false, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - }, - ); - }); - }); - - test("decline before timeout window ends -> decline", () => { - withTestScheduler(({ schedule, expectObservable }) => { - withCallViewModel( - {}, - (vm, rtcSession) => { - // Notify at 10ms with 50ms lifetime, decline at 40ms with matching id - schedule(" 10ms r 29ms d", { - r: () => { - rtcSession.emit( - MatrixRTCSessionEvent.DidSendCallNotification, - mockRingEvent("$decl1", 50), - mockLegacyRingEvent, - ); - }, - d: () => { - // Emit decline timeline event with id matching the notification - rtcSession.room.emit( - MatrixRoomEvent.Timeline, - new MatrixEvent({ - type: EventType.RTCDecline, - content: { - "m.relates_to": { - rel_type: "m.reference", - event_id: "$decl1", - }, - }, - }), - rtcSession.room, - undefined, - false, - {} as IRoomTimelineData, - ); - }, - }); - expectObservable(vm.callPickupState$).toBe("a 9ms b 29ms e", { - a: "unknown", - b: "ringing", - e: "decline", - }); - }, - { - waitForCallPickup: true, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - }, - ); - }); - }); - - test("decline after timeout window ends -> stays timeout", () => { - withTestScheduler(({ schedule, expectObservable }) => { - withCallViewModel( - {}, - (vm, rtcSession) => { - // Notify at 10ms with 20ms lifetime (timeout at 30ms), decline at 40ms - schedule(" 10ms r 20ms t 10ms d", { - r: () => { - rtcSession.emit( - MatrixRTCSessionEvent.DidSendCallNotification, - mockRingEvent("$decl2", 20), - mockLegacyRingEvent, - ); - }, - t: () => {}, - d: () => { - rtcSession.room.emit( - MatrixRoomEvent.Timeline, - new MatrixEvent({ - event_id: "$decl2", - type: "m.rtc.decline", - }), - rtcSession.room, - undefined, - false, - {} as IRoomTimelineData, - ); - }, - }); - expectObservable(vm.callPickupState$).toBe("a 9ms b 19ms c", { - a: "unknown", - b: "ringing", - c: "timeout", - }); - }, - { - waitForCallPickup: true, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - }, - ); - }); - }); - - function testStaysRinging(declineEvent: Partial): void { - withTestScheduler(({ schedule, expectObservable }) => { - withCallViewModel( - {}, - (vm, rtcSession) => { - // Notify at 10ms with id A, decline arrives at 20ms with id B - schedule(" 10ms r 10ms d", { - r: () => { - rtcSession.emit( - MatrixRTCSessionEvent.DidSendCallNotification, - mockRingEvent("$right", 50), - mockLegacyRingEvent, - ); - }, - d: () => { - rtcSession.room.emit( - MatrixRoomEvent.Timeline, - new MatrixEvent(declineEvent), - rtcSession.room, - undefined, - false, - {} as IRoomTimelineData, - ); - }, - }); - // We assert up to 21ms to see the ringing at 10ms and no change at 20ms - expectObservable(vm.callPickupState$, "21ms !").toBe("a 9ms b", { - a: "unknown", - b: "ringing", - }); - }, - { - waitForCallPickup: true, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - }, - ); - }); - } - - test("decline with wrong id is ignored (stays ringing)", () => { - testStaysRinging({ - event_id: "$wrong", - type: "m.rtc.decline", - sender: local.userId, - }); - }); - - test("decline with sender being the local user is ignored (stays ringing)", () => { - testStaysRinging({ - event_id: "$right", - type: "m.rtc.decline", - sender: alice.userId, - }); - }); }); - test("audio output changes when toggling earpiece mode", () => { + it.skip("audio output changes when toggling earpiece mode", () => { withTestScheduler(({ schedule, expectObservable }) => { getUrlParams.mockReturnValue({ controlledAudioDevices: true }); vi.mocked(ComponentsCore.createMediaDeviceObserver).mockReturnValue( @@ -1428,7 +1235,7 @@ describe("CallViewModel", () => { }); }); - test("media tracks are paused while reconnecting to MatrixRTC", () => { + it.skip("media tracks are paused while reconnecting to MatrixRTC", () => { withTestScheduler(({ schedule, expectObservable }) => { const trackRunning$ = new BehaviorSubject(true); const originalPublications = localParticipant.trackPublications; diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 72f10034..ff4a6269 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -199,14 +199,14 @@ export class CallViewModel { // memberships$ private memberships$ = createMemberships$(this.scope, this.matrixRTCSession); + // ------------------------------------------------------------------------ + // matrixLivekitMembers$ AND localMembership + private membershipsAndTransports = membershipsAndTransports$( this.scope, this.memberships$, ); - // ------------------------------------------------------------------------ - // matrixLivekitMembers$ AND localMembership - private localTransport$ = createLocalTransport$({ scope: this.scope, memberships$: this.memberships$, @@ -575,6 +575,8 @@ export class CallViewModel { ); // only public to expose to the view. + // TODO if we are in "unknown" state we need a loading rendering (or empty screen) + // Otherwise it looks like we already connected and only than the ringing starts which is weird. public readonly callPickupState$ = this.callLifecycle.callPickupState$; public readonly leaveSoundEffect$ = combineLatest([ diff --git a/src/state/CallViewModel/CallViewModelTestUtils.test.ts b/src/state/CallViewModel/CallViewModelTestUtils.ts similarity index 98% rename from src/state/CallViewModel/CallViewModelTestUtils.test.ts rename to src/state/CallViewModel/CallViewModelTestUtils.ts index 6cae1bfa..81671971 100644 --- a/src/state/CallViewModel/CallViewModelTestUtils.test.ts +++ b/src/state/CallViewModel/CallViewModelTestUtils.ts @@ -26,6 +26,7 @@ import { E2eeType } from "../../e2ee/e2eeType"; import { type RaisedHandInfo, type ReactionInfo } from "../../reactions"; import { CallViewModel, type CallViewModelOptions } from "./CallViewModel"; import { + mockConfig, mockLivekitRoom, mockLocalParticipant, mockMatrixRoom, @@ -49,6 +50,10 @@ import { type Behavior, constant } from "../Behavior"; import { type ProcessorState } from "../../livekit/TrackProcessorContext"; import { type MediaDevices } from "../MediaDevices"; +mockConfig({ + livekit: { livekit_service_url: "http://my-default-service-url.com" }, +}); + const carol = local; const dave = mockMatrixRoomMember(daveRTLRtcMember, { rawDisplayName: "Dave" }); diff --git a/src/state/CallViewModelWidget.test.ts b/src/state/CallViewModelWidget.test.ts index 7947cc1a..afcf69ba 100644 --- a/src/state/CallViewModelWidget.test.ts +++ b/src/state/CallViewModelWidget.test.ts @@ -5,15 +5,18 @@ Copyright 2025 Element Creations Ltd. Please see LICENSE in the repository root for full details. */ -import { test, vi, expect } from "vitest"; +import { it, vi, expect } from "vitest"; import EventEmitter from "events"; +// import * as ComponentsCore from "@livekit/components-core"; +import { withCallViewModel } from "./CallViewModel/CallViewModelTestUtils.ts"; +import { type CallViewModel } from "./CallViewModel/CallViewModel.ts"; import { constant } from "./Behavior.ts"; import { aliceParticipant, localRtcMember } from "../utils/test-fixtures.ts"; import { ElementWidgetActions, widget } from "../widget.ts"; import { E2eeType } from "../e2ee/e2eeType.ts"; -import { type CallViewModel } from "./CallViewModel/CallViewModel.ts"; -import { withCallViewModel } from "./CallViewModel/CallViewModelTestUtils.test"; + +vi.mock("@livekit/components-core", { spy: true }); vi.mock("../widget", () => ({ ElementWidgetActions: { @@ -31,7 +34,7 @@ vi.mock("../widget", () => ({ }, })); -test("expect leave when ElementWidgetActions.HangupCall is called", async () => { +it("expect leave when ElementWidgetActions.HangupCall is called", async () => { const pr = Promise.withResolvers(); withCallViewModel( { From 494ed7c126257a98963251b3e8d387811b644aa5 Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 14 Nov 2025 14:15:58 -0500 Subject: [PATCH 57/75] Fix the MatrixRTC mode developer controls They weren't responding properly to clicks, and make more sense as radio buttons. --- locales/en/app.json | 1 + src/settings/DeveloperSettingsTab.tsx | 115 +++++++++++++++----------- 2 files changed, 68 insertions(+), 48 deletions(-) diff --git a/locales/en/app.json b/locales/en/app.json index 2b9eae5a..55394718 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -73,6 +73,7 @@ "livekit_sfu": "LiveKit SFU: {{url}}", "matrix_id": "Matrix ID: {{id}}", "matrixRTCMode": { + "title": "MatrixRTC mode", "Comptibility": { "description": "Compatible with homeservers that do not support sticky events (but all other EC clients are v0.17.0 or later)", "label": "Compatibility: state events & multi SFU" diff --git a/src/settings/DeveloperSettingsTab.tsx b/src/settings/DeveloperSettingsTab.tsx index a8f485b6..60d9028d 100644 --- a/src/settings/DeveloperSettingsTab.tsx +++ b/src/settings/DeveloperSettingsTab.tsx @@ -12,6 +12,7 @@ import { useEffect, useMemo, useState, + useId, } from "react"; import { useTranslation } from "react-i18next"; import { @@ -19,6 +20,14 @@ import { type MatrixClient, } from "matrix-js-sdk"; import { logger } from "matrix-js-sdk/lib/logger"; +import { + Root as Form, + Heading, + HelpMessage, + InlineField, + Label, + RadioControl, +} from "@vector-im/compound-web"; import { FieldRow, InputField } from "../input/Input"; import { @@ -60,6 +69,13 @@ export const DeveloperSettingsTab: FC = ({ client, livekitRooms }) => { }, [client]); const [matrixRTCMode, setMatrixRTCMode] = useSetting(matrixRTCModeSetting); + const matrixRTCModeRadioGroup = useId(); + const onMatrixRTCModeChange = useCallback( + (e: ChangeEvent) => { + setMatrixRTCMode(e.target.value as MatrixRTCMode); + }, + [setMatrixRTCMode], + ); const [showConnectionStats, setShowConnectionStats] = useSetting( showConnectionStatsSetting, @@ -85,7 +101,7 @@ export const DeveloperSettingsTab: FC = ({ client, livekitRooms }) => { }, [livekitRooms]); return ( - <> +

{t("developer_mode.hostname", { hostname: window.location.hostname || "unknown", @@ -142,52 +158,6 @@ export const DeveloperSettingsTab: FC = ({ client, livekitRooms }) => { } /> - - ) => { - if (event.target.checked) setMatrixRTCMode(MatrixRTCMode.Legacy); - }, - [setMatrixRTCMode], - )} - /> - ) => { - if (event.target.checked) - setMatrixRTCMode(MatrixRTCMode.Compatibil); - }, - [setMatrixRTCMode], - )} - /> - ) => { - if (event.target.checked) - setMatrixRTCMode(MatrixRTCMode.Matrix_2_0); - }, - [setMatrixRTCMode], - )} - /> - = ({ client, livekitRooms }) => { )} />{" "} + + {t("developer_mode.matrixRTCMode.title")} + + + } + > + + + {t("developer_mode.matrixRTCMode.Legacy.description")} + + + + } + > + + + {t("developer_mode.matrixRTCMode.Comptibility.description")} + + + + } + > + + + {t("developer_mode.matrixRTCMode.Matrix_2_0.description")} + + {livekitRooms?.map((livekitRoom) => ( <>

@@ -254,6 +273,6 @@ export const DeveloperSettingsTab: FC = ({ client, livekitRooms }) => { 
{JSON.stringify(import.meta.env, null, 2)}

{t("developer_mode.url_params")}

{JSON.stringify(urlParams, null, 2)}
- + ); }; From 53aaa011b769b3ee18945ac9ee5565cbc71700a1 Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 14 Nov 2025 16:41:18 -0500 Subject: [PATCH 58/75] Remove stray comma This comma was showing up in the bottom left corner of the UI. --- src/main.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.tsx b/src/main.tsx index e41aaff8..946e0238 100644 --- a/src/main.tsx +++ b/src/main.tsx @@ -62,7 +62,7 @@ Initializer.initBeforeReact() .then(() => { root.render( - , + , ); }) From bb746dc8dc886f7cf0b9b83421d2813a173422ab Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 14 Nov 2025 18:45:02 -0500 Subject: [PATCH 59/75] Fix a resource leak in the track processor code --- src/livekit/TrackProcessorContext.tsx | 14 +++++++++----- src/state/CallViewModel/localMember/Publisher.ts | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/livekit/TrackProcessorContext.tsx b/src/livekit/TrackProcessorContext.tsx index bb13d5dc..02888466 100644 --- a/src/livekit/TrackProcessorContext.tsx +++ b/src/livekit/TrackProcessorContext.tsx @@ -28,6 +28,7 @@ import { } from "../settings/settings"; import { BlurBackgroundTransformer } from "./BlurBackgroundTransformer"; import { type Behavior } from "../state/Behavior"; +import { type ObservableScope } from "../state/ObservableScope"; //TODO-MULTI-SFU: This is not yet fully there. // it is a combination of exposing observable and react hooks. @@ -63,13 +64,17 @@ export function useTrackProcessorObservable$(): Observable { return state$; } +/** + * Updates your video tracks to always use the given processor. + */ export const trackProcessorSync = ( + scope: ObservableScope, videoTrack$: Behavior, processor$: Behavior, ): void => { - // TODO-MULTI-SFU: Bind to an ObservableScope to avoid leaking resources. - combineLatest([videoTrack$, processor$]).subscribe( - ([videoTrack, processorState]) => { + combineLatest([videoTrack$, processor$]) + .pipe(scope.bind()) + .subscribe(([videoTrack, processorState]) => { if (!processorState) return; if (!videoTrack) return; const { processor } = processorState; @@ -79,8 +84,7 @@ export const trackProcessorSync = ( if (!processor && videoTrack.getProcessor()) { void videoTrack.stopProcessor(); } - }, - ); + }); }; export const useTrackProcessorSync = ( diff --git a/src/state/CallViewModel/localMember/Publisher.ts b/src/state/CallViewModel/localMember/Publisher.ts index ff4afbd6..7be612e3 100644 --- a/src/state/CallViewModel/localMember/Publisher.ts +++ b/src/state/CallViewModel/localMember/Publisher.ts @@ -320,6 +320,6 @@ export class Publisher { ), null, ); - trackProcessorSync(track$, trackerProcessorState$); + trackProcessorSync(scope, track$, trackerProcessorState$); } } From cf8298cd69d4be6cfe4966c8cd54aa6d1871866c Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 14 Nov 2025 18:57:39 -0500 Subject: [PATCH 60/75] Fix lints --- locales/en/app.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/locales/en/app.json b/locales/en/app.json index 55394718..ff26fbe3 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -73,7 +73,6 @@ "livekit_sfu": "LiveKit SFU: {{url}}", "matrix_id": "Matrix ID: {{id}}", "matrixRTCMode": { - "title": "MatrixRTC mode", "Comptibility": { "description": "Compatible with homeservers that do not support sticky events (but all other EC clients are v0.17.0 or later)", "label": "Compatibility: state events & multi SFU" @@ -85,7 +84,8 @@ "Matrix_2_0": { "description": "Compatible only with homservers supporting sticky events and all EC clients v0.17.0 or later", "label": "Matrix 2.0: sticky events & multi SFU" - } + }, + "title": "MatrixRTC mode" }, "mute_all_audio": "Mute all audio (participants, reactions, join sounds)", "show_connection_stats": "Show connection statistics", From 9ad029d4ff27437b6e363d20d1eda82e437d0f90 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 17 Nov 2025 02:42:33 +0000 Subject: [PATCH 61/75] Update dependency livekit-client to v2.15.15 --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 601c6c66..07e3bac3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10110,8 +10110,8 @@ __metadata: linkType: hard "livekit-client@npm:^2.13.0": - version: 2.15.14 - resolution: "livekit-client@npm:2.15.14" + version: 2.15.16 + resolution: "livekit-client@npm:2.15.16" dependencies: "@livekit/mutex": "npm:1.1.1" "@livekit/protocol": "npm:1.42.2" @@ -10125,7 +10125,7 @@ __metadata: webrtc-adapter: "npm:^9.0.1" peerDependencies: "@types/dom-mediacapture-record": ^1 - checksum: 10c0/619a5932cbb2a7e797344f800db76d529e33f35029e48e0c50a4bccaf7e912fed617360ad745b30b6870b3de4318d714f26b02b7f61a0f0acd442d21f67fb417 + checksum: 10c0/22c8f80a31f6e4d860c6e0a54e3f424852f95bcb9a6d53f3090ecbc147836299205793765ec036aba9d2b18354e2df94d05161ac9c93e52469b18b1ac8af5045 languageName: node linkType: hard From 348f0dbfa81ebcc08ff532ee65f8312024bdaac9 Mon Sep 17 00:00:00 2001 From: Timo K Date: Mon, 17 Nov 2025 11:37:58 +0100 Subject: [PATCH 62/75] CallViewModel structure change. Clear input and outputs. --- src/state/CallViewModel/CallViewModel.ts | 2100 +++++++++-------- .../CallViewModel/CallViewModelFunction.ts | 1400 +++++++++++ src/state/CallViewModel/CallViewModelOld.ts | 1335 +++++++++++ 3 files changed, 3830 insertions(+), 1005 deletions(-) create mode 100644 src/state/CallViewModel/CallViewModelFunction.ts create mode 100644 src/state/CallViewModel/CallViewModelOld.ts diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index ff4a6269..ff2ed0cb 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -12,7 +12,7 @@ import { type Room as LivekitRoom, type RoomOptions, } from "livekit-client"; -import { type Room as MatrixRoom } from "matrix-js-sdk"; +import { type RoomMember, type Room as MatrixRoom } from "matrix-js-sdk"; import { combineLatest, distinctUntilChanged, @@ -106,6 +106,7 @@ import { type MatrixLivekitMember, } from "./remoteMembers/MatrixLivekitMembers.ts"; import { + type AutoLeaveReason, createCallNotificationLifecycle$, createReceivedDecline$, createSentCallNotification$, @@ -165,6 +166,11 @@ type AudioLivekitItem = { participants: string[]; url: string; }; + +type JoinReturn = ReturnType< + ReturnType["requestConnect"] +>; + /** * A view model providing all the application logic needed to show the in-call * UI (may eventually be expanded to cover the lobby and feedback screens in the @@ -174,463 +180,562 @@ type AudioLivekitItem = { // state and LiveKit state. We use the common terminology of room "members", RTC // "memberships", and LiveKit "participants". export class CallViewModel { - private readonly userId = this.matrixRoom.client.getUserId()!; - private readonly deviceId = this.matrixRoom.client.getDeviceId()!; + // lifecycle + public autoLeave$: Observable; + public callPickupState$: Behavior< + "unknown" | "ringing" | "timeout" | "decline" | "success" | null + >; + public leave$: Observable<"user" | AutoLeaveReason>; + public hangup: () => void; - private readonly livekitKeyProvider = getE2eeKeyProvider( - this.options.encryptionSystem, - this.matrixRTCSession, - ); + // joining + public join: () => JoinReturn; - // Each hbar seperates a block of input variables required for the CallViewModel to function. - // The outputs of this block is written under the hbar. - // - // For mocking purposes it is recommended to only mock the functions creating those outputs. - // All other fields are just temp computations for the mentioned output. - // The class does not need anything except the values underneath the bar. - // The creation of the values under the bar are all tested independently and testing the callViewModel Should - // not test their cretation. Call view model only needs: - // - memberships$ via createMemberships$ - // - localMembership via createLocalMembership$ - // - callLifecycle via createCallNotificationLifecycle$ - // - matrixMemberMetadataStore via createMatrixMemberMetadata$ + // screen sharing + public toggleScreenSharing: (() => void) | null; + public sharingScreen$: Behavior; - // ------------------------------------------------------------------------ - // memberships$ - private memberships$ = createMemberships$(this.scope, this.matrixRTCSession); + // UI interactions + public tapScreen: () => void; + public tapControls: () => void; + public hoverScreen: () => void; + public unhoverScreen: () => void; - // ------------------------------------------------------------------------ - // matrixLivekitMembers$ AND localMembership + // errors + public configError$: Behavior; - private membershipsAndTransports = membershipsAndTransports$( - this.scope, - this.memberships$, - ); + // participants and counts + public participantCount$: Behavior; + public audioParticipants$: Behavior; + public handsRaised$: Behavior>; + public reactions$: Behavior>; + public isOneOnOneWith$: Behavior | null>; + public localUserIsAlone$: Behavior; + // sounds and events + public joinSoundEffect$: Observable; + public leaveSoundEffect$: Observable; + public newHandRaised$: Observable<{ value: number; playSounds: boolean }>; + public newScreenShare$: Observable<{ value: number; playSounds: boolean }>; + public audibleReactions$: Observable; + public visibleReactions$: Behavior< + { sender: string; emoji: string; startX: number }[] + >; - private localTransport$ = createLocalTransport$({ - scope: this.scope, - memberships$: this.memberships$, - client: this.matrixRoom.client, - roomId: this.matrixRoom.roomId, - useOldestMember$: this.scope.behavior( - matrixRTCMode.value$.pipe(map((v) => v === MatrixRTCMode.Legacy)), - ), - }); + // window/layout + public windowMode$: Behavior; + public spotlightExpanded$: Behavior; + public toggleSpotlightExpanded$: Behavior<(() => void) | null>; + public gridMode$: Behavior; + public setGridMode: (value: GridMode) => void; - private connectionFactory = new ECConnectionFactory( - this.matrixRoom.client, - this.mediaDevices, - this.trackProcessorState$, - this.livekitKeyProvider, - getUrlParams().controlledAudioDevices, - this.options.livekitRoomFactory, - ); + // media view models and layout + public grid$: Behavior; + public spotlight$: Behavior; + public pip$: Behavior; + public layout$: Behavior; + public tileStoreGeneration$: Behavior; + public showSpotlightIndicators$: Behavior; + public showSpeakingIndicators$: Behavior; - private connectionManager = createConnectionManager$({ - scope: this.scope, - connectionFactory: this.connectionFactory, - inputTransports$: this.scope.behavior( - combineLatest( - [this.localTransport$, this.membershipsAndTransports.transports$], - (localTransport, transports) => { - const localTransportAsArray = localTransport ? [localTransport] : []; - return transports.mapInner((transports) => [ - ...localTransportAsArray, - ...transports, - ]); - }, + // header/footer visibility + public showHeader$: Behavior; + public showFooter$: Behavior; + + // audio routing + public earpieceMode$: Behavior; + public audioOutputSwitcher$: Behavior<{ + targetOutput: "earpiece" | "speaker"; + switch: () => void; + } | null>; + + // connection state + public reconnecting$: Behavior; + + // THIS has to be the last public field declaration + public constructor( + scope: ObservableScope, + // A call is permanently tied to a single Matrix room + matrixRTCSession: MatrixRTCSession, + matrixRoom: MatrixRoom, + mediaDevices: MediaDevices, + muteStates: MuteStates, + options: CallViewModelOptions, + handsRaisedSubject$: Observable>, + reactionsSubject$: Observable>, + trackProcessorState$: Behavior, + ) { + const userId = matrixRoom.client.getUserId()!; + const deviceId = matrixRoom.client.getDeviceId()!; + + const livekitKeyProvider = getE2eeKeyProvider( + options.encryptionSystem, + matrixRTCSession, + ); + + // Each hbar seperates a block of input variables required for the CallViewModel to function. + // The outputs of this block is written under the hbar. + // + // For mocking purposes it is recommended to only mock the functions creating those outputs. + // All other fields are just temp computations for the mentioned output. + // The class does not need anything except the values underneath the bar. + // The creation of the values under the bar are all tested independently and testing the callViewModel Should + // not test their cretation. Call view model only needs: + // - memberships$ via createMemberships$ + // - localMembership via createLocalMembership$ + // - callLifecycle via createCallNotificationLifecycle$ + // - matrixMemberMetadataStore via createMatrixMemberMetadata$ + + // ------------------------------------------------------------------------ + // memberships$ + const memberships$ = createMemberships$(scope, matrixRTCSession); + + // ------------------------------------------------------------------------ + // matrixLivekitMembers$ AND localMembership + + const membershipsAndTransports = membershipsAndTransports$( + scope, + memberships$, + ); + + const localTransport$ = createLocalTransport$({ + scope: scope, + memberships$: memberships$, + client: matrixRoom.client, + roomId: matrixRoom.roomId, + useOldestMember$: scope.behavior( + matrixRTCMode.value$.pipe(map((v) => v === MatrixRTCMode.Legacy)), ), - ), - logger: logger, - }); + }); - private matrixLivekitMembers$ = createMatrixLivekitMembers$({ - scope: this.scope, - membershipsWithTransport$: - this.membershipsAndTransports.membershipsWithTransport$, - connectionManager: this.connectionManager, - }); + const connectionFactory = new ECConnectionFactory( + matrixRoom.client, + mediaDevices, + trackProcessorState$, + livekitKeyProvider, + getUrlParams().controlledAudioDevices, + options.livekitRoomFactory, + ); - private connectOptions$ = this.scope.behavior( - matrixRTCMode.value$.pipe( - map((mode) => ({ - encryptMedia: this.livekitKeyProvider !== undefined, - // TODO. This might need to get called again on each cahnge of matrixRTCMode... - matrixRTCMode: mode, - })), - ), - ); - - private localMembership = createLocalMembership$({ - scope: this.scope, - muteStates: this.muteStates, - mediaDevices: this.mediaDevices, - connectionManager: this.connectionManager, - matrixRTCSession: this.matrixRTCSession, - matrixRoom: this.matrixRoom, - localTransport$: this.localTransport$, - trackProcessorState$: this.trackProcessorState$, - widget, - options: this.connectOptions$, - logger: logger.getChild(`[${Date.now()}]`), - }); - - private localRtcMembership$ = this.scope.behavior( - this.memberships$.pipe( - map( - (memberships) => - memberships.value.find( - (membership) => - membership.userId === this.userId && - membership.deviceId === this.deviceId, - ) ?? null, + const connectionManager = createConnectionManager$({ + scope: scope, + connectionFactory: connectionFactory, + inputTransports$: scope.behavior( + combineLatest( + [localTransport$, membershipsAndTransports.transports$], + (localTransport, transports) => { + const localTransportAsArray = localTransport + ? [localTransport] + : []; + return transports.mapInner((transports) => [ + ...localTransportAsArray, + ...transports, + ]); + }, + ), ), - ), - ); + logger: logger, + }); - private localMatrixLivekitMemberUninitialized = { - membership$: this.localRtcMembership$, - participant$: this.localMembership.participant$, - connection$: this.localMembership.connection$, - userId: this.userId, - }; + const matrixLivekitMembers$ = createMatrixLivekitMembers$({ + scope: scope, + membershipsWithTransport$: + membershipsAndTransports.membershipsWithTransport$, + connectionManager: connectionManager, + }); - private localMatrixLivekitMember$: Behavior = - this.scope.behavior( - this.localRtcMembership$.pipe( - switchMap((membership) => { - if (!membership) return of(null); - return of( - // casting is save here since we know that localRtcMembership$ is !== null since we reached this case. - this.localMatrixLivekitMemberUninitialized as MatrixLivekitMember, + const connectOptions$ = scope.behavior( + matrixRTCMode.value$.pipe( + map((mode) => ({ + encryptMedia: livekitKeyProvider !== undefined, + // TODO. This might need to get called again on each cahnge of matrixRTCMode... + matrixRTCMode: mode, + })), + ), + ); + + const localMembership = createLocalMembership$({ + scope: scope, + muteStates: muteStates, + mediaDevices: mediaDevices, + connectionManager: connectionManager, + matrixRTCSession: matrixRTCSession, + matrixRoom: matrixRoom, + localTransport$: localTransport$, + trackProcessorState$: trackProcessorState$, + widget, + options: connectOptions$, + logger: logger.getChild(`[${Date.now()}]`), + }); + + const localRtcMembership$ = scope.behavior( + memberships$.pipe( + map( + (memberships) => + memberships.value.find( + (membership) => + membership.userId === userId && + membership.deviceId === deviceId, + ) ?? null, + ), + ), + ); + + const localMatrixLivekitMemberUninitialized = { + membership$: localRtcMembership$, + participant$: localMembership.participant$, + connection$: localMembership.connection$, + userId: userId, + }; + + const localMatrixLivekitMember$: Behavior = + scope.behavior( + localRtcMembership$.pipe( + switchMap((membership) => { + if (!membership) return of(null); + return of( + // casting is save here since we know that localRtcMembership$ is !== null since we reached this case. + localMatrixLivekitMemberUninitialized as MatrixLivekitMember, + ); + }), + ), + ); + + // ------------------------------------------------------------------------ + // callLifecycle + + const callLifecycle = createCallNotificationLifecycle$({ + scope: scope, + memberships$: memberships$, + sentCallNotification$: createSentCallNotification$( + scope, + matrixRTCSession, + ), + receivedDecline$: createReceivedDecline$(matrixRoom), + options: options, + localUser: { userId: userId, deviceId: deviceId }, + }); + + // ------------------------------------------------------------------------ + // matrixMemberMetadataStore + + const matrixRoomMembers$ = createRoomMembers$(scope, matrixRoom); + const matrixMemberMetadataStore = createMatrixMemberMetadata$( + scope, + scope.behavior(memberships$.pipe(map((mems) => mems.value))), + matrixRoomMembers$, + ); + + /** + * Returns the Member {userId, getMxcAvatarUrl, rawDisplayName} of the other user in the call, if it's a one-on-one call. + */ + const isOneOnOneWith$ = scope.behavior( + matrixRoomMembers$.pipe( + map((roomMembersMap) => { + const otherMembers = Array.from(roomMembersMap.values()).filter( + (member) => member.userId !== userId, ); + return otherMembers.length === 1 ? otherMembers[0] : null; }), ), ); - // ------------------------------------------------------------------------ - // callLifecycle - - private callLifecycle = createCallNotificationLifecycle$({ - scope: this.scope, - memberships$: this.memberships$, - sentCallNotification$: createSentCallNotification$( - this.scope, - this.matrixRTCSession, - ), - receivedDecline$: createReceivedDecline$(this.matrixRoom), - options: this.options, - localUser: { userId: this.userId, deviceId: this.deviceId }, - }); - public autoLeave$ = this.callLifecycle.autoLeave$; - - // ------------------------------------------------------------------------ - // matrixMemberMetadataStore - - private matrixMemberMetadataStore = createMatrixMemberMetadata$( - this.scope, - this.scope.behavior(this.memberships$.pipe(map((mems) => mems.value))), - createRoomMembers$(this.scope, this.matrixRoom), - ); - - /** - * If there is a configuration error with the call (e.g. misconfigured E2EE). - * This is a fatal error that prevents the call from being created/joined. - * Should render a blocking error screen. - */ - public get configError$(): Behavior { - return this.localMembership.configError$; - } - - public join = this.localMembership.requestConnect; - - // CODESMELL? - // This is functionally the same Observable as leave$, except here it's - // hoisted to the top of the class. This enables the cyclic dependency between - // leave$ -> autoLeave$ -> callPickupState$ -> livekitConnectionState$ -> - // localConnection$ -> transports$ -> joined$ -> leave$. - private readonly leaveHoisted$ = new Subject< - "user" | "timeout" | "decline" | "allOthersLeft" - >(); - - /** - * Whether various media/event sources should pretend to be disconnected from - * all network input, even if their connection still technically works. - */ - // We do this when the app is in the 'reconnecting' state, because it might be - // that the LiveKit connection is still functional while the homeserver is - // down, for example, and we want to avoid making people worry that the app is - // in a split-brained state. - // DISCUSSION own membership manager ALSO this probably can be simplifis - public reconnecting$ = this.localMembership.reconnecting$; - private readonly pretendToBeDisconnected$ = this.reconnecting$; - - public readonly audioParticipants$ = this.scope.behavior( - this.matrixLivekitMembers$.pipe( - switchMap((membersWithEpoch) => { - const members = membersWithEpoch.value; - const a$ = combineLatest( - members.map((member) => - combineLatest([member.connection$, member.participant$]).pipe( - map(([connection, participant]) => { - // do not render audio for local participant - if (!connection || !participant || participant.isLocal) - return null; - const livekitRoom = connection.livekitRoom; - const url = connection.transport.livekit_service_url; - - return { url, livekitRoom, participant: participant.identity }; - }), - ), - ), - ); - return a$; - }), - map((members) => - members.reduce((acc, curr) => { - if (!curr) return acc; - - const existing = acc.find((item) => item.url === curr.url); - if (existing) { - existing.participants.push(curr.participant); - } else { - acc.push({ - livekitRoom: curr.livekitRoom, - participants: [curr.participant], - url: curr.url, - }); - } - return acc; - }, []), - ), - ), - [], - ); - - public readonly handsRaised$ = this.scope.behavior( - this.handsRaisedSubject$.pipe(pauseWhen(this.pretendToBeDisconnected$)), - ); - - public readonly reactions$ = this.scope.behavior( - this.reactionsSubject$.pipe( - map((v) => - Object.fromEntries( - Object.entries(v).map(([a, { reactionOption }]) => [ - a, - reactionOption, - ]), + const localUserIsAlone$ = scope.behavior( + matrixRoomMembers$.pipe( + map( + (roomMembersMap) => + roomMembersMap.size === 1 && + roomMembersMap.get(userId) !== undefined, ), ), - pauseWhen(this.pretendToBeDisconnected$), - ), - ); + ); - /** - * List of user media (camera feeds) that we want tiles for. - */ - // TODO this also needs the local participant to be added. - private readonly userMedia$ = this.scope.behavior( - combineLatest([ - this.localMatrixLivekitMember$, - this.matrixLivekitMembers$, - duplicateTiles.value$, - ]).pipe( - // Generate a collection of MediaItems from the list of expected (whether - // present or missing) LiveKit participants. - generateItems( - function* ([ - localMatrixLivekitMember, - { value: matrixLivekitMembers }, - duplicateTiles, - ]) { - let localParticipantId = undefined; - // add local member if available - if (localMatrixLivekitMember) { - const { userId, participant$, connection$, membership$ } = - localMatrixLivekitMember; - localParticipantId = `${userId}:${membership$.value.deviceId}`; // should be membership$.value.membershipID which is not optional - // const participantId = membership$.value.membershipID; - if (localParticipantId) { + // CODESMELL? + // This is functionally the same Observable as leave$, except here it's + // hoisted to the top of the class. This enables the cyclic dependency between + // leave$ -> autoLeave$ -> callPickupState$ -> livekitConnectionState$ -> + // localConnection$ -> transports$ -> joined$ -> leave$. + const leaveHoisted$ = new Subject< + "user" | "timeout" | "decline" | "allOthersLeft" + >(); + + /** + * Whether various media/event sources should pretend to be disconnected from + * all network input, even if their connection still technically works. + */ + // We do this when the app is in the 'reconnecting' state, because it might be + // that the LiveKit connection is still functional while the homeserver is + // down, for example, and we want to avoid making people worry that the app is + // in a split-brained state. + // DISCUSSION own membership manager ALSO this probably can be simplifis + /*PUBLIC*/ const reconnecting$ = localMembership.reconnecting$; + const pretendToBeDisconnected$ = reconnecting$; + + /*PUBLIC*/ const audioParticipants$ = scope.behavior( + matrixLivekitMembers$.pipe( + switchMap((membersWithEpoch) => { + const members = membersWithEpoch.value; + const a$ = combineLatest( + members.map((member) => + combineLatest([member.connection$, member.participant$]).pipe( + map(([connection, participant]) => { + // do not render audio for local participant + if (!connection || !participant || participant.isLocal) + return null; + const livekitRoom = connection.livekitRoom; + const url = connection.transport.livekit_service_url; + + return { + url, + livekitRoom, + participant: participant.identity, + }; + }), + ), + ), + ); + return a$; + }), + map((members) => + members.reduce((acc, curr) => { + if (!curr) return acc; + + const existing = acc.find((item) => item.url === curr.url); + if (existing) { + existing.participants.push(curr.participant); + } else { + acc.push({ + livekitRoom: curr.livekitRoom, + participants: [curr.participant], + url: curr.url, + }); + } + return acc; + }, []), + ), + ), + [], + ); + + /*PUBLIC*/ const handsRaised$ = scope.behavior( + handsRaisedSubject$.pipe(pauseWhen(pretendToBeDisconnected$)), + ); + + /*PUBLIC*/ const reactions$ = scope.behavior( + reactionsSubject$.pipe( + map((v) => + Object.fromEntries( + Object.entries(v).map(([a, { reactionOption }]) => [ + a, + reactionOption, + ]), + ), + ), + pauseWhen(pretendToBeDisconnected$), + ), + ); + + /** + * List of user media (camera feeds) that we want tiles for. + */ + // TODO this also needs the local participant to be added. + const userMedia$ = scope.behavior( + combineLatest([ + localMatrixLivekitMember$, + matrixLivekitMembers$, + duplicateTiles.value$, + ]).pipe( + // Generate a collection of MediaItems from the list of expected (whether + // present or missing) LiveKit participants. + generateItems( + function* ([ + localMatrixLivekitMember, + { value: matrixLivekitMembers }, + duplicateTiles, + ]) { + let localParticipantId = undefined; + // add local member if available + if (localMatrixLivekitMember) { + const { userId, participant$, connection$, membership$ } = + localMatrixLivekitMember; + localParticipantId = `${userId}:${membership$.value.deviceId}`; // should be membership$.value.membershipID which is not optional + // const participantId = membership$.value.membershipID; + if (localParticipantId) { + for (let dup = 0; dup < 1 + duplicateTiles; dup++) { + yield { + keys: [ + dup, + localParticipantId, + userId, + participant$, + connection$, + ], + data: undefined, + }; + } + } + } + // add remote members that are available + for (const { + userId, + participant$, + connection$, + membership$, + } of matrixLivekitMembers) { + const participantId = `${userId}:${membership$.value.deviceId}`; + if (participantId === localParticipantId) continue; + // const participantId = membership$.value?.identity; for (let dup = 0; dup < 1 + duplicateTiles; dup++) { yield { - keys: [ - dup, - localParticipantId, - userId, - participant$, - connection$, - ], + keys: [dup, participantId, userId, participant$, connection$], data: undefined, }; } } - } - // add remote members that are available - for (const { + }, + ( + scope, + _data$, + dup, + participantId, userId, participant$, connection$, - membership$, - } of matrixLivekitMembers) { - const participantId = `${userId}:${membership$.value.deviceId}`; - if (participantId === localParticipantId) continue; - // const participantId = membership$.value?.identity; - for (let dup = 0; dup < 1 + duplicateTiles; dup++) { - yield { - keys: [dup, participantId, userId, participant$, connection$], - data: undefined, - }; - } - } - }, - ( - scope, - _data$, - dup, - participantId, - userId, - participant$, - connection$, - ) => { - const livekitRoom$ = scope.behavior( - connection$.pipe(map((c) => c?.livekitRoom)), - ); - const focusUrl$ = scope.behavior( - connection$.pipe(map((c) => c?.transport.livekit_service_url)), - ); - const displayName$ = scope.behavior( - this.matrixMemberMetadataStore - .createDisplayNameBehavior$(userId) - .pipe(map((name) => name ?? userId)), - ); + ) => { + const livekitRoom$ = scope.behavior( + connection$.pipe(map((c) => c?.livekitRoom)), + ); + const focusUrl$ = scope.behavior( + connection$.pipe(map((c) => c?.transport.livekit_service_url)), + ); + const displayName$ = scope.behavior( + matrixMemberMetadataStore + .createDisplayNameBehavior$(userId) + .pipe(map((name) => name ?? userId)), + ); - return new UserMedia( - scope, - `${participantId}:${dup}`, - userId, - participant$, - this.options.encryptionSystem, - livekitRoom$, - focusUrl$, - this.mediaDevices, - this.pretendToBeDisconnected$, - displayName$, - this.matrixMemberMetadataStore.createAvatarUrlBehavior$(userId), - this.handsRaised$.pipe(map((v) => v[participantId]?.time ?? null)), - this.reactions$.pipe(map((v) => v[participantId] ?? undefined)), - ); - }, + return new UserMedia( + scope, + `${participantId}:${dup}`, + userId, + participant$, + options.encryptionSystem, + livekitRoom$, + focusUrl$, + mediaDevices, + pretendToBeDisconnected$, + displayName$, + matrixMemberMetadataStore.createAvatarUrlBehavior$(userId), + handsRaised$.pipe(map((v) => v[participantId]?.time ?? null)), + reactions$.pipe(map((v) => v[participantId] ?? undefined)), + ); + }, + ), ), - ), - ); + ); - /** - * List of all media items (user media and screen share media) that we want - * tiles for. - */ - private readonly mediaItems$ = this.scope.behavior( - this.userMedia$.pipe( - switchMap((userMedia) => - userMedia.length === 0 - ? of([]) - : combineLatest( - userMedia.map((m) => m.screenShares$), - (...screenShares) => [...userMedia, ...screenShares.flat(1)], - ), + /** + * List of all media items (user media and screen share media) that we want + * tiles for. + */ + const mediaItems$ = scope.behavior( + userMedia$.pipe( + switchMap((userMedia) => + userMedia.length === 0 + ? of([]) + : combineLatest( + userMedia.map((m) => m.screenShares$), + (...screenShares) => [...userMedia, ...screenShares.flat(1)], + ), + ), ), - ), - ); + ); - /** - * List of MediaItems that we want to display, that are of type ScreenShare - */ - private readonly screenShares$ = this.scope.behavior( - this.mediaItems$.pipe( - map((mediaItems) => - mediaItems.filter((m): m is ScreenShare => m instanceof ScreenShare), + /** + * List of MediaItems that we want to display, that are of type ScreenShare + */ + const screenShares$ = scope.behavior( + mediaItems$.pipe( + map((mediaItems) => + mediaItems.filter((m): m is ScreenShare => m instanceof ScreenShare), + ), ), - ), - ); + ); - public readonly joinSoundEffect$ = this.userMedia$.pipe( - pairwise(), - filter( - ([prev, current]) => - current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && - current.length > prev.length, - ), - map(() => {}), - throttleTime(THROTTLE_SOUND_EFFECT_MS), - ); + /*PUBLIC*/ const joinSoundEffect$ = userMedia$.pipe( + pairwise(), + filter( + ([prev, current]) => + current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && + current.length > prev.length, + ), + map(() => {}), + throttleTime(THROTTLE_SOUND_EFFECT_MS), + ); - /** - * The number of participants currently in the call. - * - * - Each participant has a corresponding MatrixRTC membership state event - * - There can be multiple participants for one Matrix user if they join from - * multiple devices. - */ - public readonly participantCount$ = this.scope.behavior( - this.matrixLivekitMembers$.pipe(map((ms) => ms.value.length)), - ); + /** + * The number of participants currently in the call. + * + * - Each participant has a corresponding MatrixRTC membership state event + * - There can be multiple participants for one Matrix user if they join from + * multiple devices. + */ + /*PUBLIC*/ const participantCount$ = scope.behavior( + matrixLivekitMembers$.pipe(map((ms) => ms.value.length)), + ); - // only public to expose to the view. - // TODO if we are in "unknown" state we need a loading rendering (or empty screen) - // Otherwise it looks like we already connected and only than the ringing starts which is weird. - public readonly callPickupState$ = this.callLifecycle.callPickupState$; + // only public to expose to the view. + // TODO if we are in "unknown" state we need a loading rendering (or empty screen) + // Otherwise it looks like we already connected and only than the ringing starts which is weird. + /*PUBLIC*/ const callPickupState$ = callLifecycle.callPickupState$; - public readonly leaveSoundEffect$ = combineLatest([ - this.callLifecycle.callPickupState$, - this.userMedia$, - ]).pipe( - // Until the call is successful, do not play a leave sound. - // If callPickupState$ is null, then we always play the sound as it will not conflict with a decline sound. - skipWhile(([c]) => c !== null && c !== "success"), - map(([, userMedia]) => userMedia), - pairwise(), - filter( - ([prev, current]) => - current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && - current.length < prev.length, - ), - map(() => {}), - throttleTime(THROTTLE_SOUND_EFFECT_MS), - ); + /*PUBLIC*/ const leaveSoundEffect$ = combineLatest([ + callLifecycle.callPickupState$, + userMedia$, + ]).pipe( + // Until the call is successful, do not play a leave sound. + // If callPickupState$ is null, then we always play the sound as it will not conflict with a decline sound. + skipWhile(([c]) => c !== null && c !== "success"), + map(([, userMedia]) => userMedia), + pairwise(), + filter( + ([prev, current]) => + current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && + current.length < prev.length, + ), + map(() => {}), + throttleTime(THROTTLE_SOUND_EFFECT_MS), + ); - private readonly userHangup$ = new Subject(); - public hangup(): void { - this.userHangup$.next(); - } + const userHangup$ = new Subject(); - private readonly widgetHangup$ = - widget === null - ? NEVER - : ( - fromEvent( - widget.lazyActions, - ElementWidgetActions.HangupCall, - ) as Observable> - ).pipe( - tap((ev) => { - widget!.api.transport.reply(ev.detail, {}); - }), - ); + const widgetHangup$ = + widget === null + ? NEVER + : ( + fromEvent( + widget.lazyActions, + ElementWidgetActions.HangupCall, + ) as Observable> + ).pipe( + tap((ev) => { + widget!.api.transport.reply(ev.detail, {}); + }), + ); - public readonly leave$: Observable< - "user" | "timeout" | "decline" | "allOthersLeft" - > = merge( - this.callLifecycle.autoLeave$, - merge(this.userHangup$, this.widgetHangup$).pipe( - map(() => "user" as const), - ), - ).pipe( - this.scope.share, - tap((reason) => this.leaveHoisted$.next(reason)), - ); + const leave$: Observable<"user" | "timeout" | "decline" | "allOthersLeft"> = + merge( + callLifecycle.autoLeave$, + merge(userHangup$, widgetHangup$).pipe(map(() => "user" as const)), + ).pipe( + scope.share, + tap((reason) => leaveHoisted$.next(reason)), + ); - private readonly spotlightSpeaker$ = - this.scope.behavior( - this.userMedia$.pipe( + const spotlightSpeaker$ = scope.behavior( + userMedia$.pipe( switchMap((mediaItems) => mediaItems.length === 0 ? of([]) @@ -665,653 +770,638 @@ export class CallViewModel { ), ); - private readonly grid$ = this.scope.behavior( - this.userMedia$.pipe( - switchMap((mediaItems) => { - const bins = mediaItems.map((m) => - m.bin$.pipe(map((bin) => [m, bin] as const)), - ); - // Sort the media by bin order and generate a tile for each one - return bins.length === 0 - ? of([]) - : combineLatest(bins, (...bins) => - bins.sort(([, bin1], [, bin2]) => bin1 - bin2).map(([m]) => m.vm), - ); - }), - distinctUntilChanged(shallowEquals), - ), - ); + const grid$ = scope.behavior( + userMedia$.pipe( + switchMap((mediaItems) => { + const bins = mediaItems.map((m) => + m.bin$.pipe(map((bin) => [m, bin] as const)), + ); + // Sort the media by bin order and generate a tile for each one + return bins.length === 0 + ? of([]) + : combineLatest(bins, (...bins) => + bins + .sort(([, bin1], [, bin2]) => bin1 - bin2) + .map(([m]) => m.vm), + ); + }), + distinctUntilChanged(shallowEquals), + ), + ); - private readonly spotlight$ = this.scope.behavior( - this.screenShares$.pipe( - switchMap((screenShares) => { - if (screenShares.length > 0) { - return of(screenShares.map((m) => m.vm)); - } + const spotlight$ = scope.behavior( + screenShares$.pipe( + switchMap((screenShares) => { + if (screenShares.length > 0) { + return of(screenShares.map((m) => m.vm)); + } - return this.spotlightSpeaker$.pipe( - map((speaker) => (speaker ? [speaker] : [])), - ); - }), - distinctUntilChanged(shallowEquals), - ), - ); + return spotlightSpeaker$.pipe( + map((speaker) => (speaker ? [speaker] : [])), + ); + }), + distinctUntilChanged(shallowEquals), + ), + ); - private readonly pip$ = this.scope.behavior( - combineLatest([ - // TODO This also needs epoch logic to dedupe the screenshares and mediaItems emits - this.screenShares$, - this.spotlightSpeaker$, - this.mediaItems$, - ]).pipe( - switchMap(([screenShares, spotlight, mediaItems]) => { - if (screenShares.length > 0) { - return this.spotlightSpeaker$; - } - if (!spotlight || spotlight.local) { - return of(null); - } + const pip$ = scope.behavior( + combineLatest([ + // TODO This also needs epoch logic to dedupe the screenshares and mediaItems emits + screenShares$, + spotlightSpeaker$, + mediaItems$, + ]).pipe( + switchMap(([screenShares, spotlight, mediaItems]) => { + if (screenShares.length > 0) { + return spotlightSpeaker$; + } + if (!spotlight || spotlight.local) { + return of(null); + } - const localUserMedia = mediaItems.find( - (m) => m.vm instanceof LocalUserMediaViewModel, - ) as UserMedia | undefined; + const localUserMedia = mediaItems.find( + (m) => m.vm instanceof LocalUserMediaViewModel, + ) as UserMedia | undefined; - const localUserMediaViewModel = localUserMedia?.vm as - | LocalUserMediaViewModel - | undefined; + const localUserMediaViewModel = localUserMedia?.vm as + | LocalUserMediaViewModel + | undefined; - if (!localUserMediaViewModel) { - return of(null); - } - return localUserMediaViewModel.alwaysShow$.pipe( - map((alwaysShow) => { - if (alwaysShow) { - return localUserMediaViewModel; - } + if (!localUserMediaViewModel) { + return of(null); + } + return localUserMediaViewModel.alwaysShow$.pipe( + map((alwaysShow) => { + if (alwaysShow) { + return localUserMediaViewModel; + } - return null; - }), - ); - }), - ), - ); + return null; + }), + ); + }), + ), + ); - private readonly hasRemoteScreenShares$: Observable = - this.spotlight$.pipe( + const hasRemoteScreenShares$: Observable = spotlight$.pipe( map((spotlight) => spotlight.some((vm) => !vm.local && vm instanceof ScreenShareViewModel), ), distinctUntilChanged(), ); - private readonly pipEnabled$ = this.scope.behavior(setPipEnabled$, false); + const pipEnabled$ = scope.behavior(setPipEnabled$, false); - private readonly naturalWindowMode$ = this.scope.behavior( - fromEvent(window, "resize").pipe( - map(() => { - const height = window.innerHeight; - const width = window.innerWidth; - if (height <= 400 && width <= 340) return "pip"; - // Our layouts for flat windows are better at adapting to a small width - // than our layouts for narrow windows are at adapting to a small height, - // so we give "flat" precedence here - if (height <= 600) return "flat"; - if (width <= 600) return "narrow"; - return "normal"; - }), - ), - "normal", - ); - - /** - * The general shape of the window. - */ - public readonly windowMode$ = this.scope.behavior( - this.pipEnabled$.pipe( - switchMap((pip) => - pip ? of("pip") : this.naturalWindowMode$, + const naturalWindowMode$ = scope.behavior( + fromEvent(window, "resize").pipe( + map(() => { + const height = window.innerHeight; + const width = window.innerWidth; + if (height <= 400 && width <= 340) return "pip"; + // Our layouts for flat windows are better at adapting to a small width + // than our layouts for narrow windows are at adapting to a small height, + // so we give "flat" precedence here + if (height <= 600) return "flat"; + if (width <= 600) return "narrow"; + return "normal"; + }), ), - ), - ); + "normal", + ); - private readonly spotlightExpandedToggle$ = new Subject(); - public readonly spotlightExpanded$ = this.scope.behavior( - this.spotlightExpandedToggle$.pipe( - accumulate(false, (expanded) => !expanded), - ), - ); + /** + * The general shape of the window. + */ + /*PUBLIC*/ const windowMode$ = scope.behavior( + pipEnabled$.pipe( + switchMap((pip) => (pip ? of("pip") : naturalWindowMode$)), + ), + ); - private readonly gridModeUserSelection$ = new Subject(); - /** - * The layout mode of the media tile grid. - */ - public readonly gridMode$ = - // If the user hasn't selected spotlight and somebody starts screen sharing, - // automatically switch to spotlight mode and reset when screen sharing ends - this.scope.behavior( - this.gridModeUserSelection$.pipe( - switchMap((userSelection) => - (userSelection === "spotlight" - ? EMPTY - : combineLatest([ - this.hasRemoteScreenShares$, - this.windowMode$, - ]).pipe( - skip(userSelection === null ? 0 : 1), - map( - ([hasScreenShares, windowMode]): GridMode => - hasScreenShares || windowMode === "flat" - ? "spotlight" - : "grid", - ), - ) - ).pipe(startWith(userSelection ?? "grid")), + const spotlightExpandedToggle$ = new Subject(); + /*PUBLIC*/ const spotlightExpanded$ = scope.behavior( + spotlightExpandedToggle$.pipe(accumulate(false, (expanded) => !expanded)), + ); + + const gridModeUserSelection$ = new Subject(); + /** + * The layout mode of the media tile grid. + */ + /*PUBLIC*/ const gridMode$ = + // If the user hasn't selected spotlight and somebody starts screen sharing, + // automatically switch to spotlight mode and reset when screen sharing ends + scope.behavior( + gridModeUserSelection$.pipe( + switchMap((userSelection) => + (userSelection === "spotlight" + ? EMPTY + : combineLatest([hasRemoteScreenShares$, windowMode$]).pipe( + skip(userSelection === null ? 0 : 1), + map( + ([hasScreenShares, windowMode]): GridMode => + hasScreenShares || windowMode === "flat" + ? "spotlight" + : "grid", + ), + ) + ).pipe(startWith(userSelection ?? "grid")), + ), ), - ), - "grid", - ); + "grid", + ); - public setGridMode(value: GridMode): void { - this.gridModeUserSelection$.next(value); - } + /*PUBLIC*/ const setGridMode = (value: GridMode): void => { + gridModeUserSelection$.next(value); + }; - private readonly gridLayoutMedia$: Observable = - combineLatest([this.grid$, this.spotlight$], (grid, spotlight) => ({ - type: "grid", - spotlight: spotlight.some((vm) => vm instanceof ScreenShareViewModel) - ? spotlight - : undefined, - grid, - })); - - private readonly spotlightLandscapeLayoutMedia$: Observable = - combineLatest([this.grid$, this.spotlight$], (grid, spotlight) => ({ - type: "spotlight-landscape", - spotlight, - grid, - })); - - private readonly spotlightPortraitLayoutMedia$: Observable = - combineLatest([this.grid$, this.spotlight$], (grid, spotlight) => ({ - type: "spotlight-portrait", - spotlight, - grid, - })); - - private readonly spotlightExpandedLayoutMedia$: Observable = - combineLatest([this.spotlight$, this.pip$], (spotlight, pip) => ({ - type: "spotlight-expanded", - spotlight, - pip: pip ?? undefined, - })); - - private readonly oneOnOneLayoutMedia$: Observable = - this.mediaItems$.pipe( - map((mediaItems) => { - if (mediaItems.length !== 2) return null; - const local = mediaItems.find((vm) => vm.vm.local)?.vm as - | LocalUserMediaViewModel - | undefined; - const remote = mediaItems.find((vm) => !vm.vm.local)?.vm as - | RemoteUserMediaViewModel - | undefined; - // There might not be a remote tile if there are screen shares, or if - // only the local user is in the call and they're using the duplicate - // tiles option - if (!remote || !local) return null; - - return { type: "one-on-one", local, remote }; + const gridLayoutMedia$: Observable = combineLatest( + [grid$, spotlight$], + (grid, spotlight) => ({ + type: "grid", + spotlight: spotlight.some((vm) => vm instanceof ScreenShareViewModel) + ? spotlight + : undefined, + grid, }), ); - private readonly pipLayoutMedia$: Observable = - this.spotlight$.pipe(map((spotlight) => ({ type: "pip", spotlight }))); + const spotlightLandscapeLayoutMedia$: Observable = + combineLatest([grid$, spotlight$], (grid, spotlight) => ({ + type: "spotlight-landscape", + spotlight, + grid, + })); - /** - * The media to be used to produce a layout. - */ - private readonly layoutMedia$ = this.scope.behavior( - this.windowMode$.pipe( - switchMap((windowMode) => { - switch (windowMode) { - case "normal": - return this.gridMode$.pipe( - switchMap((gridMode) => { - switch (gridMode) { - case "grid": - return this.oneOnOneLayoutMedia$.pipe( - switchMap((oneOnOne) => - oneOnOne === null - ? this.gridLayoutMedia$ - : of(oneOnOne), - ), - ); - case "spotlight": - return this.spotlightExpanded$.pipe( - switchMap((expanded) => - expanded - ? this.spotlightExpandedLayoutMedia$ - : this.spotlightLandscapeLayoutMedia$, - ), - ); - } - }), - ); - case "narrow": - return this.oneOnOneLayoutMedia$.pipe( - switchMap((oneOnOne) => - oneOnOne === null - ? combineLatest( - [this.grid$, this.spotlight$], - (grid, spotlight) => + const spotlightPortraitLayoutMedia$: Observable = + combineLatest([grid$, spotlight$], (grid, spotlight) => ({ + type: "spotlight-portrait", + spotlight, + grid, + })); + + const spotlightExpandedLayoutMedia$: Observable = + combineLatest([spotlight$, pip$], (spotlight, pip) => ({ + type: "spotlight-expanded", + spotlight, + pip: pip ?? undefined, + })); + + const oneOnOneLayoutMedia$: Observable = + mediaItems$.pipe( + map((mediaItems) => { + if (mediaItems.length !== 2) return null; + const local = mediaItems.find((vm) => vm.vm.local)?.vm as + | LocalUserMediaViewModel + | undefined; + const remote = mediaItems.find((vm) => !vm.vm.local)?.vm as + | RemoteUserMediaViewModel + | undefined; + // There might not be a remote tile if there are screen shares, or if + // only the local user is in the call and they're using the duplicate + // tiles option + if (!remote || !local) return null; + + return { type: "one-on-one", local, remote }; + }), + ); + + const pipLayoutMedia$: Observable = spotlight$.pipe( + map((spotlight) => ({ type: "pip", spotlight })), + ); + + /** + * The media to be used to produce a layout. + */ + const layoutMedia$ = scope.behavior( + windowMode$.pipe( + switchMap((windowMode) => { + switch (windowMode) { + case "normal": + return gridMode$.pipe( + switchMap((gridMode) => { + switch (gridMode) { + case "grid": + return oneOnOneLayoutMedia$.pipe( + switchMap((oneOnOne) => + oneOnOne === null ? gridLayoutMedia$ : of(oneOnOne), + ), + ); + case "spotlight": + return spotlightExpanded$.pipe( + switchMap((expanded) => + expanded + ? spotlightExpandedLayoutMedia$ + : spotlightLandscapeLayoutMedia$, + ), + ); + } + }), + ); + case "narrow": + return oneOnOneLayoutMedia$.pipe( + switchMap((oneOnOne) => + oneOnOne === null + ? combineLatest([grid$, spotlight$], (grid, spotlight) => grid.length > smallMobileCallThreshold || spotlight.some( (vm) => vm instanceof ScreenShareViewModel, ) - ? this.spotlightPortraitLayoutMedia$ - : this.gridLayoutMedia$, - ).pipe(switchAll()) - : // The expanded spotlight layout makes for a better one-on-one - // experience in narrow windows - this.spotlightExpandedLayoutMedia$, - ), - ); - case "flat": - return this.gridMode$.pipe( - switchMap((gridMode) => { - switch (gridMode) { - case "grid": - // Yes, grid mode actually gets you a "spotlight" layout in - // this window mode. - return this.spotlightLandscapeLayoutMedia$; - case "spotlight": - return this.spotlightExpandedLayoutMedia$; - } - }), - ); - case "pip": - return this.pipLayoutMedia$; - } - }), - ), - ); + ? spotlightPortraitLayoutMedia$ + : gridLayoutMedia$, + ).pipe(switchAll()) + : // The expanded spotlight layout makes for a better one-on-one + // experience in narrow windows + spotlightExpandedLayoutMedia$, + ), + ); + case "flat": + return gridMode$.pipe( + switchMap((gridMode) => { + switch (gridMode) { + case "grid": + // Yes, grid mode actually gets you a "spotlight" layout in + // this window mode. + return spotlightLandscapeLayoutMedia$; + case "spotlight": + return spotlightExpandedLayoutMedia$; + } + }), + ); + case "pip": + return pipLayoutMedia$; + } + }), + ), + ); - // There is a cyclical dependency here: the layout algorithms want to know - // which tiles are on screen, but to know which tiles are on screen we have to - // first render a layout. To deal with this we assume initially that no tiles - // are visible, and loop the data back into the layouts with a Subject. - private readonly visibleTiles$ = new Subject(); - private readonly setVisibleTiles = (value: number): void => - this.visibleTiles$.next(value); + // There is a cyclical dependency here: the layout algorithms want to know + // which tiles are on screen, but to know which tiles are on screen we have to + // first render a layout. To deal with this we assume initially that no tiles + // are visible, and loop the data back into the layouts with a Subject. + const visibleTiles$ = new Subject(); + const setVisibleTiles = (value: number): void => visibleTiles$.next(value); - private readonly layoutInternals$ = this.scope.behavior< - LayoutScanState & { layout: Layout } - >( - combineLatest([ - this.layoutMedia$, - this.visibleTiles$.pipe(startWith(0), distinctUntilChanged()), - ]).pipe( - scan< - [LayoutMedia, number], - LayoutScanState & { layout: Layout }, - LayoutScanState - >( - ({ tiles: prevTiles }, [media, visibleTiles]) => { - let layout: Layout; - let newTiles: TileStore; - switch (media.type) { - case "grid": + const layoutInternals$ = scope.behavior< + LayoutScanState & { layout: Layout } + >( + combineLatest([ + layoutMedia$, + visibleTiles$.pipe(startWith(0), distinctUntilChanged()), + ]).pipe( + scan< + [LayoutMedia, number], + LayoutScanState & { layout: Layout }, + LayoutScanState + >( + ({ tiles: prevTiles }, [media, visibleTiles]) => { + let layout: Layout; + let newTiles: TileStore; + switch (media.type) { + case "grid": + case "spotlight-landscape": + case "spotlight-portrait": + [layout, newTiles] = gridLikeLayout( + media, + visibleTiles, + setVisibleTiles, + prevTiles, + ); + break; + case "spotlight-expanded": + [layout, newTiles] = spotlightExpandedLayout(media, prevTiles); + break; + case "one-on-one": + [layout, newTiles] = oneOnOneLayout(media, prevTiles); + break; + case "pip": + [layout, newTiles] = pipLayout(media, prevTiles); + break; + } + + return { layout, tiles: newTiles }; + }, + { layout: null, tiles: TileStore.empty() }, + ), + ), + ); + + /** + * The layout of tiles in the call interface. + */ + /*PUBLIC*/ const layout$ = scope.behavior( + layoutInternals$.pipe(map(({ layout }) => layout)), + ); + + /** + * The current generation of the tile store, exposed for debugging purposes. + */ + /*PUBLIC*/ const tileStoreGeneration$ = scope.behavior( + layoutInternals$.pipe(map(({ tiles }) => tiles.generation)), + ); + + /*PUBLIC*/ const showSpotlightIndicators$ = scope.behavior( + layout$.pipe(map((l) => l.type !== "grid")), + ); + + /*PUBLIC*/ const showSpeakingIndicators$ = scope.behavior( + layout$.pipe( + switchMap((l) => { + switch (l.type) { case "spotlight-landscape": case "spotlight-portrait": - [layout, newTiles] = gridLikeLayout( - media, - visibleTiles, - this.setVisibleTiles, - prevTiles, + // If the spotlight is showing the active speaker, we can do without + // speaking indicators as they're a redundant visual cue. But if + // screen sharing feeds are in the spotlight we still need them. + return l.spotlight.media$.pipe( + map((models: MediaViewModel[]) => + models.some((m) => m instanceof ScreenShareViewModel), + ), ); - break; + // In expanded spotlight layout, the active speaker is always shown in + // the picture-in-picture tile so there is no need for speaking + // indicators. And in one-on-one layout there's no question as to who is + // speaking. case "spotlight-expanded": - [layout, newTiles] = spotlightExpandedLayout(media, prevTiles); - break; case "one-on-one": - [layout, newTiles] = oneOnOneLayout(media, prevTiles); - break; - case "pip": - [layout, newTiles] = pipLayout(media, prevTiles); - break; + return of(false); + default: + return of(true); } + }), + ), + ); - return { layout, tiles: newTiles }; + /*PUBLIC*/ const toggleSpotlightExpanded$ = scope.behavior< + (() => void) | null + >( + windowMode$.pipe( + switchMap((mode) => + mode === "normal" + ? layout$.pipe( + map( + (l) => + l.type === "spotlight-landscape" || + l.type === "spotlight-expanded", + ), + ) + : of(false), + ), + distinctUntilChanged(), + map((enabled) => + enabled ? (): void => spotlightExpandedToggle$.next() : null, + ), + ), + ); + + const screenTap$ = new Subject(); + const controlsTap$ = new Subject(); + const screenHover$ = new Subject(); + const screenUnhover$ = new Subject(); + + /*PUBLIC*/ const showHeader$ = scope.behavior( + windowMode$.pipe(map((mode) => mode !== "pip" && mode !== "flat")), + ); + + /*PUBLIC*/ const showFooter$ = scope.behavior( + windowMode$.pipe( + switchMap((mode) => { + switch (mode) { + case "pip": + return of(false); + case "normal": + case "narrow": + return of(true); + case "flat": + // Sadly Firefox has some layering glitches that prevent the footer + // from appearing properly. They happen less often if we never hide + // the footer. + if (isFirefox()) return of(true); + // Show/hide the footer in response to interactions + return merge( + screenTap$.pipe(map(() => "tap screen" as const)), + controlsTap$.pipe(map(() => "tap controls" as const)), + screenHover$.pipe(map(() => "hover" as const)), + ).pipe( + switchScan((state, interaction) => { + switch (interaction) { + case "tap screen": + return state + ? // Toggle visibility on tap + of(false) + : // Hide after a timeout + timer(showFooterMs).pipe( + map(() => false), + startWith(true), + ); + case "tap controls": + // The user is interacting with things, so reset the timeout + return timer(showFooterMs).pipe( + map(() => false), + startWith(true), + ); + case "hover": + // Show on hover and hide after a timeout + return race( + timer(showFooterMs), + screenUnhover$.pipe(take(1)), + ).pipe( + map(() => false), + startWith(true), + ); + } + }, false), + startWith(false), + ); + } + }), + ), + ); + + /** + * Whether audio is currently being output through the earpiece. + */ + /*PUBLIC*/ const earpieceMode$ = scope.behavior( + combineLatest( + [ + mediaDevices.audioOutput.available$, + mediaDevices.audioOutput.selected$, + ], + (available, selected) => + selected !== undefined && + available.get(selected.id)?.type === "earpiece", + ), + ); + + /** + * Callback to toggle between the earpiece and the loudspeaker. + * + * This will be `null` in case the target does not exist in the list + * of available audio outputs. + */ + /*PUBLIC*/ const audioOutputSwitcher$ = scope.behavior<{ + targetOutput: "earpiece" | "speaker"; + switch: () => void; + } | null>( + combineLatest( + [ + mediaDevices.audioOutput.available$, + mediaDevices.audioOutput.selected$, + ], + (available, selected) => { + const selectionType = selected && available.get(selected.id)?.type; + + // If we are in any output mode other than speaker switch to speaker. + const newSelectionType: "earpiece" | "speaker" = + selectionType === "speaker" ? "earpiece" : "speaker"; + const newSelection = [...available].find( + ([, d]) => d.type === newSelectionType, + ); + if (newSelection === undefined) return null; + + const [id] = newSelection; + return { + targetOutput: newSelectionType, + switch: (): void => mediaDevices.audioOutput.select(id), + }; }, - { layout: null, tiles: TileStore.empty() }, ), - ), - ); + ); - /** - * The layout of tiles in the call interface. - */ - public readonly layout$ = this.scope.behavior( - this.layoutInternals$.pipe(map(({ layout }) => layout)), - ); - - /** - * The current generation of the tile store, exposed for debugging purposes. - */ - public readonly tileStoreGeneration$ = this.scope.behavior( - this.layoutInternals$.pipe(map(({ tiles }) => tiles.generation)), - ); - - public showSpotlightIndicators$ = this.scope.behavior( - this.layout$.pipe(map((l) => l.type !== "grid")), - ); - - public showSpeakingIndicators$ = this.scope.behavior( - this.layout$.pipe( - switchMap((l) => { - switch (l.type) { - case "spotlight-landscape": - case "spotlight-portrait": - // If the spotlight is showing the active speaker, we can do without - // speaking indicators as they're a redundant visual cue. But if - // screen sharing feeds are in the spotlight we still need them. - return l.spotlight.media$.pipe( - map((models: MediaViewModel[]) => - models.some((m) => m instanceof ScreenShareViewModel), - ), - ); - // In expanded spotlight layout, the active speaker is always shown in - // the picture-in-picture tile so there is no need for speaking - // indicators. And in one-on-one layout there's no question as to who is - // speaking. - case "spotlight-expanded": - case "one-on-one": - return of(false); - default: - return of(true); - } - }), - ), - ); - - public readonly toggleSpotlightExpanded$ = this.scope.behavior< - (() => void) | null - >( - this.windowMode$.pipe( - switchMap((mode) => - mode === "normal" - ? this.layout$.pipe( - map( - (l) => - l.type === "spotlight-landscape" || - l.type === "spotlight-expanded", - ), - ) - : of(false), + /** + * Emits an array of reactions that should be visible on the screen. + */ + // DISCUSSION move this into a reaction file + // const {visibleReactions$, audibleReactions$} = reactionsObservables$(showReactionSetting$, ) + const visibleReactions$ = scope.behavior( + showReactions.value$.pipe( + switchMap((show) => (show ? reactions$ : of({}))), + scan< + Record, + { sender: string; emoji: string; startX: number }[] + >((acc, latest) => { + const newSet: { sender: string; emoji: string; startX: number }[] = + []; + for (const [sender, reaction] of Object.entries(latest)) { + const startX = + acc.find((v) => v.sender === sender && v.emoji)?.startX ?? + Math.ceil(Math.random() * 80) + 10; + newSet.push({ sender, emoji: reaction.emoji, startX }); + } + return newSet; + }, []), ), - distinctUntilChanged(), - map((enabled) => - enabled ? (): void => this.spotlightExpandedToggle$.next() : null, + ); + + /** + * Emits an array of reactions that should be played. + */ + const audibleReactions$ = playReactionsSound.value$.pipe( + switchMap((show) => + show ? reactions$ : of>({}), ), - ), - ); + map((reactions) => Object.values(reactions).map((v) => v.name)), + scan( + (acc, latest) => { + return { + playing: latest.filter( + (v) => acc.playing.includes(v) || acc.newSounds.includes(v), + ), + newSounds: latest.filter( + (v) => !acc.playing.includes(v) && !acc.newSounds.includes(v), + ), + }; + }, + { playing: [], newSounds: [] }, + ), + map((v) => v.newSounds), + ); - private readonly screenTap$ = new Subject(); - private readonly controlsTap$ = new Subject(); - private readonly screenHover$ = new Subject(); - private readonly screenUnhover$ = new Subject(); + const newHandRaised$ = handsRaised$.pipe( + map((v) => Object.keys(v).length), + scan( + (acc, newValue) => ({ + value: newValue, + playSounds: newValue > acc.value, + }), + { value: 0, playSounds: false }, + ), + filter((v) => v.playSounds), + ); - /** - * Callback for when the user taps the call view. - */ - public tapScreen(): void { - this.screenTap$.next(); - } + const newScreenShare$ = screenShares$.pipe( + map((v) => v.length), + scan( + (acc, newValue) => ({ + value: newValue, + playSounds: newValue > acc.value, + }), + { value: 0, playSounds: false }, + ), + filter((v) => v.playSounds), + ); - /** - * Callback for when the user taps the call's controls. - */ - public tapControls(): void { - this.controlsTap$.next(); - } + /** + * Whether we are sharing our screen. + */ + // reassigned here to make it publicly accessible + const sharingScreen$ = localMembership.sharingScreen$; - /** - * Callback for when the user hovers over the call view. - */ - public hoverScreen(): void { - this.screenHover$.next(); - } + /** + * Callback to toggle screen sharing. If null, screen sharing is not possible. + */ + // reassigned here to make it publicly accessible + const toggleScreenSharing = localMembership.toggleScreenSharing; - /** - * Callback for when the user stops hovering over the call view. - */ - public unhoverScreen(): void { - this.screenUnhover$.next(); - } + const join = localMembership.requestConnect; + join(); // TODO-MULTI-SFU: Use this view model for the lobby as well, and only call this once 'join' is clicked? - public readonly showHeader$ = this.scope.behavior( - this.windowMode$.pipe(map((mode) => mode !== "pip" && mode !== "flat")), - ); + this.autoLeave$ = callLifecycle.autoLeave$; + this.callPickupState$ = callPickupState$; + this.leave$ = leave$; + this.hangup = (): void => userHangup$.next(); + this.join = join; + this.toggleScreenSharing = toggleScreenSharing; + this.sharingScreen$ = sharingScreen$; - public readonly showFooter$ = this.scope.behavior( - this.windowMode$.pipe( - switchMap((mode) => { - switch (mode) { - case "pip": - return of(false); - case "normal": - case "narrow": - return of(true); - case "flat": - // Sadly Firefox has some layering glitches that prevent the footer - // from appearing properly. They happen less often if we never hide - // the footer. - if (isFirefox()) return of(true); - // Show/hide the footer in response to interactions - return merge( - this.screenTap$.pipe(map(() => "tap screen" as const)), - this.controlsTap$.pipe(map(() => "tap controls" as const)), - this.screenHover$.pipe(map(() => "hover" as const)), - ).pipe( - switchScan((state, interaction) => { - switch (interaction) { - case "tap screen": - return state - ? // Toggle visibility on tap - of(false) - : // Hide after a timeout - timer(showFooterMs).pipe( - map(() => false), - startWith(true), - ); - case "tap controls": - // The user is interacting with things, so reset the timeout - return timer(showFooterMs).pipe( - map(() => false), - startWith(true), - ); - case "hover": - // Show on hover and hide after a timeout - return race( - timer(showFooterMs), - this.screenUnhover$.pipe(take(1)), - ).pipe( - map(() => false), - startWith(true), - ); - } - }, false), - startWith(false), - ); - } - }), - ), - ); + this.tapScreen = (): void => screenTap$.next(); + this.tapControls = (): void => controlsTap$.next(); + this.hoverScreen = (): void => screenHover$.next(); + this.unhoverScreen = (): void => screenUnhover$.next(); - /** - * Whether audio is currently being output through the earpiece. - */ - public readonly earpieceMode$ = this.scope.behavior( - combineLatest( - [ - this.mediaDevices.audioOutput.available$, - this.mediaDevices.audioOutput.selected$, - ], - (available, selected) => - selected !== undefined && - available.get(selected.id)?.type === "earpiece", - ), - ); + this.configError$ = localMembership.configError$; + this.participantCount$ = participantCount$; + this.audioParticipants$ = audioParticipants$; + this.isOneOnOneWith$ = isOneOnOneWith$; + this.localUserIsAlone$ = localUserIsAlone$; - /** - * Callback to toggle between the earpiece and the loudspeaker. - * - * This will be `null` in case the target does not exist in the list - * of available audio outputs. - */ - public readonly audioOutputSwitcher$ = this.scope.behavior<{ - targetOutput: "earpiece" | "speaker"; - switch: () => void; - } | null>( - combineLatest( - [ - this.mediaDevices.audioOutput.available$, - this.mediaDevices.audioOutput.selected$, - ], - (available, selected) => { - const selectionType = selected && available.get(selected.id)?.type; + this.handsRaised$ = handsRaised$; + this.reactions$ = reactions$; + this.joinSoundEffect$ = joinSoundEffect$; + this.leaveSoundEffect$ = leaveSoundEffect$; + this.newHandRaised$ = newHandRaised$; + this.newScreenShare$ = newScreenShare$; + this.audibleReactions$ = audibleReactions$; + this.visibleReactions$ = visibleReactions$; - // If we are in any output mode other than speaker switch to speaker. - const newSelectionType: "earpiece" | "speaker" = - selectionType === "speaker" ? "earpiece" : "speaker"; - const newSelection = [...available].find( - ([, d]) => d.type === newSelectionType, - ); - if (newSelection === undefined) return null; - - const [id] = newSelection; - return { - targetOutput: newSelectionType, - switch: (): void => this.mediaDevices.audioOutput.select(id), - }; - }, - ), - ); - - /** - * Emits an array of reactions that should be visible on the screen. - */ - // DISCUSSION move this into a reaction file - // const {visibleReactions$, audibleReactions$} = reactionsObservables$(showReactionSetting$, ) - public readonly visibleReactions$ = this.scope.behavior( - showReactions.value$.pipe( - switchMap((show) => (show ? this.reactions$ : of({}))), - scan< - Record, - { sender: string; emoji: string; startX: number }[] - >((acc, latest) => { - const newSet: { sender: string; emoji: string; startX: number }[] = []; - for (const [sender, reaction] of Object.entries(latest)) { - const startX = - acc.find((v) => v.sender === sender && v.emoji)?.startX ?? - Math.ceil(Math.random() * 80) + 10; - newSet.push({ sender, emoji: reaction.emoji, startX }); - } - return newSet; - }, []), - ), - ); - - /** - * Emits an array of reactions that should be played. - */ - public readonly audibleReactions$ = playReactionsSound.value$.pipe( - switchMap((show) => - show ? this.reactions$ : of>({}), - ), - map((reactions) => Object.values(reactions).map((v) => v.name)), - scan( - (acc, latest) => { - return { - playing: latest.filter( - (v) => acc.playing.includes(v) || acc.newSounds.includes(v), - ), - newSounds: latest.filter( - (v) => !acc.playing.includes(v) && !acc.newSounds.includes(v), - ), - }; - }, - { playing: [], newSounds: [] }, - ), - map((v) => v.newSounds), - ); - - /** - * Emits an event every time a new hand is raised in - * the call. - */ - public readonly newHandRaised$ = this.handsRaised$.pipe( - map((v) => Object.keys(v).length), - scan( - (acc, newValue) => ({ - value: newValue, - playSounds: newValue > acc.value, - }), - { value: 0, playSounds: false }, - ), - filter((v) => v.playSounds), - ); - - /** - * Emits an event every time a new screenshare is started in - * the call. - */ - public readonly newScreenShare$ = this.screenShares$.pipe( - map((v) => v.length), - scan( - (acc, newValue) => ({ - value: newValue, - playSounds: newValue > acc.value, - }), - { value: 0, playSounds: false }, - ), - filter((v) => v.playSounds), - ); - - /** - * Whether we are sharing our screen. - */ - // reassigned here to make it publicly accessible - public readonly sharingScreen$ = this.localMembership.sharingScreen$; - - /** - * Callback to toggle screen sharing. If null, screen sharing is not possible. - */ - // reassigned here to make it publicly accessible - public readonly toggleScreenSharing = - this.localMembership.toggleScreenSharing; - - public constructor( - private readonly scope: ObservableScope, - // A call is permanently tied to a single Matrix room - private readonly matrixRTCSession: MatrixRTCSession, - private readonly matrixRoom: MatrixRoom, - private readonly mediaDevices: MediaDevices, - private readonly muteStates: MuteStates, - private readonly options: CallViewModelOptions, - private readonly handsRaisedSubject$: Observable< - Record - >, - private readonly reactionsSubject$: Observable< - Record - >, - private readonly trackProcessorState$: Behavior, - ) { - // Join automatically - this.join(); // TODO-MULTI-SFU: Use this view model for the lobby as well, and only call this once 'join' is clicked? + this.windowMode$ = windowMode$; + this.spotlightExpanded$ = spotlightExpanded$; + this.toggleSpotlightExpanded$ = toggleSpotlightExpanded$; + this.gridMode$ = gridMode$; + this.setGridMode = setGridMode; + this.grid$ = grid$; + this.spotlight$ = spotlight$; + this.pip$ = pip$; + this.layout$ = layout$; + this.tileStoreGeneration$ = tileStoreGeneration$; + this.showSpotlightIndicators$ = showSpotlightIndicators$; + this.showSpeakingIndicators$ = showSpeakingIndicators$; + this.showHeader$ = showHeader$; + this.showFooter$ = showFooter$; + this.earpieceMode$ = earpieceMode$; + this.audioOutputSwitcher$ = audioOutputSwitcher$; + this.reconnecting$ = reconnecting$; } } - // TODO-MULTI-SFU // Setup and update the keyProvider which was create by `createRoom` was a thing before. Now we never update if the E2EEsystem changes // do we need this? diff --git a/src/state/CallViewModel/CallViewModelFunction.ts b/src/state/CallViewModel/CallViewModelFunction.ts new file mode 100644 index 00000000..20e37395 --- /dev/null +++ b/src/state/CallViewModel/CallViewModelFunction.ts @@ -0,0 +1,1400 @@ +/* +Copyright 2023, 2024, 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + type BaseKeyProvider, + type ConnectionState, + ExternalE2EEKeyProvider, + type Room as LivekitRoom, + type RoomOptions, +} from "livekit-client"; +import { type RoomMember, type Room as MatrixRoom } from "matrix-js-sdk"; +import { + combineLatest, + distinctUntilChanged, + EMPTY, + filter, + fromEvent, + map, + merge, + NEVER, + type Observable, + of, + pairwise, + race, + scan, + skip, + skipWhile, + startWith, + Subject, + switchAll, + switchMap, + switchScan, + take, + tap, + throttleTime, + timer, +} from "rxjs"; +import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; +import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; +import { type IWidgetApiRequest } from "matrix-widget-api"; + +import { + LocalUserMediaViewModel, + type MediaViewModel, + type RemoteUserMediaViewModel, + ScreenShareViewModel, + type UserMediaViewModel, +} from "../MediaViewModel"; +import { accumulate, generateItems, pauseWhen } from "../../utils/observable"; +import { + duplicateTiles, + MatrixRTCMode, + matrixRTCMode, + playReactionsSound, + showReactions, +} from "../../settings/settings"; +import { isFirefox } from "../../Platform"; +import { setPipEnabled$ } from "../../controls"; +import { TileStore } from "../TileStore"; +import { gridLikeLayout } from "../GridLikeLayout"; +import { spotlightExpandedLayout } from "../SpotlightExpandedLayout"; +import { oneOnOneLayout } from "../OneOnOneLayout"; +import { pipLayout } from "../PipLayout"; +import { type EncryptionSystem } from "../../e2ee/sharedKeyManagement"; +import { + type RaisedHandInfo, + type ReactionInfo, + type ReactionOption, +} from "../../reactions"; +import { shallowEquals } from "../../utils/array"; +import { type MediaDevices } from "../MediaDevices"; +import { type Behavior } from "../Behavior"; +import { E2eeType } from "../../e2ee/e2eeType"; +import { MatrixKeyProvider } from "../../e2ee/matrixKeyProvider"; +import { type MuteStates } from "../MuteStates"; +import { getUrlParams } from "../../UrlParams"; +import { type ProcessorState } from "../../livekit/TrackProcessorContext"; +import { ElementWidgetActions, widget } from "../../widget"; +import { UserMedia } from "../UserMedia.ts"; +import { ScreenShare } from "../ScreenShare.ts"; +import { + type GridLayoutMedia, + type Layout, + type LayoutMedia, + type OneOnOneLayoutMedia, + type SpotlightExpandedLayoutMedia, + type SpotlightLandscapeLayoutMedia, + type SpotlightPortraitLayoutMedia, +} from "../layout-types.ts"; +import { type ElementCallError } from "../../utils/errors.ts"; +import { type ObservableScope } from "../ObservableScope.ts"; +import { createLocalMembership$ } from "./localMember/LocalMembership.ts"; +import { createLocalTransport$ } from "./localMember/LocalTransport.ts"; +import { + createMemberships$, + membershipsAndTransports$, +} from "../SessionBehaviors.ts"; +import { ECConnectionFactory } from "./remoteMembers/ConnectionFactory.ts"; +import { createConnectionManager$ } from "./remoteMembers/ConnectionManager.ts"; +import { + createMatrixLivekitMembers$, + type MatrixLivekitMember, +} from "./remoteMembers/MatrixLivekitMembers.ts"; +import { + type AutoLeaveReason, + createCallNotificationLifecycle$, + createReceivedDecline$, + createSentCallNotification$, +} from "./CallNotificationLifecycle.ts"; +import { + createMatrixMemberMetadata$, + createRoomMembers$, +} from "./remoteMembers/MatrixMemberMetadata.ts"; + +const logger = rootLogger.getChild("[CallViewModel]"); +//TODO +// Larger rename +// member,membership -> rtcMember +// participant -> livekitParticipant +// matrixLivekitItem -> callMember +// js-sdk +// callMembership -> rtcMembership +export interface CallViewModelOptions { + encryptionSystem: EncryptionSystem; + autoLeaveWhenOthersLeft?: boolean; + /** + * If the call is started in a way where we want it to behave like a telephone usecase + * If we sent a notification event, we want the ui to show a ringing state + */ + waitForCallPickup?: boolean; + /** Optional factory to create LiveKit rooms, mainly for testing purposes. */ + livekitRoomFactory?: (options?: RoomOptions) => LivekitRoom; + /** Optional behavior overriding the local connection state, mainly for testing purposes. */ + connectionState$?: Behavior; +} + +// Do not play any sounds if the participant count has exceeded this +// number. +export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8; +export const THROTTLE_SOUND_EFFECT_MS = 500; + +// This is the number of participants that we think constitutes a "small" call +// on mobile. No spotlight tile should be shown below this threshold. +const smallMobileCallThreshold = 3; + +// How long the footer should be shown for when hovering over or interacting +// with the interface +const showFooterMs = 4000; + +export type GridMode = "grid" | "spotlight"; + +export type WindowMode = "normal" | "narrow" | "flat" | "pip"; + +interface LayoutScanState { + layout: Layout | null; + tiles: TileStore; +} + +type MediaItem = UserMedia | ScreenShare; +type AudioLivekitItem = { + livekitRoom: LivekitRoom; + participants: string[]; + url: string; +}; + +type JoinReturn = ReturnType< + ReturnType["requestConnect"] +>; + +/** + * A view model providing all the application logic needed to show the in-call + * UI (may eventually be expanded to cover the lobby and feedback screens in the + * future). + */ +// Throughout this class and related code we must distinguish between MatrixRTC +// state and LiveKit state. We use the common terminology of room "members", RTC +// "memberships", and LiveKit "participants". +export function createCallViewModel$( + scope: ObservableScope, + // A call is permanently tied to a single Matrix room + matrixRTCSession: MatrixRTCSession, + matrixRoom: MatrixRoom, + mediaDevices: MediaDevices, + muteStates: MuteStates, + options: CallViewModelOptions, + handsRaisedSubject$: Observable>, + reactionsSubject$: Observable>, + trackProcessorState$: Behavior, +): { + // lifecycle + autoLeave$: Observable; + callPickupState$: Behavior< + "unknown" | "ringing" | "timeout" | "decline" | "success" | null + >; + leave$: Observable<"user" | AutoLeaveReason>; + hangup: () => void; + + // joining + join: () => JoinReturn; + + // screen sharing + toggleScreenSharing: (() => void) | null; + sharingScreen$: Behavior; + + // UI interactions + tapScreen: () => void; + tapControls: () => void; + hoverScreen: () => void; + unhoverScreen: () => void; + + // errors + configError$: Behavior; + + // participants and counts + participantCount$: Behavior; + audioParticipants$: Behavior; + handsRaised$: Behavior>; + reactions$: Behavior>; + isOneOnOneWith$: Behavior | null>; + localUserIsAlone$: Behavior; + // sounds and events + joinSoundEffect$: Observable; + leaveSoundEffect$: Observable; + newHandRaised$: Observable<{ value: number; playSounds: boolean }>; + newScreenShare$: Observable<{ value: number; playSounds: boolean }>; + audibleReactions$: Observable; + visibleReactions$: Behavior< + { sender: string; emoji: string; startX: number }[] + >; + + // window/layout + windowMode$: Behavior; + spotlightExpanded$: Behavior; + toggleSpotlightExpanded$: Behavior<(() => void) | null>; + gridMode$: Behavior; + setGridMode: (value: GridMode) => void; + + // media view models and layout + grid$: Behavior; + spotlight$: Behavior; + pip$: Behavior; + layout$: Behavior; + tileStoreGeneration$: Behavior; + showSpotlightIndicators$: Behavior; + showSpeakingIndicators$: Behavior; + + // header/footer visibility + showHeader$: Behavior; + showFooter$: Behavior; + + // audio routing + earpieceMode$: Behavior; + audioOutputSwitcher$: Behavior<{ + targetOutput: "earpiece" | "speaker"; + switch: () => void; + } | null>; + + // connection state + reconnecting$: Behavior; +} { + const userId = matrixRoom.client.getUserId()!; + const deviceId = matrixRoom.client.getDeviceId()!; + + const livekitKeyProvider = getE2eeKeyProvider( + options.encryptionSystem, + matrixRTCSession, + ); + + // Each hbar seperates a block of input variables required for the CallViewModel to function. + // The outputs of this block is written under the hbar. + // + // For mocking purposes it is recommended to only mock the functions creating those outputs. + // All other fields are just temp computations for the mentioned output. + // The class does not need anything except the values underneath the bar. + // The creation of the values under the bar are all tested independently and testing the callViewModel Should + // not test their cretation. Call view model only needs: + // - memberships$ via createMemberships$ + // - localMembership via createLocalMembership$ + // - callLifecycle via createCallNotificationLifecycle$ + // - matrixMemberMetadataStore via createMatrixMemberMetadata$ + + // ------------------------------------------------------------------------ + // memberships$ + const memberships$ = createMemberships$(scope, matrixRTCSession); + + // ------------------------------------------------------------------------ + // matrixLivekitMembers$ AND localMembership + + const membershipsAndTransports = membershipsAndTransports$( + scope, + memberships$, + ); + + const localTransport$ = createLocalTransport$({ + scope: scope, + memberships$: memberships$, + client: matrixRoom.client, + roomId: matrixRoom.roomId, + useOldestMember$: scope.behavior( + matrixRTCMode.value$.pipe(map((v) => v === MatrixRTCMode.Legacy)), + ), + }); + + const connectionFactory = new ECConnectionFactory( + matrixRoom.client, + mediaDevices, + trackProcessorState$, + livekitKeyProvider, + getUrlParams().controlledAudioDevices, + options.livekitRoomFactory, + ); + + const connectionManager = createConnectionManager$({ + scope: scope, + connectionFactory: connectionFactory, + inputTransports$: scope.behavior( + combineLatest( + [localTransport$, membershipsAndTransports.transports$], + (localTransport, transports) => { + const localTransportAsArray = localTransport ? [localTransport] : []; + return transports.mapInner((transports) => [ + ...localTransportAsArray, + ...transports, + ]); + }, + ), + ), + logger: logger, + }); + + const matrixLivekitMembers$ = createMatrixLivekitMembers$({ + scope: scope, + membershipsWithTransport$: + membershipsAndTransports.membershipsWithTransport$, + connectionManager: connectionManager, + }); + + const connectOptions$ = scope.behavior( + matrixRTCMode.value$.pipe( + map((mode) => ({ + encryptMedia: livekitKeyProvider !== undefined, + // TODO. This might need to get called again on each cahnge of matrixRTCMode... + matrixRTCMode: mode, + })), + ), + ); + + const localMembership = createLocalMembership$({ + scope: scope, + muteStates: muteStates, + mediaDevices: mediaDevices, + connectionManager: connectionManager, + matrixRTCSession: matrixRTCSession, + matrixRoom: matrixRoom, + localTransport$: localTransport$, + trackProcessorState$: trackProcessorState$, + widget, + options: connectOptions$, + logger: logger.getChild(`[${Date.now()}]`), + }); + + const localRtcMembership$ = scope.behavior( + memberships$.pipe( + map( + (memberships) => + memberships.value.find( + (membership) => + membership.userId === userId && membership.deviceId === deviceId, + ) ?? null, + ), + ), + ); + + const localMatrixLivekitMemberUninitialized = { + membership$: localRtcMembership$, + participant$: localMembership.participant$, + connection$: localMembership.connection$, + userId: userId, + }; + + const localMatrixLivekitMember$: Behavior = + scope.behavior( + localRtcMembership$.pipe( + switchMap((membership) => { + if (!membership) return of(null); + return of( + // casting is save here since we know that localRtcMembership$ is !== null since we reached this case. + localMatrixLivekitMemberUninitialized as MatrixLivekitMember, + ); + }), + ), + ); + + // ------------------------------------------------------------------------ + // callLifecycle + + const callLifecycle = createCallNotificationLifecycle$({ + scope: scope, + memberships$: memberships$, + sentCallNotification$: createSentCallNotification$(scope, matrixRTCSession), + receivedDecline$: createReceivedDecline$(matrixRoom), + options: options, + localUser: { userId: userId, deviceId: deviceId }, + }); + + // ------------------------------------------------------------------------ + // matrixMemberMetadataStore + + const matrixRoomMembers$ = createRoomMembers$(scope, matrixRoom); + const matrixMemberMetadataStore = createMatrixMemberMetadata$( + scope, + scope.behavior(memberships$.pipe(map((mems) => mems.value))), + matrixRoomMembers$, + ); + + /** + * Returns the Member {userId, getMxcAvatarUrl, rawDisplayName} of the other user in the call, if it's a one-on-one call. + */ + const isOneOnOneWith$ = scope.behavior( + matrixRoomMembers$.pipe( + map((roomMembersMap) => { + const otherMembers = Array.from(roomMembersMap.values()).filter( + (member) => member.userId !== userId, + ); + return otherMembers.length === 1 ? otherMembers[0] : null; + }), + ), + ); + + const localUserIsAlone$ = scope.behavior( + matrixRoomMembers$.pipe( + map( + (roomMembersMap) => + roomMembersMap.size === 1 && roomMembersMap.get(userId) !== undefined, + ), + ), + ); + + // CODESMELL? + // This is functionally the same Observable as leave$, except here it's + // hoisted to the top of the class. This enables the cyclic dependency between + // leave$ -> autoLeave$ -> callPickupState$ -> livekitConnectionState$ -> + // localConnection$ -> transports$ -> joined$ -> leave$. + const leaveHoisted$ = new Subject< + "user" | "timeout" | "decline" | "allOthersLeft" + >(); + + /** + * Whether various media/event sources should pretend to be disconnected from + * all network input, even if their connection still technically works. + */ + // We do this when the app is in the 'reconnecting' state, because it might be + // that the LiveKit connection is still functional while the homeserver is + // down, for example, and we want to avoid making people worry that the app is + // in a split-brained state. + // DISCUSSION own membership manager ALSO this probably can be simplifis + /*PUBLIC*/ const reconnecting$ = localMembership.reconnecting$; + const pretendToBeDisconnected$ = reconnecting$; + + /*PUBLIC*/ const audioParticipants$ = scope.behavior( + matrixLivekitMembers$.pipe( + switchMap((membersWithEpoch) => { + const members = membersWithEpoch.value; + const a$ = combineLatest( + members.map((member) => + combineLatest([member.connection$, member.participant$]).pipe( + map(([connection, participant]) => { + // do not render audio for local participant + if (!connection || !participant || participant.isLocal) + return null; + const livekitRoom = connection.livekitRoom; + const url = connection.transport.livekit_service_url; + + return { url, livekitRoom, participant: participant.identity }; + }), + ), + ), + ); + return a$; + }), + map((members) => + members.reduce((acc, curr) => { + if (!curr) return acc; + + const existing = acc.find((item) => item.url === curr.url); + if (existing) { + existing.participants.push(curr.participant); + } else { + acc.push({ + livekitRoom: curr.livekitRoom, + participants: [curr.participant], + url: curr.url, + }); + } + return acc; + }, []), + ), + ), + [], + ); + + /*PUBLIC*/ const handsRaised$ = scope.behavior( + handsRaisedSubject$.pipe(pauseWhen(pretendToBeDisconnected$)), + ); + + /*PUBLIC*/ const reactions$ = scope.behavior( + reactionsSubject$.pipe( + map((v) => + Object.fromEntries( + Object.entries(v).map(([a, { reactionOption }]) => [ + a, + reactionOption, + ]), + ), + ), + pauseWhen(pretendToBeDisconnected$), + ), + ); + + /** + * List of user media (camera feeds) that we want tiles for. + */ + // TODO this also needs the local participant to be added. + const userMedia$ = scope.behavior( + combineLatest([ + localMatrixLivekitMember$, + matrixLivekitMembers$, + duplicateTiles.value$, + ]).pipe( + // Generate a collection of MediaItems from the list of expected (whether + // present or missing) LiveKit participants. + generateItems( + function* ([ + localMatrixLivekitMember, + { value: matrixLivekitMembers }, + duplicateTiles, + ]) { + let localParticipantId = undefined; + // add local member if available + if (localMatrixLivekitMember) { + const { userId, participant$, connection$, membership$ } = + localMatrixLivekitMember; + localParticipantId = `${userId}:${membership$.value.deviceId}`; // should be membership$.value.membershipID which is not optional + // const participantId = membership$.value.membershipID; + if (localParticipantId) { + for (let dup = 0; dup < 1 + duplicateTiles; dup++) { + yield { + keys: [ + dup, + localParticipantId, + userId, + participant$, + connection$, + ], + data: undefined, + }; + } + } + } + // add remote members that are available + for (const { + userId, + participant$, + connection$, + membership$, + } of matrixLivekitMembers) { + const participantId = `${userId}:${membership$.value.deviceId}`; + if (participantId === localParticipantId) continue; + // const participantId = membership$.value?.identity; + for (let dup = 0; dup < 1 + duplicateTiles; dup++) { + yield { + keys: [dup, participantId, userId, participant$, connection$], + data: undefined, + }; + } + } + }, + ( + scope, + _data$, + dup, + participantId, + userId, + participant$, + connection$, + ) => { + const livekitRoom$ = scope.behavior( + connection$.pipe(map((c) => c?.livekitRoom)), + ); + const focusUrl$ = scope.behavior( + connection$.pipe(map((c) => c?.transport.livekit_service_url)), + ); + const displayName$ = scope.behavior( + matrixMemberMetadataStore + .createDisplayNameBehavior$(userId) + .pipe(map((name) => name ?? userId)), + ); + + return new UserMedia( + scope, + `${participantId}:${dup}`, + userId, + participant$, + options.encryptionSystem, + livekitRoom$, + focusUrl$, + mediaDevices, + pretendToBeDisconnected$, + displayName$, + matrixMemberMetadataStore.createAvatarUrlBehavior$(userId), + handsRaised$.pipe(map((v) => v[participantId]?.time ?? null)), + reactions$.pipe(map((v) => v[participantId] ?? undefined)), + ); + }, + ), + ), + ); + + /** + * List of all media items (user media and screen share media) that we want + * tiles for. + */ + const mediaItems$ = scope.behavior( + userMedia$.pipe( + switchMap((userMedia) => + userMedia.length === 0 + ? of([]) + : combineLatest( + userMedia.map((m) => m.screenShares$), + (...screenShares) => [...userMedia, ...screenShares.flat(1)], + ), + ), + ), + ); + + /** + * List of MediaItems that we want to display, that are of type ScreenShare + */ + const screenShares$ = scope.behavior( + mediaItems$.pipe( + map((mediaItems) => + mediaItems.filter((m): m is ScreenShare => m instanceof ScreenShare), + ), + ), + ); + + /*PUBLIC*/ const joinSoundEffect$ = userMedia$.pipe( + pairwise(), + filter( + ([prev, current]) => + current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && + current.length > prev.length, + ), + map(() => {}), + throttleTime(THROTTLE_SOUND_EFFECT_MS), + ); + + /** + * The number of participants currently in the call. + * + * - Each participant has a corresponding MatrixRTC membership state event + * - There can be multiple participants for one Matrix user if they join from + * multiple devices. + */ + /*PUBLIC*/ const participantCount$ = scope.behavior( + matrixLivekitMembers$.pipe(map((ms) => ms.value.length)), + ); + + // only public to expose to the view. + // TODO if we are in "unknown" state we need a loading rendering (or empty screen) + // Otherwise it looks like we already connected and only than the ringing starts which is weird. + /*PUBLIC*/ const callPickupState$ = callLifecycle.callPickupState$; + + /*PUBLIC*/ const leaveSoundEffect$ = combineLatest([ + callLifecycle.callPickupState$, + userMedia$, + ]).pipe( + // Until the call is successful, do not play a leave sound. + // If callPickupState$ is null, then we always play the sound as it will not conflict with a decline sound. + skipWhile(([c]) => c !== null && c !== "success"), + map(([, userMedia]) => userMedia), + pairwise(), + filter( + ([prev, current]) => + current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && + current.length < prev.length, + ), + map(() => {}), + throttleTime(THROTTLE_SOUND_EFFECT_MS), + ); + + const userHangup$ = new Subject(); + + const widgetHangup$ = + widget === null + ? NEVER + : ( + fromEvent( + widget.lazyActions, + ElementWidgetActions.HangupCall, + ) as Observable> + ).pipe( + tap((ev) => { + widget!.api.transport.reply(ev.detail, {}); + }), + ); + + const leave$: Observable<"user" | "timeout" | "decline" | "allOthersLeft"> = + merge( + callLifecycle.autoLeave$, + merge(userHangup$, widgetHangup$).pipe(map(() => "user" as const)), + ).pipe( + scope.share, + tap((reason) => leaveHoisted$.next(reason)), + ); + + const spotlightSpeaker$ = scope.behavior( + userMedia$.pipe( + switchMap((mediaItems) => + mediaItems.length === 0 + ? of([]) + : combineLatest( + mediaItems.map((m) => + m.vm.speaking$.pipe(map((s) => [m, s] as const)), + ), + ), + ), + scan<(readonly [UserMedia, boolean])[], UserMedia | undefined, null>( + (prev, mediaItems) => { + // Only remote users that are still in the call should be sticky + const [stickyMedia, stickySpeaking] = + (!prev?.vm.local && mediaItems.find(([m]) => m === prev)) || []; + // Decide who to spotlight: + // If the previous speaker is still speaking, stick with them rather + // than switching eagerly to someone else + return stickySpeaking + ? stickyMedia! + : // Otherwise, select any remote user who is speaking + (mediaItems.find(([m, s]) => !m.vm.local && s)?.[0] ?? + // Otherwise, stick with the person who was last speaking + stickyMedia ?? + // Otherwise, spotlight an arbitrary remote user + mediaItems.find(([m]) => !m.vm.local)?.[0] ?? + // Otherwise, spotlight the local user + mediaItems.find(([m]) => m.vm.local)?.[0]); + }, + null, + ), + map((speaker) => speaker?.vm ?? null), + ), + ); + + const grid$ = scope.behavior( + userMedia$.pipe( + switchMap((mediaItems) => { + const bins = mediaItems.map((m) => + m.bin$.pipe(map((bin) => [m, bin] as const)), + ); + // Sort the media by bin order and generate a tile for each one + return bins.length === 0 + ? of([]) + : combineLatest(bins, (...bins) => + bins.sort(([, bin1], [, bin2]) => bin1 - bin2).map(([m]) => m.vm), + ); + }), + distinctUntilChanged(shallowEquals), + ), + ); + + const spotlight$ = scope.behavior( + screenShares$.pipe( + switchMap((screenShares) => { + if (screenShares.length > 0) { + return of(screenShares.map((m) => m.vm)); + } + + return spotlightSpeaker$.pipe( + map((speaker) => (speaker ? [speaker] : [])), + ); + }), + distinctUntilChanged(shallowEquals), + ), + ); + + const pip$ = scope.behavior( + combineLatest([ + // TODO This also needs epoch logic to dedupe the screenshares and mediaItems emits + screenShares$, + spotlightSpeaker$, + mediaItems$, + ]).pipe( + switchMap(([screenShares, spotlight, mediaItems]) => { + if (screenShares.length > 0) { + return spotlightSpeaker$; + } + if (!spotlight || spotlight.local) { + return of(null); + } + + const localUserMedia = mediaItems.find( + (m) => m.vm instanceof LocalUserMediaViewModel, + ) as UserMedia | undefined; + + const localUserMediaViewModel = localUserMedia?.vm as + | LocalUserMediaViewModel + | undefined; + + if (!localUserMediaViewModel) { + return of(null); + } + return localUserMediaViewModel.alwaysShow$.pipe( + map((alwaysShow) => { + if (alwaysShow) { + return localUserMediaViewModel; + } + + return null; + }), + ); + }), + ), + ); + + const hasRemoteScreenShares$: Observable = spotlight$.pipe( + map((spotlight) => + spotlight.some((vm) => !vm.local && vm instanceof ScreenShareViewModel), + ), + distinctUntilChanged(), + ); + + const pipEnabled$ = scope.behavior(setPipEnabled$, false); + + const naturalWindowMode$ = scope.behavior( + fromEvent(window, "resize").pipe( + map(() => { + const height = window.innerHeight; + const width = window.innerWidth; + if (height <= 400 && width <= 340) return "pip"; + // Our layouts for flat windows are better at adapting to a small width + // than our layouts for narrow windows are at adapting to a small height, + // so we give "flat" precedence here + if (height <= 600) return "flat"; + if (width <= 600) return "narrow"; + return "normal"; + }), + ), + "normal", + ); + + /** + * The general shape of the window. + */ + /*PUBLIC*/ const windowMode$ = scope.behavior( + pipEnabled$.pipe( + switchMap((pip) => (pip ? of("pip") : naturalWindowMode$)), + ), + ); + + const spotlightExpandedToggle$ = new Subject(); + /*PUBLIC*/ const spotlightExpanded$ = scope.behavior( + spotlightExpandedToggle$.pipe(accumulate(false, (expanded) => !expanded)), + ); + + const gridModeUserSelection$ = new Subject(); + /** + * The layout mode of the media tile grid. + */ + /*PUBLIC*/ const gridMode$ = + // If the user hasn't selected spotlight and somebody starts screen sharing, + // automatically switch to spotlight mode and reset when screen sharing ends + scope.behavior( + gridModeUserSelection$.pipe( + switchMap((userSelection) => + (userSelection === "spotlight" + ? EMPTY + : combineLatest([hasRemoteScreenShares$, windowMode$]).pipe( + skip(userSelection === null ? 0 : 1), + map( + ([hasScreenShares, windowMode]): GridMode => + hasScreenShares || windowMode === "flat" + ? "spotlight" + : "grid", + ), + ) + ).pipe(startWith(userSelection ?? "grid")), + ), + ), + "grid", + ); + + /*PUBLIC*/ const setGridMode = (value: GridMode): void => { + gridModeUserSelection$.next(value); + }; + + const gridLayoutMedia$: Observable = combineLatest( + [grid$, spotlight$], + (grid, spotlight) => ({ + type: "grid", + spotlight: spotlight.some((vm) => vm instanceof ScreenShareViewModel) + ? spotlight + : undefined, + grid, + }), + ); + + const spotlightLandscapeLayoutMedia$: Observable = + combineLatest([grid$, spotlight$], (grid, spotlight) => ({ + type: "spotlight-landscape", + spotlight, + grid, + })); + + const spotlightPortraitLayoutMedia$: Observable = + combineLatest([grid$, spotlight$], (grid, spotlight) => ({ + type: "spotlight-portrait", + spotlight, + grid, + })); + + const spotlightExpandedLayoutMedia$: Observable = + combineLatest([spotlight$, pip$], (spotlight, pip) => ({ + type: "spotlight-expanded", + spotlight, + pip: pip ?? undefined, + })); + + const oneOnOneLayoutMedia$: Observable = + mediaItems$.pipe( + map((mediaItems) => { + if (mediaItems.length !== 2) return null; + const local = mediaItems.find((vm) => vm.vm.local)?.vm as + | LocalUserMediaViewModel + | undefined; + const remote = mediaItems.find((vm) => !vm.vm.local)?.vm as + | RemoteUserMediaViewModel + | undefined; + // There might not be a remote tile if there are screen shares, or if + // only the local user is in the call and they're using the duplicate + // tiles option + if (!remote || !local) return null; + + return { type: "one-on-one", local, remote }; + }), + ); + + const pipLayoutMedia$: Observable = spotlight$.pipe( + map((spotlight) => ({ type: "pip", spotlight })), + ); + + /** + * The media to be used to produce a layout. + */ + const layoutMedia$ = scope.behavior( + windowMode$.pipe( + switchMap((windowMode) => { + switch (windowMode) { + case "normal": + return gridMode$.pipe( + switchMap((gridMode) => { + switch (gridMode) { + case "grid": + return oneOnOneLayoutMedia$.pipe( + switchMap((oneOnOne) => + oneOnOne === null ? gridLayoutMedia$ : of(oneOnOne), + ), + ); + case "spotlight": + return spotlightExpanded$.pipe( + switchMap((expanded) => + expanded + ? spotlightExpandedLayoutMedia$ + : spotlightLandscapeLayoutMedia$, + ), + ); + } + }), + ); + case "narrow": + return oneOnOneLayoutMedia$.pipe( + switchMap((oneOnOne) => + oneOnOne === null + ? combineLatest([grid$, spotlight$], (grid, spotlight) => + grid.length > smallMobileCallThreshold || + spotlight.some((vm) => vm instanceof ScreenShareViewModel) + ? spotlightPortraitLayoutMedia$ + : gridLayoutMedia$, + ).pipe(switchAll()) + : // The expanded spotlight layout makes for a better one-on-one + // experience in narrow windows + spotlightExpandedLayoutMedia$, + ), + ); + case "flat": + return gridMode$.pipe( + switchMap((gridMode) => { + switch (gridMode) { + case "grid": + // Yes, grid mode actually gets you a "spotlight" layout in + // this window mode. + return spotlightLandscapeLayoutMedia$; + case "spotlight": + return spotlightExpandedLayoutMedia$; + } + }), + ); + case "pip": + return pipLayoutMedia$; + } + }), + ), + ); + + // There is a cyclical dependency here: the layout algorithms want to know + // which tiles are on screen, but to know which tiles are on screen we have to + // first render a layout. To deal with this we assume initially that no tiles + // are visible, and loop the data back into the layouts with a Subject. + const visibleTiles$ = new Subject(); + const setVisibleTiles = (value: number): void => visibleTiles$.next(value); + + const layoutInternals$ = scope.behavior( + combineLatest([ + layoutMedia$, + visibleTiles$.pipe(startWith(0), distinctUntilChanged()), + ]).pipe( + scan< + [LayoutMedia, number], + LayoutScanState & { layout: Layout }, + LayoutScanState + >( + ({ tiles: prevTiles }, [media, visibleTiles]) => { + let layout: Layout; + let newTiles: TileStore; + switch (media.type) { + case "grid": + case "spotlight-landscape": + case "spotlight-portrait": + [layout, newTiles] = gridLikeLayout( + media, + visibleTiles, + setVisibleTiles, + prevTiles, + ); + break; + case "spotlight-expanded": + [layout, newTiles] = spotlightExpandedLayout(media, prevTiles); + break; + case "one-on-one": + [layout, newTiles] = oneOnOneLayout(media, prevTiles); + break; + case "pip": + [layout, newTiles] = pipLayout(media, prevTiles); + break; + } + + return { layout, tiles: newTiles }; + }, + { layout: null, tiles: TileStore.empty() }, + ), + ), + ); + + /** + * The layout of tiles in the call interface. + */ + /*PUBLIC*/ const layout$ = scope.behavior( + layoutInternals$.pipe(map(({ layout }) => layout)), + ); + + /** + * The current generation of the tile store, exposed for debugging purposes. + */ + /*PUBLIC*/ const tileStoreGeneration$ = scope.behavior( + layoutInternals$.pipe(map(({ tiles }) => tiles.generation)), + ); + + /*PUBLIC*/ const showSpotlightIndicators$ = scope.behavior( + layout$.pipe(map((l) => l.type !== "grid")), + ); + + /*PUBLIC*/ const showSpeakingIndicators$ = scope.behavior( + layout$.pipe( + switchMap((l) => { + switch (l.type) { + case "spotlight-landscape": + case "spotlight-portrait": + // If the spotlight is showing the active speaker, we can do without + // speaking indicators as they're a redundant visual cue. But if + // screen sharing feeds are in the spotlight we still need them. + return l.spotlight.media$.pipe( + map((models: MediaViewModel[]) => + models.some((m) => m instanceof ScreenShareViewModel), + ), + ); + // In expanded spotlight layout, the active speaker is always shown in + // the picture-in-picture tile so there is no need for speaking + // indicators. And in one-on-one layout there's no question as to who is + // speaking. + case "spotlight-expanded": + case "one-on-one": + return of(false); + default: + return of(true); + } + }), + ), + ); + + /*PUBLIC*/ const toggleSpotlightExpanded$ = scope.behavior< + (() => void) | null + >( + windowMode$.pipe( + switchMap((mode) => + mode === "normal" + ? layout$.pipe( + map( + (l) => + l.type === "spotlight-landscape" || + l.type === "spotlight-expanded", + ), + ) + : of(false), + ), + distinctUntilChanged(), + map((enabled) => + enabled ? (): void => spotlightExpandedToggle$.next() : null, + ), + ), + ); + + const screenTap$ = new Subject(); + const controlsTap$ = new Subject(); + const screenHover$ = new Subject(); + const screenUnhover$ = new Subject(); + + /*PUBLIC*/ const showHeader$ = scope.behavior( + windowMode$.pipe(map((mode) => mode !== "pip" && mode !== "flat")), + ); + + /*PUBLIC*/ const showFooter$ = scope.behavior( + windowMode$.pipe( + switchMap((mode) => { + switch (mode) { + case "pip": + return of(false); + case "normal": + case "narrow": + return of(true); + case "flat": + // Sadly Firefox has some layering glitches that prevent the footer + // from appearing properly. They happen less often if we never hide + // the footer. + if (isFirefox()) return of(true); + // Show/hide the footer in response to interactions + return merge( + screenTap$.pipe(map(() => "tap screen" as const)), + controlsTap$.pipe(map(() => "tap controls" as const)), + screenHover$.pipe(map(() => "hover" as const)), + ).pipe( + switchScan((state, interaction) => { + switch (interaction) { + case "tap screen": + return state + ? // Toggle visibility on tap + of(false) + : // Hide after a timeout + timer(showFooterMs).pipe( + map(() => false), + startWith(true), + ); + case "tap controls": + // The user is interacting with things, so reset the timeout + return timer(showFooterMs).pipe( + map(() => false), + startWith(true), + ); + case "hover": + // Show on hover and hide after a timeout + return race( + timer(showFooterMs), + screenUnhover$.pipe(take(1)), + ).pipe( + map(() => false), + startWith(true), + ); + } + }, false), + startWith(false), + ); + } + }), + ), + ); + + /** + * Whether audio is currently being output through the earpiece. + */ + /*PUBLIC*/ const earpieceMode$ = scope.behavior( + combineLatest( + [mediaDevices.audioOutput.available$, mediaDevices.audioOutput.selected$], + (available, selected) => + selected !== undefined && + available.get(selected.id)?.type === "earpiece", + ), + ); + + /** + * Callback to toggle between the earpiece and the loudspeaker. + * + * This will be `null` in case the target does not exist in the list + * of available audio outputs. + */ + /*PUBLIC*/ const audioOutputSwitcher$ = scope.behavior<{ + targetOutput: "earpiece" | "speaker"; + switch: () => void; + } | null>( + combineLatest( + [mediaDevices.audioOutput.available$, mediaDevices.audioOutput.selected$], + (available, selected) => { + const selectionType = selected && available.get(selected.id)?.type; + + // If we are in any output mode other than speaker switch to speaker. + const newSelectionType: "earpiece" | "speaker" = + selectionType === "speaker" ? "earpiece" : "speaker"; + const newSelection = [...available].find( + ([, d]) => d.type === newSelectionType, + ); + if (newSelection === undefined) return null; + + const [id] = newSelection; + return { + targetOutput: newSelectionType, + switch: (): void => mediaDevices.audioOutput.select(id), + }; + }, + ), + ); + + /** + * Emits an array of reactions that should be visible on the screen. + */ + // DISCUSSION move this into a reaction file + // const {visibleReactions$, audibleReactions$} = reactionsObservables$(showReactionSetting$, ) + const visibleReactions$ = scope.behavior( + showReactions.value$.pipe( + switchMap((show) => (show ? reactions$ : of({}))), + scan< + Record, + { sender: string; emoji: string; startX: number }[] + >((acc, latest) => { + const newSet: { sender: string; emoji: string; startX: number }[] = []; + for (const [sender, reaction] of Object.entries(latest)) { + const startX = + acc.find((v) => v.sender === sender && v.emoji)?.startX ?? + Math.ceil(Math.random() * 80) + 10; + newSet.push({ sender, emoji: reaction.emoji, startX }); + } + return newSet; + }, []), + ), + ); + + /** + * Emits an array of reactions that should be played. + */ + const audibleReactions$ = playReactionsSound.value$.pipe( + switchMap((show) => + show ? reactions$ : of>({}), + ), + map((reactions) => Object.values(reactions).map((v) => v.name)), + scan( + (acc, latest) => { + return { + playing: latest.filter( + (v) => acc.playing.includes(v) || acc.newSounds.includes(v), + ), + newSounds: latest.filter( + (v) => !acc.playing.includes(v) && !acc.newSounds.includes(v), + ), + }; + }, + { playing: [], newSounds: [] }, + ), + map((v) => v.newSounds), + ); + + const newHandRaised$ = handsRaised$.pipe( + map((v) => Object.keys(v).length), + scan( + (acc, newValue) => ({ + value: newValue, + playSounds: newValue > acc.value, + }), + { value: 0, playSounds: false }, + ), + filter((v) => v.playSounds), + ); + + const newScreenShare$ = screenShares$.pipe( + map((v) => v.length), + scan( + (acc, newValue) => ({ + value: newValue, + playSounds: newValue > acc.value, + }), + { value: 0, playSounds: false }, + ), + filter((v) => v.playSounds), + ); + + /** + * Whether we are sharing our screen. + */ + // reassigned here to make it publicly accessible + const sharingScreen$ = localMembership.sharingScreen$; + + /** + * Callback to toggle screen sharing. If null, screen sharing is not possible. + */ + // reassigned here to make it publicly accessible + const toggleScreenSharing = localMembership.toggleScreenSharing; + + const join = localMembership.requestConnect; + join(); // TODO-MULTI-SFU: Use this view model for the lobby as well, and only call this once 'join' is clicked? + + return { + autoLeave$: callLifecycle.autoLeave$, + callPickupState$, + leave$, + hangup: () => userHangup$.next(), + join, + toggleScreenSharing, + sharingScreen$, + + tapScreen: () => screenTap$.next(), + tapControls: () => controlsTap$.next(), + hoverScreen: () => screenHover$.next(), + unhoverScreen: () => screenUnhover$.next(), + + configError$: localMembership.configError$, + participantCount$, + audioParticipants$, + isOneOnOneWith$, + localUserIsAlone$, + + handsRaised$, + reactions$, + joinSoundEffect$, + leaveSoundEffect$, + newHandRaised$, + newScreenShare$, + audibleReactions$, + visibleReactions$, + + windowMode$, + spotlightExpanded$, + toggleSpotlightExpanded$, + gridMode$, + setGridMode, + grid$, + spotlight$, + pip$, + layout$, + tileStoreGeneration$, + showSpotlightIndicators$, + showSpeakingIndicators$, + showHeader$, + showFooter$, + earpieceMode$, + audioOutputSwitcher$, + reconnecting$, + }; +} +// TODO-MULTI-SFU // Setup and update the keyProvider which was create by `createRoom` was a thing before. Now we never update if the E2EEsystem changes +// do we need this? + +function getE2eeKeyProvider( + e2eeSystem: EncryptionSystem, + rtcSession: MatrixRTCSession, +): BaseKeyProvider | undefined { + if (e2eeSystem.kind === E2eeType.NONE) return undefined; + + if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) { + const keyProvider = new MatrixKeyProvider(); + keyProvider.setRTCSession(rtcSession); + return keyProvider; + } else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) { + const keyProvider = new ExternalE2EEKeyProvider(); + keyProvider + .setKey(e2eeSystem.secret) + .catch((e) => logger.error("Failed to set shared key for E2EE", e)); + return keyProvider; + } +} diff --git a/src/state/CallViewModel/CallViewModelOld.ts b/src/state/CallViewModel/CallViewModelOld.ts new file mode 100644 index 00000000..ff4a6269 --- /dev/null +++ b/src/state/CallViewModel/CallViewModelOld.ts @@ -0,0 +1,1335 @@ +/* +Copyright 2023, 2024, 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + type BaseKeyProvider, + type ConnectionState, + ExternalE2EEKeyProvider, + type Room as LivekitRoom, + type RoomOptions, +} from "livekit-client"; +import { type Room as MatrixRoom } from "matrix-js-sdk"; +import { + combineLatest, + distinctUntilChanged, + EMPTY, + filter, + fromEvent, + map, + merge, + NEVER, + type Observable, + of, + pairwise, + race, + scan, + skip, + skipWhile, + startWith, + Subject, + switchAll, + switchMap, + switchScan, + take, + tap, + throttleTime, + timer, +} from "rxjs"; +import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; +import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; +import { type IWidgetApiRequest } from "matrix-widget-api"; + +import { + LocalUserMediaViewModel, + type MediaViewModel, + type RemoteUserMediaViewModel, + ScreenShareViewModel, + type UserMediaViewModel, +} from "../MediaViewModel"; +import { accumulate, generateItems, pauseWhen } from "../../utils/observable"; +import { + duplicateTiles, + MatrixRTCMode, + matrixRTCMode, + playReactionsSound, + showReactions, +} from "../../settings/settings"; +import { isFirefox } from "../../Platform"; +import { setPipEnabled$ } from "../../controls"; +import { TileStore } from "../TileStore"; +import { gridLikeLayout } from "../GridLikeLayout"; +import { spotlightExpandedLayout } from "../SpotlightExpandedLayout"; +import { oneOnOneLayout } from "../OneOnOneLayout"; +import { pipLayout } from "../PipLayout"; +import { type EncryptionSystem } from "../../e2ee/sharedKeyManagement"; +import { + type RaisedHandInfo, + type ReactionInfo, + type ReactionOption, +} from "../../reactions"; +import { shallowEquals } from "../../utils/array"; +import { type MediaDevices } from "../MediaDevices"; +import { type Behavior } from "../Behavior"; +import { E2eeType } from "../../e2ee/e2eeType"; +import { MatrixKeyProvider } from "../../e2ee/matrixKeyProvider"; +import { type MuteStates } from "../MuteStates"; +import { getUrlParams } from "../../UrlParams"; +import { type ProcessorState } from "../../livekit/TrackProcessorContext"; +import { ElementWidgetActions, widget } from "../../widget"; +import { UserMedia } from "../UserMedia.ts"; +import { ScreenShare } from "../ScreenShare.ts"; +import { + type GridLayoutMedia, + type Layout, + type LayoutMedia, + type OneOnOneLayoutMedia, + type SpotlightExpandedLayoutMedia, + type SpotlightLandscapeLayoutMedia, + type SpotlightPortraitLayoutMedia, +} from "../layout-types.ts"; +import { type ElementCallError } from "../../utils/errors.ts"; +import { type ObservableScope } from "../ObservableScope.ts"; +import { createLocalMembership$ } from "./localMember/LocalMembership.ts"; +import { createLocalTransport$ } from "./localMember/LocalTransport.ts"; +import { + createMemberships$, + membershipsAndTransports$, +} from "../SessionBehaviors.ts"; +import { ECConnectionFactory } from "./remoteMembers/ConnectionFactory.ts"; +import { createConnectionManager$ } from "./remoteMembers/ConnectionManager.ts"; +import { + createMatrixLivekitMembers$, + type MatrixLivekitMember, +} from "./remoteMembers/MatrixLivekitMembers.ts"; +import { + createCallNotificationLifecycle$, + createReceivedDecline$, + createSentCallNotification$, +} from "./CallNotificationLifecycle.ts"; +import { + createMatrixMemberMetadata$, + createRoomMembers$, +} from "./remoteMembers/MatrixMemberMetadata.ts"; + +const logger = rootLogger.getChild("[CallViewModel]"); +//TODO +// Larger rename +// member,membership -> rtcMember +// participant -> livekitParticipant +// matrixLivekitItem -> callMember +// js-sdk +// callMembership -> rtcMembership +export interface CallViewModelOptions { + encryptionSystem: EncryptionSystem; + autoLeaveWhenOthersLeft?: boolean; + /** + * If the call is started in a way where we want it to behave like a telephone usecase + * If we sent a notification event, we want the ui to show a ringing state + */ + waitForCallPickup?: boolean; + /** Optional factory to create LiveKit rooms, mainly for testing purposes. */ + livekitRoomFactory?: (options?: RoomOptions) => LivekitRoom; + /** Optional behavior overriding the local connection state, mainly for testing purposes. */ + connectionState$?: Behavior; +} + +// Do not play any sounds if the participant count has exceeded this +// number. +export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8; +export const THROTTLE_SOUND_EFFECT_MS = 500; + +// This is the number of participants that we think constitutes a "small" call +// on mobile. No spotlight tile should be shown below this threshold. +const smallMobileCallThreshold = 3; + +// How long the footer should be shown for when hovering over or interacting +// with the interface +const showFooterMs = 4000; + +export type GridMode = "grid" | "spotlight"; + +export type WindowMode = "normal" | "narrow" | "flat" | "pip"; + +interface LayoutScanState { + layout: Layout | null; + tiles: TileStore; +} + +type MediaItem = UserMedia | ScreenShare; +type AudioLivekitItem = { + livekitRoom: LivekitRoom; + participants: string[]; + url: string; +}; +/** + * A view model providing all the application logic needed to show the in-call + * UI (may eventually be expanded to cover the lobby and feedback screens in the + * future). + */ +// Throughout this class and related code we must distinguish between MatrixRTC +// state and LiveKit state. We use the common terminology of room "members", RTC +// "memberships", and LiveKit "participants". +export class CallViewModel { + private readonly userId = this.matrixRoom.client.getUserId()!; + private readonly deviceId = this.matrixRoom.client.getDeviceId()!; + + private readonly livekitKeyProvider = getE2eeKeyProvider( + this.options.encryptionSystem, + this.matrixRTCSession, + ); + + // Each hbar seperates a block of input variables required for the CallViewModel to function. + // The outputs of this block is written under the hbar. + // + // For mocking purposes it is recommended to only mock the functions creating those outputs. + // All other fields are just temp computations for the mentioned output. + // The class does not need anything except the values underneath the bar. + // The creation of the values under the bar are all tested independently and testing the callViewModel Should + // not test their cretation. Call view model only needs: + // - memberships$ via createMemberships$ + // - localMembership via createLocalMembership$ + // - callLifecycle via createCallNotificationLifecycle$ + // - matrixMemberMetadataStore via createMatrixMemberMetadata$ + + // ------------------------------------------------------------------------ + // memberships$ + private memberships$ = createMemberships$(this.scope, this.matrixRTCSession); + + // ------------------------------------------------------------------------ + // matrixLivekitMembers$ AND localMembership + + private membershipsAndTransports = membershipsAndTransports$( + this.scope, + this.memberships$, + ); + + private localTransport$ = createLocalTransport$({ + scope: this.scope, + memberships$: this.memberships$, + client: this.matrixRoom.client, + roomId: this.matrixRoom.roomId, + useOldestMember$: this.scope.behavior( + matrixRTCMode.value$.pipe(map((v) => v === MatrixRTCMode.Legacy)), + ), + }); + + private connectionFactory = new ECConnectionFactory( + this.matrixRoom.client, + this.mediaDevices, + this.trackProcessorState$, + this.livekitKeyProvider, + getUrlParams().controlledAudioDevices, + this.options.livekitRoomFactory, + ); + + private connectionManager = createConnectionManager$({ + scope: this.scope, + connectionFactory: this.connectionFactory, + inputTransports$: this.scope.behavior( + combineLatest( + [this.localTransport$, this.membershipsAndTransports.transports$], + (localTransport, transports) => { + const localTransportAsArray = localTransport ? [localTransport] : []; + return transports.mapInner((transports) => [ + ...localTransportAsArray, + ...transports, + ]); + }, + ), + ), + logger: logger, + }); + + private matrixLivekitMembers$ = createMatrixLivekitMembers$({ + scope: this.scope, + membershipsWithTransport$: + this.membershipsAndTransports.membershipsWithTransport$, + connectionManager: this.connectionManager, + }); + + private connectOptions$ = this.scope.behavior( + matrixRTCMode.value$.pipe( + map((mode) => ({ + encryptMedia: this.livekitKeyProvider !== undefined, + // TODO. This might need to get called again on each cahnge of matrixRTCMode... + matrixRTCMode: mode, + })), + ), + ); + + private localMembership = createLocalMembership$({ + scope: this.scope, + muteStates: this.muteStates, + mediaDevices: this.mediaDevices, + connectionManager: this.connectionManager, + matrixRTCSession: this.matrixRTCSession, + matrixRoom: this.matrixRoom, + localTransport$: this.localTransport$, + trackProcessorState$: this.trackProcessorState$, + widget, + options: this.connectOptions$, + logger: logger.getChild(`[${Date.now()}]`), + }); + + private localRtcMembership$ = this.scope.behavior( + this.memberships$.pipe( + map( + (memberships) => + memberships.value.find( + (membership) => + membership.userId === this.userId && + membership.deviceId === this.deviceId, + ) ?? null, + ), + ), + ); + + private localMatrixLivekitMemberUninitialized = { + membership$: this.localRtcMembership$, + participant$: this.localMembership.participant$, + connection$: this.localMembership.connection$, + userId: this.userId, + }; + + private localMatrixLivekitMember$: Behavior = + this.scope.behavior( + this.localRtcMembership$.pipe( + switchMap((membership) => { + if (!membership) return of(null); + return of( + // casting is save here since we know that localRtcMembership$ is !== null since we reached this case. + this.localMatrixLivekitMemberUninitialized as MatrixLivekitMember, + ); + }), + ), + ); + + // ------------------------------------------------------------------------ + // callLifecycle + + private callLifecycle = createCallNotificationLifecycle$({ + scope: this.scope, + memberships$: this.memberships$, + sentCallNotification$: createSentCallNotification$( + this.scope, + this.matrixRTCSession, + ), + receivedDecline$: createReceivedDecline$(this.matrixRoom), + options: this.options, + localUser: { userId: this.userId, deviceId: this.deviceId }, + }); + public autoLeave$ = this.callLifecycle.autoLeave$; + + // ------------------------------------------------------------------------ + // matrixMemberMetadataStore + + private matrixMemberMetadataStore = createMatrixMemberMetadata$( + this.scope, + this.scope.behavior(this.memberships$.pipe(map((mems) => mems.value))), + createRoomMembers$(this.scope, this.matrixRoom), + ); + + /** + * If there is a configuration error with the call (e.g. misconfigured E2EE). + * This is a fatal error that prevents the call from being created/joined. + * Should render a blocking error screen. + */ + public get configError$(): Behavior { + return this.localMembership.configError$; + } + + public join = this.localMembership.requestConnect; + + // CODESMELL? + // This is functionally the same Observable as leave$, except here it's + // hoisted to the top of the class. This enables the cyclic dependency between + // leave$ -> autoLeave$ -> callPickupState$ -> livekitConnectionState$ -> + // localConnection$ -> transports$ -> joined$ -> leave$. + private readonly leaveHoisted$ = new Subject< + "user" | "timeout" | "decline" | "allOthersLeft" + >(); + + /** + * Whether various media/event sources should pretend to be disconnected from + * all network input, even if their connection still technically works. + */ + // We do this when the app is in the 'reconnecting' state, because it might be + // that the LiveKit connection is still functional while the homeserver is + // down, for example, and we want to avoid making people worry that the app is + // in a split-brained state. + // DISCUSSION own membership manager ALSO this probably can be simplifis + public reconnecting$ = this.localMembership.reconnecting$; + private readonly pretendToBeDisconnected$ = this.reconnecting$; + + public readonly audioParticipants$ = this.scope.behavior( + this.matrixLivekitMembers$.pipe( + switchMap((membersWithEpoch) => { + const members = membersWithEpoch.value; + const a$ = combineLatest( + members.map((member) => + combineLatest([member.connection$, member.participant$]).pipe( + map(([connection, participant]) => { + // do not render audio for local participant + if (!connection || !participant || participant.isLocal) + return null; + const livekitRoom = connection.livekitRoom; + const url = connection.transport.livekit_service_url; + + return { url, livekitRoom, participant: participant.identity }; + }), + ), + ), + ); + return a$; + }), + map((members) => + members.reduce((acc, curr) => { + if (!curr) return acc; + + const existing = acc.find((item) => item.url === curr.url); + if (existing) { + existing.participants.push(curr.participant); + } else { + acc.push({ + livekitRoom: curr.livekitRoom, + participants: [curr.participant], + url: curr.url, + }); + } + return acc; + }, []), + ), + ), + [], + ); + + public readonly handsRaised$ = this.scope.behavior( + this.handsRaisedSubject$.pipe(pauseWhen(this.pretendToBeDisconnected$)), + ); + + public readonly reactions$ = this.scope.behavior( + this.reactionsSubject$.pipe( + map((v) => + Object.fromEntries( + Object.entries(v).map(([a, { reactionOption }]) => [ + a, + reactionOption, + ]), + ), + ), + pauseWhen(this.pretendToBeDisconnected$), + ), + ); + + /** + * List of user media (camera feeds) that we want tiles for. + */ + // TODO this also needs the local participant to be added. + private readonly userMedia$ = this.scope.behavior( + combineLatest([ + this.localMatrixLivekitMember$, + this.matrixLivekitMembers$, + duplicateTiles.value$, + ]).pipe( + // Generate a collection of MediaItems from the list of expected (whether + // present or missing) LiveKit participants. + generateItems( + function* ([ + localMatrixLivekitMember, + { value: matrixLivekitMembers }, + duplicateTiles, + ]) { + let localParticipantId = undefined; + // add local member if available + if (localMatrixLivekitMember) { + const { userId, participant$, connection$, membership$ } = + localMatrixLivekitMember; + localParticipantId = `${userId}:${membership$.value.deviceId}`; // should be membership$.value.membershipID which is not optional + // const participantId = membership$.value.membershipID; + if (localParticipantId) { + for (let dup = 0; dup < 1 + duplicateTiles; dup++) { + yield { + keys: [ + dup, + localParticipantId, + userId, + participant$, + connection$, + ], + data: undefined, + }; + } + } + } + // add remote members that are available + for (const { + userId, + participant$, + connection$, + membership$, + } of matrixLivekitMembers) { + const participantId = `${userId}:${membership$.value.deviceId}`; + if (participantId === localParticipantId) continue; + // const participantId = membership$.value?.identity; + for (let dup = 0; dup < 1 + duplicateTiles; dup++) { + yield { + keys: [dup, participantId, userId, participant$, connection$], + data: undefined, + }; + } + } + }, + ( + scope, + _data$, + dup, + participantId, + userId, + participant$, + connection$, + ) => { + const livekitRoom$ = scope.behavior( + connection$.pipe(map((c) => c?.livekitRoom)), + ); + const focusUrl$ = scope.behavior( + connection$.pipe(map((c) => c?.transport.livekit_service_url)), + ); + const displayName$ = scope.behavior( + this.matrixMemberMetadataStore + .createDisplayNameBehavior$(userId) + .pipe(map((name) => name ?? userId)), + ); + + return new UserMedia( + scope, + `${participantId}:${dup}`, + userId, + participant$, + this.options.encryptionSystem, + livekitRoom$, + focusUrl$, + this.mediaDevices, + this.pretendToBeDisconnected$, + displayName$, + this.matrixMemberMetadataStore.createAvatarUrlBehavior$(userId), + this.handsRaised$.pipe(map((v) => v[participantId]?.time ?? null)), + this.reactions$.pipe(map((v) => v[participantId] ?? undefined)), + ); + }, + ), + ), + ); + + /** + * List of all media items (user media and screen share media) that we want + * tiles for. + */ + private readonly mediaItems$ = this.scope.behavior( + this.userMedia$.pipe( + switchMap((userMedia) => + userMedia.length === 0 + ? of([]) + : combineLatest( + userMedia.map((m) => m.screenShares$), + (...screenShares) => [...userMedia, ...screenShares.flat(1)], + ), + ), + ), + ); + + /** + * List of MediaItems that we want to display, that are of type ScreenShare + */ + private readonly screenShares$ = this.scope.behavior( + this.mediaItems$.pipe( + map((mediaItems) => + mediaItems.filter((m): m is ScreenShare => m instanceof ScreenShare), + ), + ), + ); + + public readonly joinSoundEffect$ = this.userMedia$.pipe( + pairwise(), + filter( + ([prev, current]) => + current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && + current.length > prev.length, + ), + map(() => {}), + throttleTime(THROTTLE_SOUND_EFFECT_MS), + ); + + /** + * The number of participants currently in the call. + * + * - Each participant has a corresponding MatrixRTC membership state event + * - There can be multiple participants for one Matrix user if they join from + * multiple devices. + */ + public readonly participantCount$ = this.scope.behavior( + this.matrixLivekitMembers$.pipe(map((ms) => ms.value.length)), + ); + + // only public to expose to the view. + // TODO if we are in "unknown" state we need a loading rendering (or empty screen) + // Otherwise it looks like we already connected and only than the ringing starts which is weird. + public readonly callPickupState$ = this.callLifecycle.callPickupState$; + + public readonly leaveSoundEffect$ = combineLatest([ + this.callLifecycle.callPickupState$, + this.userMedia$, + ]).pipe( + // Until the call is successful, do not play a leave sound. + // If callPickupState$ is null, then we always play the sound as it will not conflict with a decline sound. + skipWhile(([c]) => c !== null && c !== "success"), + map(([, userMedia]) => userMedia), + pairwise(), + filter( + ([prev, current]) => + current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && + current.length < prev.length, + ), + map(() => {}), + throttleTime(THROTTLE_SOUND_EFFECT_MS), + ); + + private readonly userHangup$ = new Subject(); + public hangup(): void { + this.userHangup$.next(); + } + + private readonly widgetHangup$ = + widget === null + ? NEVER + : ( + fromEvent( + widget.lazyActions, + ElementWidgetActions.HangupCall, + ) as Observable> + ).pipe( + tap((ev) => { + widget!.api.transport.reply(ev.detail, {}); + }), + ); + + public readonly leave$: Observable< + "user" | "timeout" | "decline" | "allOthersLeft" + > = merge( + this.callLifecycle.autoLeave$, + merge(this.userHangup$, this.widgetHangup$).pipe( + map(() => "user" as const), + ), + ).pipe( + this.scope.share, + tap((reason) => this.leaveHoisted$.next(reason)), + ); + + private readonly spotlightSpeaker$ = + this.scope.behavior( + this.userMedia$.pipe( + switchMap((mediaItems) => + mediaItems.length === 0 + ? of([]) + : combineLatest( + mediaItems.map((m) => + m.vm.speaking$.pipe(map((s) => [m, s] as const)), + ), + ), + ), + scan<(readonly [UserMedia, boolean])[], UserMedia | undefined, null>( + (prev, mediaItems) => { + // Only remote users that are still in the call should be sticky + const [stickyMedia, stickySpeaking] = + (!prev?.vm.local && mediaItems.find(([m]) => m === prev)) || []; + // Decide who to spotlight: + // If the previous speaker is still speaking, stick with them rather + // than switching eagerly to someone else + return stickySpeaking + ? stickyMedia! + : // Otherwise, select any remote user who is speaking + (mediaItems.find(([m, s]) => !m.vm.local && s)?.[0] ?? + // Otherwise, stick with the person who was last speaking + stickyMedia ?? + // Otherwise, spotlight an arbitrary remote user + mediaItems.find(([m]) => !m.vm.local)?.[0] ?? + // Otherwise, spotlight the local user + mediaItems.find(([m]) => m.vm.local)?.[0]); + }, + null, + ), + map((speaker) => speaker?.vm ?? null), + ), + ); + + private readonly grid$ = this.scope.behavior( + this.userMedia$.pipe( + switchMap((mediaItems) => { + const bins = mediaItems.map((m) => + m.bin$.pipe(map((bin) => [m, bin] as const)), + ); + // Sort the media by bin order and generate a tile for each one + return bins.length === 0 + ? of([]) + : combineLatest(bins, (...bins) => + bins.sort(([, bin1], [, bin2]) => bin1 - bin2).map(([m]) => m.vm), + ); + }), + distinctUntilChanged(shallowEquals), + ), + ); + + private readonly spotlight$ = this.scope.behavior( + this.screenShares$.pipe( + switchMap((screenShares) => { + if (screenShares.length > 0) { + return of(screenShares.map((m) => m.vm)); + } + + return this.spotlightSpeaker$.pipe( + map((speaker) => (speaker ? [speaker] : [])), + ); + }), + distinctUntilChanged(shallowEquals), + ), + ); + + private readonly pip$ = this.scope.behavior( + combineLatest([ + // TODO This also needs epoch logic to dedupe the screenshares and mediaItems emits + this.screenShares$, + this.spotlightSpeaker$, + this.mediaItems$, + ]).pipe( + switchMap(([screenShares, spotlight, mediaItems]) => { + if (screenShares.length > 0) { + return this.spotlightSpeaker$; + } + if (!spotlight || spotlight.local) { + return of(null); + } + + const localUserMedia = mediaItems.find( + (m) => m.vm instanceof LocalUserMediaViewModel, + ) as UserMedia | undefined; + + const localUserMediaViewModel = localUserMedia?.vm as + | LocalUserMediaViewModel + | undefined; + + if (!localUserMediaViewModel) { + return of(null); + } + return localUserMediaViewModel.alwaysShow$.pipe( + map((alwaysShow) => { + if (alwaysShow) { + return localUserMediaViewModel; + } + + return null; + }), + ); + }), + ), + ); + + private readonly hasRemoteScreenShares$: Observable = + this.spotlight$.pipe( + map((spotlight) => + spotlight.some((vm) => !vm.local && vm instanceof ScreenShareViewModel), + ), + distinctUntilChanged(), + ); + + private readonly pipEnabled$ = this.scope.behavior(setPipEnabled$, false); + + private readonly naturalWindowMode$ = this.scope.behavior( + fromEvent(window, "resize").pipe( + map(() => { + const height = window.innerHeight; + const width = window.innerWidth; + if (height <= 400 && width <= 340) return "pip"; + // Our layouts for flat windows are better at adapting to a small width + // than our layouts for narrow windows are at adapting to a small height, + // so we give "flat" precedence here + if (height <= 600) return "flat"; + if (width <= 600) return "narrow"; + return "normal"; + }), + ), + "normal", + ); + + /** + * The general shape of the window. + */ + public readonly windowMode$ = this.scope.behavior( + this.pipEnabled$.pipe( + switchMap((pip) => + pip ? of("pip") : this.naturalWindowMode$, + ), + ), + ); + + private readonly spotlightExpandedToggle$ = new Subject(); + public readonly spotlightExpanded$ = this.scope.behavior( + this.spotlightExpandedToggle$.pipe( + accumulate(false, (expanded) => !expanded), + ), + ); + + private readonly gridModeUserSelection$ = new Subject(); + /** + * The layout mode of the media tile grid. + */ + public readonly gridMode$ = + // If the user hasn't selected spotlight and somebody starts screen sharing, + // automatically switch to spotlight mode and reset when screen sharing ends + this.scope.behavior( + this.gridModeUserSelection$.pipe( + switchMap((userSelection) => + (userSelection === "spotlight" + ? EMPTY + : combineLatest([ + this.hasRemoteScreenShares$, + this.windowMode$, + ]).pipe( + skip(userSelection === null ? 0 : 1), + map( + ([hasScreenShares, windowMode]): GridMode => + hasScreenShares || windowMode === "flat" + ? "spotlight" + : "grid", + ), + ) + ).pipe(startWith(userSelection ?? "grid")), + ), + ), + "grid", + ); + + public setGridMode(value: GridMode): void { + this.gridModeUserSelection$.next(value); + } + + private readonly gridLayoutMedia$: Observable = + combineLatest([this.grid$, this.spotlight$], (grid, spotlight) => ({ + type: "grid", + spotlight: spotlight.some((vm) => vm instanceof ScreenShareViewModel) + ? spotlight + : undefined, + grid, + })); + + private readonly spotlightLandscapeLayoutMedia$: Observable = + combineLatest([this.grid$, this.spotlight$], (grid, spotlight) => ({ + type: "spotlight-landscape", + spotlight, + grid, + })); + + private readonly spotlightPortraitLayoutMedia$: Observable = + combineLatest([this.grid$, this.spotlight$], (grid, spotlight) => ({ + type: "spotlight-portrait", + spotlight, + grid, + })); + + private readonly spotlightExpandedLayoutMedia$: Observable = + combineLatest([this.spotlight$, this.pip$], (spotlight, pip) => ({ + type: "spotlight-expanded", + spotlight, + pip: pip ?? undefined, + })); + + private readonly oneOnOneLayoutMedia$: Observable = + this.mediaItems$.pipe( + map((mediaItems) => { + if (mediaItems.length !== 2) return null; + const local = mediaItems.find((vm) => vm.vm.local)?.vm as + | LocalUserMediaViewModel + | undefined; + const remote = mediaItems.find((vm) => !vm.vm.local)?.vm as + | RemoteUserMediaViewModel + | undefined; + // There might not be a remote tile if there are screen shares, or if + // only the local user is in the call and they're using the duplicate + // tiles option + if (!remote || !local) return null; + + return { type: "one-on-one", local, remote }; + }), + ); + + private readonly pipLayoutMedia$: Observable = + this.spotlight$.pipe(map((spotlight) => ({ type: "pip", spotlight }))); + + /** + * The media to be used to produce a layout. + */ + private readonly layoutMedia$ = this.scope.behavior( + this.windowMode$.pipe( + switchMap((windowMode) => { + switch (windowMode) { + case "normal": + return this.gridMode$.pipe( + switchMap((gridMode) => { + switch (gridMode) { + case "grid": + return this.oneOnOneLayoutMedia$.pipe( + switchMap((oneOnOne) => + oneOnOne === null + ? this.gridLayoutMedia$ + : of(oneOnOne), + ), + ); + case "spotlight": + return this.spotlightExpanded$.pipe( + switchMap((expanded) => + expanded + ? this.spotlightExpandedLayoutMedia$ + : this.spotlightLandscapeLayoutMedia$, + ), + ); + } + }), + ); + case "narrow": + return this.oneOnOneLayoutMedia$.pipe( + switchMap((oneOnOne) => + oneOnOne === null + ? combineLatest( + [this.grid$, this.spotlight$], + (grid, spotlight) => + grid.length > smallMobileCallThreshold || + spotlight.some( + (vm) => vm instanceof ScreenShareViewModel, + ) + ? this.spotlightPortraitLayoutMedia$ + : this.gridLayoutMedia$, + ).pipe(switchAll()) + : // The expanded spotlight layout makes for a better one-on-one + // experience in narrow windows + this.spotlightExpandedLayoutMedia$, + ), + ); + case "flat": + return this.gridMode$.pipe( + switchMap((gridMode) => { + switch (gridMode) { + case "grid": + // Yes, grid mode actually gets you a "spotlight" layout in + // this window mode. + return this.spotlightLandscapeLayoutMedia$; + case "spotlight": + return this.spotlightExpandedLayoutMedia$; + } + }), + ); + case "pip": + return this.pipLayoutMedia$; + } + }), + ), + ); + + // There is a cyclical dependency here: the layout algorithms want to know + // which tiles are on screen, but to know which tiles are on screen we have to + // first render a layout. To deal with this we assume initially that no tiles + // are visible, and loop the data back into the layouts with a Subject. + private readonly visibleTiles$ = new Subject(); + private readonly setVisibleTiles = (value: number): void => + this.visibleTiles$.next(value); + + private readonly layoutInternals$ = this.scope.behavior< + LayoutScanState & { layout: Layout } + >( + combineLatest([ + this.layoutMedia$, + this.visibleTiles$.pipe(startWith(0), distinctUntilChanged()), + ]).pipe( + scan< + [LayoutMedia, number], + LayoutScanState & { layout: Layout }, + LayoutScanState + >( + ({ tiles: prevTiles }, [media, visibleTiles]) => { + let layout: Layout; + let newTiles: TileStore; + switch (media.type) { + case "grid": + case "spotlight-landscape": + case "spotlight-portrait": + [layout, newTiles] = gridLikeLayout( + media, + visibleTiles, + this.setVisibleTiles, + prevTiles, + ); + break; + case "spotlight-expanded": + [layout, newTiles] = spotlightExpandedLayout(media, prevTiles); + break; + case "one-on-one": + [layout, newTiles] = oneOnOneLayout(media, prevTiles); + break; + case "pip": + [layout, newTiles] = pipLayout(media, prevTiles); + break; + } + + return { layout, tiles: newTiles }; + }, + { layout: null, tiles: TileStore.empty() }, + ), + ), + ); + + /** + * The layout of tiles in the call interface. + */ + public readonly layout$ = this.scope.behavior( + this.layoutInternals$.pipe(map(({ layout }) => layout)), + ); + + /** + * The current generation of the tile store, exposed for debugging purposes. + */ + public readonly tileStoreGeneration$ = this.scope.behavior( + this.layoutInternals$.pipe(map(({ tiles }) => tiles.generation)), + ); + + public showSpotlightIndicators$ = this.scope.behavior( + this.layout$.pipe(map((l) => l.type !== "grid")), + ); + + public showSpeakingIndicators$ = this.scope.behavior( + this.layout$.pipe( + switchMap((l) => { + switch (l.type) { + case "spotlight-landscape": + case "spotlight-portrait": + // If the spotlight is showing the active speaker, we can do without + // speaking indicators as they're a redundant visual cue. But if + // screen sharing feeds are in the spotlight we still need them. + return l.spotlight.media$.pipe( + map((models: MediaViewModel[]) => + models.some((m) => m instanceof ScreenShareViewModel), + ), + ); + // In expanded spotlight layout, the active speaker is always shown in + // the picture-in-picture tile so there is no need for speaking + // indicators. And in one-on-one layout there's no question as to who is + // speaking. + case "spotlight-expanded": + case "one-on-one": + return of(false); + default: + return of(true); + } + }), + ), + ); + + public readonly toggleSpotlightExpanded$ = this.scope.behavior< + (() => void) | null + >( + this.windowMode$.pipe( + switchMap((mode) => + mode === "normal" + ? this.layout$.pipe( + map( + (l) => + l.type === "spotlight-landscape" || + l.type === "spotlight-expanded", + ), + ) + : of(false), + ), + distinctUntilChanged(), + map((enabled) => + enabled ? (): void => this.spotlightExpandedToggle$.next() : null, + ), + ), + ); + + private readonly screenTap$ = new Subject(); + private readonly controlsTap$ = new Subject(); + private readonly screenHover$ = new Subject(); + private readonly screenUnhover$ = new Subject(); + + /** + * Callback for when the user taps the call view. + */ + public tapScreen(): void { + this.screenTap$.next(); + } + + /** + * Callback for when the user taps the call's controls. + */ + public tapControls(): void { + this.controlsTap$.next(); + } + + /** + * Callback for when the user hovers over the call view. + */ + public hoverScreen(): void { + this.screenHover$.next(); + } + + /** + * Callback for when the user stops hovering over the call view. + */ + public unhoverScreen(): void { + this.screenUnhover$.next(); + } + + public readonly showHeader$ = this.scope.behavior( + this.windowMode$.pipe(map((mode) => mode !== "pip" && mode !== "flat")), + ); + + public readonly showFooter$ = this.scope.behavior( + this.windowMode$.pipe( + switchMap((mode) => { + switch (mode) { + case "pip": + return of(false); + case "normal": + case "narrow": + return of(true); + case "flat": + // Sadly Firefox has some layering glitches that prevent the footer + // from appearing properly. They happen less often if we never hide + // the footer. + if (isFirefox()) return of(true); + // Show/hide the footer in response to interactions + return merge( + this.screenTap$.pipe(map(() => "tap screen" as const)), + this.controlsTap$.pipe(map(() => "tap controls" as const)), + this.screenHover$.pipe(map(() => "hover" as const)), + ).pipe( + switchScan((state, interaction) => { + switch (interaction) { + case "tap screen": + return state + ? // Toggle visibility on tap + of(false) + : // Hide after a timeout + timer(showFooterMs).pipe( + map(() => false), + startWith(true), + ); + case "tap controls": + // The user is interacting with things, so reset the timeout + return timer(showFooterMs).pipe( + map(() => false), + startWith(true), + ); + case "hover": + // Show on hover and hide after a timeout + return race( + timer(showFooterMs), + this.screenUnhover$.pipe(take(1)), + ).pipe( + map(() => false), + startWith(true), + ); + } + }, false), + startWith(false), + ); + } + }), + ), + ); + + /** + * Whether audio is currently being output through the earpiece. + */ + public readonly earpieceMode$ = this.scope.behavior( + combineLatest( + [ + this.mediaDevices.audioOutput.available$, + this.mediaDevices.audioOutput.selected$, + ], + (available, selected) => + selected !== undefined && + available.get(selected.id)?.type === "earpiece", + ), + ); + + /** + * Callback to toggle between the earpiece and the loudspeaker. + * + * This will be `null` in case the target does not exist in the list + * of available audio outputs. + */ + public readonly audioOutputSwitcher$ = this.scope.behavior<{ + targetOutput: "earpiece" | "speaker"; + switch: () => void; + } | null>( + combineLatest( + [ + this.mediaDevices.audioOutput.available$, + this.mediaDevices.audioOutput.selected$, + ], + (available, selected) => { + const selectionType = selected && available.get(selected.id)?.type; + + // If we are in any output mode other than speaker switch to speaker. + const newSelectionType: "earpiece" | "speaker" = + selectionType === "speaker" ? "earpiece" : "speaker"; + const newSelection = [...available].find( + ([, d]) => d.type === newSelectionType, + ); + if (newSelection === undefined) return null; + + const [id] = newSelection; + return { + targetOutput: newSelectionType, + switch: (): void => this.mediaDevices.audioOutput.select(id), + }; + }, + ), + ); + + /** + * Emits an array of reactions that should be visible on the screen. + */ + // DISCUSSION move this into a reaction file + // const {visibleReactions$, audibleReactions$} = reactionsObservables$(showReactionSetting$, ) + public readonly visibleReactions$ = this.scope.behavior( + showReactions.value$.pipe( + switchMap((show) => (show ? this.reactions$ : of({}))), + scan< + Record, + { sender: string; emoji: string; startX: number }[] + >((acc, latest) => { + const newSet: { sender: string; emoji: string; startX: number }[] = []; + for (const [sender, reaction] of Object.entries(latest)) { + const startX = + acc.find((v) => v.sender === sender && v.emoji)?.startX ?? + Math.ceil(Math.random() * 80) + 10; + newSet.push({ sender, emoji: reaction.emoji, startX }); + } + return newSet; + }, []), + ), + ); + + /** + * Emits an array of reactions that should be played. + */ + public readonly audibleReactions$ = playReactionsSound.value$.pipe( + switchMap((show) => + show ? this.reactions$ : of>({}), + ), + map((reactions) => Object.values(reactions).map((v) => v.name)), + scan( + (acc, latest) => { + return { + playing: latest.filter( + (v) => acc.playing.includes(v) || acc.newSounds.includes(v), + ), + newSounds: latest.filter( + (v) => !acc.playing.includes(v) && !acc.newSounds.includes(v), + ), + }; + }, + { playing: [], newSounds: [] }, + ), + map((v) => v.newSounds), + ); + + /** + * Emits an event every time a new hand is raised in + * the call. + */ + public readonly newHandRaised$ = this.handsRaised$.pipe( + map((v) => Object.keys(v).length), + scan( + (acc, newValue) => ({ + value: newValue, + playSounds: newValue > acc.value, + }), + { value: 0, playSounds: false }, + ), + filter((v) => v.playSounds), + ); + + /** + * Emits an event every time a new screenshare is started in + * the call. + */ + public readonly newScreenShare$ = this.screenShares$.pipe( + map((v) => v.length), + scan( + (acc, newValue) => ({ + value: newValue, + playSounds: newValue > acc.value, + }), + { value: 0, playSounds: false }, + ), + filter((v) => v.playSounds), + ); + + /** + * Whether we are sharing our screen. + */ + // reassigned here to make it publicly accessible + public readonly sharingScreen$ = this.localMembership.sharingScreen$; + + /** + * Callback to toggle screen sharing. If null, screen sharing is not possible. + */ + // reassigned here to make it publicly accessible + public readonly toggleScreenSharing = + this.localMembership.toggleScreenSharing; + + public constructor( + private readonly scope: ObservableScope, + // A call is permanently tied to a single Matrix room + private readonly matrixRTCSession: MatrixRTCSession, + private readonly matrixRoom: MatrixRoom, + private readonly mediaDevices: MediaDevices, + private readonly muteStates: MuteStates, + private readonly options: CallViewModelOptions, + private readonly handsRaisedSubject$: Observable< + Record + >, + private readonly reactionsSubject$: Observable< + Record + >, + private readonly trackProcessorState$: Behavior, + ) { + // Join automatically + this.join(); // TODO-MULTI-SFU: Use this view model for the lobby as well, and only call this once 'join' is clicked? + } +} + +// TODO-MULTI-SFU // Setup and update the keyProvider which was create by `createRoom` was a thing before. Now we never update if the E2EEsystem changes +// do we need this? + +function getE2eeKeyProvider( + e2eeSystem: EncryptionSystem, + rtcSession: MatrixRTCSession, +): BaseKeyProvider | undefined { + if (e2eeSystem.kind === E2eeType.NONE) return undefined; + + if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) { + const keyProvider = new MatrixKeyProvider(); + keyProvider.setRTCSession(rtcSession); + return keyProvider; + } else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) { + const keyProvider = new ExternalE2EEKeyProvider(); + keyProvider + .setKey(e2eeSystem.secret) + .catch((e) => logger.error("Failed to set shared key for E2EE", e)); + return keyProvider; + } +} From 7ab90d3c1f152a890d7ce73fc22005fbc2eeec81 Mon Sep 17 00:00:00 2001 From: Timo K Date: Mon, 17 Nov 2025 12:00:28 +0100 Subject: [PATCH 63/75] Delete unused files --- .../CallViewModel/CallViewModelFunction.ts | 1400 ----------------- src/state/CallViewModel/CallViewModelOld.ts | 1335 ---------------- 2 files changed, 2735 deletions(-) delete mode 100644 src/state/CallViewModel/CallViewModelFunction.ts delete mode 100644 src/state/CallViewModel/CallViewModelOld.ts diff --git a/src/state/CallViewModel/CallViewModelFunction.ts b/src/state/CallViewModel/CallViewModelFunction.ts deleted file mode 100644 index 20e37395..00000000 --- a/src/state/CallViewModel/CallViewModelFunction.ts +++ /dev/null @@ -1,1400 +0,0 @@ -/* -Copyright 2023, 2024, 2025 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - type BaseKeyProvider, - type ConnectionState, - ExternalE2EEKeyProvider, - type Room as LivekitRoom, - type RoomOptions, -} from "livekit-client"; -import { type RoomMember, type Room as MatrixRoom } from "matrix-js-sdk"; -import { - combineLatest, - distinctUntilChanged, - EMPTY, - filter, - fromEvent, - map, - merge, - NEVER, - type Observable, - of, - pairwise, - race, - scan, - skip, - skipWhile, - startWith, - Subject, - switchAll, - switchMap, - switchScan, - take, - tap, - throttleTime, - timer, -} from "rxjs"; -import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; -import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; -import { type IWidgetApiRequest } from "matrix-widget-api"; - -import { - LocalUserMediaViewModel, - type MediaViewModel, - type RemoteUserMediaViewModel, - ScreenShareViewModel, - type UserMediaViewModel, -} from "../MediaViewModel"; -import { accumulate, generateItems, pauseWhen } from "../../utils/observable"; -import { - duplicateTiles, - MatrixRTCMode, - matrixRTCMode, - playReactionsSound, - showReactions, -} from "../../settings/settings"; -import { isFirefox } from "../../Platform"; -import { setPipEnabled$ } from "../../controls"; -import { TileStore } from "../TileStore"; -import { gridLikeLayout } from "../GridLikeLayout"; -import { spotlightExpandedLayout } from "../SpotlightExpandedLayout"; -import { oneOnOneLayout } from "../OneOnOneLayout"; -import { pipLayout } from "../PipLayout"; -import { type EncryptionSystem } from "../../e2ee/sharedKeyManagement"; -import { - type RaisedHandInfo, - type ReactionInfo, - type ReactionOption, -} from "../../reactions"; -import { shallowEquals } from "../../utils/array"; -import { type MediaDevices } from "../MediaDevices"; -import { type Behavior } from "../Behavior"; -import { E2eeType } from "../../e2ee/e2eeType"; -import { MatrixKeyProvider } from "../../e2ee/matrixKeyProvider"; -import { type MuteStates } from "../MuteStates"; -import { getUrlParams } from "../../UrlParams"; -import { type ProcessorState } from "../../livekit/TrackProcessorContext"; -import { ElementWidgetActions, widget } from "../../widget"; -import { UserMedia } from "../UserMedia.ts"; -import { ScreenShare } from "../ScreenShare.ts"; -import { - type GridLayoutMedia, - type Layout, - type LayoutMedia, - type OneOnOneLayoutMedia, - type SpotlightExpandedLayoutMedia, - type SpotlightLandscapeLayoutMedia, - type SpotlightPortraitLayoutMedia, -} from "../layout-types.ts"; -import { type ElementCallError } from "../../utils/errors.ts"; -import { type ObservableScope } from "../ObservableScope.ts"; -import { createLocalMembership$ } from "./localMember/LocalMembership.ts"; -import { createLocalTransport$ } from "./localMember/LocalTransport.ts"; -import { - createMemberships$, - membershipsAndTransports$, -} from "../SessionBehaviors.ts"; -import { ECConnectionFactory } from "./remoteMembers/ConnectionFactory.ts"; -import { createConnectionManager$ } from "./remoteMembers/ConnectionManager.ts"; -import { - createMatrixLivekitMembers$, - type MatrixLivekitMember, -} from "./remoteMembers/MatrixLivekitMembers.ts"; -import { - type AutoLeaveReason, - createCallNotificationLifecycle$, - createReceivedDecline$, - createSentCallNotification$, -} from "./CallNotificationLifecycle.ts"; -import { - createMatrixMemberMetadata$, - createRoomMembers$, -} from "./remoteMembers/MatrixMemberMetadata.ts"; - -const logger = rootLogger.getChild("[CallViewModel]"); -//TODO -// Larger rename -// member,membership -> rtcMember -// participant -> livekitParticipant -// matrixLivekitItem -> callMember -// js-sdk -// callMembership -> rtcMembership -export interface CallViewModelOptions { - encryptionSystem: EncryptionSystem; - autoLeaveWhenOthersLeft?: boolean; - /** - * If the call is started in a way where we want it to behave like a telephone usecase - * If we sent a notification event, we want the ui to show a ringing state - */ - waitForCallPickup?: boolean; - /** Optional factory to create LiveKit rooms, mainly for testing purposes. */ - livekitRoomFactory?: (options?: RoomOptions) => LivekitRoom; - /** Optional behavior overriding the local connection state, mainly for testing purposes. */ - connectionState$?: Behavior; -} - -// Do not play any sounds if the participant count has exceeded this -// number. -export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8; -export const THROTTLE_SOUND_EFFECT_MS = 500; - -// This is the number of participants that we think constitutes a "small" call -// on mobile. No spotlight tile should be shown below this threshold. -const smallMobileCallThreshold = 3; - -// How long the footer should be shown for when hovering over or interacting -// with the interface -const showFooterMs = 4000; - -export type GridMode = "grid" | "spotlight"; - -export type WindowMode = "normal" | "narrow" | "flat" | "pip"; - -interface LayoutScanState { - layout: Layout | null; - tiles: TileStore; -} - -type MediaItem = UserMedia | ScreenShare; -type AudioLivekitItem = { - livekitRoom: LivekitRoom; - participants: string[]; - url: string; -}; - -type JoinReturn = ReturnType< - ReturnType["requestConnect"] ->; - -/** - * A view model providing all the application logic needed to show the in-call - * UI (may eventually be expanded to cover the lobby and feedback screens in the - * future). - */ -// Throughout this class and related code we must distinguish between MatrixRTC -// state and LiveKit state. We use the common terminology of room "members", RTC -// "memberships", and LiveKit "participants". -export function createCallViewModel$( - scope: ObservableScope, - // A call is permanently tied to a single Matrix room - matrixRTCSession: MatrixRTCSession, - matrixRoom: MatrixRoom, - mediaDevices: MediaDevices, - muteStates: MuteStates, - options: CallViewModelOptions, - handsRaisedSubject$: Observable>, - reactionsSubject$: Observable>, - trackProcessorState$: Behavior, -): { - // lifecycle - autoLeave$: Observable; - callPickupState$: Behavior< - "unknown" | "ringing" | "timeout" | "decline" | "success" | null - >; - leave$: Observable<"user" | AutoLeaveReason>; - hangup: () => void; - - // joining - join: () => JoinReturn; - - // screen sharing - toggleScreenSharing: (() => void) | null; - sharingScreen$: Behavior; - - // UI interactions - tapScreen: () => void; - tapControls: () => void; - hoverScreen: () => void; - unhoverScreen: () => void; - - // errors - configError$: Behavior; - - // participants and counts - participantCount$: Behavior; - audioParticipants$: Behavior; - handsRaised$: Behavior>; - reactions$: Behavior>; - isOneOnOneWith$: Behavior | null>; - localUserIsAlone$: Behavior; - // sounds and events - joinSoundEffect$: Observable; - leaveSoundEffect$: Observable; - newHandRaised$: Observable<{ value: number; playSounds: boolean }>; - newScreenShare$: Observable<{ value: number; playSounds: boolean }>; - audibleReactions$: Observable; - visibleReactions$: Behavior< - { sender: string; emoji: string; startX: number }[] - >; - - // window/layout - windowMode$: Behavior; - spotlightExpanded$: Behavior; - toggleSpotlightExpanded$: Behavior<(() => void) | null>; - gridMode$: Behavior; - setGridMode: (value: GridMode) => void; - - // media view models and layout - grid$: Behavior; - spotlight$: Behavior; - pip$: Behavior; - layout$: Behavior; - tileStoreGeneration$: Behavior; - showSpotlightIndicators$: Behavior; - showSpeakingIndicators$: Behavior; - - // header/footer visibility - showHeader$: Behavior; - showFooter$: Behavior; - - // audio routing - earpieceMode$: Behavior; - audioOutputSwitcher$: Behavior<{ - targetOutput: "earpiece" | "speaker"; - switch: () => void; - } | null>; - - // connection state - reconnecting$: Behavior; -} { - const userId = matrixRoom.client.getUserId()!; - const deviceId = matrixRoom.client.getDeviceId()!; - - const livekitKeyProvider = getE2eeKeyProvider( - options.encryptionSystem, - matrixRTCSession, - ); - - // Each hbar seperates a block of input variables required for the CallViewModel to function. - // The outputs of this block is written under the hbar. - // - // For mocking purposes it is recommended to only mock the functions creating those outputs. - // All other fields are just temp computations for the mentioned output. - // The class does not need anything except the values underneath the bar. - // The creation of the values under the bar are all tested independently and testing the callViewModel Should - // not test their cretation. Call view model only needs: - // - memberships$ via createMemberships$ - // - localMembership via createLocalMembership$ - // - callLifecycle via createCallNotificationLifecycle$ - // - matrixMemberMetadataStore via createMatrixMemberMetadata$ - - // ------------------------------------------------------------------------ - // memberships$ - const memberships$ = createMemberships$(scope, matrixRTCSession); - - // ------------------------------------------------------------------------ - // matrixLivekitMembers$ AND localMembership - - const membershipsAndTransports = membershipsAndTransports$( - scope, - memberships$, - ); - - const localTransport$ = createLocalTransport$({ - scope: scope, - memberships$: memberships$, - client: matrixRoom.client, - roomId: matrixRoom.roomId, - useOldestMember$: scope.behavior( - matrixRTCMode.value$.pipe(map((v) => v === MatrixRTCMode.Legacy)), - ), - }); - - const connectionFactory = new ECConnectionFactory( - matrixRoom.client, - mediaDevices, - trackProcessorState$, - livekitKeyProvider, - getUrlParams().controlledAudioDevices, - options.livekitRoomFactory, - ); - - const connectionManager = createConnectionManager$({ - scope: scope, - connectionFactory: connectionFactory, - inputTransports$: scope.behavior( - combineLatest( - [localTransport$, membershipsAndTransports.transports$], - (localTransport, transports) => { - const localTransportAsArray = localTransport ? [localTransport] : []; - return transports.mapInner((transports) => [ - ...localTransportAsArray, - ...transports, - ]); - }, - ), - ), - logger: logger, - }); - - const matrixLivekitMembers$ = createMatrixLivekitMembers$({ - scope: scope, - membershipsWithTransport$: - membershipsAndTransports.membershipsWithTransport$, - connectionManager: connectionManager, - }); - - const connectOptions$ = scope.behavior( - matrixRTCMode.value$.pipe( - map((mode) => ({ - encryptMedia: livekitKeyProvider !== undefined, - // TODO. This might need to get called again on each cahnge of matrixRTCMode... - matrixRTCMode: mode, - })), - ), - ); - - const localMembership = createLocalMembership$({ - scope: scope, - muteStates: muteStates, - mediaDevices: mediaDevices, - connectionManager: connectionManager, - matrixRTCSession: matrixRTCSession, - matrixRoom: matrixRoom, - localTransport$: localTransport$, - trackProcessorState$: trackProcessorState$, - widget, - options: connectOptions$, - logger: logger.getChild(`[${Date.now()}]`), - }); - - const localRtcMembership$ = scope.behavior( - memberships$.pipe( - map( - (memberships) => - memberships.value.find( - (membership) => - membership.userId === userId && membership.deviceId === deviceId, - ) ?? null, - ), - ), - ); - - const localMatrixLivekitMemberUninitialized = { - membership$: localRtcMembership$, - participant$: localMembership.participant$, - connection$: localMembership.connection$, - userId: userId, - }; - - const localMatrixLivekitMember$: Behavior = - scope.behavior( - localRtcMembership$.pipe( - switchMap((membership) => { - if (!membership) return of(null); - return of( - // casting is save here since we know that localRtcMembership$ is !== null since we reached this case. - localMatrixLivekitMemberUninitialized as MatrixLivekitMember, - ); - }), - ), - ); - - // ------------------------------------------------------------------------ - // callLifecycle - - const callLifecycle = createCallNotificationLifecycle$({ - scope: scope, - memberships$: memberships$, - sentCallNotification$: createSentCallNotification$(scope, matrixRTCSession), - receivedDecline$: createReceivedDecline$(matrixRoom), - options: options, - localUser: { userId: userId, deviceId: deviceId }, - }); - - // ------------------------------------------------------------------------ - // matrixMemberMetadataStore - - const matrixRoomMembers$ = createRoomMembers$(scope, matrixRoom); - const matrixMemberMetadataStore = createMatrixMemberMetadata$( - scope, - scope.behavior(memberships$.pipe(map((mems) => mems.value))), - matrixRoomMembers$, - ); - - /** - * Returns the Member {userId, getMxcAvatarUrl, rawDisplayName} of the other user in the call, if it's a one-on-one call. - */ - const isOneOnOneWith$ = scope.behavior( - matrixRoomMembers$.pipe( - map((roomMembersMap) => { - const otherMembers = Array.from(roomMembersMap.values()).filter( - (member) => member.userId !== userId, - ); - return otherMembers.length === 1 ? otherMembers[0] : null; - }), - ), - ); - - const localUserIsAlone$ = scope.behavior( - matrixRoomMembers$.pipe( - map( - (roomMembersMap) => - roomMembersMap.size === 1 && roomMembersMap.get(userId) !== undefined, - ), - ), - ); - - // CODESMELL? - // This is functionally the same Observable as leave$, except here it's - // hoisted to the top of the class. This enables the cyclic dependency between - // leave$ -> autoLeave$ -> callPickupState$ -> livekitConnectionState$ -> - // localConnection$ -> transports$ -> joined$ -> leave$. - const leaveHoisted$ = new Subject< - "user" | "timeout" | "decline" | "allOthersLeft" - >(); - - /** - * Whether various media/event sources should pretend to be disconnected from - * all network input, even if their connection still technically works. - */ - // We do this when the app is in the 'reconnecting' state, because it might be - // that the LiveKit connection is still functional while the homeserver is - // down, for example, and we want to avoid making people worry that the app is - // in a split-brained state. - // DISCUSSION own membership manager ALSO this probably can be simplifis - /*PUBLIC*/ const reconnecting$ = localMembership.reconnecting$; - const pretendToBeDisconnected$ = reconnecting$; - - /*PUBLIC*/ const audioParticipants$ = scope.behavior( - matrixLivekitMembers$.pipe( - switchMap((membersWithEpoch) => { - const members = membersWithEpoch.value; - const a$ = combineLatest( - members.map((member) => - combineLatest([member.connection$, member.participant$]).pipe( - map(([connection, participant]) => { - // do not render audio for local participant - if (!connection || !participant || participant.isLocal) - return null; - const livekitRoom = connection.livekitRoom; - const url = connection.transport.livekit_service_url; - - return { url, livekitRoom, participant: participant.identity }; - }), - ), - ), - ); - return a$; - }), - map((members) => - members.reduce((acc, curr) => { - if (!curr) return acc; - - const existing = acc.find((item) => item.url === curr.url); - if (existing) { - existing.participants.push(curr.participant); - } else { - acc.push({ - livekitRoom: curr.livekitRoom, - participants: [curr.participant], - url: curr.url, - }); - } - return acc; - }, []), - ), - ), - [], - ); - - /*PUBLIC*/ const handsRaised$ = scope.behavior( - handsRaisedSubject$.pipe(pauseWhen(pretendToBeDisconnected$)), - ); - - /*PUBLIC*/ const reactions$ = scope.behavior( - reactionsSubject$.pipe( - map((v) => - Object.fromEntries( - Object.entries(v).map(([a, { reactionOption }]) => [ - a, - reactionOption, - ]), - ), - ), - pauseWhen(pretendToBeDisconnected$), - ), - ); - - /** - * List of user media (camera feeds) that we want tiles for. - */ - // TODO this also needs the local participant to be added. - const userMedia$ = scope.behavior( - combineLatest([ - localMatrixLivekitMember$, - matrixLivekitMembers$, - duplicateTiles.value$, - ]).pipe( - // Generate a collection of MediaItems from the list of expected (whether - // present or missing) LiveKit participants. - generateItems( - function* ([ - localMatrixLivekitMember, - { value: matrixLivekitMembers }, - duplicateTiles, - ]) { - let localParticipantId = undefined; - // add local member if available - if (localMatrixLivekitMember) { - const { userId, participant$, connection$, membership$ } = - localMatrixLivekitMember; - localParticipantId = `${userId}:${membership$.value.deviceId}`; // should be membership$.value.membershipID which is not optional - // const participantId = membership$.value.membershipID; - if (localParticipantId) { - for (let dup = 0; dup < 1 + duplicateTiles; dup++) { - yield { - keys: [ - dup, - localParticipantId, - userId, - participant$, - connection$, - ], - data: undefined, - }; - } - } - } - // add remote members that are available - for (const { - userId, - participant$, - connection$, - membership$, - } of matrixLivekitMembers) { - const participantId = `${userId}:${membership$.value.deviceId}`; - if (participantId === localParticipantId) continue; - // const participantId = membership$.value?.identity; - for (let dup = 0; dup < 1 + duplicateTiles; dup++) { - yield { - keys: [dup, participantId, userId, participant$, connection$], - data: undefined, - }; - } - } - }, - ( - scope, - _data$, - dup, - participantId, - userId, - participant$, - connection$, - ) => { - const livekitRoom$ = scope.behavior( - connection$.pipe(map((c) => c?.livekitRoom)), - ); - const focusUrl$ = scope.behavior( - connection$.pipe(map((c) => c?.transport.livekit_service_url)), - ); - const displayName$ = scope.behavior( - matrixMemberMetadataStore - .createDisplayNameBehavior$(userId) - .pipe(map((name) => name ?? userId)), - ); - - return new UserMedia( - scope, - `${participantId}:${dup}`, - userId, - participant$, - options.encryptionSystem, - livekitRoom$, - focusUrl$, - mediaDevices, - pretendToBeDisconnected$, - displayName$, - matrixMemberMetadataStore.createAvatarUrlBehavior$(userId), - handsRaised$.pipe(map((v) => v[participantId]?.time ?? null)), - reactions$.pipe(map((v) => v[participantId] ?? undefined)), - ); - }, - ), - ), - ); - - /** - * List of all media items (user media and screen share media) that we want - * tiles for. - */ - const mediaItems$ = scope.behavior( - userMedia$.pipe( - switchMap((userMedia) => - userMedia.length === 0 - ? of([]) - : combineLatest( - userMedia.map((m) => m.screenShares$), - (...screenShares) => [...userMedia, ...screenShares.flat(1)], - ), - ), - ), - ); - - /** - * List of MediaItems that we want to display, that are of type ScreenShare - */ - const screenShares$ = scope.behavior( - mediaItems$.pipe( - map((mediaItems) => - mediaItems.filter((m): m is ScreenShare => m instanceof ScreenShare), - ), - ), - ); - - /*PUBLIC*/ const joinSoundEffect$ = userMedia$.pipe( - pairwise(), - filter( - ([prev, current]) => - current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && - current.length > prev.length, - ), - map(() => {}), - throttleTime(THROTTLE_SOUND_EFFECT_MS), - ); - - /** - * The number of participants currently in the call. - * - * - Each participant has a corresponding MatrixRTC membership state event - * - There can be multiple participants for one Matrix user if they join from - * multiple devices. - */ - /*PUBLIC*/ const participantCount$ = scope.behavior( - matrixLivekitMembers$.pipe(map((ms) => ms.value.length)), - ); - - // only public to expose to the view. - // TODO if we are in "unknown" state we need a loading rendering (or empty screen) - // Otherwise it looks like we already connected and only than the ringing starts which is weird. - /*PUBLIC*/ const callPickupState$ = callLifecycle.callPickupState$; - - /*PUBLIC*/ const leaveSoundEffect$ = combineLatest([ - callLifecycle.callPickupState$, - userMedia$, - ]).pipe( - // Until the call is successful, do not play a leave sound. - // If callPickupState$ is null, then we always play the sound as it will not conflict with a decline sound. - skipWhile(([c]) => c !== null && c !== "success"), - map(([, userMedia]) => userMedia), - pairwise(), - filter( - ([prev, current]) => - current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && - current.length < prev.length, - ), - map(() => {}), - throttleTime(THROTTLE_SOUND_EFFECT_MS), - ); - - const userHangup$ = new Subject(); - - const widgetHangup$ = - widget === null - ? NEVER - : ( - fromEvent( - widget.lazyActions, - ElementWidgetActions.HangupCall, - ) as Observable> - ).pipe( - tap((ev) => { - widget!.api.transport.reply(ev.detail, {}); - }), - ); - - const leave$: Observable<"user" | "timeout" | "decline" | "allOthersLeft"> = - merge( - callLifecycle.autoLeave$, - merge(userHangup$, widgetHangup$).pipe(map(() => "user" as const)), - ).pipe( - scope.share, - tap((reason) => leaveHoisted$.next(reason)), - ); - - const spotlightSpeaker$ = scope.behavior( - userMedia$.pipe( - switchMap((mediaItems) => - mediaItems.length === 0 - ? of([]) - : combineLatest( - mediaItems.map((m) => - m.vm.speaking$.pipe(map((s) => [m, s] as const)), - ), - ), - ), - scan<(readonly [UserMedia, boolean])[], UserMedia | undefined, null>( - (prev, mediaItems) => { - // Only remote users that are still in the call should be sticky - const [stickyMedia, stickySpeaking] = - (!prev?.vm.local && mediaItems.find(([m]) => m === prev)) || []; - // Decide who to spotlight: - // If the previous speaker is still speaking, stick with them rather - // than switching eagerly to someone else - return stickySpeaking - ? stickyMedia! - : // Otherwise, select any remote user who is speaking - (mediaItems.find(([m, s]) => !m.vm.local && s)?.[0] ?? - // Otherwise, stick with the person who was last speaking - stickyMedia ?? - // Otherwise, spotlight an arbitrary remote user - mediaItems.find(([m]) => !m.vm.local)?.[0] ?? - // Otherwise, spotlight the local user - mediaItems.find(([m]) => m.vm.local)?.[0]); - }, - null, - ), - map((speaker) => speaker?.vm ?? null), - ), - ); - - const grid$ = scope.behavior( - userMedia$.pipe( - switchMap((mediaItems) => { - const bins = mediaItems.map((m) => - m.bin$.pipe(map((bin) => [m, bin] as const)), - ); - // Sort the media by bin order and generate a tile for each one - return bins.length === 0 - ? of([]) - : combineLatest(bins, (...bins) => - bins.sort(([, bin1], [, bin2]) => bin1 - bin2).map(([m]) => m.vm), - ); - }), - distinctUntilChanged(shallowEquals), - ), - ); - - const spotlight$ = scope.behavior( - screenShares$.pipe( - switchMap((screenShares) => { - if (screenShares.length > 0) { - return of(screenShares.map((m) => m.vm)); - } - - return spotlightSpeaker$.pipe( - map((speaker) => (speaker ? [speaker] : [])), - ); - }), - distinctUntilChanged(shallowEquals), - ), - ); - - const pip$ = scope.behavior( - combineLatest([ - // TODO This also needs epoch logic to dedupe the screenshares and mediaItems emits - screenShares$, - spotlightSpeaker$, - mediaItems$, - ]).pipe( - switchMap(([screenShares, spotlight, mediaItems]) => { - if (screenShares.length > 0) { - return spotlightSpeaker$; - } - if (!spotlight || spotlight.local) { - return of(null); - } - - const localUserMedia = mediaItems.find( - (m) => m.vm instanceof LocalUserMediaViewModel, - ) as UserMedia | undefined; - - const localUserMediaViewModel = localUserMedia?.vm as - | LocalUserMediaViewModel - | undefined; - - if (!localUserMediaViewModel) { - return of(null); - } - return localUserMediaViewModel.alwaysShow$.pipe( - map((alwaysShow) => { - if (alwaysShow) { - return localUserMediaViewModel; - } - - return null; - }), - ); - }), - ), - ); - - const hasRemoteScreenShares$: Observable = spotlight$.pipe( - map((spotlight) => - spotlight.some((vm) => !vm.local && vm instanceof ScreenShareViewModel), - ), - distinctUntilChanged(), - ); - - const pipEnabled$ = scope.behavior(setPipEnabled$, false); - - const naturalWindowMode$ = scope.behavior( - fromEvent(window, "resize").pipe( - map(() => { - const height = window.innerHeight; - const width = window.innerWidth; - if (height <= 400 && width <= 340) return "pip"; - // Our layouts for flat windows are better at adapting to a small width - // than our layouts for narrow windows are at adapting to a small height, - // so we give "flat" precedence here - if (height <= 600) return "flat"; - if (width <= 600) return "narrow"; - return "normal"; - }), - ), - "normal", - ); - - /** - * The general shape of the window. - */ - /*PUBLIC*/ const windowMode$ = scope.behavior( - pipEnabled$.pipe( - switchMap((pip) => (pip ? of("pip") : naturalWindowMode$)), - ), - ); - - const spotlightExpandedToggle$ = new Subject(); - /*PUBLIC*/ const spotlightExpanded$ = scope.behavior( - spotlightExpandedToggle$.pipe(accumulate(false, (expanded) => !expanded)), - ); - - const gridModeUserSelection$ = new Subject(); - /** - * The layout mode of the media tile grid. - */ - /*PUBLIC*/ const gridMode$ = - // If the user hasn't selected spotlight and somebody starts screen sharing, - // automatically switch to spotlight mode and reset when screen sharing ends - scope.behavior( - gridModeUserSelection$.pipe( - switchMap((userSelection) => - (userSelection === "spotlight" - ? EMPTY - : combineLatest([hasRemoteScreenShares$, windowMode$]).pipe( - skip(userSelection === null ? 0 : 1), - map( - ([hasScreenShares, windowMode]): GridMode => - hasScreenShares || windowMode === "flat" - ? "spotlight" - : "grid", - ), - ) - ).pipe(startWith(userSelection ?? "grid")), - ), - ), - "grid", - ); - - /*PUBLIC*/ const setGridMode = (value: GridMode): void => { - gridModeUserSelection$.next(value); - }; - - const gridLayoutMedia$: Observable = combineLatest( - [grid$, spotlight$], - (grid, spotlight) => ({ - type: "grid", - spotlight: spotlight.some((vm) => vm instanceof ScreenShareViewModel) - ? spotlight - : undefined, - grid, - }), - ); - - const spotlightLandscapeLayoutMedia$: Observable = - combineLatest([grid$, spotlight$], (grid, spotlight) => ({ - type: "spotlight-landscape", - spotlight, - grid, - })); - - const spotlightPortraitLayoutMedia$: Observable = - combineLatest([grid$, spotlight$], (grid, spotlight) => ({ - type: "spotlight-portrait", - spotlight, - grid, - })); - - const spotlightExpandedLayoutMedia$: Observable = - combineLatest([spotlight$, pip$], (spotlight, pip) => ({ - type: "spotlight-expanded", - spotlight, - pip: pip ?? undefined, - })); - - const oneOnOneLayoutMedia$: Observable = - mediaItems$.pipe( - map((mediaItems) => { - if (mediaItems.length !== 2) return null; - const local = mediaItems.find((vm) => vm.vm.local)?.vm as - | LocalUserMediaViewModel - | undefined; - const remote = mediaItems.find((vm) => !vm.vm.local)?.vm as - | RemoteUserMediaViewModel - | undefined; - // There might not be a remote tile if there are screen shares, or if - // only the local user is in the call and they're using the duplicate - // tiles option - if (!remote || !local) return null; - - return { type: "one-on-one", local, remote }; - }), - ); - - const pipLayoutMedia$: Observable = spotlight$.pipe( - map((spotlight) => ({ type: "pip", spotlight })), - ); - - /** - * The media to be used to produce a layout. - */ - const layoutMedia$ = scope.behavior( - windowMode$.pipe( - switchMap((windowMode) => { - switch (windowMode) { - case "normal": - return gridMode$.pipe( - switchMap((gridMode) => { - switch (gridMode) { - case "grid": - return oneOnOneLayoutMedia$.pipe( - switchMap((oneOnOne) => - oneOnOne === null ? gridLayoutMedia$ : of(oneOnOne), - ), - ); - case "spotlight": - return spotlightExpanded$.pipe( - switchMap((expanded) => - expanded - ? spotlightExpandedLayoutMedia$ - : spotlightLandscapeLayoutMedia$, - ), - ); - } - }), - ); - case "narrow": - return oneOnOneLayoutMedia$.pipe( - switchMap((oneOnOne) => - oneOnOne === null - ? combineLatest([grid$, spotlight$], (grid, spotlight) => - grid.length > smallMobileCallThreshold || - spotlight.some((vm) => vm instanceof ScreenShareViewModel) - ? spotlightPortraitLayoutMedia$ - : gridLayoutMedia$, - ).pipe(switchAll()) - : // The expanded spotlight layout makes for a better one-on-one - // experience in narrow windows - spotlightExpandedLayoutMedia$, - ), - ); - case "flat": - return gridMode$.pipe( - switchMap((gridMode) => { - switch (gridMode) { - case "grid": - // Yes, grid mode actually gets you a "spotlight" layout in - // this window mode. - return spotlightLandscapeLayoutMedia$; - case "spotlight": - return spotlightExpandedLayoutMedia$; - } - }), - ); - case "pip": - return pipLayoutMedia$; - } - }), - ), - ); - - // There is a cyclical dependency here: the layout algorithms want to know - // which tiles are on screen, but to know which tiles are on screen we have to - // first render a layout. To deal with this we assume initially that no tiles - // are visible, and loop the data back into the layouts with a Subject. - const visibleTiles$ = new Subject(); - const setVisibleTiles = (value: number): void => visibleTiles$.next(value); - - const layoutInternals$ = scope.behavior( - combineLatest([ - layoutMedia$, - visibleTiles$.pipe(startWith(0), distinctUntilChanged()), - ]).pipe( - scan< - [LayoutMedia, number], - LayoutScanState & { layout: Layout }, - LayoutScanState - >( - ({ tiles: prevTiles }, [media, visibleTiles]) => { - let layout: Layout; - let newTiles: TileStore; - switch (media.type) { - case "grid": - case "spotlight-landscape": - case "spotlight-portrait": - [layout, newTiles] = gridLikeLayout( - media, - visibleTiles, - setVisibleTiles, - prevTiles, - ); - break; - case "spotlight-expanded": - [layout, newTiles] = spotlightExpandedLayout(media, prevTiles); - break; - case "one-on-one": - [layout, newTiles] = oneOnOneLayout(media, prevTiles); - break; - case "pip": - [layout, newTiles] = pipLayout(media, prevTiles); - break; - } - - return { layout, tiles: newTiles }; - }, - { layout: null, tiles: TileStore.empty() }, - ), - ), - ); - - /** - * The layout of tiles in the call interface. - */ - /*PUBLIC*/ const layout$ = scope.behavior( - layoutInternals$.pipe(map(({ layout }) => layout)), - ); - - /** - * The current generation of the tile store, exposed for debugging purposes. - */ - /*PUBLIC*/ const tileStoreGeneration$ = scope.behavior( - layoutInternals$.pipe(map(({ tiles }) => tiles.generation)), - ); - - /*PUBLIC*/ const showSpotlightIndicators$ = scope.behavior( - layout$.pipe(map((l) => l.type !== "grid")), - ); - - /*PUBLIC*/ const showSpeakingIndicators$ = scope.behavior( - layout$.pipe( - switchMap((l) => { - switch (l.type) { - case "spotlight-landscape": - case "spotlight-portrait": - // If the spotlight is showing the active speaker, we can do without - // speaking indicators as they're a redundant visual cue. But if - // screen sharing feeds are in the spotlight we still need them. - return l.spotlight.media$.pipe( - map((models: MediaViewModel[]) => - models.some((m) => m instanceof ScreenShareViewModel), - ), - ); - // In expanded spotlight layout, the active speaker is always shown in - // the picture-in-picture tile so there is no need for speaking - // indicators. And in one-on-one layout there's no question as to who is - // speaking. - case "spotlight-expanded": - case "one-on-one": - return of(false); - default: - return of(true); - } - }), - ), - ); - - /*PUBLIC*/ const toggleSpotlightExpanded$ = scope.behavior< - (() => void) | null - >( - windowMode$.pipe( - switchMap((mode) => - mode === "normal" - ? layout$.pipe( - map( - (l) => - l.type === "spotlight-landscape" || - l.type === "spotlight-expanded", - ), - ) - : of(false), - ), - distinctUntilChanged(), - map((enabled) => - enabled ? (): void => spotlightExpandedToggle$.next() : null, - ), - ), - ); - - const screenTap$ = new Subject(); - const controlsTap$ = new Subject(); - const screenHover$ = new Subject(); - const screenUnhover$ = new Subject(); - - /*PUBLIC*/ const showHeader$ = scope.behavior( - windowMode$.pipe(map((mode) => mode !== "pip" && mode !== "flat")), - ); - - /*PUBLIC*/ const showFooter$ = scope.behavior( - windowMode$.pipe( - switchMap((mode) => { - switch (mode) { - case "pip": - return of(false); - case "normal": - case "narrow": - return of(true); - case "flat": - // Sadly Firefox has some layering glitches that prevent the footer - // from appearing properly. They happen less often if we never hide - // the footer. - if (isFirefox()) return of(true); - // Show/hide the footer in response to interactions - return merge( - screenTap$.pipe(map(() => "tap screen" as const)), - controlsTap$.pipe(map(() => "tap controls" as const)), - screenHover$.pipe(map(() => "hover" as const)), - ).pipe( - switchScan((state, interaction) => { - switch (interaction) { - case "tap screen": - return state - ? // Toggle visibility on tap - of(false) - : // Hide after a timeout - timer(showFooterMs).pipe( - map(() => false), - startWith(true), - ); - case "tap controls": - // The user is interacting with things, so reset the timeout - return timer(showFooterMs).pipe( - map(() => false), - startWith(true), - ); - case "hover": - // Show on hover and hide after a timeout - return race( - timer(showFooterMs), - screenUnhover$.pipe(take(1)), - ).pipe( - map(() => false), - startWith(true), - ); - } - }, false), - startWith(false), - ); - } - }), - ), - ); - - /** - * Whether audio is currently being output through the earpiece. - */ - /*PUBLIC*/ const earpieceMode$ = scope.behavior( - combineLatest( - [mediaDevices.audioOutput.available$, mediaDevices.audioOutput.selected$], - (available, selected) => - selected !== undefined && - available.get(selected.id)?.type === "earpiece", - ), - ); - - /** - * Callback to toggle between the earpiece and the loudspeaker. - * - * This will be `null` in case the target does not exist in the list - * of available audio outputs. - */ - /*PUBLIC*/ const audioOutputSwitcher$ = scope.behavior<{ - targetOutput: "earpiece" | "speaker"; - switch: () => void; - } | null>( - combineLatest( - [mediaDevices.audioOutput.available$, mediaDevices.audioOutput.selected$], - (available, selected) => { - const selectionType = selected && available.get(selected.id)?.type; - - // If we are in any output mode other than speaker switch to speaker. - const newSelectionType: "earpiece" | "speaker" = - selectionType === "speaker" ? "earpiece" : "speaker"; - const newSelection = [...available].find( - ([, d]) => d.type === newSelectionType, - ); - if (newSelection === undefined) return null; - - const [id] = newSelection; - return { - targetOutput: newSelectionType, - switch: (): void => mediaDevices.audioOutput.select(id), - }; - }, - ), - ); - - /** - * Emits an array of reactions that should be visible on the screen. - */ - // DISCUSSION move this into a reaction file - // const {visibleReactions$, audibleReactions$} = reactionsObservables$(showReactionSetting$, ) - const visibleReactions$ = scope.behavior( - showReactions.value$.pipe( - switchMap((show) => (show ? reactions$ : of({}))), - scan< - Record, - { sender: string; emoji: string; startX: number }[] - >((acc, latest) => { - const newSet: { sender: string; emoji: string; startX: number }[] = []; - for (const [sender, reaction] of Object.entries(latest)) { - const startX = - acc.find((v) => v.sender === sender && v.emoji)?.startX ?? - Math.ceil(Math.random() * 80) + 10; - newSet.push({ sender, emoji: reaction.emoji, startX }); - } - return newSet; - }, []), - ), - ); - - /** - * Emits an array of reactions that should be played. - */ - const audibleReactions$ = playReactionsSound.value$.pipe( - switchMap((show) => - show ? reactions$ : of>({}), - ), - map((reactions) => Object.values(reactions).map((v) => v.name)), - scan( - (acc, latest) => { - return { - playing: latest.filter( - (v) => acc.playing.includes(v) || acc.newSounds.includes(v), - ), - newSounds: latest.filter( - (v) => !acc.playing.includes(v) && !acc.newSounds.includes(v), - ), - }; - }, - { playing: [], newSounds: [] }, - ), - map((v) => v.newSounds), - ); - - const newHandRaised$ = handsRaised$.pipe( - map((v) => Object.keys(v).length), - scan( - (acc, newValue) => ({ - value: newValue, - playSounds: newValue > acc.value, - }), - { value: 0, playSounds: false }, - ), - filter((v) => v.playSounds), - ); - - const newScreenShare$ = screenShares$.pipe( - map((v) => v.length), - scan( - (acc, newValue) => ({ - value: newValue, - playSounds: newValue > acc.value, - }), - { value: 0, playSounds: false }, - ), - filter((v) => v.playSounds), - ); - - /** - * Whether we are sharing our screen. - */ - // reassigned here to make it publicly accessible - const sharingScreen$ = localMembership.sharingScreen$; - - /** - * Callback to toggle screen sharing. If null, screen sharing is not possible. - */ - // reassigned here to make it publicly accessible - const toggleScreenSharing = localMembership.toggleScreenSharing; - - const join = localMembership.requestConnect; - join(); // TODO-MULTI-SFU: Use this view model for the lobby as well, and only call this once 'join' is clicked? - - return { - autoLeave$: callLifecycle.autoLeave$, - callPickupState$, - leave$, - hangup: () => userHangup$.next(), - join, - toggleScreenSharing, - sharingScreen$, - - tapScreen: () => screenTap$.next(), - tapControls: () => controlsTap$.next(), - hoverScreen: () => screenHover$.next(), - unhoverScreen: () => screenUnhover$.next(), - - configError$: localMembership.configError$, - participantCount$, - audioParticipants$, - isOneOnOneWith$, - localUserIsAlone$, - - handsRaised$, - reactions$, - joinSoundEffect$, - leaveSoundEffect$, - newHandRaised$, - newScreenShare$, - audibleReactions$, - visibleReactions$, - - windowMode$, - spotlightExpanded$, - toggleSpotlightExpanded$, - gridMode$, - setGridMode, - grid$, - spotlight$, - pip$, - layout$, - tileStoreGeneration$, - showSpotlightIndicators$, - showSpeakingIndicators$, - showHeader$, - showFooter$, - earpieceMode$, - audioOutputSwitcher$, - reconnecting$, - }; -} -// TODO-MULTI-SFU // Setup and update the keyProvider which was create by `createRoom` was a thing before. Now we never update if the E2EEsystem changes -// do we need this? - -function getE2eeKeyProvider( - e2eeSystem: EncryptionSystem, - rtcSession: MatrixRTCSession, -): BaseKeyProvider | undefined { - if (e2eeSystem.kind === E2eeType.NONE) return undefined; - - if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) { - const keyProvider = new MatrixKeyProvider(); - keyProvider.setRTCSession(rtcSession); - return keyProvider; - } else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) { - const keyProvider = new ExternalE2EEKeyProvider(); - keyProvider - .setKey(e2eeSystem.secret) - .catch((e) => logger.error("Failed to set shared key for E2EE", e)); - return keyProvider; - } -} diff --git a/src/state/CallViewModel/CallViewModelOld.ts b/src/state/CallViewModel/CallViewModelOld.ts deleted file mode 100644 index ff4a6269..00000000 --- a/src/state/CallViewModel/CallViewModelOld.ts +++ /dev/null @@ -1,1335 +0,0 @@ -/* -Copyright 2023, 2024, 2025 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - type BaseKeyProvider, - type ConnectionState, - ExternalE2EEKeyProvider, - type Room as LivekitRoom, - type RoomOptions, -} from "livekit-client"; -import { type Room as MatrixRoom } from "matrix-js-sdk"; -import { - combineLatest, - distinctUntilChanged, - EMPTY, - filter, - fromEvent, - map, - merge, - NEVER, - type Observable, - of, - pairwise, - race, - scan, - skip, - skipWhile, - startWith, - Subject, - switchAll, - switchMap, - switchScan, - take, - tap, - throttleTime, - timer, -} from "rxjs"; -import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; -import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; -import { type IWidgetApiRequest } from "matrix-widget-api"; - -import { - LocalUserMediaViewModel, - type MediaViewModel, - type RemoteUserMediaViewModel, - ScreenShareViewModel, - type UserMediaViewModel, -} from "../MediaViewModel"; -import { accumulate, generateItems, pauseWhen } from "../../utils/observable"; -import { - duplicateTiles, - MatrixRTCMode, - matrixRTCMode, - playReactionsSound, - showReactions, -} from "../../settings/settings"; -import { isFirefox } from "../../Platform"; -import { setPipEnabled$ } from "../../controls"; -import { TileStore } from "../TileStore"; -import { gridLikeLayout } from "../GridLikeLayout"; -import { spotlightExpandedLayout } from "../SpotlightExpandedLayout"; -import { oneOnOneLayout } from "../OneOnOneLayout"; -import { pipLayout } from "../PipLayout"; -import { type EncryptionSystem } from "../../e2ee/sharedKeyManagement"; -import { - type RaisedHandInfo, - type ReactionInfo, - type ReactionOption, -} from "../../reactions"; -import { shallowEquals } from "../../utils/array"; -import { type MediaDevices } from "../MediaDevices"; -import { type Behavior } from "../Behavior"; -import { E2eeType } from "../../e2ee/e2eeType"; -import { MatrixKeyProvider } from "../../e2ee/matrixKeyProvider"; -import { type MuteStates } from "../MuteStates"; -import { getUrlParams } from "../../UrlParams"; -import { type ProcessorState } from "../../livekit/TrackProcessorContext"; -import { ElementWidgetActions, widget } from "../../widget"; -import { UserMedia } from "../UserMedia.ts"; -import { ScreenShare } from "../ScreenShare.ts"; -import { - type GridLayoutMedia, - type Layout, - type LayoutMedia, - type OneOnOneLayoutMedia, - type SpotlightExpandedLayoutMedia, - type SpotlightLandscapeLayoutMedia, - type SpotlightPortraitLayoutMedia, -} from "../layout-types.ts"; -import { type ElementCallError } from "../../utils/errors.ts"; -import { type ObservableScope } from "../ObservableScope.ts"; -import { createLocalMembership$ } from "./localMember/LocalMembership.ts"; -import { createLocalTransport$ } from "./localMember/LocalTransport.ts"; -import { - createMemberships$, - membershipsAndTransports$, -} from "../SessionBehaviors.ts"; -import { ECConnectionFactory } from "./remoteMembers/ConnectionFactory.ts"; -import { createConnectionManager$ } from "./remoteMembers/ConnectionManager.ts"; -import { - createMatrixLivekitMembers$, - type MatrixLivekitMember, -} from "./remoteMembers/MatrixLivekitMembers.ts"; -import { - createCallNotificationLifecycle$, - createReceivedDecline$, - createSentCallNotification$, -} from "./CallNotificationLifecycle.ts"; -import { - createMatrixMemberMetadata$, - createRoomMembers$, -} from "./remoteMembers/MatrixMemberMetadata.ts"; - -const logger = rootLogger.getChild("[CallViewModel]"); -//TODO -// Larger rename -// member,membership -> rtcMember -// participant -> livekitParticipant -// matrixLivekitItem -> callMember -// js-sdk -// callMembership -> rtcMembership -export interface CallViewModelOptions { - encryptionSystem: EncryptionSystem; - autoLeaveWhenOthersLeft?: boolean; - /** - * If the call is started in a way where we want it to behave like a telephone usecase - * If we sent a notification event, we want the ui to show a ringing state - */ - waitForCallPickup?: boolean; - /** Optional factory to create LiveKit rooms, mainly for testing purposes. */ - livekitRoomFactory?: (options?: RoomOptions) => LivekitRoom; - /** Optional behavior overriding the local connection state, mainly for testing purposes. */ - connectionState$?: Behavior; -} - -// Do not play any sounds if the participant count has exceeded this -// number. -export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8; -export const THROTTLE_SOUND_EFFECT_MS = 500; - -// This is the number of participants that we think constitutes a "small" call -// on mobile. No spotlight tile should be shown below this threshold. -const smallMobileCallThreshold = 3; - -// How long the footer should be shown for when hovering over or interacting -// with the interface -const showFooterMs = 4000; - -export type GridMode = "grid" | "spotlight"; - -export type WindowMode = "normal" | "narrow" | "flat" | "pip"; - -interface LayoutScanState { - layout: Layout | null; - tiles: TileStore; -} - -type MediaItem = UserMedia | ScreenShare; -type AudioLivekitItem = { - livekitRoom: LivekitRoom; - participants: string[]; - url: string; -}; -/** - * A view model providing all the application logic needed to show the in-call - * UI (may eventually be expanded to cover the lobby and feedback screens in the - * future). - */ -// Throughout this class and related code we must distinguish between MatrixRTC -// state and LiveKit state. We use the common terminology of room "members", RTC -// "memberships", and LiveKit "participants". -export class CallViewModel { - private readonly userId = this.matrixRoom.client.getUserId()!; - private readonly deviceId = this.matrixRoom.client.getDeviceId()!; - - private readonly livekitKeyProvider = getE2eeKeyProvider( - this.options.encryptionSystem, - this.matrixRTCSession, - ); - - // Each hbar seperates a block of input variables required for the CallViewModel to function. - // The outputs of this block is written under the hbar. - // - // For mocking purposes it is recommended to only mock the functions creating those outputs. - // All other fields are just temp computations for the mentioned output. - // The class does not need anything except the values underneath the bar. - // The creation of the values under the bar are all tested independently and testing the callViewModel Should - // not test their cretation. Call view model only needs: - // - memberships$ via createMemberships$ - // - localMembership via createLocalMembership$ - // - callLifecycle via createCallNotificationLifecycle$ - // - matrixMemberMetadataStore via createMatrixMemberMetadata$ - - // ------------------------------------------------------------------------ - // memberships$ - private memberships$ = createMemberships$(this.scope, this.matrixRTCSession); - - // ------------------------------------------------------------------------ - // matrixLivekitMembers$ AND localMembership - - private membershipsAndTransports = membershipsAndTransports$( - this.scope, - this.memberships$, - ); - - private localTransport$ = createLocalTransport$({ - scope: this.scope, - memberships$: this.memberships$, - client: this.matrixRoom.client, - roomId: this.matrixRoom.roomId, - useOldestMember$: this.scope.behavior( - matrixRTCMode.value$.pipe(map((v) => v === MatrixRTCMode.Legacy)), - ), - }); - - private connectionFactory = new ECConnectionFactory( - this.matrixRoom.client, - this.mediaDevices, - this.trackProcessorState$, - this.livekitKeyProvider, - getUrlParams().controlledAudioDevices, - this.options.livekitRoomFactory, - ); - - private connectionManager = createConnectionManager$({ - scope: this.scope, - connectionFactory: this.connectionFactory, - inputTransports$: this.scope.behavior( - combineLatest( - [this.localTransport$, this.membershipsAndTransports.transports$], - (localTransport, transports) => { - const localTransportAsArray = localTransport ? [localTransport] : []; - return transports.mapInner((transports) => [ - ...localTransportAsArray, - ...transports, - ]); - }, - ), - ), - logger: logger, - }); - - private matrixLivekitMembers$ = createMatrixLivekitMembers$({ - scope: this.scope, - membershipsWithTransport$: - this.membershipsAndTransports.membershipsWithTransport$, - connectionManager: this.connectionManager, - }); - - private connectOptions$ = this.scope.behavior( - matrixRTCMode.value$.pipe( - map((mode) => ({ - encryptMedia: this.livekitKeyProvider !== undefined, - // TODO. This might need to get called again on each cahnge of matrixRTCMode... - matrixRTCMode: mode, - })), - ), - ); - - private localMembership = createLocalMembership$({ - scope: this.scope, - muteStates: this.muteStates, - mediaDevices: this.mediaDevices, - connectionManager: this.connectionManager, - matrixRTCSession: this.matrixRTCSession, - matrixRoom: this.matrixRoom, - localTransport$: this.localTransport$, - trackProcessorState$: this.trackProcessorState$, - widget, - options: this.connectOptions$, - logger: logger.getChild(`[${Date.now()}]`), - }); - - private localRtcMembership$ = this.scope.behavior( - this.memberships$.pipe( - map( - (memberships) => - memberships.value.find( - (membership) => - membership.userId === this.userId && - membership.deviceId === this.deviceId, - ) ?? null, - ), - ), - ); - - private localMatrixLivekitMemberUninitialized = { - membership$: this.localRtcMembership$, - participant$: this.localMembership.participant$, - connection$: this.localMembership.connection$, - userId: this.userId, - }; - - private localMatrixLivekitMember$: Behavior = - this.scope.behavior( - this.localRtcMembership$.pipe( - switchMap((membership) => { - if (!membership) return of(null); - return of( - // casting is save here since we know that localRtcMembership$ is !== null since we reached this case. - this.localMatrixLivekitMemberUninitialized as MatrixLivekitMember, - ); - }), - ), - ); - - // ------------------------------------------------------------------------ - // callLifecycle - - private callLifecycle = createCallNotificationLifecycle$({ - scope: this.scope, - memberships$: this.memberships$, - sentCallNotification$: createSentCallNotification$( - this.scope, - this.matrixRTCSession, - ), - receivedDecline$: createReceivedDecline$(this.matrixRoom), - options: this.options, - localUser: { userId: this.userId, deviceId: this.deviceId }, - }); - public autoLeave$ = this.callLifecycle.autoLeave$; - - // ------------------------------------------------------------------------ - // matrixMemberMetadataStore - - private matrixMemberMetadataStore = createMatrixMemberMetadata$( - this.scope, - this.scope.behavior(this.memberships$.pipe(map((mems) => mems.value))), - createRoomMembers$(this.scope, this.matrixRoom), - ); - - /** - * If there is a configuration error with the call (e.g. misconfigured E2EE). - * This is a fatal error that prevents the call from being created/joined. - * Should render a blocking error screen. - */ - public get configError$(): Behavior { - return this.localMembership.configError$; - } - - public join = this.localMembership.requestConnect; - - // CODESMELL? - // This is functionally the same Observable as leave$, except here it's - // hoisted to the top of the class. This enables the cyclic dependency between - // leave$ -> autoLeave$ -> callPickupState$ -> livekitConnectionState$ -> - // localConnection$ -> transports$ -> joined$ -> leave$. - private readonly leaveHoisted$ = new Subject< - "user" | "timeout" | "decline" | "allOthersLeft" - >(); - - /** - * Whether various media/event sources should pretend to be disconnected from - * all network input, even if their connection still technically works. - */ - // We do this when the app is in the 'reconnecting' state, because it might be - // that the LiveKit connection is still functional while the homeserver is - // down, for example, and we want to avoid making people worry that the app is - // in a split-brained state. - // DISCUSSION own membership manager ALSO this probably can be simplifis - public reconnecting$ = this.localMembership.reconnecting$; - private readonly pretendToBeDisconnected$ = this.reconnecting$; - - public readonly audioParticipants$ = this.scope.behavior( - this.matrixLivekitMembers$.pipe( - switchMap((membersWithEpoch) => { - const members = membersWithEpoch.value; - const a$ = combineLatest( - members.map((member) => - combineLatest([member.connection$, member.participant$]).pipe( - map(([connection, participant]) => { - // do not render audio for local participant - if (!connection || !participant || participant.isLocal) - return null; - const livekitRoom = connection.livekitRoom; - const url = connection.transport.livekit_service_url; - - return { url, livekitRoom, participant: participant.identity }; - }), - ), - ), - ); - return a$; - }), - map((members) => - members.reduce((acc, curr) => { - if (!curr) return acc; - - const existing = acc.find((item) => item.url === curr.url); - if (existing) { - existing.participants.push(curr.participant); - } else { - acc.push({ - livekitRoom: curr.livekitRoom, - participants: [curr.participant], - url: curr.url, - }); - } - return acc; - }, []), - ), - ), - [], - ); - - public readonly handsRaised$ = this.scope.behavior( - this.handsRaisedSubject$.pipe(pauseWhen(this.pretendToBeDisconnected$)), - ); - - public readonly reactions$ = this.scope.behavior( - this.reactionsSubject$.pipe( - map((v) => - Object.fromEntries( - Object.entries(v).map(([a, { reactionOption }]) => [ - a, - reactionOption, - ]), - ), - ), - pauseWhen(this.pretendToBeDisconnected$), - ), - ); - - /** - * List of user media (camera feeds) that we want tiles for. - */ - // TODO this also needs the local participant to be added. - private readonly userMedia$ = this.scope.behavior( - combineLatest([ - this.localMatrixLivekitMember$, - this.matrixLivekitMembers$, - duplicateTiles.value$, - ]).pipe( - // Generate a collection of MediaItems from the list of expected (whether - // present or missing) LiveKit participants. - generateItems( - function* ([ - localMatrixLivekitMember, - { value: matrixLivekitMembers }, - duplicateTiles, - ]) { - let localParticipantId = undefined; - // add local member if available - if (localMatrixLivekitMember) { - const { userId, participant$, connection$, membership$ } = - localMatrixLivekitMember; - localParticipantId = `${userId}:${membership$.value.deviceId}`; // should be membership$.value.membershipID which is not optional - // const participantId = membership$.value.membershipID; - if (localParticipantId) { - for (let dup = 0; dup < 1 + duplicateTiles; dup++) { - yield { - keys: [ - dup, - localParticipantId, - userId, - participant$, - connection$, - ], - data: undefined, - }; - } - } - } - // add remote members that are available - for (const { - userId, - participant$, - connection$, - membership$, - } of matrixLivekitMembers) { - const participantId = `${userId}:${membership$.value.deviceId}`; - if (participantId === localParticipantId) continue; - // const participantId = membership$.value?.identity; - for (let dup = 0; dup < 1 + duplicateTiles; dup++) { - yield { - keys: [dup, participantId, userId, participant$, connection$], - data: undefined, - }; - } - } - }, - ( - scope, - _data$, - dup, - participantId, - userId, - participant$, - connection$, - ) => { - const livekitRoom$ = scope.behavior( - connection$.pipe(map((c) => c?.livekitRoom)), - ); - const focusUrl$ = scope.behavior( - connection$.pipe(map((c) => c?.transport.livekit_service_url)), - ); - const displayName$ = scope.behavior( - this.matrixMemberMetadataStore - .createDisplayNameBehavior$(userId) - .pipe(map((name) => name ?? userId)), - ); - - return new UserMedia( - scope, - `${participantId}:${dup}`, - userId, - participant$, - this.options.encryptionSystem, - livekitRoom$, - focusUrl$, - this.mediaDevices, - this.pretendToBeDisconnected$, - displayName$, - this.matrixMemberMetadataStore.createAvatarUrlBehavior$(userId), - this.handsRaised$.pipe(map((v) => v[participantId]?.time ?? null)), - this.reactions$.pipe(map((v) => v[participantId] ?? undefined)), - ); - }, - ), - ), - ); - - /** - * List of all media items (user media and screen share media) that we want - * tiles for. - */ - private readonly mediaItems$ = this.scope.behavior( - this.userMedia$.pipe( - switchMap((userMedia) => - userMedia.length === 0 - ? of([]) - : combineLatest( - userMedia.map((m) => m.screenShares$), - (...screenShares) => [...userMedia, ...screenShares.flat(1)], - ), - ), - ), - ); - - /** - * List of MediaItems that we want to display, that are of type ScreenShare - */ - private readonly screenShares$ = this.scope.behavior( - this.mediaItems$.pipe( - map((mediaItems) => - mediaItems.filter((m): m is ScreenShare => m instanceof ScreenShare), - ), - ), - ); - - public readonly joinSoundEffect$ = this.userMedia$.pipe( - pairwise(), - filter( - ([prev, current]) => - current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && - current.length > prev.length, - ), - map(() => {}), - throttleTime(THROTTLE_SOUND_EFFECT_MS), - ); - - /** - * The number of participants currently in the call. - * - * - Each participant has a corresponding MatrixRTC membership state event - * - There can be multiple participants for one Matrix user if they join from - * multiple devices. - */ - public readonly participantCount$ = this.scope.behavior( - this.matrixLivekitMembers$.pipe(map((ms) => ms.value.length)), - ); - - // only public to expose to the view. - // TODO if we are in "unknown" state we need a loading rendering (or empty screen) - // Otherwise it looks like we already connected and only than the ringing starts which is weird. - public readonly callPickupState$ = this.callLifecycle.callPickupState$; - - public readonly leaveSoundEffect$ = combineLatest([ - this.callLifecycle.callPickupState$, - this.userMedia$, - ]).pipe( - // Until the call is successful, do not play a leave sound. - // If callPickupState$ is null, then we always play the sound as it will not conflict with a decline sound. - skipWhile(([c]) => c !== null && c !== "success"), - map(([, userMedia]) => userMedia), - pairwise(), - filter( - ([prev, current]) => - current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && - current.length < prev.length, - ), - map(() => {}), - throttleTime(THROTTLE_SOUND_EFFECT_MS), - ); - - private readonly userHangup$ = new Subject(); - public hangup(): void { - this.userHangup$.next(); - } - - private readonly widgetHangup$ = - widget === null - ? NEVER - : ( - fromEvent( - widget.lazyActions, - ElementWidgetActions.HangupCall, - ) as Observable> - ).pipe( - tap((ev) => { - widget!.api.transport.reply(ev.detail, {}); - }), - ); - - public readonly leave$: Observable< - "user" | "timeout" | "decline" | "allOthersLeft" - > = merge( - this.callLifecycle.autoLeave$, - merge(this.userHangup$, this.widgetHangup$).pipe( - map(() => "user" as const), - ), - ).pipe( - this.scope.share, - tap((reason) => this.leaveHoisted$.next(reason)), - ); - - private readonly spotlightSpeaker$ = - this.scope.behavior( - this.userMedia$.pipe( - switchMap((mediaItems) => - mediaItems.length === 0 - ? of([]) - : combineLatest( - mediaItems.map((m) => - m.vm.speaking$.pipe(map((s) => [m, s] as const)), - ), - ), - ), - scan<(readonly [UserMedia, boolean])[], UserMedia | undefined, null>( - (prev, mediaItems) => { - // Only remote users that are still in the call should be sticky - const [stickyMedia, stickySpeaking] = - (!prev?.vm.local && mediaItems.find(([m]) => m === prev)) || []; - // Decide who to spotlight: - // If the previous speaker is still speaking, stick with them rather - // than switching eagerly to someone else - return stickySpeaking - ? stickyMedia! - : // Otherwise, select any remote user who is speaking - (mediaItems.find(([m, s]) => !m.vm.local && s)?.[0] ?? - // Otherwise, stick with the person who was last speaking - stickyMedia ?? - // Otherwise, spotlight an arbitrary remote user - mediaItems.find(([m]) => !m.vm.local)?.[0] ?? - // Otherwise, spotlight the local user - mediaItems.find(([m]) => m.vm.local)?.[0]); - }, - null, - ), - map((speaker) => speaker?.vm ?? null), - ), - ); - - private readonly grid$ = this.scope.behavior( - this.userMedia$.pipe( - switchMap((mediaItems) => { - const bins = mediaItems.map((m) => - m.bin$.pipe(map((bin) => [m, bin] as const)), - ); - // Sort the media by bin order and generate a tile for each one - return bins.length === 0 - ? of([]) - : combineLatest(bins, (...bins) => - bins.sort(([, bin1], [, bin2]) => bin1 - bin2).map(([m]) => m.vm), - ); - }), - distinctUntilChanged(shallowEquals), - ), - ); - - private readonly spotlight$ = this.scope.behavior( - this.screenShares$.pipe( - switchMap((screenShares) => { - if (screenShares.length > 0) { - return of(screenShares.map((m) => m.vm)); - } - - return this.spotlightSpeaker$.pipe( - map((speaker) => (speaker ? [speaker] : [])), - ); - }), - distinctUntilChanged(shallowEquals), - ), - ); - - private readonly pip$ = this.scope.behavior( - combineLatest([ - // TODO This also needs epoch logic to dedupe the screenshares and mediaItems emits - this.screenShares$, - this.spotlightSpeaker$, - this.mediaItems$, - ]).pipe( - switchMap(([screenShares, spotlight, mediaItems]) => { - if (screenShares.length > 0) { - return this.spotlightSpeaker$; - } - if (!spotlight || spotlight.local) { - return of(null); - } - - const localUserMedia = mediaItems.find( - (m) => m.vm instanceof LocalUserMediaViewModel, - ) as UserMedia | undefined; - - const localUserMediaViewModel = localUserMedia?.vm as - | LocalUserMediaViewModel - | undefined; - - if (!localUserMediaViewModel) { - return of(null); - } - return localUserMediaViewModel.alwaysShow$.pipe( - map((alwaysShow) => { - if (alwaysShow) { - return localUserMediaViewModel; - } - - return null; - }), - ); - }), - ), - ); - - private readonly hasRemoteScreenShares$: Observable = - this.spotlight$.pipe( - map((spotlight) => - spotlight.some((vm) => !vm.local && vm instanceof ScreenShareViewModel), - ), - distinctUntilChanged(), - ); - - private readonly pipEnabled$ = this.scope.behavior(setPipEnabled$, false); - - private readonly naturalWindowMode$ = this.scope.behavior( - fromEvent(window, "resize").pipe( - map(() => { - const height = window.innerHeight; - const width = window.innerWidth; - if (height <= 400 && width <= 340) return "pip"; - // Our layouts for flat windows are better at adapting to a small width - // than our layouts for narrow windows are at adapting to a small height, - // so we give "flat" precedence here - if (height <= 600) return "flat"; - if (width <= 600) return "narrow"; - return "normal"; - }), - ), - "normal", - ); - - /** - * The general shape of the window. - */ - public readonly windowMode$ = this.scope.behavior( - this.pipEnabled$.pipe( - switchMap((pip) => - pip ? of("pip") : this.naturalWindowMode$, - ), - ), - ); - - private readonly spotlightExpandedToggle$ = new Subject(); - public readonly spotlightExpanded$ = this.scope.behavior( - this.spotlightExpandedToggle$.pipe( - accumulate(false, (expanded) => !expanded), - ), - ); - - private readonly gridModeUserSelection$ = new Subject(); - /** - * The layout mode of the media tile grid. - */ - public readonly gridMode$ = - // If the user hasn't selected spotlight and somebody starts screen sharing, - // automatically switch to spotlight mode and reset when screen sharing ends - this.scope.behavior( - this.gridModeUserSelection$.pipe( - switchMap((userSelection) => - (userSelection === "spotlight" - ? EMPTY - : combineLatest([ - this.hasRemoteScreenShares$, - this.windowMode$, - ]).pipe( - skip(userSelection === null ? 0 : 1), - map( - ([hasScreenShares, windowMode]): GridMode => - hasScreenShares || windowMode === "flat" - ? "spotlight" - : "grid", - ), - ) - ).pipe(startWith(userSelection ?? "grid")), - ), - ), - "grid", - ); - - public setGridMode(value: GridMode): void { - this.gridModeUserSelection$.next(value); - } - - private readonly gridLayoutMedia$: Observable = - combineLatest([this.grid$, this.spotlight$], (grid, spotlight) => ({ - type: "grid", - spotlight: spotlight.some((vm) => vm instanceof ScreenShareViewModel) - ? spotlight - : undefined, - grid, - })); - - private readonly spotlightLandscapeLayoutMedia$: Observable = - combineLatest([this.grid$, this.spotlight$], (grid, spotlight) => ({ - type: "spotlight-landscape", - spotlight, - grid, - })); - - private readonly spotlightPortraitLayoutMedia$: Observable = - combineLatest([this.grid$, this.spotlight$], (grid, spotlight) => ({ - type: "spotlight-portrait", - spotlight, - grid, - })); - - private readonly spotlightExpandedLayoutMedia$: Observable = - combineLatest([this.spotlight$, this.pip$], (spotlight, pip) => ({ - type: "spotlight-expanded", - spotlight, - pip: pip ?? undefined, - })); - - private readonly oneOnOneLayoutMedia$: Observable = - this.mediaItems$.pipe( - map((mediaItems) => { - if (mediaItems.length !== 2) return null; - const local = mediaItems.find((vm) => vm.vm.local)?.vm as - | LocalUserMediaViewModel - | undefined; - const remote = mediaItems.find((vm) => !vm.vm.local)?.vm as - | RemoteUserMediaViewModel - | undefined; - // There might not be a remote tile if there are screen shares, or if - // only the local user is in the call and they're using the duplicate - // tiles option - if (!remote || !local) return null; - - return { type: "one-on-one", local, remote }; - }), - ); - - private readonly pipLayoutMedia$: Observable = - this.spotlight$.pipe(map((spotlight) => ({ type: "pip", spotlight }))); - - /** - * The media to be used to produce a layout. - */ - private readonly layoutMedia$ = this.scope.behavior( - this.windowMode$.pipe( - switchMap((windowMode) => { - switch (windowMode) { - case "normal": - return this.gridMode$.pipe( - switchMap((gridMode) => { - switch (gridMode) { - case "grid": - return this.oneOnOneLayoutMedia$.pipe( - switchMap((oneOnOne) => - oneOnOne === null - ? this.gridLayoutMedia$ - : of(oneOnOne), - ), - ); - case "spotlight": - return this.spotlightExpanded$.pipe( - switchMap((expanded) => - expanded - ? this.spotlightExpandedLayoutMedia$ - : this.spotlightLandscapeLayoutMedia$, - ), - ); - } - }), - ); - case "narrow": - return this.oneOnOneLayoutMedia$.pipe( - switchMap((oneOnOne) => - oneOnOne === null - ? combineLatest( - [this.grid$, this.spotlight$], - (grid, spotlight) => - grid.length > smallMobileCallThreshold || - spotlight.some( - (vm) => vm instanceof ScreenShareViewModel, - ) - ? this.spotlightPortraitLayoutMedia$ - : this.gridLayoutMedia$, - ).pipe(switchAll()) - : // The expanded spotlight layout makes for a better one-on-one - // experience in narrow windows - this.spotlightExpandedLayoutMedia$, - ), - ); - case "flat": - return this.gridMode$.pipe( - switchMap((gridMode) => { - switch (gridMode) { - case "grid": - // Yes, grid mode actually gets you a "spotlight" layout in - // this window mode. - return this.spotlightLandscapeLayoutMedia$; - case "spotlight": - return this.spotlightExpandedLayoutMedia$; - } - }), - ); - case "pip": - return this.pipLayoutMedia$; - } - }), - ), - ); - - // There is a cyclical dependency here: the layout algorithms want to know - // which tiles are on screen, but to know which tiles are on screen we have to - // first render a layout. To deal with this we assume initially that no tiles - // are visible, and loop the data back into the layouts with a Subject. - private readonly visibleTiles$ = new Subject(); - private readonly setVisibleTiles = (value: number): void => - this.visibleTiles$.next(value); - - private readonly layoutInternals$ = this.scope.behavior< - LayoutScanState & { layout: Layout } - >( - combineLatest([ - this.layoutMedia$, - this.visibleTiles$.pipe(startWith(0), distinctUntilChanged()), - ]).pipe( - scan< - [LayoutMedia, number], - LayoutScanState & { layout: Layout }, - LayoutScanState - >( - ({ tiles: prevTiles }, [media, visibleTiles]) => { - let layout: Layout; - let newTiles: TileStore; - switch (media.type) { - case "grid": - case "spotlight-landscape": - case "spotlight-portrait": - [layout, newTiles] = gridLikeLayout( - media, - visibleTiles, - this.setVisibleTiles, - prevTiles, - ); - break; - case "spotlight-expanded": - [layout, newTiles] = spotlightExpandedLayout(media, prevTiles); - break; - case "one-on-one": - [layout, newTiles] = oneOnOneLayout(media, prevTiles); - break; - case "pip": - [layout, newTiles] = pipLayout(media, prevTiles); - break; - } - - return { layout, tiles: newTiles }; - }, - { layout: null, tiles: TileStore.empty() }, - ), - ), - ); - - /** - * The layout of tiles in the call interface. - */ - public readonly layout$ = this.scope.behavior( - this.layoutInternals$.pipe(map(({ layout }) => layout)), - ); - - /** - * The current generation of the tile store, exposed for debugging purposes. - */ - public readonly tileStoreGeneration$ = this.scope.behavior( - this.layoutInternals$.pipe(map(({ tiles }) => tiles.generation)), - ); - - public showSpotlightIndicators$ = this.scope.behavior( - this.layout$.pipe(map((l) => l.type !== "grid")), - ); - - public showSpeakingIndicators$ = this.scope.behavior( - this.layout$.pipe( - switchMap((l) => { - switch (l.type) { - case "spotlight-landscape": - case "spotlight-portrait": - // If the spotlight is showing the active speaker, we can do without - // speaking indicators as they're a redundant visual cue. But if - // screen sharing feeds are in the spotlight we still need them. - return l.spotlight.media$.pipe( - map((models: MediaViewModel[]) => - models.some((m) => m instanceof ScreenShareViewModel), - ), - ); - // In expanded spotlight layout, the active speaker is always shown in - // the picture-in-picture tile so there is no need for speaking - // indicators. And in one-on-one layout there's no question as to who is - // speaking. - case "spotlight-expanded": - case "one-on-one": - return of(false); - default: - return of(true); - } - }), - ), - ); - - public readonly toggleSpotlightExpanded$ = this.scope.behavior< - (() => void) | null - >( - this.windowMode$.pipe( - switchMap((mode) => - mode === "normal" - ? this.layout$.pipe( - map( - (l) => - l.type === "spotlight-landscape" || - l.type === "spotlight-expanded", - ), - ) - : of(false), - ), - distinctUntilChanged(), - map((enabled) => - enabled ? (): void => this.spotlightExpandedToggle$.next() : null, - ), - ), - ); - - private readonly screenTap$ = new Subject(); - private readonly controlsTap$ = new Subject(); - private readonly screenHover$ = new Subject(); - private readonly screenUnhover$ = new Subject(); - - /** - * Callback for when the user taps the call view. - */ - public tapScreen(): void { - this.screenTap$.next(); - } - - /** - * Callback for when the user taps the call's controls. - */ - public tapControls(): void { - this.controlsTap$.next(); - } - - /** - * Callback for when the user hovers over the call view. - */ - public hoverScreen(): void { - this.screenHover$.next(); - } - - /** - * Callback for when the user stops hovering over the call view. - */ - public unhoverScreen(): void { - this.screenUnhover$.next(); - } - - public readonly showHeader$ = this.scope.behavior( - this.windowMode$.pipe(map((mode) => mode !== "pip" && mode !== "flat")), - ); - - public readonly showFooter$ = this.scope.behavior( - this.windowMode$.pipe( - switchMap((mode) => { - switch (mode) { - case "pip": - return of(false); - case "normal": - case "narrow": - return of(true); - case "flat": - // Sadly Firefox has some layering glitches that prevent the footer - // from appearing properly. They happen less often if we never hide - // the footer. - if (isFirefox()) return of(true); - // Show/hide the footer in response to interactions - return merge( - this.screenTap$.pipe(map(() => "tap screen" as const)), - this.controlsTap$.pipe(map(() => "tap controls" as const)), - this.screenHover$.pipe(map(() => "hover" as const)), - ).pipe( - switchScan((state, interaction) => { - switch (interaction) { - case "tap screen": - return state - ? // Toggle visibility on tap - of(false) - : // Hide after a timeout - timer(showFooterMs).pipe( - map(() => false), - startWith(true), - ); - case "tap controls": - // The user is interacting with things, so reset the timeout - return timer(showFooterMs).pipe( - map(() => false), - startWith(true), - ); - case "hover": - // Show on hover and hide after a timeout - return race( - timer(showFooterMs), - this.screenUnhover$.pipe(take(1)), - ).pipe( - map(() => false), - startWith(true), - ); - } - }, false), - startWith(false), - ); - } - }), - ), - ); - - /** - * Whether audio is currently being output through the earpiece. - */ - public readonly earpieceMode$ = this.scope.behavior( - combineLatest( - [ - this.mediaDevices.audioOutput.available$, - this.mediaDevices.audioOutput.selected$, - ], - (available, selected) => - selected !== undefined && - available.get(selected.id)?.type === "earpiece", - ), - ); - - /** - * Callback to toggle between the earpiece and the loudspeaker. - * - * This will be `null` in case the target does not exist in the list - * of available audio outputs. - */ - public readonly audioOutputSwitcher$ = this.scope.behavior<{ - targetOutput: "earpiece" | "speaker"; - switch: () => void; - } | null>( - combineLatest( - [ - this.mediaDevices.audioOutput.available$, - this.mediaDevices.audioOutput.selected$, - ], - (available, selected) => { - const selectionType = selected && available.get(selected.id)?.type; - - // If we are in any output mode other than speaker switch to speaker. - const newSelectionType: "earpiece" | "speaker" = - selectionType === "speaker" ? "earpiece" : "speaker"; - const newSelection = [...available].find( - ([, d]) => d.type === newSelectionType, - ); - if (newSelection === undefined) return null; - - const [id] = newSelection; - return { - targetOutput: newSelectionType, - switch: (): void => this.mediaDevices.audioOutput.select(id), - }; - }, - ), - ); - - /** - * Emits an array of reactions that should be visible on the screen. - */ - // DISCUSSION move this into a reaction file - // const {visibleReactions$, audibleReactions$} = reactionsObservables$(showReactionSetting$, ) - public readonly visibleReactions$ = this.scope.behavior( - showReactions.value$.pipe( - switchMap((show) => (show ? this.reactions$ : of({}))), - scan< - Record, - { sender: string; emoji: string; startX: number }[] - >((acc, latest) => { - const newSet: { sender: string; emoji: string; startX: number }[] = []; - for (const [sender, reaction] of Object.entries(latest)) { - const startX = - acc.find((v) => v.sender === sender && v.emoji)?.startX ?? - Math.ceil(Math.random() * 80) + 10; - newSet.push({ sender, emoji: reaction.emoji, startX }); - } - return newSet; - }, []), - ), - ); - - /** - * Emits an array of reactions that should be played. - */ - public readonly audibleReactions$ = playReactionsSound.value$.pipe( - switchMap((show) => - show ? this.reactions$ : of>({}), - ), - map((reactions) => Object.values(reactions).map((v) => v.name)), - scan( - (acc, latest) => { - return { - playing: latest.filter( - (v) => acc.playing.includes(v) || acc.newSounds.includes(v), - ), - newSounds: latest.filter( - (v) => !acc.playing.includes(v) && !acc.newSounds.includes(v), - ), - }; - }, - { playing: [], newSounds: [] }, - ), - map((v) => v.newSounds), - ); - - /** - * Emits an event every time a new hand is raised in - * the call. - */ - public readonly newHandRaised$ = this.handsRaised$.pipe( - map((v) => Object.keys(v).length), - scan( - (acc, newValue) => ({ - value: newValue, - playSounds: newValue > acc.value, - }), - { value: 0, playSounds: false }, - ), - filter((v) => v.playSounds), - ); - - /** - * Emits an event every time a new screenshare is started in - * the call. - */ - public readonly newScreenShare$ = this.screenShares$.pipe( - map((v) => v.length), - scan( - (acc, newValue) => ({ - value: newValue, - playSounds: newValue > acc.value, - }), - { value: 0, playSounds: false }, - ), - filter((v) => v.playSounds), - ); - - /** - * Whether we are sharing our screen. - */ - // reassigned here to make it publicly accessible - public readonly sharingScreen$ = this.localMembership.sharingScreen$; - - /** - * Callback to toggle screen sharing. If null, screen sharing is not possible. - */ - // reassigned here to make it publicly accessible - public readonly toggleScreenSharing = - this.localMembership.toggleScreenSharing; - - public constructor( - private readonly scope: ObservableScope, - // A call is permanently tied to a single Matrix room - private readonly matrixRTCSession: MatrixRTCSession, - private readonly matrixRoom: MatrixRoom, - private readonly mediaDevices: MediaDevices, - private readonly muteStates: MuteStates, - private readonly options: CallViewModelOptions, - private readonly handsRaisedSubject$: Observable< - Record - >, - private readonly reactionsSubject$: Observable< - Record - >, - private readonly trackProcessorState$: Behavior, - ) { - // Join automatically - this.join(); // TODO-MULTI-SFU: Use this view model for the lobby as well, and only call this once 'join' is clicked? - } -} - -// TODO-MULTI-SFU // Setup and update the keyProvider which was create by `createRoom` was a thing before. Now we never update if the E2EEsystem changes -// do we need this? - -function getE2eeKeyProvider( - e2eeSystem: EncryptionSystem, - rtcSession: MatrixRTCSession, -): BaseKeyProvider | undefined { - if (e2eeSystem.kind === E2eeType.NONE) return undefined; - - if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) { - const keyProvider = new MatrixKeyProvider(); - keyProvider.setRTCSession(rtcSession); - return keyProvider; - } else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) { - const keyProvider = new ExternalE2EEKeyProvider(); - keyProvider - .setKey(e2eeSystem.secret) - .catch((e) => logger.error("Failed to set shared key for E2EE", e)); - return keyProvider; - } -} From e9f400e9f15c568080331ee7399a248499775383 Mon Sep 17 00:00:00 2001 From: Timo K Date: Mon, 17 Nov 2025 14:30:16 +0100 Subject: [PATCH 64/75] remove ReturnType --- src/state/CallViewModel/CallViewModel.ts | 11 +++++------ .../CallViewModel/localMember/LocalMembership.ts | 1 + 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index ff2ed0cb..0fd71d3d 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -93,7 +93,10 @@ import { } from "../layout-types.ts"; import { type ElementCallError } from "../../utils/errors.ts"; import { type ObservableScope } from "../ObservableScope.ts"; -import { createLocalMembership$ } from "./localMember/LocalMembership.ts"; +import { + createLocalMembership$, + type LocalMemberConnectionState, +} from "./localMember/LocalMembership.ts"; import { createLocalTransport$ } from "./localMember/LocalTransport.ts"; import { createMemberships$, @@ -167,10 +170,6 @@ type AudioLivekitItem = { url: string; }; -type JoinReturn = ReturnType< - ReturnType["requestConnect"] ->; - /** * A view model providing all the application logic needed to show the in-call * UI (may eventually be expanded to cover the lobby and feedback screens in the @@ -189,7 +188,7 @@ export class CallViewModel { public hangup: () => void; // joining - public join: () => JoinReturn; + public join: () => LocalMemberConnectionState; // screen sharing public toggleScreenSharing: (() => void) | null; diff --git a/src/state/CallViewModel/localMember/LocalMembership.ts b/src/state/CallViewModel/localMember/LocalMembership.ts index ae6c1c7c..d9a3887b 100644 --- a/src/state/CallViewModel/localMember/LocalMembership.ts +++ b/src/state/CallViewModel/localMember/LocalMembership.ts @@ -335,6 +335,7 @@ export const createLocalMembership$ = ({ }); combineLatest([localTransport$, connectRequested$]).subscribe( + // TODO reconnect on options change. ([transport, connectRequested]) => { if ( transport === null || From 4414fe3c8243021f733c5265c1fde1551e32a75b Mon Sep 17 00:00:00 2001 From: Timo K Date: Mon, 17 Nov 2025 14:39:24 +0100 Subject: [PATCH 65/75] remove all `/*PUBLIC*/` --- src/state/CallViewModel/CallViewModel.ts | 44 +++++++++++------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 0fd71d3d..3e3d8190 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -469,10 +469,10 @@ export class CallViewModel { // down, for example, and we want to avoid making people worry that the app is // in a split-brained state. // DISCUSSION own membership manager ALSO this probably can be simplifis - /*PUBLIC*/ const reconnecting$ = localMembership.reconnecting$; + const reconnecting$ = localMembership.reconnecting$; const pretendToBeDisconnected$ = reconnecting$; - /*PUBLIC*/ const audioParticipants$ = scope.behavior( + const audioParticipants$ = scope.behavior( matrixLivekitMembers$.pipe( switchMap((membersWithEpoch) => { const members = membersWithEpoch.value; @@ -518,11 +518,11 @@ export class CallViewModel { [], ); - /*PUBLIC*/ const handsRaised$ = scope.behavior( + const handsRaised$ = scope.behavior( handsRaisedSubject$.pipe(pauseWhen(pretendToBeDisconnected$)), ); - /*PUBLIC*/ const reactions$ = scope.behavior( + const reactions$ = scope.behavior( reactionsSubject$.pipe( map((v) => Object.fromEntries( @@ -663,7 +663,7 @@ export class CallViewModel { ), ); - /*PUBLIC*/ const joinSoundEffect$ = userMedia$.pipe( + const joinSoundEffect$ = userMedia$.pipe( pairwise(), filter( ([prev, current]) => @@ -681,16 +681,16 @@ export class CallViewModel { * - There can be multiple participants for one Matrix user if they join from * multiple devices. */ - /*PUBLIC*/ const participantCount$ = scope.behavior( + const participantCount$ = scope.behavior( matrixLivekitMembers$.pipe(map((ms) => ms.value.length)), ); // only public to expose to the view. // TODO if we are in "unknown" state we need a loading rendering (or empty screen) // Otherwise it looks like we already connected and only than the ringing starts which is weird. - /*PUBLIC*/ const callPickupState$ = callLifecycle.callPickupState$; + const callPickupState$ = callLifecycle.callPickupState$; - /*PUBLIC*/ const leaveSoundEffect$ = combineLatest([ + const leaveSoundEffect$ = combineLatest([ callLifecycle.callPickupState$, userMedia$, ]).pipe( @@ -871,14 +871,14 @@ export class CallViewModel { /** * The general shape of the window. */ - /*PUBLIC*/ const windowMode$ = scope.behavior( + const windowMode$ = scope.behavior( pipEnabled$.pipe( switchMap((pip) => (pip ? of("pip") : naturalWindowMode$)), ), ); const spotlightExpandedToggle$ = new Subject(); - /*PUBLIC*/ const spotlightExpanded$ = scope.behavior( + const spotlightExpanded$ = scope.behavior( spotlightExpandedToggle$.pipe(accumulate(false, (expanded) => !expanded)), ); @@ -886,7 +886,7 @@ export class CallViewModel { /** * The layout mode of the media tile grid. */ - /*PUBLIC*/ const gridMode$ = + const gridMode$ = // If the user hasn't selected spotlight and somebody starts screen sharing, // automatically switch to spotlight mode and reset when screen sharing ends scope.behavior( @@ -909,7 +909,7 @@ export class CallViewModel { "grid", ); - /*PUBLIC*/ const setGridMode = (value: GridMode): void => { + const setGridMode = (value: GridMode): void => { gridModeUserSelection$.next(value); }; @@ -1087,22 +1087,22 @@ export class CallViewModel { /** * The layout of tiles in the call interface. */ - /*PUBLIC*/ const layout$ = scope.behavior( + const layout$ = scope.behavior( layoutInternals$.pipe(map(({ layout }) => layout)), ); /** * The current generation of the tile store, exposed for debugging purposes. */ - /*PUBLIC*/ const tileStoreGeneration$ = scope.behavior( + const tileStoreGeneration$ = scope.behavior( layoutInternals$.pipe(map(({ tiles }) => tiles.generation)), ); - /*PUBLIC*/ const showSpotlightIndicators$ = scope.behavior( + const showSpotlightIndicators$ = scope.behavior( layout$.pipe(map((l) => l.type !== "grid")), ); - /*PUBLIC*/ const showSpeakingIndicators$ = scope.behavior( + const showSpeakingIndicators$ = scope.behavior( layout$.pipe( switchMap((l) => { switch (l.type) { @@ -1130,9 +1130,7 @@ export class CallViewModel { ), ); - /*PUBLIC*/ const toggleSpotlightExpanded$ = scope.behavior< - (() => void) | null - >( + const toggleSpotlightExpanded$ = scope.behavior<(() => void) | null>( windowMode$.pipe( switchMap((mode) => mode === "normal" @@ -1157,11 +1155,11 @@ export class CallViewModel { const screenHover$ = new Subject(); const screenUnhover$ = new Subject(); - /*PUBLIC*/ const showHeader$ = scope.behavior( + const showHeader$ = scope.behavior( windowMode$.pipe(map((mode) => mode !== "pip" && mode !== "flat")), ); - /*PUBLIC*/ const showFooter$ = scope.behavior( + const showFooter$ = scope.behavior( windowMode$.pipe( switchMap((mode) => { switch (mode) { @@ -1219,7 +1217,7 @@ export class CallViewModel { /** * Whether audio is currently being output through the earpiece. */ - /*PUBLIC*/ const earpieceMode$ = scope.behavior( + const earpieceMode$ = scope.behavior( combineLatest( [ mediaDevices.audioOutput.available$, @@ -1237,7 +1235,7 @@ export class CallViewModel { * This will be `null` in case the target does not exist in the list * of available audio outputs. */ - /*PUBLIC*/ const audioOutputSwitcher$ = scope.behavior<{ + const audioOutputSwitcher$ = scope.behavior<{ targetOutput: "earpiece" | "speaker"; switch: () => void; } | null>( From f518111887048e382cdf92fdaf95d8cd6434b692 Mon Sep 17 00:00:00 2001 From: Timo K Date: Mon, 17 Nov 2025 14:55:00 +0100 Subject: [PATCH 66/75] add back docstrings --- src/reactions/ReactionsReader.ts | 1 + src/state/CallViewModel/CallViewModel.ts | 78 ++++++++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/src/reactions/ReactionsReader.ts b/src/reactions/ReactionsReader.ts index c1f78b51..74b47c77 100644 --- a/src/reactions/ReactionsReader.ts +++ b/src/reactions/ReactionsReader.ts @@ -266,6 +266,7 @@ export class ReactionsReader { ); return; } + // TODO refactor to use memer id `membershipEvent.membershipID` (needs to happen in combination with other memberId refactors) const identifier = `${membershipEvent.userId}:${membershipEvent.deviceId}`; if (!content.emoji) { diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 3e3d8190..063a953e 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -181,32 +181,68 @@ type AudioLivekitItem = { export class CallViewModel { // lifecycle public autoLeave$: Observable; + // TODO if we are in "unknown" state we need a loading rendering (or empty screen) + // Otherwise it looks like we already connected and only than the ringing starts which is weird. public callPickupState$: Behavior< "unknown" | "ringing" | "timeout" | "decline" | "success" | null >; public leave$: Observable<"user" | AutoLeaveReason>; + /** Call to initiate hangup. Use in conbination with connectino state track the async hangup process. */ public hangup: () => void; // joining public join: () => LocalMemberConnectionState; // screen sharing + /** + * Callback to toggle screen sharing. If null, screen sharing is not possible. + */ public toggleScreenSharing: (() => void) | null; + /** + * Whether we are sharing our screen. + */ public sharingScreen$: Behavior; // UI interactions + /** + * Callback for when the user taps the call view. + */ public tapScreen: () => void; + /** + * Callback for when the user taps the call's controls. + */ public tapControls: () => void; + /** + * Callback for when the user hovers over the call view. + */ public hoverScreen: () => void; + /** + * Callback for when the user stops hovering over the call view. + */ public unhoverScreen: () => void; // errors + /** + * If there is a configuration error with the call (e.g. misconfigured E2EE). + * This is a fatal error that prevents the call from being created/joined. + * Should render a blocking error screen. + */ public configError$: Behavior; // participants and counts + /** + * The number of participants currently in the call. + * + * - Each participant has a corresponding MatrixRTC membership state event + * - There can be multiple participants for one Matrix user if they join from + * multiple devices. + */ public participantCount$: Behavior; + /** Participants sorted by livekit room so they can be used in the audio rendering */ public audioParticipants$: Behavior; + /** List of participants raising their hand */ public handsRaised$: Behavior>; + /** List of reactions. Keys are: membership.membershipId (currently predefined as: `${membershipEvent.userId}:${membershipEvent.deviceId}`)*/ public reactions$: Behavior>; public isOneOnOneWith$: Behavior; public leaveSoundEffect$: Observable; + /** + * Emits an event every time a new hand is raised in + * the call. + */ public newHandRaised$: Observable<{ value: number; playSounds: boolean }>; + /** + * Emits an event every time a new screenshare is started in + * the call. + */ public newScreenShare$: Observable<{ value: number; playSounds: boolean }>; + /** + * Emits an array of reactions that should be played. + */ public audibleReactions$: Observable; + /** + * Emits an array of reactions that should be visible on the screen. + */ + // DISCUSSION move this into a reaction file public visibleReactions$: Behavior< { sender: string; emoji: string; startX: number }[] >; // window/layout + /** + * The general shape of the window. + */ public windowMode$: Behavior; public spotlightExpanded$: Behavior; public toggleSpotlightExpanded$: Behavior<(() => void) | null>; @@ -234,7 +288,13 @@ export class CallViewModel { public grid$: Behavior; public spotlight$: Behavior; public pip$: Behavior; + /** + * The layout of tiles in the call interface. + */ public layout$: Behavior; + /** + * The current generation of the tile store, exposed for debugging purposes. + */ public tileStoreGeneration$: Behavior; public showSpotlightIndicators$: Behavior; public showSpeakingIndicators$: Behavior; @@ -244,13 +304,31 @@ export class CallViewModel { public showFooter$: Behavior; // audio routing + /** + * Whether audio is currently being output through the earpiece. + */ public earpieceMode$: Behavior; + /** + * Callback to toggle between the earpiece and the loudspeaker. + * + * This will be `null` in case the target does not exist in the list + * of available audio outputs. + */ public audioOutputSwitcher$: Behavior<{ targetOutput: "earpiece" | "speaker"; switch: () => void; } | null>; // connection state + /** + * Whether various media/event sources should pretend to be disconnected from + * all network input, even if their connection still technically works. + */ + // We do this when the app is in the 'reconnecting' state, because it might be + // that the LiveKit connection is still functional while the homeserver is + // down, for example, and we want to avoid making people worry that the app is + // in a split-brained state. + // DISCUSSION own membership manager ALSO this probably can be simplifis public reconnecting$: Behavior; // THIS has to be the last public field declaration From b51df36a06306e6e38ca7b5944f31f0cdd79f6d4 Mon Sep 17 00:00:00 2001 From: Timo <16718859+toger5@users.noreply.github.com> Date: Mon, 17 Nov 2025 17:42:37 +0100 Subject: [PATCH 67/75] Move one-on-one layout into CallViewModel (#3567) * move ononone layout into CallViewModel * move even more into the vm. * tests --- src/room/InCallView.tsx | 37 ++------- src/state/CallViewModel/CallViewModel.ts | 75 +++++++++++-------- .../CallViewModel/CallViewModelTestUtils.ts | 1 + .../MatrixMemberMetadata.test.ts | 4 + .../remoteMembers/MatrixMemberMetadata.ts | 36 ++++++++- src/utils/displayname.test.ts | 12 +-- src/utils/test-viewmodel.ts | 2 + 7 files changed, 98 insertions(+), 69 deletions(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 1474fb81..1f410adb 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -248,7 +248,6 @@ export const InCallView: FC = ({ () => void toggleRaisedHand(), ); - // const allLivekitRooms = useBehavior(vm.allLivekitRooms$); const audioParticipants = useBehavior(vm.audioParticipants$); const participantCount = useBehavior(vm.participantCount$); const reconnecting = useBehavior(vm.reconnecting$); @@ -263,6 +262,7 @@ export const InCallView: FC = ({ const audioOutputSwitcher = useBehavior(vm.audioOutputSwitcher$); const sharingScreen = useBehavior(vm.sharingScreen$); + const ringOverlay = useBehavior(vm.ringOverlay$); const fatalCallError = useBehavior(vm.configError$); // Stop the rendering and throw for the error boundary if (fatalCallError) throw fatalCallError; @@ -299,47 +299,26 @@ export const InCallView: FC = ({ // Waiting UI overlay const waitingOverlay: JSX.Element | null = useMemo(() => { - // No overlay if not in ringing state - if (callPickupState !== "ringing") return null; - - // Use room state for other participants data (the one that we likely want to reach) - // TODO: this screams it wants to be a behavior in the vm. - const roomOthers = [ - ...matrixRoom.getMembersWithMembership("join"), - ...matrixRoom.getMembersWithMembership("invite"), - ].filter((m) => m.userId !== client.getUserId()); - // Yield if there are not other members in the room. - if (roomOthers.length === 0) return null; - - const otherMember = roomOthers.length > 0 ? roomOthers[0] : undefined; - const isOneOnOne = roomOthers.length === 1 && otherMember; - const text = isOneOnOne - ? `Waiting for ${otherMember.name ?? otherMember.userId} to join…` - : "Waiting for other participants…"; - const avatarMxc = isOneOnOne - ? (otherMember.getMxcAvatarUrl?.() ?? undefined) - : (matrixRoom.getMxcAvatarUrl() ?? undefined); - - return ( + return ringOverlay ? (
- {text} + {ringOverlay.text}
- ); - }, [callPickupState, client, matrixRoom]); + ) : null; + }, [ringOverlay]); // Ideally we could detect taps by listening for click events and checking // that the pointerType of the event is "touch", but this isn't yet supported diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 063a953e..f1332286 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -12,7 +12,7 @@ import { type Room as LivekitRoom, type RoomOptions, } from "livekit-client"; -import { type RoomMember, type Room as MatrixRoom } from "matrix-js-sdk"; +import { type Room as MatrixRoom } from "matrix-js-sdk"; import { combineLatest, distinctUntilChanged, @@ -115,6 +115,7 @@ import { createSentCallNotification$, } from "./CallNotificationLifecycle.ts"; import { + createDMMember$, createMatrixMemberMetadata$, createRoomMembers$, } from "./remoteMembers/MatrixMemberMetadata.ts"; @@ -244,11 +245,14 @@ export class CallViewModel { public handsRaised$: Behavior>; /** List of reactions. Keys are: membership.membershipId (currently predefined as: `${membershipEvent.userId}:${membershipEvent.deviceId}`)*/ public reactions$: Behavior>; - public isOneOnOneWith$: Behavior | null>; - public localUserIsAlone$: Behavior; + + public ringOverlay$: Behavior; // sounds and events public joinSoundEffect$: Observable; public leaveSoundEffect$: Observable; @@ -483,7 +487,9 @@ export class CallViewModel { // ------------------------------------------------------------------------ // callLifecycle - const callLifecycle = createCallNotificationLifecycle$({ + // TODO if we are in "unknown" state we need a loading rendering (or empty screen) + // Otherwise it looks like we already connected and only than the ringing starts which is weird. + const { callPickupState$, autoLeave$ } = createCallNotificationLifecycle$({ scope: scope, memberships$: memberships$, sentCallNotification$: createSentCallNotification$( @@ -505,21 +511,8 @@ export class CallViewModel { matrixRoomMembers$, ); - /** - * Returns the Member {userId, getMxcAvatarUrl, rawDisplayName} of the other user in the call, if it's a one-on-one call. - */ - const isOneOnOneWith$ = scope.behavior( - matrixRoomMembers$.pipe( - map((roomMembersMap) => { - const otherMembers = Array.from(roomMembersMap.values()).filter( - (member) => member.userId !== userId, - ); - return otherMembers.length === 1 ? otherMembers[0] : null; - }), - ), - ); - - const localUserIsAlone$ = scope.behavior( + const dmMember$ = createDMMember$(scope, matrixRoomMembers$, matrixRoom); + const noUserToCallInRoom$ = scope.behavior( matrixRoomMembers$.pipe( map( (roomMembersMap) => @@ -529,6 +522,30 @@ export class CallViewModel { ), ); + const ringOverlay$ = scope.behavior( + combineLatest([noUserToCallInRoom$, dmMember$, callPickupState$]).pipe( + map(([noUserToCallInRoom, dmMember, callPickupState]) => { + // No overlay if not in ringing state + if (callPickupState !== "ringing" || noUserToCallInRoom) return null; + + const name = dmMember ? dmMember.rawDisplayName : matrixRoom.name; + const id = dmMember ? dmMember.userId : matrixRoom.roomId; + const text = dmMember + ? `Waiting for ${name} to join…` + : "Waiting for other participants…"; + const avatarMxc = dmMember + ? (dmMember.getMxcAvatarUrl?.() ?? undefined) + : (matrixRoom.getMxcAvatarUrl() ?? undefined); + return { + name: name ?? id, + idForAvatar: id, + text, + avatarMxc, + }; + }), + ), + ); + // CODESMELL? // This is functionally the same Observable as leave$, except here it's // hoisted to the top of the class. This enables the cyclic dependency between @@ -763,13 +780,8 @@ export class CallViewModel { matrixLivekitMembers$.pipe(map((ms) => ms.value.length)), ); - // only public to expose to the view. - // TODO if we are in "unknown" state we need a loading rendering (or empty screen) - // Otherwise it looks like we already connected and only than the ringing starts which is weird. - const callPickupState$ = callLifecycle.callPickupState$; - const leaveSoundEffect$ = combineLatest([ - callLifecycle.callPickupState$, + callPickupState$, userMedia$, ]).pipe( // Until the call is successful, do not play a leave sound. @@ -804,7 +816,7 @@ export class CallViewModel { const leave$: Observable<"user" | "timeout" | "decline" | "allOthersLeft"> = merge( - callLifecycle.autoLeave$, + autoLeave$, merge(userHangup$, widgetHangup$).pipe(map(() => "user" as const)), ).pipe( scope.share, @@ -1430,8 +1442,9 @@ export class CallViewModel { const join = localMembership.requestConnect; join(); // TODO-MULTI-SFU: Use this view model for the lobby as well, and only call this once 'join' is clicked? - this.autoLeave$ = callLifecycle.autoLeave$; + this.autoLeave$ = autoLeave$; this.callPickupState$ = callPickupState$; + this.ringOverlay$ = ringOverlay$; this.leave$ = leave$; this.hangup = (): void => userHangup$.next(); this.join = join; @@ -1446,8 +1459,6 @@ export class CallViewModel { this.configError$ = localMembership.configError$; this.participantCount$ = participantCount$; this.audioParticipants$ = audioParticipants$; - this.isOneOnOneWith$ = isOneOnOneWith$; - this.localUserIsAlone$ = localUserIsAlone$; this.handsRaised$ = handsRaised$; this.reactions$ = reactions$; diff --git a/src/state/CallViewModel/CallViewModelTestUtils.ts b/src/state/CallViewModel/CallViewModelTestUtils.ts index 81671971..fefc57a0 100644 --- a/src/state/CallViewModel/CallViewModelTestUtils.ts +++ b/src/state/CallViewModel/CallViewModelTestUtils.ts @@ -122,6 +122,7 @@ export function withCallViewModel( } })() as Partial as MatrixClient, getMembers: () => Array.from(roomMembers.values()), + getMembersWithMembership: () => Array.from(roomMembers.values()), }); const rtcSession = new MockRTCSession(room, []).withMemberships(rtcMembers$); const participantsSpy = vi diff --git a/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.test.ts b/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.test.ts index bc4d329c..d27fe4c9 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.test.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.test.ts @@ -58,6 +58,10 @@ describe("MatrixMemberMetadata", () => { const members = Array.from(fakeMembersMap.values()); return members; }), + getMembersWithMembership: vi.fn().mockImplementation(() => { + const members = Array.from(fakeMembersMap.values()); + return members; + }), } as unknown as MatrixRoom; }); diff --git a/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.ts b/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.ts index ad603708..91363f90 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.ts @@ -9,7 +9,10 @@ import { type RoomMember, RoomStateEvent } from "matrix-js-sdk"; import { combineLatest, fromEvent, map } from "rxjs"; import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; -import { type Room as MatrixRoom } from "matrix-js-sdk/lib/matrix"; +import { + KnownMembership, + type Room as MatrixRoom, +} from "matrix-js-sdk/lib/matrix"; // eslint-disable-next-line rxjs/no-internal import { type ObservableScope } from "../../ObservableScope"; @@ -26,7 +29,10 @@ export type RoomMemberMap = Map< Pick >; export function roomToMembersMap(matrixRoom: MatrixRoom): RoomMemberMap { - return matrixRoom.getMembers().reduce((acc, member) => { + const members = matrixRoom + .getMembersWithMembership(KnownMembership.Join) + .concat(matrixRoom.getMembersWithMembership(KnownMembership.Invite)); + return members.reduce((acc, member) => { acc.set(member.userId, { userId: member.userId, getMxcAvatarUrl: member.getMxcAvatarUrl.bind(member), @@ -47,6 +53,32 @@ export function createRoomMembers$( roomToMembersMap(matrixRoom), ); } + +/** + * creates the member that this DM is with in case it is a DM (two members) otherwise null + */ +export function createDMMember$( + scope: ObservableScope, + roomMembers$: Behavior, + matrixRoom: MatrixRoom, +): Behavior | null> { + // We cannot use the normal direct check from matrix since we do not have access to the account data. + // use primitive member count === 2 check instead. + return scope.behavior( + roomMembers$.pipe( + map((membersMap) => { + // primitive appraoch do to no access to account data. + const isDM = membersMap.size === 2; + if (!isDM) return null; + return matrixRoom.getMember(matrixRoom.guessDMUserId()); + }), + ), + ); +} + /** * Displayname for each member of the call. This will disambiguate * any displayname that clashes with another member. Only members diff --git a/src/utils/displayname.test.ts b/src/utils/displayname.test.ts index 442b928a..fa732f19 100644 --- a/src/utils/displayname.test.ts +++ b/src/utils/displayname.test.ts @@ -25,7 +25,7 @@ import { roomToMembersMap } from "../state/CallViewModel/remoteMembers/MatrixMem describe("shouldDisambiguate", () => { test("should not disambiguate a solo member", () => { const room = mockMatrixRoom({ - getMembers: () => [], + getMembersWithMembership: () => [], }); expect(shouldDisambiguate(alice, [], roomToMembersMap(room))).toEqual( false, @@ -33,7 +33,7 @@ describe("shouldDisambiguate", () => { }); test("should not disambiguate a member with an empty displayname", () => { const room = mockMatrixRoom({ - getMembers: () => [alice, aliceDoppelganger], + getMembersWithMembership: () => [alice, aliceDoppelganger], }); expect( shouldDisambiguate( @@ -44,14 +44,14 @@ describe("shouldDisambiguate", () => { ).toEqual(false); }); test("should disambiguate a member with RTL characters", () => { - const room = mockMatrixRoom({ getMembers: () => [] }); + const room = mockMatrixRoom({ getMembersWithMembership: () => [] }); expect(shouldDisambiguate(daveRTL, [], roomToMembersMap(room))).toEqual( true, ); }); test("should disambiguate a member with a matching displayname", () => { const room = mockMatrixRoom({ - getMembers: () => [alice, aliceDoppelganger], + getMembersWithMembership: () => [alice, aliceDoppelganger], }); expect( shouldDisambiguate( @@ -70,7 +70,7 @@ describe("shouldDisambiguate", () => { }); test("should disambiguate a member with a matching displayname with hidden spaces", () => { const room = mockMatrixRoom({ - getMembers: () => [bob, bobZeroWidthSpace], + getMembersWithMembership: () => [bob, bobZeroWidthSpace], }); expect( shouldDisambiguate( @@ -91,7 +91,7 @@ describe("shouldDisambiguate", () => { "should disambiguate a member with a displayname containing a mxid-like string '%s'", (rawDisplayName) => { const room = mockMatrixRoom({ - getMembers: () => [alice, aliceDoppelganger], + getMembersWithMembership: () => [alice, aliceDoppelganger], }); expect( shouldDisambiguate( diff --git a/src/utils/test-viewmodel.ts b/src/utils/test-viewmodel.ts index 314b6aa9..990c43f8 100644 --- a/src/utils/test-viewmodel.ts +++ b/src/utils/test-viewmodel.ts @@ -83,6 +83,8 @@ export function getBasicRTCSession( } as Partial as MatrixClient, getMember: (userId) => matrixRoomMembers.get(userId) ?? null, getMembers: () => Array.from(matrixRoomMembers.values()), + getMembersWithMembership: () => Array.from(matrixRoomMembers.values()), + guessDMUserId: vitest.fn(), roomId: matrixRoomId, on: vitest .fn() From 16e1c59e114aa806357450f6f500d35eea021df8 Mon Sep 17 00:00:00 2001 From: Valere Fedronic Date: Mon, 17 Nov 2025 18:22:37 +0100 Subject: [PATCH 68/75] playwright: Fix error boundary mgmt or openId errors (#3570) * playwright: Fix error boundary mgmt or openId errors * do not use tap for important logic * fix lint --------- Co-authored-by: Timo K --- .../localMember/LocalMembership.ts | 32 +++++++++++++++++-- .../localMember/LocalTransport.ts | 14 +++++--- 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/src/state/CallViewModel/localMember/LocalMembership.ts b/src/state/CallViewModel/localMember/LocalMembership.ts index d9a3887b..26269985 100644 --- a/src/state/CallViewModel/localMember/LocalMembership.ts +++ b/src/state/CallViewModel/localMember/LocalMembership.ts @@ -22,6 +22,7 @@ import { ClientEvent, SyncState, type Room as MatrixRoom } from "matrix-js-sdk"; import { BehaviorSubject, combineLatest, + distinctUntilChanged, fromEvent, map, type Observable, @@ -41,7 +42,7 @@ import { type MuteStates } from "../../MuteStates"; import { type ProcessorState } from "../../../livekit/TrackProcessorContext"; import { type MediaDevices } from "../../MediaDevices"; import { and$ } from "../../../utils/observable"; -import { type ElementCallError } from "../../../utils/errors"; +import { ElementCallError, UnknownCallError } from "../../../utils/errors"; import { ElementWidgetActions, widget, @@ -52,7 +53,10 @@ import { getUrlParams } from "../../../UrlParams.ts"; import { PosthogAnalytics } from "../../../analytics/PosthogAnalytics.ts"; import { MatrixRTCMode } from "../../../settings/settings.ts"; import { Config } from "../../../config/Config.ts"; -import { type Connection } from "../remoteMembers/Connection.ts"; +import { + type Connection, + type ConnectionState, +} from "../remoteMembers/Connection.ts"; export enum LivekitState { Uninitialized = "uninitialized", @@ -446,8 +450,8 @@ export const createLocalMembership$ = ({ scope.reconcile(localTransport$, async (advertised) => { if (advertised !== null && advertised !== undefined) { try { - configError$.next(null); await enterRTCSession(matrixRTCSession, advertised, options.value); + configError$.next(null); } catch (e) { logger.error("Error entering RTC session", e); } @@ -477,6 +481,28 @@ export const createLocalMembership$ = ({ } }); + localConnection$ + .pipe( + distinctUntilChanged(), + switchMap((c) => + c === null ? of({ state: "Initialized" } as ConnectionState) : c.state$, + ), + map((s) => { + logger.trace(`Local connection state update: ${s.state}`); + if (s.state == "FailedToStart") { + return s.error instanceof ElementCallError + ? s.error + : new UnknownCallError(s.error); + } else { + return null; + } + }), + scope.bind(), + ) + .subscribe((fatalError) => { + configError$.next(fatalError); + }); + /** * Whether the user is currently sharing their screen. */ diff --git a/src/state/CallViewModel/localMember/LocalTransport.ts b/src/state/CallViewModel/localMember/LocalTransport.ts index 94c89deb..83c997e8 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.ts @@ -166,10 +166,14 @@ async function makeTransport( ): Promise { const transport = await makeTransportInternal(client, roomId); // this will call the jwt/sfu/get endpoint to pre create the livekit room. - await getSFUConfigWithOpenID( - client, - transport.livekit_service_url, - transport.livekit_alias, - ); + try { + await getSFUConfigWithOpenID( + client, + transport.livekit_service_url, + transport.livekit_alias, + ); + } catch (e) { + logger.warn(`Failed to get SFU config for transport: ${e}`); + } return transport; } From 2e2c799f7295d97feed57044fbe0e2ff4202d70f Mon Sep 17 00:00:00 2001 From: Timo K Date: Mon, 17 Nov 2025 18:22:25 +0100 Subject: [PATCH 69/75] convert CallViewModel into create function pattern. (with much more minimal changes thanks to the intermediate class refactor) --- src/room/InCallView.tsx | 5 +- src/state/CallViewModel/CallViewModel.test.ts | 4 +- src/state/CallViewModel/CallViewModel.ts | 2216 ++++++++--------- .../CallViewModel/CallViewModelTestUtils.ts | 8 +- src/utils/test-viewmodel.ts | 5 +- 5 files changed, 1115 insertions(+), 1123 deletions(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 1f410adb..10b86770 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -59,7 +59,8 @@ import { type MatrixInfo } from "./VideoPreview"; import { InviteButton } from "../button/InviteButton"; import { LayoutToggle } from "./LayoutToggle"; import { - CallViewModel, + type CallViewModel, + createCallViewModel$, type GridMode, } from "../state/CallViewModel/CallViewModel.ts"; import { Grid, type TileProps } from "../grid/Grid"; @@ -128,7 +129,7 @@ export const ActiveCall: FC = (props) => { const reactionsReader = new ReactionsReader(scope, props.rtcSession); const { autoLeaveWhenOthersLeft, waitForCallPickup, sendNotificationType } = urlParams; - const vm = new CallViewModel( + const vm = createCallViewModel$( scope, props.rtcSession, props.matrixRoom, diff --git a/src/state/CallViewModel/CallViewModel.test.ts b/src/state/CallViewModel/CallViewModel.test.ts index eb8d438d..9e28687e 100644 --- a/src/state/CallViewModel/CallViewModel.test.ts +++ b/src/state/CallViewModel/CallViewModel.test.ts @@ -37,7 +37,7 @@ import { import { deepCompare } from "matrix-js-sdk/lib/utils"; import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery"; -import { CallViewModel } from "./CallViewModel"; +import { createCallViewModel$ } from "./CallViewModel"; import { type Layout } from "../layout-types.ts"; import { mockLocalParticipant, @@ -277,7 +277,7 @@ describe("CallViewModel", () => { vi.spyOn(AutoDiscovery, "getRawClientConfig").mockResolvedValue({}); - const callVM = new CallViewModel( + const callVM = createCallViewModel$( testScope(), fakeRtcSession.asMockedSession(), matrixRoom, diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index f1332286..62e4ae44 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -172,55 +172,56 @@ type AudioLivekitItem = { }; /** - * A view model providing all the application logic needed to show the in-call - * UI (may eventually be expanded to cover the lobby and feedback screens in the - * future). + * The return of createCallViewModel$ + * This interface represents the root snapshot for the call view. Snapshots in EC with rxjs behave like snapshot trees. + * They are a list of observables and objects containing observables to allow for a very granular update mechanism. + * + * This allows to have one huge call view model that represents the entire view without a unnecessary amount of updates. + * + * (Mocking this interface should allow building a full view in all states.) */ -// Throughout this class and related code we must distinguish between MatrixRTC -// state and LiveKit state. We use the common terminology of room "members", RTC -// "memberships", and LiveKit "participants". -export class CallViewModel { +export interface CallViewModel { // lifecycle - public autoLeave$: Observable; + autoLeave$: Observable; // TODO if we are in "unknown" state we need a loading rendering (or empty screen) // Otherwise it looks like we already connected and only than the ringing starts which is weird. - public callPickupState$: Behavior< + callPickupState$: Behavior< "unknown" | "ringing" | "timeout" | "decline" | "success" | null >; - public leave$: Observable<"user" | AutoLeaveReason>; + leave$: Observable<"user" | AutoLeaveReason>; /** Call to initiate hangup. Use in conbination with connectino state track the async hangup process. */ - public hangup: () => void; + hangup: () => void; // joining - public join: () => LocalMemberConnectionState; + join: () => LocalMemberConnectionState; // screen sharing /** * Callback to toggle screen sharing. If null, screen sharing is not possible. */ - public toggleScreenSharing: (() => void) | null; + toggleScreenSharing: (() => void) | null; /** * Whether we are sharing our screen. */ - public sharingScreen$: Behavior; + sharingScreen$: Behavior; // UI interactions /** * Callback for when the user taps the call view. */ - public tapScreen: () => void; + tapScreen: () => void; /** * Callback for when the user taps the call's controls. */ - public tapControls: () => void; + tapControls: () => void; /** * Callback for when the user hovers over the call view. */ - public hoverScreen: () => void; + hoverScreen: () => void; /** * Callback for when the user stops hovering over the call view. */ - public unhoverScreen: () => void; + unhoverScreen: () => void; // errors /** @@ -228,7 +229,7 @@ export class CallViewModel { * This is a fatal error that prevents the call from being created/joined. * Should render a blocking error screen. */ - public configError$: Behavior; + configError$: Behavior; // participants and counts /** @@ -238,15 +239,15 @@ export class CallViewModel { * - There can be multiple participants for one Matrix user if they join from * multiple devices. */ - public participantCount$: Behavior; + participantCount$: Behavior; /** Participants sorted by livekit room so they can be used in the audio rendering */ - public audioParticipants$: Behavior; + audioParticipants$: Behavior; /** List of participants raising their hand */ - public handsRaised$: Behavior>; + handsRaised$: Behavior>; /** List of reactions. Keys are: membership.membershipId (currently predefined as: `${membershipEvent.userId}:${membershipEvent.deviceId}`)*/ - public reactions$: Behavior>; + reactions$: Behavior>; - public ringOverlay$: Behavior; // sounds and events - public joinSoundEffect$: Observable; - public leaveSoundEffect$: Observable; + joinSoundEffect$: Observable; + leaveSoundEffect$: Observable; /** * Emits an event every time a new hand is raised in * the call. */ - public newHandRaised$: Observable<{ value: number; playSounds: boolean }>; + newHandRaised$: Observable<{ value: number; playSounds: boolean }>; /** * Emits an event every time a new screenshare is started in * the call. */ - public newScreenShare$: Observable<{ value: number; playSounds: boolean }>; + newScreenShare$: Observable<{ value: number; playSounds: boolean }>; /** * Emits an array of reactions that should be played. */ - public audibleReactions$: Observable; + audibleReactions$: Observable; /** * Emits an array of reactions that should be visible on the screen. */ // DISCUSSION move this into a reaction file - public visibleReactions$: Behavior< + visibleReactions$: Behavior< { sender: string; emoji: string; startX: number }[] >; @@ -282,43 +283,43 @@ export class CallViewModel { /** * The general shape of the window. */ - public windowMode$: Behavior; - public spotlightExpanded$: Behavior; - public toggleSpotlightExpanded$: Behavior<(() => void) | null>; - public gridMode$: Behavior; - public setGridMode: (value: GridMode) => void; + windowMode$: Behavior; + spotlightExpanded$: Behavior; + toggleSpotlightExpanded$: Behavior<(() => void) | null>; + gridMode$: Behavior; + setGridMode: (value: GridMode) => void; // media view models and layout - public grid$: Behavior; - public spotlight$: Behavior; - public pip$: Behavior; + grid$: Behavior; + spotlight$: Behavior; + pip$: Behavior; /** * The layout of tiles in the call interface. */ - public layout$: Behavior; + layout$: Behavior; /** * The current generation of the tile store, exposed for debugging purposes. */ - public tileStoreGeneration$: Behavior; - public showSpotlightIndicators$: Behavior; - public showSpeakingIndicators$: Behavior; + tileStoreGeneration$: Behavior; + showSpotlightIndicators$: Behavior; + showSpeakingIndicators$: Behavior; // header/footer visibility - public showHeader$: Behavior; - public showFooter$: Behavior; + showHeader$: Behavior; + showFooter$: Behavior; // audio routing /** * Whether audio is currently being output through the earpiece. */ - public earpieceMode$: Behavior; + earpieceMode$: Behavior; /** * Callback to toggle between the earpiece and the loudspeaker. * * This will be `null` in case the target does not exist in the list * of available audio outputs. */ - public audioOutputSwitcher$: Behavior<{ + audioOutputSwitcher$: Behavior<{ targetOutput: "earpiece" | "speaker"; switch: () => void; } | null>; @@ -333,1160 +334,1145 @@ export class CallViewModel { // down, for example, and we want to avoid making people worry that the app is // in a split-brained state. // DISCUSSION own membership manager ALSO this probably can be simplifis - public reconnecting$: Behavior; + reconnecting$: Behavior; +} +/** + * A view model providing all the application logic needed to show the in-call + * UI (may eventually be expanded to cover the lobby and feedback screens in the + * future). + */ +// Throughout this class and related code we must distinguish between MatrixRTC +// state and LiveKit state. We use the common terminology of room "members", RTC +// "memberships", and LiveKit "participants". +export function createCallViewModel$( + scope: ObservableScope, + // A call is permanently tied to a single Matrix room + matrixRTCSession: MatrixRTCSession, + matrixRoom: MatrixRoom, + mediaDevices: MediaDevices, + muteStates: MuteStates, + options: CallViewModelOptions, + handsRaisedSubject$: Observable>, + reactionsSubject$: Observable>, + trackProcessorState$: Behavior, +): CallViewModel { + const userId = matrixRoom.client.getUserId()!; + const deviceId = matrixRoom.client.getDeviceId()!; - // THIS has to be the last public field declaration - public constructor( - scope: ObservableScope, - // A call is permanently tied to a single Matrix room - matrixRTCSession: MatrixRTCSession, - matrixRoom: MatrixRoom, - mediaDevices: MediaDevices, - muteStates: MuteStates, - options: CallViewModelOptions, - handsRaisedSubject$: Observable>, - reactionsSubject$: Observable>, - trackProcessorState$: Behavior, - ) { - const userId = matrixRoom.client.getUserId()!; - const deviceId = matrixRoom.client.getDeviceId()!; + const livekitKeyProvider = getE2eeKeyProvider( + options.encryptionSystem, + matrixRTCSession, + ); - const livekitKeyProvider = getE2eeKeyProvider( - options.encryptionSystem, - matrixRTCSession, - ); + // Each hbar seperates a block of input variables required for the CallViewModel to function. + // The outputs of this block is written under the hbar. + // + // For mocking purposes it is recommended to only mock the functions creating those outputs. + // All other fields are just temp computations for the mentioned output. + // The class does not need anything except the values underneath the bar. + // The creation of the values under the bar are all tested independently and testing the callViewModel Should + // not test their cretation. Call view model only needs: + // - memberships$ via createMemberships$ + // - localMembership via createLocalMembership$ + // - callLifecycle via createCallNotificationLifecycle$ + // - matrixMemberMetadataStore via createMatrixMemberMetadata$ - // Each hbar seperates a block of input variables required for the CallViewModel to function. - // The outputs of this block is written under the hbar. - // - // For mocking purposes it is recommended to only mock the functions creating those outputs. - // All other fields are just temp computations for the mentioned output. - // The class does not need anything except the values underneath the bar. - // The creation of the values under the bar are all tested independently and testing the callViewModel Should - // not test their cretation. Call view model only needs: - // - memberships$ via createMemberships$ - // - localMembership via createLocalMembership$ - // - callLifecycle via createCallNotificationLifecycle$ - // - matrixMemberMetadataStore via createMatrixMemberMetadata$ + // ------------------------------------------------------------------------ + // memberships$ + const memberships$ = createMemberships$(scope, matrixRTCSession); - // ------------------------------------------------------------------------ - // memberships$ - const memberships$ = createMemberships$(scope, matrixRTCSession); + // ------------------------------------------------------------------------ + // matrixLivekitMembers$ AND localMembership - // ------------------------------------------------------------------------ - // matrixLivekitMembers$ AND localMembership + const membershipsAndTransports = membershipsAndTransports$( + scope, + memberships$, + ); - const membershipsAndTransports = membershipsAndTransports$( - scope, - memberships$, - ); + const localTransport$ = createLocalTransport$({ + scope: scope, + memberships$: memberships$, + client: matrixRoom.client, + roomId: matrixRoom.roomId, + useOldestMember$: scope.behavior( + matrixRTCMode.value$.pipe(map((v) => v === MatrixRTCMode.Legacy)), + ), + }); - const localTransport$ = createLocalTransport$({ - scope: scope, - memberships$: memberships$, - client: matrixRoom.client, - roomId: matrixRoom.roomId, - useOldestMember$: scope.behavior( - matrixRTCMode.value$.pipe(map((v) => v === MatrixRTCMode.Legacy)), + const connectionFactory = new ECConnectionFactory( + matrixRoom.client, + mediaDevices, + trackProcessorState$, + livekitKeyProvider, + getUrlParams().controlledAudioDevices, + options.livekitRoomFactory, + ); + + const connectionManager = createConnectionManager$({ + scope: scope, + connectionFactory: connectionFactory, + inputTransports$: scope.behavior( + combineLatest( + [localTransport$, membershipsAndTransports.transports$], + (localTransport, transports) => { + const localTransportAsArray = localTransport ? [localTransport] : []; + return transports.mapInner((transports) => [ + ...localTransportAsArray, + ...transports, + ]); + }, ), - }); + ), + logger: logger, + }); - const connectionFactory = new ECConnectionFactory( - matrixRoom.client, - mediaDevices, - trackProcessorState$, - livekitKeyProvider, - getUrlParams().controlledAudioDevices, - options.livekitRoomFactory, - ); + const matrixLivekitMembers$ = createMatrixLivekitMembers$({ + scope: scope, + membershipsWithTransport$: + membershipsAndTransports.membershipsWithTransport$, + connectionManager: connectionManager, + }); - const connectionManager = createConnectionManager$({ - scope: scope, - connectionFactory: connectionFactory, - inputTransports$: scope.behavior( - combineLatest( - [localTransport$, membershipsAndTransports.transports$], - (localTransport, transports) => { - const localTransportAsArray = localTransport - ? [localTransport] - : []; - return transports.mapInner((transports) => [ - ...localTransportAsArray, - ...transports, - ]); - }, - ), + const connectOptions$ = scope.behavior( + matrixRTCMode.value$.pipe( + map((mode) => ({ + encryptMedia: livekitKeyProvider !== undefined, + // TODO. This might need to get called again on each cahnge of matrixRTCMode... + matrixRTCMode: mode, + })), + ), + ); + + const localMembership = createLocalMembership$({ + scope: scope, + muteStates: muteStates, + mediaDevices: mediaDevices, + connectionManager: connectionManager, + matrixRTCSession: matrixRTCSession, + matrixRoom: matrixRoom, + localTransport$: localTransport$, + trackProcessorState$: trackProcessorState$, + widget, + options: connectOptions$, + logger: logger.getChild(`[${Date.now()}]`), + }); + + const localRtcMembership$ = scope.behavior( + memberships$.pipe( + map( + (memberships) => + memberships.value.find( + (membership) => + membership.userId === userId && membership.deviceId === deviceId, + ) ?? null, ), - logger: logger, - }); + ), + ); - const matrixLivekitMembers$ = createMatrixLivekitMembers$({ - scope: scope, - membershipsWithTransport$: - membershipsAndTransports.membershipsWithTransport$, - connectionManager: connectionManager, - }); + const localMatrixLivekitMemberUninitialized = { + membership$: localRtcMembership$, + participant$: localMembership.participant$, + connection$: localMembership.connection$, + userId: userId, + }; - const connectOptions$ = scope.behavior( - matrixRTCMode.value$.pipe( - map((mode) => ({ - encryptMedia: livekitKeyProvider !== undefined, - // TODO. This might need to get called again on each cahnge of matrixRTCMode... - matrixRTCMode: mode, - })), - ), - ); - - const localMembership = createLocalMembership$({ - scope: scope, - muteStates: muteStates, - mediaDevices: mediaDevices, - connectionManager: connectionManager, - matrixRTCSession: matrixRTCSession, - matrixRoom: matrixRoom, - localTransport$: localTransport$, - trackProcessorState$: trackProcessorState$, - widget, - options: connectOptions$, - logger: logger.getChild(`[${Date.now()}]`), - }); - - const localRtcMembership$ = scope.behavior( - memberships$.pipe( - map( - (memberships) => - memberships.value.find( - (membership) => - membership.userId === userId && - membership.deviceId === deviceId, - ) ?? null, - ), - ), - ); - - const localMatrixLivekitMemberUninitialized = { - membership$: localRtcMembership$, - participant$: localMembership.participant$, - connection$: localMembership.connection$, - userId: userId, - }; - - const localMatrixLivekitMember$: Behavior = - scope.behavior( - localRtcMembership$.pipe( - switchMap((membership) => { - if (!membership) return of(null); - return of( - // casting is save here since we know that localRtcMembership$ is !== null since we reached this case. - localMatrixLivekitMemberUninitialized as MatrixLivekitMember, - ); - }), - ), - ); - - // ------------------------------------------------------------------------ - // callLifecycle - - // TODO if we are in "unknown" state we need a loading rendering (or empty screen) - // Otherwise it looks like we already connected and only than the ringing starts which is weird. - const { callPickupState$, autoLeave$ } = createCallNotificationLifecycle$({ - scope: scope, - memberships$: memberships$, - sentCallNotification$: createSentCallNotification$( - scope, - matrixRTCSession, - ), - receivedDecline$: createReceivedDecline$(matrixRoom), - options: options, - localUser: { userId: userId, deviceId: deviceId }, - }); - - // ------------------------------------------------------------------------ - // matrixMemberMetadataStore - - const matrixRoomMembers$ = createRoomMembers$(scope, matrixRoom); - const matrixMemberMetadataStore = createMatrixMemberMetadata$( - scope, - scope.behavior(memberships$.pipe(map((mems) => mems.value))), - matrixRoomMembers$, - ); - - const dmMember$ = createDMMember$(scope, matrixRoomMembers$, matrixRoom); - const noUserToCallInRoom$ = scope.behavior( - matrixRoomMembers$.pipe( - map( - (roomMembersMap) => - roomMembersMap.size === 1 && - roomMembersMap.get(userId) !== undefined, - ), - ), - ); - - const ringOverlay$ = scope.behavior( - combineLatest([noUserToCallInRoom$, dmMember$, callPickupState$]).pipe( - map(([noUserToCallInRoom, dmMember, callPickupState]) => { - // No overlay if not in ringing state - if (callPickupState !== "ringing" || noUserToCallInRoom) return null; - - const name = dmMember ? dmMember.rawDisplayName : matrixRoom.name; - const id = dmMember ? dmMember.userId : matrixRoom.roomId; - const text = dmMember - ? `Waiting for ${name} to join…` - : "Waiting for other participants…"; - const avatarMxc = dmMember - ? (dmMember.getMxcAvatarUrl?.() ?? undefined) - : (matrixRoom.getMxcAvatarUrl() ?? undefined); - return { - name: name ?? id, - idForAvatar: id, - text, - avatarMxc, - }; - }), - ), - ); - - // CODESMELL? - // This is functionally the same Observable as leave$, except here it's - // hoisted to the top of the class. This enables the cyclic dependency between - // leave$ -> autoLeave$ -> callPickupState$ -> livekitConnectionState$ -> - // localConnection$ -> transports$ -> joined$ -> leave$. - const leaveHoisted$ = new Subject< - "user" | "timeout" | "decline" | "allOthersLeft" - >(); - - /** - * Whether various media/event sources should pretend to be disconnected from - * all network input, even if their connection still technically works. - */ - // We do this when the app is in the 'reconnecting' state, because it might be - // that the LiveKit connection is still functional while the homeserver is - // down, for example, and we want to avoid making people worry that the app is - // in a split-brained state. - // DISCUSSION own membership manager ALSO this probably can be simplifis - const reconnecting$ = localMembership.reconnecting$; - const pretendToBeDisconnected$ = reconnecting$; - - const audioParticipants$ = scope.behavior( - matrixLivekitMembers$.pipe( - switchMap((membersWithEpoch) => { - const members = membersWithEpoch.value; - const a$ = combineLatest( - members.map((member) => - combineLatest([member.connection$, member.participant$]).pipe( - map(([connection, participant]) => { - // do not render audio for local participant - if (!connection || !participant || participant.isLocal) - return null; - const livekitRoom = connection.livekitRoom; - const url = connection.transport.livekit_service_url; - - return { - url, - livekitRoom, - participant: participant.identity, - }; - }), - ), - ), + const localMatrixLivekitMember$: Behavior = + scope.behavior( + localRtcMembership$.pipe( + switchMap((membership) => { + if (!membership) return of(null); + return of( + // casting is save here since we know that localRtcMembership$ is !== null since we reached this case. + localMatrixLivekitMemberUninitialized as MatrixLivekitMember, ); - return a$; }), - map((members) => - members.reduce((acc, curr) => { - if (!curr) return acc; - - const existing = acc.find((item) => item.url === curr.url); - if (existing) { - existing.participants.push(curr.participant); - } else { - acc.push({ - livekitRoom: curr.livekitRoom, - participants: [curr.participant], - url: curr.url, - }); - } - return acc; - }, []), - ), ), - [], ); - const handsRaised$ = scope.behavior( - handsRaisedSubject$.pipe(pauseWhen(pretendToBeDisconnected$)), - ); + // ------------------------------------------------------------------------ + // callLifecycle - const reactions$ = scope.behavior( - reactionsSubject$.pipe( - map((v) => - Object.fromEntries( - Object.entries(v).map(([a, { reactionOption }]) => [ - a, - reactionOption, - ]), + // TODO if we are in "unknown" state we need a loading rendering (or empty screen) + // Otherwise it looks like we already connected and only than the ringing starts which is weird. + const { callPickupState$, autoLeave$ } = createCallNotificationLifecycle$({ + scope: scope, + memberships$: memberships$, + sentCallNotification$: createSentCallNotification$(scope, matrixRTCSession), + receivedDecline$: createReceivedDecline$(matrixRoom), + options: options, + localUser: { userId: userId, deviceId: deviceId }, + }); + + // ------------------------------------------------------------------------ + // matrixMemberMetadataStore + + const matrixRoomMembers$ = createRoomMembers$(scope, matrixRoom); + const matrixMemberMetadataStore = createMatrixMemberMetadata$( + scope, + scope.behavior(memberships$.pipe(map((mems) => mems.value))), + matrixRoomMembers$, + ); + + const dmMember$ = createDMMember$(scope, matrixRoomMembers$, matrixRoom); + const noUserToCallInRoom$ = scope.behavior( + matrixRoomMembers$.pipe( + map( + (roomMembersMap) => + roomMembersMap.size === 1 && roomMembersMap.get(userId) !== undefined, + ), + ), + ); + + const ringOverlay$ = scope.behavior( + combineLatest([noUserToCallInRoom$, dmMember$, callPickupState$]).pipe( + map(([noUserToCallInRoom, dmMember, callPickupState]) => { + // No overlay if not in ringing state + if (callPickupState !== "ringing" || noUserToCallInRoom) return null; + + const name = dmMember ? dmMember.rawDisplayName : matrixRoom.name; + const id = dmMember ? dmMember.userId : matrixRoom.roomId; + const text = dmMember + ? `Waiting for ${name} to join…` + : "Waiting for other participants…"; + const avatarMxc = dmMember + ? (dmMember.getMxcAvatarUrl?.() ?? undefined) + : (matrixRoom.getMxcAvatarUrl() ?? undefined); + return { + name: name ?? id, + idForAvatar: id, + text, + avatarMxc, + }; + }), + ), + ); + + // CODESMELL? + // This is functionally the same Observable as leave$, except here it's + // hoisted to the top of the class. This enables the cyclic dependency between + // leave$ -> autoLeave$ -> callPickupState$ -> livekitConnectionState$ -> + // localConnection$ -> transports$ -> joined$ -> leave$. + const leaveHoisted$ = new Subject< + "user" | "timeout" | "decline" | "allOthersLeft" + >(); + + /** + * Whether various media/event sources should pretend to be disconnected from + * all network input, even if their connection still technically works. + */ + // We do this when the app is in the 'reconnecting' state, because it might be + // that the LiveKit connection is still functional while the homeserver is + // down, for example, and we want to avoid making people worry that the app is + // in a split-brained state. + // DISCUSSION own membership manager ALSO this probably can be simplifis + const reconnecting$ = localMembership.reconnecting$; + const pretendToBeDisconnected$ = reconnecting$; + + const audioParticipants$ = scope.behavior( + matrixLivekitMembers$.pipe( + switchMap((membersWithEpoch) => { + const members = membersWithEpoch.value; + const a$ = combineLatest( + members.map((member) => + combineLatest([member.connection$, member.participant$]).pipe( + map(([connection, participant]) => { + // do not render audio for local participant + if (!connection || !participant || participant.isLocal) + return null; + const livekitRoom = connection.livekitRoom; + const url = connection.transport.livekit_service_url; + + return { + url, + livekitRoom, + participant: participant.identity, + }; + }), + ), ), - ), - pauseWhen(pretendToBeDisconnected$), - ), - ); + ); + return a$; + }), + map((members) => + members.reduce((acc, curr) => { + if (!curr) return acc; - /** - * List of user media (camera feeds) that we want tiles for. - */ - // TODO this also needs the local participant to be added. - const userMedia$ = scope.behavior( - combineLatest([ - localMatrixLivekitMember$, - matrixLivekitMembers$, - duplicateTiles.value$, - ]).pipe( - // Generate a collection of MediaItems from the list of expected (whether - // present or missing) LiveKit participants. - generateItems( - function* ([ - localMatrixLivekitMember, - { value: matrixLivekitMembers }, - duplicateTiles, - ]) { - let localParticipantId = undefined; - // add local member if available - if (localMatrixLivekitMember) { - const { userId, participant$, connection$, membership$ } = - localMatrixLivekitMember; - localParticipantId = `${userId}:${membership$.value.deviceId}`; // should be membership$.value.membershipID which is not optional - // const participantId = membership$.value.membershipID; - if (localParticipantId) { - for (let dup = 0; dup < 1 + duplicateTiles; dup++) { - yield { - keys: [ - dup, - localParticipantId, - userId, - participant$, - connection$, - ], - data: undefined, - }; - } - } - } - // add remote members that are available - for (const { - userId, - participant$, - connection$, - membership$, - } of matrixLivekitMembers) { - const participantId = `${userId}:${membership$.value.deviceId}`; - if (participantId === localParticipantId) continue; - // const participantId = membership$.value?.identity; + const existing = acc.find((item) => item.url === curr.url); + if (existing) { + existing.participants.push(curr.participant); + } else { + acc.push({ + livekitRoom: curr.livekitRoom, + participants: [curr.participant], + url: curr.url, + }); + } + return acc; + }, []), + ), + ), + [], + ); + + const handsRaised$ = scope.behavior( + handsRaisedSubject$.pipe(pauseWhen(pretendToBeDisconnected$)), + ); + + const reactions$ = scope.behavior( + reactionsSubject$.pipe( + map((v) => + Object.fromEntries( + Object.entries(v).map(([a, { reactionOption }]) => [ + a, + reactionOption, + ]), + ), + ), + pauseWhen(pretendToBeDisconnected$), + ), + ); + + /** + * List of user media (camera feeds) that we want tiles for. + */ + // TODO this also needs the local participant to be added. + const userMedia$ = scope.behavior( + combineLatest([ + localMatrixLivekitMember$, + matrixLivekitMembers$, + duplicateTiles.value$, + ]).pipe( + // Generate a collection of MediaItems from the list of expected (whether + // present or missing) LiveKit participants. + generateItems( + function* ([ + localMatrixLivekitMember, + { value: matrixLivekitMembers }, + duplicateTiles, + ]) { + let localParticipantId = undefined; + // add local member if available + if (localMatrixLivekitMember) { + const { userId, participant$, connection$, membership$ } = + localMatrixLivekitMember; + localParticipantId = `${userId}:${membership$.value.deviceId}`; // should be membership$.value.membershipID which is not optional + // const participantId = membership$.value.membershipID; + if (localParticipantId) { for (let dup = 0; dup < 1 + duplicateTiles; dup++) { yield { - keys: [dup, participantId, userId, participant$, connection$], + keys: [ + dup, + localParticipantId, + userId, + participant$, + connection$, + ], data: undefined, }; } } - }, - ( - scope, - _data$, - dup, - participantId, + } + // add remote members that are available + for (const { userId, participant$, connection$, - ) => { - const livekitRoom$ = scope.behavior( - connection$.pipe(map((c) => c?.livekitRoom)), - ); - const focusUrl$ = scope.behavior( - connection$.pipe(map((c) => c?.transport.livekit_service_url)), - ); - const displayName$ = scope.behavior( - matrixMemberMetadataStore - .createDisplayNameBehavior$(userId) - .pipe(map((name) => name ?? userId)), - ); + membership$, + } of matrixLivekitMembers) { + const participantId = `${userId}:${membership$.value.deviceId}`; + if (participantId === localParticipantId) continue; + // const participantId = membership$.value?.identity; + for (let dup = 0; dup < 1 + duplicateTiles; dup++) { + yield { + keys: [dup, participantId, userId, participant$, connection$], + data: undefined, + }; + } + } + }, + ( + scope, + _data$, + dup, + participantId, + userId, + participant$, + connection$, + ) => { + const livekitRoom$ = scope.behavior( + connection$.pipe(map((c) => c?.livekitRoom)), + ); + const focusUrl$ = scope.behavior( + connection$.pipe(map((c) => c?.transport.livekit_service_url)), + ); + const displayName$ = scope.behavior( + matrixMemberMetadataStore + .createDisplayNameBehavior$(userId) + .pipe(map((name) => name ?? userId)), + ); - return new UserMedia( - scope, - `${participantId}:${dup}`, - userId, - participant$, - options.encryptionSystem, - livekitRoom$, - focusUrl$, - mediaDevices, - pretendToBeDisconnected$, - displayName$, - matrixMemberMetadataStore.createAvatarUrlBehavior$(userId), - handsRaised$.pipe(map((v) => v[participantId]?.time ?? null)), - reactions$.pipe(map((v) => v[participantId] ?? undefined)), - ); - }, - ), + return new UserMedia( + scope, + `${participantId}:${dup}`, + userId, + participant$, + options.encryptionSystem, + livekitRoom$, + focusUrl$, + mediaDevices, + pretendToBeDisconnected$, + displayName$, + matrixMemberMetadataStore.createAvatarUrlBehavior$(userId), + handsRaised$.pipe(map((v) => v[participantId]?.time ?? null)), + reactions$.pipe(map((v) => v[participantId] ?? undefined)), + ); + }, ), + ), + ); + + /** + * List of all media items (user media and screen share media) that we want + * tiles for. + */ + const mediaItems$ = scope.behavior( + userMedia$.pipe( + switchMap((userMedia) => + userMedia.length === 0 + ? of([]) + : combineLatest( + userMedia.map((m) => m.screenShares$), + (...screenShares) => [...userMedia, ...screenShares.flat(1)], + ), + ), + ), + ); + + /** + * List of MediaItems that we want to display, that are of type ScreenShare + */ + const screenShares$ = scope.behavior( + mediaItems$.pipe( + map((mediaItems) => + mediaItems.filter((m): m is ScreenShare => m instanceof ScreenShare), + ), + ), + ); + + const joinSoundEffect$ = userMedia$.pipe( + pairwise(), + filter( + ([prev, current]) => + current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && + current.length > prev.length, + ), + map(() => {}), + throttleTime(THROTTLE_SOUND_EFFECT_MS), + ); + + /** + * The number of participants currently in the call. + * + * - Each participant has a corresponding MatrixRTC membership state event + * - There can be multiple participants for one Matrix user if they join from + * multiple devices. + */ + const participantCount$ = scope.behavior( + matrixLivekitMembers$.pipe(map((ms) => ms.value.length)), + ); + + const leaveSoundEffect$ = combineLatest([callPickupState$, userMedia$]).pipe( + // Until the call is successful, do not play a leave sound. + // If callPickupState$ is null, then we always play the sound as it will not conflict with a decline sound. + skipWhile(([c]) => c !== null && c !== "success"), + map(([, userMedia]) => userMedia), + pairwise(), + filter( + ([prev, current]) => + current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && + current.length < prev.length, + ), + map(() => {}), + throttleTime(THROTTLE_SOUND_EFFECT_MS), + ); + + const userHangup$ = new Subject(); + + const widgetHangup$ = + widget === null + ? NEVER + : ( + fromEvent( + widget.lazyActions, + ElementWidgetActions.HangupCall, + ) as Observable> + ).pipe( + tap((ev) => { + widget!.api.transport.reply(ev.detail, {}); + }), + ); + + const leave$: Observable<"user" | "timeout" | "decline" | "allOthersLeft"> = + merge( + autoLeave$, + merge(userHangup$, widgetHangup$).pipe(map(() => "user" as const)), + ).pipe( + scope.share, + tap((reason) => leaveHoisted$.next(reason)), ); - /** - * List of all media items (user media and screen share media) that we want - * tiles for. - */ - const mediaItems$ = scope.behavior( - userMedia$.pipe( - switchMap((userMedia) => - userMedia.length === 0 - ? of([]) - : combineLatest( - userMedia.map((m) => m.screenShares$), - (...screenShares) => [...userMedia, ...screenShares.flat(1)], + const spotlightSpeaker$ = scope.behavior( + userMedia$.pipe( + switchMap((mediaItems) => + mediaItems.length === 0 + ? of([]) + : combineLatest( + mediaItems.map((m) => + m.vm.speaking$.pipe(map((s) => [m, s] as const)), ), - ), + ), ), - ); - - /** - * List of MediaItems that we want to display, that are of type ScreenShare - */ - const screenShares$ = scope.behavior( - mediaItems$.pipe( - map((mediaItems) => - mediaItems.filter((m): m is ScreenShare => m instanceof ScreenShare), - ), + scan<(readonly [UserMedia, boolean])[], UserMedia | undefined, null>( + (prev, mediaItems) => { + // Only remote users that are still in the call should be sticky + const [stickyMedia, stickySpeaking] = + (!prev?.vm.local && mediaItems.find(([m]) => m === prev)) || []; + // Decide who to spotlight: + // If the previous speaker is still speaking, stick with them rather + // than switching eagerly to someone else + return stickySpeaking + ? stickyMedia! + : // Otherwise, select any remote user who is speaking + (mediaItems.find(([m, s]) => !m.vm.local && s)?.[0] ?? + // Otherwise, stick with the person who was last speaking + stickyMedia ?? + // Otherwise, spotlight an arbitrary remote user + mediaItems.find(([m]) => !m.vm.local)?.[0] ?? + // Otherwise, spotlight the local user + mediaItems.find(([m]) => m.vm.local)?.[0]); + }, + null, ), - ); + map((speaker) => speaker?.vm ?? null), + ), + ); - const joinSoundEffect$ = userMedia$.pipe( - pairwise(), - filter( - ([prev, current]) => - current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && - current.length > prev.length, - ), - map(() => {}), - throttleTime(THROTTLE_SOUND_EFFECT_MS), - ); + const grid$ = scope.behavior( + userMedia$.pipe( + switchMap((mediaItems) => { + const bins = mediaItems.map((m) => + m.bin$.pipe(map((bin) => [m, bin] as const)), + ); + // Sort the media by bin order and generate a tile for each one + return bins.length === 0 + ? of([]) + : combineLatest(bins, (...bins) => + bins.sort(([, bin1], [, bin2]) => bin1 - bin2).map(([m]) => m.vm), + ); + }), + distinctUntilChanged(shallowEquals), + ), + ); - /** - * The number of participants currently in the call. - * - * - Each participant has a corresponding MatrixRTC membership state event - * - There can be multiple participants for one Matrix user if they join from - * multiple devices. - */ - const participantCount$ = scope.behavior( - matrixLivekitMembers$.pipe(map((ms) => ms.value.length)), - ); + const spotlight$ = scope.behavior( + screenShares$.pipe( + switchMap((screenShares) => { + if (screenShares.length > 0) { + return of(screenShares.map((m) => m.vm)); + } - const leaveSoundEffect$ = combineLatest([ - callPickupState$, - userMedia$, + return spotlightSpeaker$.pipe( + map((speaker) => (speaker ? [speaker] : [])), + ); + }), + distinctUntilChanged(shallowEquals), + ), + ); + + const pip$ = scope.behavior( + combineLatest([ + // TODO This also needs epoch logic to dedupe the screenshares and mediaItems emits + screenShares$, + spotlightSpeaker$, + mediaItems$, ]).pipe( - // Until the call is successful, do not play a leave sound. - // If callPickupState$ is null, then we always play the sound as it will not conflict with a decline sound. - skipWhile(([c]) => c !== null && c !== "success"), - map(([, userMedia]) => userMedia), - pairwise(), - filter( - ([prev, current]) => - current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && - current.length < prev.length, - ), - map(() => {}), - throttleTime(THROTTLE_SOUND_EFFECT_MS), - ); + switchMap(([screenShares, spotlight, mediaItems]) => { + if (screenShares.length > 0) { + return spotlightSpeaker$; + } + if (!spotlight || spotlight.local) { + return of(null); + } - const userHangup$ = new Subject(); + const localUserMedia = mediaItems.find( + (m) => m.vm instanceof LocalUserMediaViewModel, + ) as UserMedia | undefined; - const widgetHangup$ = - widget === null - ? NEVER - : ( - fromEvent( - widget.lazyActions, - ElementWidgetActions.HangupCall, - ) as Observable> - ).pipe( - tap((ev) => { - widget!.api.transport.reply(ev.detail, {}); - }), - ); + const localUserMediaViewModel = localUserMedia?.vm as + | LocalUserMediaViewModel + | undefined; - const leave$: Observable<"user" | "timeout" | "decline" | "allOthersLeft"> = - merge( - autoLeave$, - merge(userHangup$, widgetHangup$).pipe(map(() => "user" as const)), - ).pipe( - scope.share, - tap((reason) => leaveHoisted$.next(reason)), - ); + if (!localUserMediaViewModel) { + return of(null); + } + return localUserMediaViewModel.alwaysShow$.pipe( + map((alwaysShow) => { + if (alwaysShow) { + return localUserMediaViewModel; + } - const spotlightSpeaker$ = scope.behavior( - userMedia$.pipe( - switchMap((mediaItems) => - mediaItems.length === 0 - ? of([]) - : combineLatest( - mediaItems.map((m) => - m.vm.speaking$.pipe(map((s) => [m, s] as const)), + return null; + }), + ); + }), + ), + ); + + const hasRemoteScreenShares$: Observable = spotlight$.pipe( + map((spotlight) => + spotlight.some((vm) => !vm.local && vm instanceof ScreenShareViewModel), + ), + distinctUntilChanged(), + ); + + const pipEnabled$ = scope.behavior(setPipEnabled$, false); + + const naturalWindowMode$ = scope.behavior( + fromEvent(window, "resize").pipe( + map(() => { + const height = window.innerHeight; + const width = window.innerWidth; + if (height <= 400 && width <= 340) return "pip"; + // Our layouts for flat windows are better at adapting to a small width + // than our layouts for narrow windows are at adapting to a small height, + // so we give "flat" precedence here + if (height <= 600) return "flat"; + if (width <= 600) return "narrow"; + return "normal"; + }), + ), + "normal", + ); + + /** + * The general shape of the window. + */ + const windowMode$ = scope.behavior( + pipEnabled$.pipe( + switchMap((pip) => (pip ? of("pip") : naturalWindowMode$)), + ), + ); + + const spotlightExpandedToggle$ = new Subject(); + const spotlightExpanded$ = scope.behavior( + spotlightExpandedToggle$.pipe(accumulate(false, (expanded) => !expanded)), + ); + + const gridModeUserSelection$ = new Subject(); + /** + * The layout mode of the media tile grid. + */ + const gridMode$ = + // If the user hasn't selected spotlight and somebody starts screen sharing, + // automatically switch to spotlight mode and reset when screen sharing ends + scope.behavior( + gridModeUserSelection$.pipe( + switchMap((userSelection) => + (userSelection === "spotlight" + ? EMPTY + : combineLatest([hasRemoteScreenShares$, windowMode$]).pipe( + skip(userSelection === null ? 0 : 1), + map( + ([hasScreenShares, windowMode]): GridMode => + hasScreenShares || windowMode === "flat" + ? "spotlight" + : "grid", ), - ), + ) + ).pipe(startWith(userSelection ?? "grid")), ), - scan<(readonly [UserMedia, boolean])[], UserMedia | undefined, null>( - (prev, mediaItems) => { - // Only remote users that are still in the call should be sticky - const [stickyMedia, stickySpeaking] = - (!prev?.vm.local && mediaItems.find(([m]) => m === prev)) || []; - // Decide who to spotlight: - // If the previous speaker is still speaking, stick with them rather - // than switching eagerly to someone else - return stickySpeaking - ? stickyMedia! - : // Otherwise, select any remote user who is speaking - (mediaItems.find(([m, s]) => !m.vm.local && s)?.[0] ?? - // Otherwise, stick with the person who was last speaking - stickyMedia ?? - // Otherwise, spotlight an arbitrary remote user - mediaItems.find(([m]) => !m.vm.local)?.[0] ?? - // Otherwise, spotlight the local user - mediaItems.find(([m]) => m.vm.local)?.[0]); - }, - null, - ), - map((speaker) => speaker?.vm ?? null), ), + "grid", ); - const grid$ = scope.behavior( - userMedia$.pipe( - switchMap((mediaItems) => { - const bins = mediaItems.map((m) => - m.bin$.pipe(map((bin) => [m, bin] as const)), - ); - // Sort the media by bin order and generate a tile for each one - return bins.length === 0 - ? of([]) - : combineLatest(bins, (...bins) => - bins - .sort(([, bin1], [, bin2]) => bin1 - bin2) - .map(([m]) => m.vm), - ); - }), - distinctUntilChanged(shallowEquals), - ), - ); + const setGridMode = (value: GridMode): void => { + gridModeUserSelection$.next(value); + }; - const spotlight$ = scope.behavior( - screenShares$.pipe( - switchMap((screenShares) => { - if (screenShares.length > 0) { - return of(screenShares.map((m) => m.vm)); - } + const gridLayoutMedia$: Observable = combineLatest( + [grid$, spotlight$], + (grid, spotlight) => ({ + type: "grid", + spotlight: spotlight.some((vm) => vm instanceof ScreenShareViewModel) + ? spotlight + : undefined, + grid, + }), + ); - return spotlightSpeaker$.pipe( - map((speaker) => (speaker ? [speaker] : [])), - ); - }), - distinctUntilChanged(shallowEquals), - ), - ); + const spotlightLandscapeLayoutMedia$: Observable = + combineLatest([grid$, spotlight$], (grid, spotlight) => ({ + type: "spotlight-landscape", + spotlight, + grid, + })); - const pip$ = scope.behavior( - combineLatest([ - // TODO This also needs epoch logic to dedupe the screenshares and mediaItems emits - screenShares$, - spotlightSpeaker$, - mediaItems$, - ]).pipe( - switchMap(([screenShares, spotlight, mediaItems]) => { - if (screenShares.length > 0) { - return spotlightSpeaker$; - } - if (!spotlight || spotlight.local) { - return of(null); - } + const spotlightPortraitLayoutMedia$: Observable = + combineLatest([grid$, spotlight$], (grid, spotlight) => ({ + type: "spotlight-portrait", + spotlight, + grid, + })); - const localUserMedia = mediaItems.find( - (m) => m.vm instanceof LocalUserMediaViewModel, - ) as UserMedia | undefined; + const spotlightExpandedLayoutMedia$: Observable = + combineLatest([spotlight$, pip$], (spotlight, pip) => ({ + type: "spotlight-expanded", + spotlight, + pip: pip ?? undefined, + })); - const localUserMediaViewModel = localUserMedia?.vm as - | LocalUserMediaViewModel - | undefined; + const oneOnOneLayoutMedia$: Observable = + mediaItems$.pipe( + map((mediaItems) => { + if (mediaItems.length !== 2) return null; + const local = mediaItems.find((vm) => vm.vm.local)?.vm as + | LocalUserMediaViewModel + | undefined; + const remote = mediaItems.find((vm) => !vm.vm.local)?.vm as + | RemoteUserMediaViewModel + | undefined; + // There might not be a remote tile if there are screen shares, or if + // only the local user is in the call and they're using the duplicate + // tiles option + if (!remote || !local) return null; - if (!localUserMediaViewModel) { - return of(null); - } - return localUserMediaViewModel.alwaysShow$.pipe( - map((alwaysShow) => { - if (alwaysShow) { - return localUserMediaViewModel; - } - - return null; - }), - ); - }), - ), - ); - - const hasRemoteScreenShares$: Observable = spotlight$.pipe( - map((spotlight) => - spotlight.some((vm) => !vm.local && vm instanceof ScreenShareViewModel), - ), - distinctUntilChanged(), - ); - - const pipEnabled$ = scope.behavior(setPipEnabled$, false); - - const naturalWindowMode$ = scope.behavior( - fromEvent(window, "resize").pipe( - map(() => { - const height = window.innerHeight; - const width = window.innerWidth; - if (height <= 400 && width <= 340) return "pip"; - // Our layouts for flat windows are better at adapting to a small width - // than our layouts for narrow windows are at adapting to a small height, - // so we give "flat" precedence here - if (height <= 600) return "flat"; - if (width <= 600) return "narrow"; - return "normal"; - }), - ), - "normal", - ); - - /** - * The general shape of the window. - */ - const windowMode$ = scope.behavior( - pipEnabled$.pipe( - switchMap((pip) => (pip ? of("pip") : naturalWindowMode$)), - ), - ); - - const spotlightExpandedToggle$ = new Subject(); - const spotlightExpanded$ = scope.behavior( - spotlightExpandedToggle$.pipe(accumulate(false, (expanded) => !expanded)), - ); - - const gridModeUserSelection$ = new Subject(); - /** - * The layout mode of the media tile grid. - */ - const gridMode$ = - // If the user hasn't selected spotlight and somebody starts screen sharing, - // automatically switch to spotlight mode and reset when screen sharing ends - scope.behavior( - gridModeUserSelection$.pipe( - switchMap((userSelection) => - (userSelection === "spotlight" - ? EMPTY - : combineLatest([hasRemoteScreenShares$, windowMode$]).pipe( - skip(userSelection === null ? 0 : 1), - map( - ([hasScreenShares, windowMode]): GridMode => - hasScreenShares || windowMode === "flat" - ? "spotlight" - : "grid", - ), - ) - ).pipe(startWith(userSelection ?? "grid")), - ), - ), - "grid", - ); - - const setGridMode = (value: GridMode): void => { - gridModeUserSelection$.next(value); - }; - - const gridLayoutMedia$: Observable = combineLatest( - [grid$, spotlight$], - (grid, spotlight) => ({ - type: "grid", - spotlight: spotlight.some((vm) => vm instanceof ScreenShareViewModel) - ? spotlight - : undefined, - grid, + return { type: "one-on-one", local, remote }; }), ); - const spotlightLandscapeLayoutMedia$: Observable = - combineLatest([grid$, spotlight$], (grid, spotlight) => ({ - type: "spotlight-landscape", - spotlight, - grid, - })); + const pipLayoutMedia$: Observable = spotlight$.pipe( + map((spotlight) => ({ type: "pip", spotlight })), + ); - const spotlightPortraitLayoutMedia$: Observable = - combineLatest([grid$, spotlight$], (grid, spotlight) => ({ - type: "spotlight-portrait", - spotlight, - grid, - })); + /** + * The media to be used to produce a layout. + */ + const layoutMedia$ = scope.behavior( + windowMode$.pipe( + switchMap((windowMode) => { + switch (windowMode) { + case "normal": + return gridMode$.pipe( + switchMap((gridMode) => { + switch (gridMode) { + case "grid": + return oneOnOneLayoutMedia$.pipe( + switchMap((oneOnOne) => + oneOnOne === null ? gridLayoutMedia$ : of(oneOnOne), + ), + ); + case "spotlight": + return spotlightExpanded$.pipe( + switchMap((expanded) => + expanded + ? spotlightExpandedLayoutMedia$ + : spotlightLandscapeLayoutMedia$, + ), + ); + } + }), + ); + case "narrow": + return oneOnOneLayoutMedia$.pipe( + switchMap((oneOnOne) => + oneOnOne === null + ? combineLatest([grid$, spotlight$], (grid, spotlight) => + grid.length > smallMobileCallThreshold || + spotlight.some((vm) => vm instanceof ScreenShareViewModel) + ? spotlightPortraitLayoutMedia$ + : gridLayoutMedia$, + ).pipe(switchAll()) + : // The expanded spotlight layout makes for a better one-on-one + // experience in narrow windows + spotlightExpandedLayoutMedia$, + ), + ); + case "flat": + return gridMode$.pipe( + switchMap((gridMode) => { + switch (gridMode) { + case "grid": + // Yes, grid mode actually gets you a "spotlight" layout in + // this window mode. + return spotlightLandscapeLayoutMedia$; + case "spotlight": + return spotlightExpandedLayoutMedia$; + } + }), + ); + case "pip": + return pipLayoutMedia$; + } + }), + ), + ); - const spotlightExpandedLayoutMedia$: Observable = - combineLatest([spotlight$, pip$], (spotlight, pip) => ({ - type: "spotlight-expanded", - spotlight, - pip: pip ?? undefined, - })); + // There is a cyclical dependency here: the layout algorithms want to know + // which tiles are on screen, but to know which tiles are on screen we have to + // first render a layout. To deal with this we assume initially that no tiles + // are visible, and loop the data back into the layouts with a Subject. + const visibleTiles$ = new Subject(); + const setVisibleTiles = (value: number): void => visibleTiles$.next(value); - const oneOnOneLayoutMedia$: Observable = - mediaItems$.pipe( - map((mediaItems) => { - if (mediaItems.length !== 2) return null; - const local = mediaItems.find((vm) => vm.vm.local)?.vm as - | LocalUserMediaViewModel - | undefined; - const remote = mediaItems.find((vm) => !vm.vm.local)?.vm as - | RemoteUserMediaViewModel - | undefined; - // There might not be a remote tile if there are screen shares, or if - // only the local user is in the call and they're using the duplicate - // tiles option - if (!remote || !local) return null; - - return { type: "one-on-one", local, remote }; - }), - ); - - const pipLayoutMedia$: Observable = spotlight$.pipe( - map((spotlight) => ({ type: "pip", spotlight })), - ); - - /** - * The media to be used to produce a layout. - */ - const layoutMedia$ = scope.behavior( - windowMode$.pipe( - switchMap((windowMode) => { - switch (windowMode) { - case "normal": - return gridMode$.pipe( - switchMap((gridMode) => { - switch (gridMode) { - case "grid": - return oneOnOneLayoutMedia$.pipe( - switchMap((oneOnOne) => - oneOnOne === null ? gridLayoutMedia$ : of(oneOnOne), - ), - ); - case "spotlight": - return spotlightExpanded$.pipe( - switchMap((expanded) => - expanded - ? spotlightExpandedLayoutMedia$ - : spotlightLandscapeLayoutMedia$, - ), - ); - } - }), - ); - case "narrow": - return oneOnOneLayoutMedia$.pipe( - switchMap((oneOnOne) => - oneOnOne === null - ? combineLatest([grid$, spotlight$], (grid, spotlight) => - grid.length > smallMobileCallThreshold || - spotlight.some( - (vm) => vm instanceof ScreenShareViewModel, - ) - ? spotlightPortraitLayoutMedia$ - : gridLayoutMedia$, - ).pipe(switchAll()) - : // The expanded spotlight layout makes for a better one-on-one - // experience in narrow windows - spotlightExpandedLayoutMedia$, - ), - ); - case "flat": - return gridMode$.pipe( - switchMap((gridMode) => { - switch (gridMode) { - case "grid": - // Yes, grid mode actually gets you a "spotlight" layout in - // this window mode. - return spotlightLandscapeLayoutMedia$; - case "spotlight": - return spotlightExpandedLayoutMedia$; - } - }), - ); - case "pip": - return pipLayoutMedia$; - } - }), - ), - ); - - // There is a cyclical dependency here: the layout algorithms want to know - // which tiles are on screen, but to know which tiles are on screen we have to - // first render a layout. To deal with this we assume initially that no tiles - // are visible, and loop the data back into the layouts with a Subject. - const visibleTiles$ = new Subject(); - const setVisibleTiles = (value: number): void => visibleTiles$.next(value); - - const layoutInternals$ = scope.behavior< - LayoutScanState & { layout: Layout } - >( - combineLatest([ - layoutMedia$, - visibleTiles$.pipe(startWith(0), distinctUntilChanged()), - ]).pipe( - scan< - [LayoutMedia, number], - LayoutScanState & { layout: Layout }, - LayoutScanState - >( - ({ tiles: prevTiles }, [media, visibleTiles]) => { - let layout: Layout; - let newTiles: TileStore; - switch (media.type) { - case "grid": - case "spotlight-landscape": - case "spotlight-portrait": - [layout, newTiles] = gridLikeLayout( - media, - visibleTiles, - setVisibleTiles, - prevTiles, - ); - break; - case "spotlight-expanded": - [layout, newTiles] = spotlightExpandedLayout(media, prevTiles); - break; - case "one-on-one": - [layout, newTiles] = oneOnOneLayout(media, prevTiles); - break; - case "pip": - [layout, newTiles] = pipLayout(media, prevTiles); - break; - } - - return { layout, tiles: newTiles }; - }, - { layout: null, tiles: TileStore.empty() }, - ), - ), - ); - - /** - * The layout of tiles in the call interface. - */ - const layout$ = scope.behavior( - layoutInternals$.pipe(map(({ layout }) => layout)), - ); - - /** - * The current generation of the tile store, exposed for debugging purposes. - */ - const tileStoreGeneration$ = scope.behavior( - layoutInternals$.pipe(map(({ tiles }) => tiles.generation)), - ); - - const showSpotlightIndicators$ = scope.behavior( - layout$.pipe(map((l) => l.type !== "grid")), - ); - - const showSpeakingIndicators$ = scope.behavior( - layout$.pipe( - switchMap((l) => { - switch (l.type) { + const layoutInternals$ = scope.behavior( + combineLatest([ + layoutMedia$, + visibleTiles$.pipe(startWith(0), distinctUntilChanged()), + ]).pipe( + scan< + [LayoutMedia, number], + LayoutScanState & { layout: Layout }, + LayoutScanState + >( + ({ tiles: prevTiles }, [media, visibleTiles]) => { + let layout: Layout; + let newTiles: TileStore; + switch (media.type) { + case "grid": case "spotlight-landscape": case "spotlight-portrait": - // If the spotlight is showing the active speaker, we can do without - // speaking indicators as they're a redundant visual cue. But if - // screen sharing feeds are in the spotlight we still need them. - return l.spotlight.media$.pipe( - map((models: MediaViewModel[]) => - models.some((m) => m instanceof ScreenShareViewModel), - ), + [layout, newTiles] = gridLikeLayout( + media, + visibleTiles, + setVisibleTiles, + prevTiles, ); - // In expanded spotlight layout, the active speaker is always shown in - // the picture-in-picture tile so there is no need for speaking - // indicators. And in one-on-one layout there's no question as to who is - // speaking. + break; case "spotlight-expanded": + [layout, newTiles] = spotlightExpandedLayout(media, prevTiles); + break; case "one-on-one": - return of(false); - default: - return of(true); - } - }), - ), - ); - - const toggleSpotlightExpanded$ = scope.behavior<(() => void) | null>( - windowMode$.pipe( - switchMap((mode) => - mode === "normal" - ? layout$.pipe( - map( - (l) => - l.type === "spotlight-landscape" || - l.type === "spotlight-expanded", - ), - ) - : of(false), - ), - distinctUntilChanged(), - map((enabled) => - enabled ? (): void => spotlightExpandedToggle$.next() : null, - ), - ), - ); - - const screenTap$ = new Subject(); - const controlsTap$ = new Subject(); - const screenHover$ = new Subject(); - const screenUnhover$ = new Subject(); - - const showHeader$ = scope.behavior( - windowMode$.pipe(map((mode) => mode !== "pip" && mode !== "flat")), - ); - - const showFooter$ = scope.behavior( - windowMode$.pipe( - switchMap((mode) => { - switch (mode) { + [layout, newTiles] = oneOnOneLayout(media, prevTiles); + break; case "pip": - return of(false); - case "normal": - case "narrow": - return of(true); - case "flat": - // Sadly Firefox has some layering glitches that prevent the footer - // from appearing properly. They happen less often if we never hide - // the footer. - if (isFirefox()) return of(true); - // Show/hide the footer in response to interactions - return merge( - screenTap$.pipe(map(() => "tap screen" as const)), - controlsTap$.pipe(map(() => "tap controls" as const)), - screenHover$.pipe(map(() => "hover" as const)), - ).pipe( - switchScan((state, interaction) => { - switch (interaction) { - case "tap screen": - return state - ? // Toggle visibility on tap - of(false) - : // Hide after a timeout - timer(showFooterMs).pipe( - map(() => false), - startWith(true), - ); - case "tap controls": - // The user is interacting with things, so reset the timeout - return timer(showFooterMs).pipe( - map(() => false), - startWith(true), - ); - case "hover": - // Show on hover and hide after a timeout - return race( - timer(showFooterMs), - screenUnhover$.pipe(take(1)), - ).pipe( - map(() => false), - startWith(true), - ); - } - }, false), - startWith(false), - ); + [layout, newTiles] = pipLayout(media, prevTiles); + break; } - }), - ), - ); - /** - * Whether audio is currently being output through the earpiece. - */ - const earpieceMode$ = scope.behavior( - combineLatest( - [ - mediaDevices.audioOutput.available$, - mediaDevices.audioOutput.selected$, - ], - (available, selected) => - selected !== undefined && - available.get(selected.id)?.type === "earpiece", - ), - ); - - /** - * Callback to toggle between the earpiece and the loudspeaker. - * - * This will be `null` in case the target does not exist in the list - * of available audio outputs. - */ - const audioOutputSwitcher$ = scope.behavior<{ - targetOutput: "earpiece" | "speaker"; - switch: () => void; - } | null>( - combineLatest( - [ - mediaDevices.audioOutput.available$, - mediaDevices.audioOutput.selected$, - ], - (available, selected) => { - const selectionType = selected && available.get(selected.id)?.type; - - // If we are in any output mode other than speaker switch to speaker. - const newSelectionType: "earpiece" | "speaker" = - selectionType === "speaker" ? "earpiece" : "speaker"; - const newSelection = [...available].find( - ([, d]) => d.type === newSelectionType, - ); - if (newSelection === undefined) return null; - - const [id] = newSelection; - return { - targetOutput: newSelectionType, - switch: (): void => mediaDevices.audioOutput.select(id), - }; + return { layout, tiles: newTiles }; }, + { layout: null, tiles: TileStore.empty() }, ), - ); + ), + ); - /** - * Emits an array of reactions that should be visible on the screen. - */ - // DISCUSSION move this into a reaction file - // const {visibleReactions$, audibleReactions$} = reactionsObservables$(showReactionSetting$, ) - const visibleReactions$ = scope.behavior( - showReactions.value$.pipe( - switchMap((show) => (show ? reactions$ : of({}))), - scan< - Record, - { sender: string; emoji: string; startX: number }[] - >((acc, latest) => { - const newSet: { sender: string; emoji: string; startX: number }[] = - []; - for (const [sender, reaction] of Object.entries(latest)) { - const startX = - acc.find((v) => v.sender === sender && v.emoji)?.startX ?? - Math.ceil(Math.random() * 80) + 10; - newSet.push({ sender, emoji: reaction.emoji, startX }); - } - return newSet; - }, []), + /** + * The layout of tiles in the call interface. + */ + const layout$ = scope.behavior( + layoutInternals$.pipe(map(({ layout }) => layout)), + ); + + /** + * The current generation of the tile store, exposed for debugging purposes. + */ + const tileStoreGeneration$ = scope.behavior( + layoutInternals$.pipe(map(({ tiles }) => tiles.generation)), + ); + + const showSpotlightIndicators$ = scope.behavior( + layout$.pipe(map((l) => l.type !== "grid")), + ); + + const showSpeakingIndicators$ = scope.behavior( + layout$.pipe( + switchMap((l) => { + switch (l.type) { + case "spotlight-landscape": + case "spotlight-portrait": + // If the spotlight is showing the active speaker, we can do without + // speaking indicators as they're a redundant visual cue. But if + // screen sharing feeds are in the spotlight we still need them. + return l.spotlight.media$.pipe( + map((models: MediaViewModel[]) => + models.some((m) => m instanceof ScreenShareViewModel), + ), + ); + // In expanded spotlight layout, the active speaker is always shown in + // the picture-in-picture tile so there is no need for speaking + // indicators. And in one-on-one layout there's no question as to who is + // speaking. + case "spotlight-expanded": + case "one-on-one": + return of(false); + default: + return of(true); + } + }), + ), + ); + + const toggleSpotlightExpanded$ = scope.behavior<(() => void) | null>( + windowMode$.pipe( + switchMap((mode) => + mode === "normal" + ? layout$.pipe( + map( + (l) => + l.type === "spotlight-landscape" || + l.type === "spotlight-expanded", + ), + ) + : of(false), ), - ); - - /** - * Emits an array of reactions that should be played. - */ - const audibleReactions$ = playReactionsSound.value$.pipe( - switchMap((show) => - show ? reactions$ : of>({}), + distinctUntilChanged(), + map((enabled) => + enabled ? (): void => spotlightExpandedToggle$.next() : null, ), - map((reactions) => Object.values(reactions).map((v) => v.name)), - scan( - (acc, latest) => { - return { - playing: latest.filter( - (v) => acc.playing.includes(v) || acc.newSounds.includes(v), - ), - newSounds: latest.filter( - (v) => !acc.playing.includes(v) && !acc.newSounds.includes(v), - ), - }; - }, - { playing: [], newSounds: [] }, - ), - map((v) => v.newSounds), - ); + ), + ); - const newHandRaised$ = handsRaised$.pipe( - map((v) => Object.keys(v).length), - scan( - (acc, newValue) => ({ - value: newValue, - playSounds: newValue > acc.value, - }), - { value: 0, playSounds: false }, - ), - filter((v) => v.playSounds), - ); + const screenTap$ = new Subject(); + const controlsTap$ = new Subject(); + const screenHover$ = new Subject(); + const screenUnhover$ = new Subject(); - const newScreenShare$ = screenShares$.pipe( - map((v) => v.length), - scan( - (acc, newValue) => ({ - value: newValue, - playSounds: newValue > acc.value, - }), - { value: 0, playSounds: false }, - ), - filter((v) => v.playSounds), - ); + const showHeader$ = scope.behavior( + windowMode$.pipe(map((mode) => mode !== "pip" && mode !== "flat")), + ); - /** - * Whether we are sharing our screen. - */ - // reassigned here to make it publicly accessible - const sharingScreen$ = localMembership.sharingScreen$; + const showFooter$ = scope.behavior( + windowMode$.pipe( + switchMap((mode) => { + switch (mode) { + case "pip": + return of(false); + case "normal": + case "narrow": + return of(true); + case "flat": + // Sadly Firefox has some layering glitches that prevent the footer + // from appearing properly. They happen less often if we never hide + // the footer. + if (isFirefox()) return of(true); + // Show/hide the footer in response to interactions + return merge( + screenTap$.pipe(map(() => "tap screen" as const)), + controlsTap$.pipe(map(() => "tap controls" as const)), + screenHover$.pipe(map(() => "hover" as const)), + ).pipe( + switchScan((state, interaction) => { + switch (interaction) { + case "tap screen": + return state + ? // Toggle visibility on tap + of(false) + : // Hide after a timeout + timer(showFooterMs).pipe( + map(() => false), + startWith(true), + ); + case "tap controls": + // The user is interacting with things, so reset the timeout + return timer(showFooterMs).pipe( + map(() => false), + startWith(true), + ); + case "hover": + // Show on hover and hide after a timeout + return race( + timer(showFooterMs), + screenUnhover$.pipe(take(1)), + ).pipe( + map(() => false), + startWith(true), + ); + } + }, false), + startWith(false), + ); + } + }), + ), + ); - /** - * Callback to toggle screen sharing. If null, screen sharing is not possible. - */ - // reassigned here to make it publicly accessible - const toggleScreenSharing = localMembership.toggleScreenSharing; + /** + * Whether audio is currently being output through the earpiece. + */ + const earpieceMode$ = scope.behavior( + combineLatest( + [mediaDevices.audioOutput.available$, mediaDevices.audioOutput.selected$], + (available, selected) => + selected !== undefined && + available.get(selected.id)?.type === "earpiece", + ), + ); - const join = localMembership.requestConnect; - join(); // TODO-MULTI-SFU: Use this view model for the lobby as well, and only call this once 'join' is clicked? + /** + * Callback to toggle between the earpiece and the loudspeaker. + * + * This will be `null` in case the target does not exist in the list + * of available audio outputs. + */ + const audioOutputSwitcher$ = scope.behavior<{ + targetOutput: "earpiece" | "speaker"; + switch: () => void; + } | null>( + combineLatest( + [mediaDevices.audioOutput.available$, mediaDevices.audioOutput.selected$], + (available, selected) => { + const selectionType = selected && available.get(selected.id)?.type; - this.autoLeave$ = autoLeave$; - this.callPickupState$ = callPickupState$; - this.ringOverlay$ = ringOverlay$; - this.leave$ = leave$; - this.hangup = (): void => userHangup$.next(); - this.join = join; - this.toggleScreenSharing = toggleScreenSharing; - this.sharingScreen$ = sharingScreen$; + // If we are in any output mode other than speaker switch to speaker. + const newSelectionType: "earpiece" | "speaker" = + selectionType === "speaker" ? "earpiece" : "speaker"; + const newSelection = [...available].find( + ([, d]) => d.type === newSelectionType, + ); + if (newSelection === undefined) return null; - this.tapScreen = (): void => screenTap$.next(); - this.tapControls = (): void => controlsTap$.next(); - this.hoverScreen = (): void => screenHover$.next(); - this.unhoverScreen = (): void => screenUnhover$.next(); + const [id] = newSelection; + return { + targetOutput: newSelectionType, + switch: (): void => mediaDevices.audioOutput.select(id), + }; + }, + ), + ); - this.configError$ = localMembership.configError$; - this.participantCount$ = participantCount$; - this.audioParticipants$ = audioParticipants$; + /** + * Emits an array of reactions that should be visible on the screen. + */ + // DISCUSSION move this into a reaction file + // const {visibleReactions$, audibleReactions$} = reactionsObservables$(showReactionSetting$, ) + const visibleReactions$ = scope.behavior( + showReactions.value$.pipe( + switchMap((show) => (show ? reactions$ : of({}))), + scan< + Record, + { sender: string; emoji: string; startX: number }[] + >((acc, latest) => { + const newSet: { sender: string; emoji: string; startX: number }[] = []; + for (const [sender, reaction] of Object.entries(latest)) { + const startX = + acc.find((v) => v.sender === sender && v.emoji)?.startX ?? + Math.ceil(Math.random() * 80) + 10; + newSet.push({ sender, emoji: reaction.emoji, startX }); + } + return newSet; + }, []), + ), + ); - this.handsRaised$ = handsRaised$; - this.reactions$ = reactions$; - this.joinSoundEffect$ = joinSoundEffect$; - this.leaveSoundEffect$ = leaveSoundEffect$; - this.newHandRaised$ = newHandRaised$; - this.newScreenShare$ = newScreenShare$; - this.audibleReactions$ = audibleReactions$; - this.visibleReactions$ = visibleReactions$; + /** + * Emits an array of reactions that should be played. + */ + const audibleReactions$ = playReactionsSound.value$.pipe( + switchMap((show) => + show ? reactions$ : of>({}), + ), + map((reactions) => Object.values(reactions).map((v) => v.name)), + scan( + (acc, latest) => { + return { + playing: latest.filter( + (v) => acc.playing.includes(v) || acc.newSounds.includes(v), + ), + newSounds: latest.filter( + (v) => !acc.playing.includes(v) && !acc.newSounds.includes(v), + ), + }; + }, + { playing: [], newSounds: [] }, + ), + map((v) => v.newSounds), + ); - this.windowMode$ = windowMode$; - this.spotlightExpanded$ = spotlightExpanded$; - this.toggleSpotlightExpanded$ = toggleSpotlightExpanded$; - this.gridMode$ = gridMode$; - this.setGridMode = setGridMode; - this.grid$ = grid$; - this.spotlight$ = spotlight$; - this.pip$ = pip$; - this.layout$ = layout$; - this.tileStoreGeneration$ = tileStoreGeneration$; - this.showSpotlightIndicators$ = showSpotlightIndicators$; - this.showSpeakingIndicators$ = showSpeakingIndicators$; - this.showHeader$ = showHeader$; - this.showFooter$ = showFooter$; - this.earpieceMode$ = earpieceMode$; - this.audioOutputSwitcher$ = audioOutputSwitcher$; - this.reconnecting$ = reconnecting$; - } + const newHandRaised$ = handsRaised$.pipe( + map((v) => Object.keys(v).length), + scan( + (acc, newValue) => ({ + value: newValue, + playSounds: newValue > acc.value, + }), + { value: 0, playSounds: false }, + ), + filter((v) => v.playSounds), + ); + + const newScreenShare$ = screenShares$.pipe( + map((v) => v.length), + scan( + (acc, newValue) => ({ + value: newValue, + playSounds: newValue > acc.value, + }), + { value: 0, playSounds: false }, + ), + filter((v) => v.playSounds), + ); + + /** + * Whether we are sharing our screen. + */ + // reassigned here to make it publicly accessible + const sharingScreen$ = localMembership.sharingScreen$; + + /** + * Callback to toggle screen sharing. If null, screen sharing is not possible. + */ + // reassigned here to make it publicly accessible + const toggleScreenSharing = localMembership.toggleScreenSharing; + + const join = localMembership.requestConnect; + // TODO-MULTI-SFU: Use this view model for the lobby as well, and only call this once 'join' is clicked? + join(); + return { + autoLeave$: autoLeave$, + callPickupState$: callPickupState$, + ringOverlay$: ringOverlay$, + leave$: leave$, + hangup: (): void => userHangup$.next(), + join: join, + toggleScreenSharing: toggleScreenSharing, + sharingScreen$: sharingScreen$, + + tapScreen: (): void => screenTap$.next(), + tapControls: (): void => controlsTap$.next(), + hoverScreen: (): void => screenHover$.next(), + unhoverScreen: (): void => screenUnhover$.next(), + + configError$: localMembership.configError$, + participantCount$: participantCount$, + audioParticipants$: audioParticipants$, + + handsRaised$: handsRaised$, + reactions$: reactions$, + joinSoundEffect$: joinSoundEffect$, + leaveSoundEffect$: leaveSoundEffect$, + newHandRaised$: newHandRaised$, + newScreenShare$: newScreenShare$, + audibleReactions$: audibleReactions$, + visibleReactions$: visibleReactions$, + + windowMode$: windowMode$, + spotlightExpanded$: spotlightExpanded$, + toggleSpotlightExpanded$: toggleSpotlightExpanded$, + gridMode$: gridMode$, + setGridMode: setGridMode, + grid$: grid$, + spotlight$: spotlight$, + pip$: pip$, + layout$: layout$, + tileStoreGeneration$: tileStoreGeneration$, + showSpotlightIndicators$: showSpotlightIndicators$, + showSpeakingIndicators$: showSpeakingIndicators$, + showHeader$: showHeader$, + showFooter$: showFooter$, + earpieceMode$: earpieceMode$, + audioOutputSwitcher$: audioOutputSwitcher$, + reconnecting$: reconnecting$, + }; } // TODO-MULTI-SFU // Setup and update the keyProvider which was create by `createRoom` was a thing before. Now we never update if the E2EEsystem changes // do we need this? diff --git a/src/state/CallViewModel/CallViewModelTestUtils.ts b/src/state/CallViewModel/CallViewModelTestUtils.ts index fefc57a0..f86921c5 100644 --- a/src/state/CallViewModel/CallViewModelTestUtils.ts +++ b/src/state/CallViewModel/CallViewModelTestUtils.ts @@ -24,7 +24,11 @@ import * as ComponentsCore from "@livekit/components-core"; import type { CallMembership } from "matrix-js-sdk/lib/matrixrtc"; import { E2eeType } from "../../e2ee/e2eeType"; import { type RaisedHandInfo, type ReactionInfo } from "../../reactions"; -import { CallViewModel, type CallViewModelOptions } from "./CallViewModel"; +import { + type CallViewModel, + createCallViewModel$, + type CallViewModelOptions, +} from "./CallViewModel"; import { mockConfig, mockLivekitRoom, @@ -154,7 +158,7 @@ export function withCallViewModel( const raisedHands$ = new BehaviorSubject>({}); const reactions$ = new BehaviorSubject>({}); - const vm = new CallViewModel( + const vm = createCallViewModel$( testScope(), rtcSession.asMockedSession(), room, diff --git a/src/utils/test-viewmodel.ts b/src/utils/test-viewmodel.ts index 990c43f8..98c45d86 100644 --- a/src/utils/test-viewmodel.ts +++ b/src/utils/test-viewmodel.ts @@ -20,7 +20,8 @@ import { ConnectionState, type Room as LivekitRoom } from "livekit-client"; import { E2eeType } from "../e2ee/e2eeType"; import { - CallViewModel, + type CallViewModel, + createCallViewModel$, type CallViewModelOptions, } from "../state/CallViewModel/CallViewModel"; import { @@ -145,7 +146,7 @@ export function getBasicCallViewModelEnvironment( // const remoteParticipants$ = of([aliceParticipant]); - const vm = new CallViewModel( + const vm = createCallViewModel$( testScope(), rtcSession.asMockedSession(), matrixRoom, From 2a19f06e4aa600b46b744e46478f8fb6f6b010a9 Mon Sep 17 00:00:00 2001 From: Timo K Date: Mon, 17 Nov 2025 19:45:14 +0100 Subject: [PATCH 70/75] reword docstring --- src/state/CallViewModel/CallViewModel.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 62e4ae44..66c9400c 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -173,7 +173,7 @@ type AudioLivekitItem = { /** * The return of createCallViewModel$ - * This interface represents the root snapshot for the call view. Snapshots in EC with rxjs behave like snapshot trees. + * this interface represents the root source of data for the call view. * They are a list of observables and objects containing observables to allow for a very granular update mechanism. * * This allows to have one huge call view model that represents the entire view without a unnecessary amount of updates. From 614762b5f87cad370136dfe32d68c888df1533a5 Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 18 Nov 2025 10:13:10 +0100 Subject: [PATCH 71/75] fixup headers --- src/state/CallViewModel/CallNotificationLifecycle.test.ts | 2 +- src/state/CallViewModel/CallNotificationLifecycle.ts | 2 +- src/state/CallViewModel/CallViewModel.test.ts | 1 + src/state/CallViewModel/CallViewModel.ts | 1 + src/state/CallViewModel/localMember/LocalMembership.test.ts | 1 + src/state/CallViewModel/localMember/LocalMembership.ts | 2 +- src/state/CallViewModel/localMember/LocalTransport.ts | 2 +- src/state/CallViewModel/localMember/Publisher.ts | 1 + src/state/CallViewModel/remoteMembers/Connection.test.ts | 1 + src/state/CallViewModel/remoteMembers/Connection.ts | 1 + src/state/CallViewModel/remoteMembers/ConnectionManager.ts | 3 --- src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.ts | 2 +- src/state/ObservableScope.test.ts | 2 +- src/state/SessionBehaviors.ts | 2 +- 14 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/state/CallViewModel/CallNotificationLifecycle.test.ts b/src/state/CallViewModel/CallNotificationLifecycle.test.ts index 330d96c9..236c126a 100644 --- a/src/state/CallViewModel/CallNotificationLifecycle.test.ts +++ b/src/state/CallViewModel/CallNotificationLifecycle.test.ts @@ -1,5 +1,5 @@ /* -Copyright 2025 New Vector Ltd. +Copyright 2025 Element Creations Ltd. SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. diff --git a/src/state/CallViewModel/CallNotificationLifecycle.ts b/src/state/CallViewModel/CallNotificationLifecycle.ts index d1257b04..2a0bf2f1 100644 --- a/src/state/CallViewModel/CallNotificationLifecycle.ts +++ b/src/state/CallViewModel/CallNotificationLifecycle.ts @@ -1,5 +1,5 @@ /* -Copyright 2025 New Vector Ltd. +Copyright 2025 Element Creations Ltd. SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. diff --git a/src/state/CallViewModel/CallViewModel.test.ts b/src/state/CallViewModel/CallViewModel.test.ts index 9e28687e..3a621f33 100644 --- a/src/state/CallViewModel/CallViewModel.test.ts +++ b/src/state/CallViewModel/CallViewModel.test.ts @@ -1,4 +1,5 @@ /* +Copyright 2025 Element Creations Ltd. Copyright 2024 New Vector Ltd. SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 66c9400c..22ab5e07 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -1,4 +1,5 @@ /* +Copyright 2025 Element Creations Ltd. Copyright 2023, 2024, 2025 New Vector Ltd. SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial diff --git a/src/state/CallViewModel/localMember/LocalMembership.test.ts b/src/state/CallViewModel/localMember/LocalMembership.test.ts index f16119e4..716740d3 100644 --- a/src/state/CallViewModel/localMember/LocalMembership.test.ts +++ b/src/state/CallViewModel/localMember/LocalMembership.test.ts @@ -1,4 +1,5 @@ /* +Copyright 2025 Element Creations Ltd. Copyright 2024 New Vector Ltd. SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial diff --git a/src/state/CallViewModel/localMember/LocalMembership.ts b/src/state/CallViewModel/localMember/LocalMembership.ts index 26269985..38d6092b 100644 --- a/src/state/CallViewModel/localMember/LocalMembership.ts +++ b/src/state/CallViewModel/localMember/LocalMembership.ts @@ -1,5 +1,5 @@ /* -Copyright 2025 New Vector Ltd. +Copyright 2025 Element Creations Ltd. SPDX-License-IdFentifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. diff --git a/src/state/CallViewModel/localMember/LocalTransport.ts b/src/state/CallViewModel/localMember/LocalTransport.ts index 83c997e8..6bb31e57 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.ts @@ -1,5 +1,5 @@ /* -Copyright 2025 New Vector Ltd. +Copyright 2025 Element Creations Ltd. SPDX-License-IdFentifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. diff --git a/src/state/CallViewModel/localMember/Publisher.ts b/src/state/CallViewModel/localMember/Publisher.ts index 7be612e3..11f35424 100644 --- a/src/state/CallViewModel/localMember/Publisher.ts +++ b/src/state/CallViewModel/localMember/Publisher.ts @@ -1,4 +1,5 @@ /* +Copyright 2025 Element Creations Ltd. Copyright 2025 New Vector Ltd. SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial diff --git a/src/state/CallViewModel/remoteMembers/Connection.test.ts b/src/state/CallViewModel/remoteMembers/Connection.test.ts index 9d48e893..3f58bcf6 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.test.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.test.ts @@ -1,4 +1,5 @@ /* +Copyright 2025 Element Creations Ltd. Copyright 2025 New Vector Ltd. SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial diff --git a/src/state/CallViewModel/remoteMembers/Connection.ts b/src/state/CallViewModel/remoteMembers/Connection.ts index 44ce4972..c17fae2b 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.ts @@ -1,4 +1,5 @@ /* +Copyright 2025 Element Creations Ltd. Copyright 2025 New Vector Ltd. SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts index e446571c..f284c9e3 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts @@ -1,6 +1,3 @@ -// TODOs: -// - make ConnectionManager its own actual class - /* Copyright 2025 Element Creations Ltd. Copyright 2025 New Vector Ltd. diff --git a/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.ts b/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.ts index 91363f90..c1a7a499 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.ts @@ -1,5 +1,5 @@ /* -Copyright 2025 New Vector Ltd. +Copyright 2025 Element Creations Ltd. SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. diff --git a/src/state/ObservableScope.test.ts b/src/state/ObservableScope.test.ts index e6c1249d..99f2b424 100644 --- a/src/state/ObservableScope.test.ts +++ b/src/state/ObservableScope.test.ts @@ -1,5 +1,5 @@ /* -Copyright 2025 New Vector Ltd. +Copyright 2025 Element Creations Ltd. SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. diff --git a/src/state/SessionBehaviors.ts b/src/state/SessionBehaviors.ts index 153ad164..e174a1cc 100644 --- a/src/state/SessionBehaviors.ts +++ b/src/state/SessionBehaviors.ts @@ -1,5 +1,5 @@ /* -Copyright 2025 New Vector Ltd. +Copyright 2025 Element Creations Ltd. SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. From 0076b54e521e759d7b4a171e357a87e7bd2a851e Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 18 Nov 2025 12:14:17 +0100 Subject: [PATCH 72/75] minor comments/TODO cleaning --- src/state/CallViewModel/CallViewModel.ts | 9 +++------ .../CallViewModel/localMember/LocalMembership.ts | 11 ++++++++--- .../remoteMembers/MatrixMemberMetadata.test.ts | 2 -- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 22ab5e07..eb270641 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -371,8 +371,8 @@ export function createCallViewModel$( // For mocking purposes it is recommended to only mock the functions creating those outputs. // All other fields are just temp computations for the mentioned output. // The class does not need anything except the values underneath the bar. - // The creation of the values under the bar are all tested independently and testing the callViewModel Should - // not test their cretation. Call view model only needs: + // The creations of the values under the bar are all tested independently and testing the callViewModel Should + // not test their creation. Call view model only needs: // - memberships$ via createMemberships$ // - localMembership via createLocalMembership$ // - callLifecycle via createCallNotificationLifecycle$ @@ -438,7 +438,7 @@ export function createCallViewModel$( matrixRTCMode.value$.pipe( map((mode) => ({ encryptMedia: livekitKeyProvider !== undefined, - // TODO. This might need to get called again on each cahnge of matrixRTCMode... + // TODO. This might need to get called again on each change of matrixRTCMode... matrixRTCMode: mode, })), ), @@ -636,7 +636,6 @@ export function createCallViewModel$( /** * List of user media (camera feeds) that we want tiles for. */ - // TODO this also needs the local participant to be added. const userMedia$ = scope.behavior( combineLatest([ localMatrixLivekitMember$, @@ -1475,8 +1474,6 @@ export function createCallViewModel$( reconnecting$: reconnecting$, }; } -// TODO-MULTI-SFU // Setup and update the keyProvider which was create by `createRoom` was a thing before. Now we never update if the E2EEsystem changes -// do we need this? function getE2eeKeyProvider( e2eeSystem: EncryptionSystem, diff --git a/src/state/CallViewModel/localMember/LocalMembership.ts b/src/state/CallViewModel/localMember/LocalMembership.ts index 38d6092b..68b34d94 100644 --- a/src/state/CallViewModel/localMember/LocalMembership.ts +++ b/src/state/CallViewModel/localMember/LocalMembership.ts @@ -262,7 +262,7 @@ export const createLocalMembership$ = ({ ); const publisher$ = new BehaviorSubject(null); - localConnection$.subscribe((connection) => { + localConnection$.pipe(scope.bind()).subscribe((connection) => { if (connection !== null && publisher$.value === null) { // TODO looks strange to not change publisher if connection changes. publisher$.next( @@ -339,7 +339,7 @@ export const createLocalMembership$ = ({ }); combineLatest([localTransport$, connectRequested$]).subscribe( - // TODO reconnect on options change. + // TODO reconnect when transport changes => create test. ([transport, connectRequested]) => { if ( transport === null || @@ -573,7 +573,12 @@ interface EnterRTCSessionOptions { } /** - * TODO! document this function properly + * Does the necessary steps to enter the RTC session on the matrix side: + * - Preparing the membership info (FOCUS to use, options) + * - Sends the matrix event to join the call, and starts the membership manager: + * - Delay events management + * - Handles retries (fails only after several attempts) + * * @param rtcSession * @param transport * @param options diff --git a/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.test.ts b/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.test.ts index d27fe4c9..6f392351 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.test.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.test.ts @@ -104,7 +104,6 @@ describe("MatrixMemberMetadata", () => { } } - // TODO this is a regression, now there the own user is not always in the map. Ask Timo if fine it("should show our own user if present in rtc session and room", () => { withTestScheduler(({ behavior, expectObservable }) => { fakeMemberWith({ @@ -517,7 +516,6 @@ describe("MatrixMemberMetadata", () => { } } - // TODO this is a regression, now there the own user is not always in the map. Ask Timo if fine it("should use avatar url from room members", () => { withTestScheduler(({ behavior, expectObservable }) => { fakeMemberWith({ From 82706cca4fda805b5ff6cca265671908f1631517 Mon Sep 17 00:00:00 2001 From: alutarius <35006594+alutarius@users.noreply.github.com> Date: Tue, 18 Nov 2025 18:46:00 +0100 Subject: [PATCH 73/75] Correct Caddy reverse proxy config example (#3572) --- docs/self-hosting.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/self-hosting.md b/docs/self-hosting.md index 7127abee..d15f2910 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -136,8 +136,8 @@ handle @jwt_service { reverse_proxy http://[::1]:8080 { header_up Host {host} header_up X-Forwarded-Server {host} - header_up X-Real-IP {remote_addr} - header_up X-Forwarded-For {remote_addr} + header_up X-Real-IP {remote_host} + header_up X-Forwarded-For {remote_host} } } @@ -146,8 +146,8 @@ handle { reverse_proxy http://localhost:7880 { header_up Host {host} header_up X-Forwarded-Server {host} - header_up X-Real-IP {remote_addr} - header_up X-Forwarded-For {remote_addr} + header_up X-Real-IP {remote_host} + header_up X-Forwarded-For {remote_host} } } ``` From 47324b52f8dd31beebbef758ad4257e6ff0a7a18 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 19 Nov 2025 20:14:37 +0100 Subject: [PATCH 74/75] Update dependency livekit-client to v2.16.0 (#3574) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 07e3bac3..8f43275a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10110,8 +10110,8 @@ __metadata: linkType: hard "livekit-client@npm:^2.13.0": - version: 2.15.16 - resolution: "livekit-client@npm:2.15.16" + version: 2.16.0 + resolution: "livekit-client@npm:2.16.0" dependencies: "@livekit/mutex": "npm:1.1.1" "@livekit/protocol": "npm:1.42.2" @@ -10125,7 +10125,7 @@ __metadata: webrtc-adapter: "npm:^9.0.1" peerDependencies: "@types/dom-mediacapture-record": ^1 - checksum: 10c0/22c8f80a31f6e4d860c6e0a54e3f424852f95bcb9a6d53f3090ecbc147836299205793765ec036aba9d2b18354e2df94d05161ac9c93e52469b18b1ac8af5045 + checksum: 10c0/5d03adc5d09efde343ab894db397529dff26117598e773b23a5df90a4fb166bde12c6bb1f2cfd1d28dbaf93fe9f275026d7abb75f2ffd2ba816393a2d58e6c7e languageName: node linkType: hard From cdc3c2b21624ffc5200c37de9662413385e7a3ff Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 19 Nov 2025 20:18:18 +0100 Subject: [PATCH 75/75] Update LiveKit components (#3573) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- yarn.lock | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/yarn.lock b/yarn.lock index 8f43275a..97ca1985 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2731,37 +2731,39 @@ __metadata: languageName: node linkType: hard -"@livekit/components-core@npm:0.12.10, @livekit/components-core@npm:^0.12.0": - version: 0.12.10 - resolution: "@livekit/components-core@npm:0.12.10" +"@livekit/components-core@npm:0.12.11, @livekit/components-core@npm:^0.12.0": + version: 0.12.11 + resolution: "@livekit/components-core@npm:0.12.11" dependencies: "@floating-ui/dom": "npm:1.6.13" loglevel: "npm:1.9.1" rxjs: "npm:7.8.2" peerDependencies: - livekit-client: ^2.13.3 + livekit-client: ^2.15.14 tslib: ^2.6.2 - checksum: 10c0/bfd84fb950f72dd037bd5329658c1e750a8ac6b8f2953ea673e3e944b8ea9d412ef9c98eb8b690052323e03c675964b162aacb00e60530cdc5187f77d21979bd + checksum: 10c0/9c2ac3d30bb8cc9067ae0b2049784f81e90e57df9eabf7edbaf3c8ceb65a63f644a4e6abeb6cc38d3ebe52663d8dbb88535e01a965011f365d5ae1f3daf86052 languageName: node linkType: hard "@livekit/components-react@npm:^2.0.0": - version: 2.9.15 - resolution: "@livekit/components-react@npm:2.9.15" + version: 2.9.16 + resolution: "@livekit/components-react@npm:2.9.16" dependencies: - "@livekit/components-core": "npm:0.12.10" + "@livekit/components-core": "npm:0.12.11" clsx: "npm:2.1.1" + events: "npm:^3.3.0" + jose: "npm:^6.0.12" usehooks-ts: "npm:3.1.1" peerDependencies: "@livekit/krisp-noise-filter": ^0.2.12 || ^0.3.0 - livekit-client: ^2.13.3 + livekit-client: ^2.15.14 react: ">=18" react-dom: ">=18" tslib: ^2.6.2 peerDependenciesMeta: "@livekit/krisp-noise-filter": optional: true - checksum: 10c0/58a93d85c3b8267d0afd00eceb4f34992ce66124f93450e828a78dd825ecc20e254c3123ed22ec33061a3728f50f4f020ff896769fb3a0ff79656ed8cf452a2b + checksum: 10c0/4ba4ff473c5a29d3107412733a6676a3b708d70684ed463e9b34cda26abb3d2f317c2828a52e730837b756de9df3fc248260d6f390aedebfb6ec96ef63c7b151 languageName: node linkType: hard @@ -9828,6 +9830,13 @@ __metadata: languageName: node linkType: hard +"jose@npm:^6.0.12": + version: 6.1.2 + resolution: "jose@npm:6.1.2" + checksum: 10c0/55f79426f43e652ed6d5de938d50f66bb0a10dcae078db81a23f8d3303e889ce226f000e815f3211f9956bb84badce10da892d130d40fe2eca658045a6f1778e + languageName: node + linkType: hard + "jose@npm:^6.1.0": version: 6.1.0 resolution: "jose@npm:6.1.0"

diff --git a/src/tile/SpotlightTile.tsx b/src/tile/SpotlightTile.tsx index 6034c846..48dd0f8c 100644 --- a/src/tile/SpotlightTile.tsx +++ b/src/tile/SpotlightTile.tsx @@ -27,7 +27,6 @@ import { useObservableRef } from "observable-hooks"; import { useTranslation } from "react-i18next"; import classNames from "classnames"; import { type TrackReferenceOrPlaceholder } from "@livekit/components-core"; -import { type RoomMember } from "matrix-js-sdk"; import FullScreenMaximiseIcon from "../icons/FullScreenMaximise.svg?react"; import FullScreenMinimiseIcon from "../icons/FullScreenMinimise.svg?react"; @@ -55,10 +54,12 @@ interface SpotlightItemBaseProps { targetHeight: number; video: TrackReferenceOrPlaceholder | undefined; videoEnabled: boolean; - member: RoomMember; + userId: string; unencryptedWarning: boolean; encryptionStatus: EncryptionStatus; + focusUrl: string | undefined; displayName: string; + mxcAvatarUrl: string | undefined; focusable: boolean; "aria-hidden"?: boolean; localParticipant: boolean; @@ -78,7 +79,7 @@ const SpotlightLocalUserMediaItem: FC = ({ ...props }) => { const mirror = useBehavior(vm.mirror$); - return ; + return ; }; SpotlightLocalUserMediaItem.displayName = "SpotlightLocalUserMediaItem"; @@ -134,7 +135,9 @@ const SpotlightItem: FC = ({ }) => { const ourRef = useRef(null); const ref = useMergedRefs(ourRef, theirRef); + const focusUrl = useBehavior(vm.focusUrl$); const displayName = useBehavior(vm.displayName$); + const mxcAvatarUrl = useBehavior(vm.mxcAvatarUrl$); const video = useBehavior(vm.video$); const videoEnabled = useBehavior(vm.videoEnabled$); const unencryptedWarning = useBehavior(vm.unencryptedWarning$); @@ -161,11 +164,13 @@ const SpotlightItem: FC = ({ className: classNames(styles.item, { [styles.snap]: snap }), targetWidth, targetHeight, - video, + video: video ?? undefined, videoEnabled, - member: vm.member, + userId: vm.userId, unencryptedWarning, + focusUrl, displayName, + mxcAvatarUrl, focusable, encryptionStatus, "aria-hidden": ariaHidden, diff --git a/src/utils/observable.test.ts b/src/utils/observable.test.ts index e039c846..d1034e7b 100644 --- a/src/utils/observable.test.ts +++ b/src/utils/observable.test.ts @@ -9,7 +9,7 @@ import { test } from "vitest"; import { Subject } from "rxjs"; import { withTestScheduler } from "./test"; -import { generateKeyed$, pauseWhen } from "./observable"; +import { generateItems, pauseWhen } from "./observable"; test("pauseWhen", () => { withTestScheduler(({ behavior, expectObservable }) => { @@ -24,7 +24,7 @@ test("pauseWhen", () => { }); }); -test("generateKeyed$ has the right output and ends scopes at the right times", () => { +test("generateItems", () => { const scope1$ = new Subject(); const scope2$ = new Subject(); const scope3$ = new Subject(); @@ -44,18 +44,27 @@ test("generateKeyed$ has the right output and ends scopes at the right times", ( const scope4Marbles = " ----yn"; expectObservable( - generateKeyed$(hot(inputMarbles), (input, createOrGet) => { - for (let i = 1; i <= +input; i++) { - createOrGet(i.toString(), (scope) => { + hot(inputMarbles).pipe( + generateItems( + function* (input) { + for (let i = 1; i <= +input; i++) { + yield { keys: [i], data: undefined }; + } + }, + (scope, data$, i) => { scopeSubjects[i - 1].next("y"); scope.onEnd(() => scopeSubjects[i - 1].next("n")); return i.toString(); - }); - } - return "abcd"[+input - 1]; - }), + }, + ), + ), subscriptionMarbles, - ).toBe(outputMarbles); + ).toBe(outputMarbles, { + a: ["1"], + b: ["1", "2"], + c: ["1", "2", "3"], + d: ["1", "2", "3", "4"], + }); expectObservable(scope1$).toBe(scope1Marbles); expectObservable(scope2$).toBe(scope2Marbles); diff --git a/src/utils/observable.ts b/src/utils/observable.ts index eb817991..053921cd 100644 --- a/src/utils/observable.ts +++ b/src/utils/observable.ts @@ -20,10 +20,12 @@ import { takeWhile, tap, withLatestFrom, + BehaviorSubject, + type OperatorFunction, } from "rxjs"; import { type Behavior } from "../state/Behavior"; -import { ObservableScope } from "../state/ObservableScope"; +import { Epoch, ObservableScope } from "../state/ObservableScope"; const nothing = Symbol("nothing"); @@ -119,70 +121,156 @@ export function pauseWhen(pause$: Behavior) { ); } +interface ItemHandle { + scope: ObservableScope; + data$: BehaviorSubject; + item: Item; +} + /** - * Maps a changing input value to an output value consisting of items that have - * automatically generated ObservableScopes tied to a key. Items will be - * automatically created when their key is requested for the first time, reused - * when the same key is requested at a later time, and destroyed (have their - * scope ended) when the key is no longer requested. + * Maps a changing input value to a collection of items that each capture some + * dynamic data and are tied to a key. Items will be automatically created when + * their key is requested for the first time, reused when the same key is + * requested at a later time, and destroyed (have their scope ended) when the + * key is no longer requested. * * @param input$ The input value to be mapped. - * @param project A function mapping input values to output values. This - * function receives an additional callback `createOrGet` which can be used - * within the function body to request that an item be generated for a certain - * key. The caller provides a factory which will be used to create the item if - * it is being requested for the first time. Otherwise, the item previously - * existing under that key will be returned. + * @param generator A generator function yielding a tuple of keys and the + * currently associated data for each item that it wants to exist. + * @param factory A function constructing an individual item, given the item's key, + * dynamic data, and an automatically managed ObservableScope for the item. */ -export function generateKeyed$( - input$: Observable, - project: ( - input: In, - createOrGet: ( - key: string, - factory: (scope: ObservableScope) => Item, - ) => Item, - ) => Out, -): Observable { - return input$.pipe( - // Keep track of the existing items over time, so we can reuse them - scan< - In, - { - items: Map; - output: Out; - }, - { items: Map } - >( - (state, data) => { - const nextItems = new Map< - string, - { item: Item; scope: ObservableScope } - >(); +export function generateItems< + Input, + Keys extends [unknown, ...unknown[]], + Data, + Item, +>( + generator: ( + input: Input, + ) => Generator<{ keys: readonly [...Keys]; data: Data }, void, void>, + factory: ( + scope: ObservableScope, + data$: Behavior, + ...keys: Keys + ) => Item, +): OperatorFunction { + return generateItemsInternal(generator, factory, (items) => items); +} - const output = project(data, (key, factory) => { - let item = state.items.get(key); - if (item === undefined) { - // First time requesting the key; create the item - const scope = new ObservableScope(); - item = { item: factory(scope), scope }; - } - nextItems.set(key, item); - return item.item; - }); - - // Destroy all items that are no longer being requested - for (const [key, { scope }] of state.items) - if (!nextItems.has(key)) scope.end(); - - return { items: nextItems, output }; - }, - { items: new Map() }, - ), - finalizeValue((state) => { - // Destroy all remaining items when no longer subscribed - for (const { scope } of state.items.values()) scope.end(); - }), - map(({ output }) => output), +/** + * Same as generateItems, but preserves epoch data. + */ +export function generateItemsWithEpoch< + Input, + Keys extends [unknown, ...unknown[]], + Data, + Item, +>( + generator: ( + input: Input, + ) => Generator<{ keys: readonly [...Keys]; data: Data }, void, void>, + factory: ( + scope: ObservableScope, + data$: Behavior, + ...keys: Keys + ) => Item, +): OperatorFunction, Epoch> { + return generateItemsInternal( + function* (input) { + yield* generator(input.value); + }, + factory, + (items, input) => new Epoch(items, input.epoch), ); } + +function generateItemsInternal< + Input, + Keys extends [unknown, ...unknown[]], + Data, + Item, + Output, +>( + generator: ( + input: Input, + ) => Generator<{ keys: readonly [...Keys]; data: Data }, void, void>, + factory: ( + scope: ObservableScope, + data$: Behavior, + ...keys: Keys + ) => Item, + project: (items: Item[], input: Input) => Output, +): OperatorFunction { + /* eslint-disable @typescript-eslint/no-explicit-any */ + return (input$) => + input$.pipe( + // Keep track of the existing items over time, so they can persist + scan< + Input, + { + map: Map; + items: Set>; + input: Input; + }, + { map: Map; items: Set> } + >( + ({ map: prevMap, items: prevItems }, input) => { + const nextMap = new Map(); + const nextItems = new Set>(); + + for (const { keys, data } of generator(input)) { + // Disable type checks for a second to grab the item out of a nested map + let i: any = prevMap; + for (const key of keys) i = i?.get(key); + let item = i as ItemHandle | undefined; + + if (item === undefined) { + // First time requesting the key; create the item + const scope = new ObservableScope(); + const data$ = new BehaviorSubject(data); + item = { scope, data$, item: factory(scope, data$, ...keys) }; + } else { + item.data$.next(data); + } + + // Likewise, disable type checks to insert the item in the nested map + let m: Map = nextMap; + for (let i = 0; i < keys.length - 1; i++) { + let inner = m.get(keys[i]); + if (inner === undefined) { + inner = new Map(); + m.set(keys[i], inner); + } + m = inner; + } + const finalKey = keys[keys.length - 1]; + if (m.has(finalKey)) + throw new Error( + `Keys must be unique (tried to generate multiple items for key ${keys})`, + ); + m.set(keys[keys.length - 1], item); + nextItems.add(item); + } + + // Destroy all items that are no longer being requested + for (const item of prevItems) + if (!nextItems.has(item)) item.scope.end(); + + return { map: nextMap, items: nextItems, input }; + }, + { map: new Map(), items: new Set() }, + ), + finalizeValue(({ items }) => { + // Destroy all remaining items when no longer subscribed + for (const { scope } of items) scope.end(); + }), + map(({ items, input }) => + project( + [...items].map(({ item }) => item), + input, + ), + ), + ); + /* eslint-enable @typescript-eslint/no-explicit-any */ +} diff --git a/src/utils/test-viewmodel.ts b/src/utils/test-viewmodel.ts index 5a0d7526..0e6c589a 100644 --- a/src/utils/test-viewmodel.ts +++ b/src/utils/test-viewmodel.ts @@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details. */ import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; -import { BehaviorSubject, of } from "rxjs"; +import { BehaviorSubject } from "rxjs"; import { vitest } from "vitest"; import { type RelationsContainer } from "matrix-js-sdk/lib/models/relations-container"; import EventEmitter from "events"; @@ -158,7 +158,7 @@ export function getBasicCallViewModelEnvironment( }, handRaisedSubject$, reactionsSubject$, - of({ processor: undefined, supported: false }), + constant({ processor: undefined, supported: false }), ); return { vm, diff --git a/src/utils/test.ts b/src/utils/test.ts index bb19f2b1..d0f09576 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -304,18 +304,20 @@ export function createLocalMedia( localParticipant: LocalParticipant, mediaDevices: MediaDevices, ): LocalUserMediaViewModel { + const member = mockMatrixRoomMember(localRtcMember, roomMember); return new LocalUserMediaViewModel( testScope(), "local", - mockMatrixRoomMember(localRtcMember, roomMember), + member.userId, constant(localParticipant), { kind: E2eeType.PER_PARTICIPANT, }, - mockLivekitRoom({ localParticipant }), - "https://rtc-example.org", + constant(mockLivekitRoom({ localParticipant })), + constant("https://rtc-example.org"), mediaDevices, - constant(roomMember.rawDisplayName ?? "nodisplayname"), + constant(member.rawDisplayName ?? "nodisplayname"), + constant(member.getMxcAvatarUrl()), constant(null), constant(null), ); @@ -339,19 +341,23 @@ export function createRemoteMedia( roomMember: Partial, participant: Partial, ): RemoteUserMediaViewModel { + const member = mockMatrixRoomMember(localRtcMember, roomMember); const remoteParticipant = mockRemoteParticipant(participant); return new RemoteUserMediaViewModel( testScope(), "remote", - mockMatrixRoomMember(localRtcMember, roomMember), + member.userId, of(remoteParticipant), { kind: E2eeType.PER_PARTICIPANT, }, - mockLivekitRoom({}, { remoteParticipants$: of([remoteParticipant]) }), - "https://rtc-example.org", + constant( + mockLivekitRoom({}, { remoteParticipants$: of([remoteParticipant]) }), + ), + constant("https://rtc-example.org"), constant(false), - constant(roomMember.rawDisplayName ?? "nodisplayname"), + constant(member.rawDisplayName ?? "nodisplayname"), + constant(member.getMxcAvatarUrl()), constant(null), constant(null), ); From 92ddc4c797ca7f292270d21b0414d6cf5198f19e Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 9 Nov 2025 01:16:39 -0500 Subject: [PATCH 37/75] Fix avatar reactivity, simplify display names tracking --- .../remoteMembers/MatrixLivekitMembers.ts | 68 ++++++++++--------- .../remoteMembers/displayname.test.ts | 59 ++++++++-------- .../remoteMembers/displayname.ts | 20 ++---- 3 files changed, 72 insertions(+), 75 deletions(-) diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts index 4aaaadd4..764862f2 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts @@ -13,15 +13,15 @@ import { type LivekitTransport, type CallMembership, } from "matrix-js-sdk/lib/matrixrtc"; -import { combineLatest, filter, map } from "rxjs"; +import { combineLatest, filter, fromEvent, map, startWith } from "rxjs"; // eslint-disable-next-line rxjs/no-internal import { type NodeStyleEventEmitter } from "rxjs/internal/observable/fromEvent"; -import { type Room as MatrixRoom } from "matrix-js-sdk"; +import { RoomStateEvent, type Room as MatrixRoom } from "matrix-js-sdk"; import { logger } from "matrix-js-sdk/lib/logger"; import { type Behavior } from "../../Behavior"; import { type IConnectionManager } from "./ConnectionManager"; -import { Epoch, mapEpoch, type ObservableScope } from "../../ObservableScope"; +import { Epoch, type ObservableScope } from "../../ObservableScope"; import { memberDisplaynames$ } from "./displayname"; import { type Connection } from "./Connection"; import { generateItemsWithEpoch } from "../../../utils/observable"; @@ -82,14 +82,17 @@ export function createMatrixLivekitMembers$({ const displaynameMap$ = memberDisplaynames$( scope, matrixRoom, - membershipsWithTransport$.pipe(mapEpoch((v) => v.map((v) => v.membership))), + scope.behavior( + membershipsWithTransport$.pipe( + map((ms) => ms.value.map((m) => m.membership)), + ), + ), ); return scope.behavior( combineLatest([ membershipsWithTransport$, connectionManager.connectionManagerData$, - displaynameMap$, ]).pipe( filter((values) => values.every((value) => value.epoch === values[0].epoch), @@ -98,15 +101,11 @@ export function createMatrixLivekitMembers$({ ([ { value: membershipsWithTransports, epoch }, { value: managerData }, - { value: displaynames }, ]) => - new Epoch( - [membershipsWithTransports, managerData, displaynames] as const, - epoch, - ), + new Epoch([membershipsWithTransports, managerData] as const, epoch), ), generateItemsWithEpoch( - function* ([membershipsWithTransports, managerData, displaynames]) { + function* ([membershipsWithTransports, managerData]) { for (const { membership, transport } of membershipsWithTransports) { // TODO! cannot use membership.membershipID yet, Currently its hardcoded by the jwt service to const participantId = /*membership.membershipID*/ `${membership.userId}:${membership.deviceId}`; @@ -116,35 +115,42 @@ export function createMatrixLivekitMembers$({ : []; const participant = participants.find((p) => p.identity == participantId) ?? null; - // This makes sense to add to the js-sdk callMembership (we only need the avatar so probably the call memberhsip just should aquire the avatar) - const member = matrixRoom.getMember(membership.userId); const connection = transport ? managerData.getConnectionForTransport(transport) : undefined; - let displayName = displaynames.get(membership.userId); - if (displayName === undefined) { - logger.warn(`No display name for user ${membership.userId}`); - displayName = ""; - } - yield { keys: [participantId, membership.userId], - data: { - membership, - participant, - connection, - displayName, - mxcAvatarUrl: member?.getMxcAvatarUrl(), - }, + data: { membership, participant, connection }, }; } }, - (scope, data$, participantId, userId) => ({ - participantId, - userId, - ...scope.splitBehavior(data$), - }), + (scope, data$, participantId, userId) => { + const member = matrixRoom.getMember(userId); + return { + participantId, + userId, + ...scope.splitBehavior(data$), + displayName$: scope.behavior( + displaynameMap$.pipe( + map((displayNames) => { + const name = displayNames.get(userId); + if (name === undefined) { + logger.warn(`No display name for user ${userId}`); + return ""; + } + return name; + }), + ), + ), + mxcAvatarUrl$: scope.behavior( + fromEvent(matrixRoom, RoomStateEvent.Members).pipe( + startWith(undefined), + map(() => member?.getMxcAvatarUrl()), + ), + ), + }; + }, ), ), ); diff --git a/src/state/CallViewModel/remoteMembers/displayname.test.ts b/src/state/CallViewModel/remoteMembers/displayname.test.ts index efbe1235..60a29a18 100644 --- a/src/state/CallViewModel/remoteMembers/displayname.test.ts +++ b/src/state/CallViewModel/remoteMembers/displayname.test.ts @@ -13,9 +13,8 @@ import { RoomStateEvent, } from "matrix-js-sdk"; import EventEmitter from "events"; -import { map } from "rxjs"; -import { ObservableScope, trackEpoch } from "../../ObservableScope.ts"; +import { ObservableScope } from "../../ObservableScope.ts"; import type { Room as MatrixRoom } from "matrix-js-sdk/lib/models/room"; import { mockCallMembership, withTestScheduler } from "../../../utils/test.ts"; import { memberDisplaynames$ } from "./displayname.ts"; @@ -86,16 +85,16 @@ afterEach(() => { // TODO this is a regression, now there the own user is not always in the map. Ask Timo if fine test.skip("should always have our own user", () => { - withTestScheduler(({ cold, schedule, expectObservable }) => { + withTestScheduler(({ behavior, expectObservable }) => { const dn$ = memberDisplaynames$( testScope, mockMatrixRoom, - cold("a", { + behavior("a", { a: [], - }).pipe(trackEpoch()), + }), ); - expectObservable(dn$.pipe(map((e) => e.value))).toBe("a", { + expectObservable(dn$).toBe("a", { a: new Map([ ["@local:example.com", "@local:example.com"], ]), @@ -116,19 +115,19 @@ function setUpBasicRoom(): void { test("should get displayName for users", () => { setUpBasicRoom(); - withTestScheduler(({ cold, schedule, expectObservable }) => { + withTestScheduler(({ behavior, expectObservable }) => { const dn$ = memberDisplaynames$( testScope, mockMatrixRoom, - cold("a", { + behavior("a", { a: [ mockCallMembership("@alice:example.com", "DEVICE1"), mockCallMembership("@bob:example.com", "DEVICE1"), ], - }).pipe(trackEpoch()), + }), ); - expectObservable(dn$.pipe(map((e) => e.value))).toBe("a", { + expectObservable(dn$).toBe("a", { a: new Map([ // ["@local:example.com", "it's a me"], ["@alice:example.com", "Alice"], @@ -139,18 +138,18 @@ test("should get displayName for users", () => { }); test("should use userId if no display name", () => { - withTestScheduler(({ cold, schedule, expectObservable }) => { + withTestScheduler(({ behavior, expectObservable }) => { setUpBasicRoom(); const dn$ = memberDisplaynames$( testScope, mockMatrixRoom, - cold("a", { + behavior("a", { a: [mockCallMembership("@no-name:foo.bar", "D000")], - }).pipe(trackEpoch()), + }), ); - expectObservable(dn$.pipe(map((e) => e.value))).toBe("a", { + expectObservable(dn$).toBe("a", { a: new Map([ // ["@local:example.com", "it's a me"], ["@no-name:foo.bar", "@no-name:foo.bar"], @@ -160,13 +159,13 @@ test("should use userId if no display name", () => { }); test("should disambiguate users with same display name", () => { - withTestScheduler(({ cold, schedule, expectObservable }) => { + withTestScheduler(({ behavior, expectObservable }) => { setUpBasicRoom(); const dn$ = memberDisplaynames$( testScope, mockMatrixRoom, - cold("a", { + behavior("a", { a: [ mockCallMembership("@bob:example.com", "DEVICE1"), mockCallMembership("@bob:example.com", "DEVICE2"), @@ -174,10 +173,10 @@ test("should disambiguate users with same display name", () => { mockCallMembership("@carl:example.com", "C000"), mockCallMembership("@evil:example.com", "E000"), ], - }).pipe(trackEpoch()), + }), ); - expectObservable(dn$.pipe(map((e) => e.value))).toBe("a", { + expectObservable(dn$).toBe("a", { a: new Map([ // ["@local:example.com", "it's a me"], ["@bob:example.com", "Bob (@bob:example.com)"], @@ -191,22 +190,22 @@ test("should disambiguate users with same display name", () => { }); test("should disambiguate when needed", () => { - withTestScheduler(({ cold, schedule, expectObservable }) => { + withTestScheduler(({ behavior, expectObservable }) => { setUpBasicRoom(); const dn$ = memberDisplaynames$( testScope, mockMatrixRoom, - cold("ab", { + behavior("ab", { a: [mockCallMembership("@bob:example.com", "DEVICE1")], b: [ mockCallMembership("@bob:example.com", "DEVICE1"), mockCallMembership("@bob:foo.bar", "BOB000"), ], - }).pipe(trackEpoch()), + }), ); - expectObservable(dn$.pipe(map((e) => e.value))).toBe("ab", { + expectObservable(dn$).toBe("ab", { a: new Map([ // ["@local:example.com", "it's a me"], ["@bob:example.com", "Bob"], @@ -221,22 +220,22 @@ test("should disambiguate when needed", () => { }); test.skip("should keep disambiguated name when other leave", () => { - withTestScheduler(({ cold, schedule, expectObservable }) => { + withTestScheduler(({ behavior, expectObservable }) => { setUpBasicRoom(); const dn$ = memberDisplaynames$( testScope, mockMatrixRoom, - cold("ab", { + behavior("ab", { a: [ mockCallMembership("@bob:example.com", "DEVICE1"), mockCallMembership("@bob:foo.bar", "BOB000"), ], b: [mockCallMembership("@bob:example.com", "DEVICE1")], - }).pipe(trackEpoch()), + }), ); - expectObservable(dn$.pipe(map((e) => e.value))).toBe("ab", { + expectObservable(dn$).toBe("ab", { a: new Map([ // ["@local:example.com", "it's a me"], ["@bob:example.com", "Bob (@bob:example.com)"], @@ -251,18 +250,18 @@ test.skip("should keep disambiguated name when other leave", () => { }); test("should disambiguate on name change", () => { - withTestScheduler(({ cold, schedule, expectObservable }) => { + withTestScheduler(({ behavior, schedule, expectObservable }) => { setUpBasicRoom(); const dn$ = memberDisplaynames$( testScope, mockMatrixRoom, - cold("a", { + behavior("a", { a: [ mockCallMembership("@bob:example.com", "B000"), mockCallMembership("@carl:example.com", "C000"), ], - }).pipe(trackEpoch()), + }), ); schedule("-a", { @@ -271,7 +270,7 @@ test("should disambiguate on name change", () => { }, }); - expectObservable(dn$.pipe(map((e) => e.value))).toBe("ab", { + expectObservable(dn$).toBe("ab", { a: new Map([ // ["@local:example.com", "it's a me"], ["@bob:example.com", "Bob"], diff --git a/src/state/CallViewModel/remoteMembers/displayname.ts b/src/state/CallViewModel/remoteMembers/displayname.ts index f56bc253..901f3613 100644 --- a/src/state/CallViewModel/remoteMembers/displayname.ts +++ b/src/state/CallViewModel/remoteMembers/displayname.ts @@ -6,20 +6,14 @@ Please see LICENSE in the repository root for full details. */ import { type RoomMember, RoomStateEvent } from "matrix-js-sdk"; -import { - combineLatest, - fromEvent, - map, - type Observable, - startWith, -} from "rxjs"; +import { combineLatest, fromEvent, map, startWith } from "rxjs"; import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; import { logger } from "matrix-js-sdk/lib/logger"; import { type Room as MatrixRoom } from "matrix-js-sdk/lib/matrix"; // eslint-disable-next-line rxjs/no-internal import { type NodeStyleEventEmitter } from "rxjs/internal/observable/fromEvent"; -import { Epoch, type ObservableScope } from "../../ObservableScope"; +import { type ObservableScope } from "../../ObservableScope"; import { calculateDisplayName, shouldDisambiguate, @@ -49,8 +43,8 @@ export const memberDisplaynames$ = ( scope: ObservableScope, matrixRoom: Pick & NodeStyleEventEmitter, // roomMember$: Behavior>; - memberships$: Observable>, -): Behavior>> => + memberships$: Behavior, +): Behavior> => scope.behavior( combineLatest([ // Handle call membership changes @@ -59,8 +53,7 @@ export const memberDisplaynames$ = ( fromEvent(matrixRoom, RoomStateEvent.Members).pipe(startWith(null)), // TODO: do we need: pauseWhen(this.pretendToBeDisconnected$), ]).pipe( - map(([epochMemberships, _displayNames]) => { - const { epoch, value: memberships } = epochMemberships; + map(([memberships, _displayNames]) => { const displaynameMap = new Map(); const room = matrixRoom; @@ -77,8 +70,7 @@ export const memberDisplaynames$ = ( calculateDisplayName(member, disambiguate), ); } - return new Epoch(displaynameMap, epoch); + return displaynameMap; }), ), - new Epoch(new Map()), ); From 5c83e0dce158aa1abb87b107fc8e24f966d77e84 Mon Sep 17 00:00:00 2001 From: Valere Date: Mon, 10 Nov 2025 10:43:53 +0100 Subject: [PATCH 38/75] test: fixup MatrixLivekitMembers tests --- .../MatrixLivekitMembers.test.ts | 468 +++++++++--------- 1 file changed, 235 insertions(+), 233 deletions(-) diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts index 60b52d69..ccf93a30 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts @@ -12,19 +12,23 @@ import { } from "matrix-js-sdk/lib/matrixrtc"; import { type Room as MatrixRoom, type RoomMember } from "matrix-js-sdk"; import { getParticipantId } from "matrix-js-sdk/lib/matrixrtc/utils"; +import { combineLatest, map, type Observable } from "rxjs"; import { type IConnectionManager } from "./ConnectionManager.ts"; import { type MatrixLivekitMember, createMatrixLivekitMembers$, - areLivekitTransportsEqual, } from "./MatrixLivekitMembers.ts"; -import { ObservableScope } from "../../ObservableScope.ts"; +import { + Epoch, + mapEpoch, + ObservableScope, + trackEpoch, +} from "../../ObservableScope.ts"; import { ConnectionManagerData } from "./ConnectionManager.ts"; import { mockCallMembership, mockRemoteParticipant, - type OurRunHelpers, withTestScheduler, } from "../../../utils/test.ts"; import { type Connection } from "./Connection.ts"; @@ -32,7 +36,28 @@ import { type Connection } from "./Connection.ts"; let testScope: ObservableScope; let mockMatrixRoom: MatrixRoom; -// The merger beeing tested +const transportA: LivekitTransport = { + type: "livekit", + livekit_service_url: "https://lk.example.org", + livekit_alias: "!alias:example.org", +}; + +const transportB: LivekitTransport = { + type: "livekit", + livekit_service_url: "https://lk.sample.com", + livekit_alias: "!alias:sample.com", +}; + +const bobMembership = mockCallMembership( + "@bob:example.org", + "DEV000", + transportA, +); +const carlMembership = mockCallMembership( + "@carl:sample.com", + "DEV111", + transportB, +); beforeEach(() => { testScope = new ObservableScope(); @@ -53,113 +78,138 @@ afterEach(() => { testScope.end(); }); +function epochMeWith$( + source$: Observable>, + me$: Observable, +): Observable> { + return combineLatest([source$, me$]).pipe( + map(([ep, cd]) => { + return new Epoch(cd, ep.epoch); + }), + ); +} + test("should signal participant not yet connected to livekit", () => { withTestScheduler(({ behavior, expectObservable }) => { - const bobMembership = { - userId: "@bob:example.org", - deviceId: "DEV000", - transports: [ - { - type: "livekit", - livekit_service_url: "https://lk.example.org", - livekit_alias: "!alias:example.org", - }, - ], - } as unknown as CallMembership; + const { memberships$, membershipsWithTransport$ } = fromMemberships$( + behavior("a", { + a: [bobMembership], + }), + ); + + const connectionManagerData$ = epochMeWith$( + memberships$, + behavior("a", { + a: new ConnectionManagerData(), + }), + ); const matrixLivekitMember$ = createMatrixLivekitMembers$({ scope: testScope, - membershipsWithTransport$: behavior("a", { - a: [ - { - membership: bobMembership, - }, - ], - }), + membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), connectionManager: { - connectionManagerData$: behavior("a", { - a: new ConnectionManagerData(), - }), - transports$: behavior("a", { a: [] }), - connections$: behavior("a", { a: [] }), - }, + connectionManagerData$: connectionManagerData$, + } as unknown as IConnectionManager, matrixRoom: mockMatrixRoom, }); - expectObservable(matrixLivekitMember$).toBe("a", { + expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", { a: expect.toSatisfy((data: MatrixLivekitMember[]) => { - return ( - data.length == 1 && - data[0].membership === bobMembership && - data[0].participant === undefined && - data[0].connection === undefined - ); + expect(data.length).toEqual(1); + expectObservable(data[0].membership$).toBe("a", { + a: bobMembership, + }); + expectObservable(data[0].participant$).toBe("a", { + a: null, + }); + expectObservable(data[0].connection$).toBe("a", { + a: null, + }); + return true; }), }); }); }); -function aConnectionManager( - data: ConnectionManagerData, - behavior: OurRunHelpers["behavior"], -): IConnectionManager { - return { - connectionManagerData$: behavior("a", { a: data }), - transports$: behavior("a", { - a: data.getConnections().map((connection) => connection.transport), +// Helper to create epoch'ed memberships$ and membershipsWithTransport$ from memberships observable. +function fromMemberships$(m$: Observable): { + memberships$: Observable>; + membershipsWithTransport$: Observable< + Epoch<{ membership: CallMembership; transport?: LivekitTransport }[]> + >; +} { + const memberships$ = m$.pipe(trackEpoch()); + const membershipsWithTransport$ = memberships$.pipe( + mapEpoch((members) => { + return members.map((m) => { + const tr = m.getTransport(m); + return { + membership: m, + transport: + tr?.type === "livekit" ? (tr as LivekitTransport) : undefined, + }; + }); }), - connections$: behavior("a", { a: data.getConnections() }), + ); + return { + memberships$, + membershipsWithTransport$, }; } test("should signal participant on a connection that is publishing", () => { withTestScheduler(({ behavior, expectObservable }) => { - const transport: LivekitTransport = { - type: "livekit", - livekit_service_url: "https://lk.example.org", - livekit_alias: "!alias:example.org", - }; - - const bobMembership = mockCallMembership( - "@bob:example.org", - "DEV000", - transport, - ); - - const connectionWithPublisher = new ConnectionManagerData(); const bobParticipantId = getParticipantId( bobMembership.userId, bobMembership.deviceId, ); + + const { memberships$, membershipsWithTransport$ } = fromMemberships$( + behavior("a", { + a: [bobMembership], + }), + ); + const connection = { - transport: transport, + transport: bobMembership.getTransport(bobMembership), } as unknown as Connection; - connectionWithPublisher.add(connection, [ + const dataWithPublisher = new ConnectionManagerData(); + dataWithPublisher.add(connection, [ mockRemoteParticipant({ identity: bobParticipantId }), ]); + + const connectionManagerData$ = epochMeWith$( + memberships$, + behavior("a", { + a: dataWithPublisher, + }), + ); + const matrixLivekitMember$ = createMatrixLivekitMembers$({ scope: testScope, - membershipsWithTransport$: behavior("a", { - a: [ - { - membership: bobMembership, - transport, - }, - ], - }), - connectionManager: aConnectionManager(connectionWithPublisher, behavior), + membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), + connectionManager: { + connectionManagerData$: connectionManagerData$, + } as unknown as IConnectionManager, matrixRoom: mockMatrixRoom, }); - expectObservable(matrixLivekitMember$).toBe("a", { + expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", { a: expect.toSatisfy((data: MatrixLivekitMember[]) => { expect(data.length).toEqual(1); - expect(data[0].participant).toBeDefined(); - expect(data[0].connection).toBeDefined(); - expect(data[0].membership).toEqual(bobMembership); - expect( - areLivekitTransportsEqual(data[0].connection!.transport, transport), - ).toBe(true); + expectObservable(data[0].membership$).toBe("a", { + a: bobMembership, + }); + expectObservable(data[0].participant$).toBe("a", { + a: expect.toSatisfy((participant) => { + expect(participant).toBeDefined(); + expect(participant!.identity).toEqual(bobParticipantId); + return true; + }), + }); + expectObservable(data[0].connection$).toBe("a", { + a: connection, + }); return true; }), }); @@ -168,47 +218,46 @@ test("should signal participant on a connection that is publishing", () => { test("should signal participant on a connection that is not publishing", () => { withTestScheduler(({ behavior, expectObservable }) => { - const transport: LivekitTransport = { - type: "livekit", - livekit_service_url: "https://lk.example.org", - livekit_alias: "!alias:example.org", - }; - - const bobMembership = mockCallMembership( - "@bob:example.org", - "DEV000", - transport, + const { memberships$, membershipsWithTransport$ } = fromMemberships$( + behavior("a", { + a: [bobMembership], + }), ); - const connectionWithPublisher = new ConnectionManagerData(); - // const bobParticipantId = getParticipantId(bobMembership.userId, bobMembership.deviceId); const connection = { - transport: transport, + transport: bobMembership.getTransport(bobMembership), } as unknown as Connection; - connectionWithPublisher.add(connection, []); + const dataWithPublisher = new ConnectionManagerData(); + dataWithPublisher.add(connection, []); + + const connectionManagerData$ = epochMeWith$( + memberships$, + behavior("a", { + a: dataWithPublisher, + }), + ); + const matrixLivekitMember$ = createMatrixLivekitMembers$({ scope: testScope, - membershipsWithTransport$: behavior("a", { - a: [ - { - membership: bobMembership, - transport, - }, - ], - }), - connectionManager: aConnectionManager(connectionWithPublisher, behavior), + membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), + connectionManager: { + connectionManagerData$: connectionManagerData$, + } as unknown as IConnectionManager, matrixRoom: mockMatrixRoom, }); - expectObservable(matrixLivekitMember$).toBe("a", { + expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", { a: expect.toSatisfy((data: MatrixLivekitMember[]) => { expect(data.length).toEqual(1); - expect(data[0].participant).not.toBeDefined(); - expect(data[0].connection).toBeDefined(); - expect(data[0].membership).toEqual(bobMembership); - expect( - areLivekitTransportsEqual(data[0].connection!.transport, transport), - ).toBe(true); + expectObservable(data[0].membership$).toBe("a", { + a: bobMembership, + }); + expectObservable(data[0].participant$).toBe("a", { + a: null, + }); + expectObservable(data[0].connection$).toBe("a", { + a: connection, + }); return true; }), }); @@ -218,22 +267,10 @@ test("should signal participant on a connection that is not publishing", () => { describe("Publication edge case", () => { test("bob is publishing in several connections", () => { withTestScheduler(({ behavior, expectObservable }) => { - const transportA: LivekitTransport = { - type: "livekit", - livekit_service_url: "https://lk.example.org", - livekit_alias: "!alias:example.org", - }; - - const transportB: LivekitTransport = { - type: "livekit", - livekit_service_url: "https://lk.sample.com", - livekit_alias: "!alias:sample.com", - }; - - const bobMembership = mockCallMembership( - "@bob:example.org", - "DEV000", - transportA, + const { memberships$, membershipsWithTransport$ } = fromMemberships$( + behavior("a", { + a: [bobMembership, carlMembership], + }), ); const connectionWithPublisher = new ConnectionManagerData(); @@ -254,60 +291,57 @@ describe("Publication edge case", () => { connectionWithPublisher.add(connectionB, [ mockRemoteParticipant({ identity: bobParticipantId }), ]); + + const connectionManagerData$ = epochMeWith$( + memberships$, + behavior("a", { + a: connectionWithPublisher, + }), + ); + const matrixLivekitMember$ = createMatrixLivekitMembers$({ scope: testScope, - membershipsWithTransport$: behavior("a", { - a: [ - { - membership: bobMembership, - transport: transportA, - }, - ], - }), - connectionManager: aConnectionManager( - connectionWithPublisher, - behavior, + membershipsWithTransport$: testScope.behavior( + membershipsWithTransport$, ), + connectionManager: { + connectionManagerData$: connectionManagerData$, + } as unknown as IConnectionManager, matrixRoom: mockMatrixRoom, }); - expectObservable(matrixLivekitMember$).toBe("a", { - a: expect.toSatisfy((data: MatrixLivekitMember[]) => { - expect(data.length).toEqual(1); - expect(data[0].participant).toBeDefined(); - expect(data[0].participant!.identity).toEqual(bobParticipantId); - expect(data[0].connection).toBeDefined(); - expect(data[0].membership).toEqual(bobMembership); - expect( - areLivekitTransportsEqual( - data[0].connection!.transport, - transportA, - ), - ).toBe(true); - return true; - }), - }); + expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe( + "a", + { + a: expect.toSatisfy((data: MatrixLivekitMember[]) => { + expect(data.length).toEqual(2); + expectObservable(data[0].membership$).toBe("a", { + a: bobMembership, + }); + expectObservable(data[0].connection$).toBe("a", { + // The real connection should be from transportA as per the membership + a: connectionA, + }); + expectObservable(data[0].participant$).toBe("a", { + a: expect.toSatisfy((participant) => { + expect(participant).toBeDefined(); + expect(participant!.identity).toEqual(bobParticipantId); + return true; + }), + }); + return true; + }), + }, + ); }); }); test("bob is publishing in the wrong connection", () => { withTestScheduler(({ behavior, expectObservable }) => { - const transportA: LivekitTransport = { - type: "livekit", - livekit_service_url: "https://lk.example.org", - livekit_alias: "!alias:example.org", - }; - - const transportB: LivekitTransport = { - type: "livekit", - livekit_service_url: "https://lk.sample.com", - livekit_alias: "!alias:sample.com", - }; - - const bobMembership = mockCallMembership( - "@bob:example.org", - "DEV000", - transportA, + const { memberships$, membershipsWithTransport$ } = fromMemberships$( + behavior("a", { + a: [bobMembership, carlMembership], + }), ); const connectionWithPublisher = new ConnectionManagerData(); @@ -315,86 +349,54 @@ describe("Publication edge case", () => { bobMembership.userId, bobMembership.deviceId, ); - const connectionA = { - transport: transportA, - } as unknown as Connection; - const connectionB = { - transport: transportB, - } as unknown as Connection; + const connectionA = { transport: transportA } as unknown as Connection; + const connectionB = { transport: transportB } as unknown as Connection; + // Bob is not publishing on A connectionWithPublisher.add(connectionA, []); + // Bob is publishing on B but his membership says A connectionWithPublisher.add(connectionB, [ mockRemoteParticipant({ identity: bobParticipantId }), ]); + + const connectionManagerData$ = epochMeWith$( + memberships$, + behavior("a", { + a: connectionWithPublisher, + }), + ); + const matrixLivekitMember$ = createMatrixLivekitMembers$({ scope: testScope, - membershipsWithTransport$: behavior("a", { - a: [ - { - membership: bobMembership, - transport: transportA, - }, - ], - }), - connectionManager: aConnectionManager( - connectionWithPublisher, - behavior, + membershipsWithTransport$: testScope.behavior( + membershipsWithTransport$, ), + connectionManager: { + connectionManagerData$: connectionManagerData$, + } as unknown as IConnectionManager, matrixRoom: mockMatrixRoom, }); - expectObservable(matrixLivekitMember$).toBe("a", { - a: expect.toSatisfy((data: MatrixLivekitMember[]) => { - expect(data.length).toEqual(1); - expect(data[0].participant).not.toBeDefined(); - expect(data[0].connection).toBeDefined(); - expect(data[0].membership).toEqual(bobMembership); - expect( - areLivekitTransportsEqual( - data[0].connection!.transport, - transportA, - ), - ).toBe(true); - return true; - }), - }); + expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe( + "a", + { + a: expect.toSatisfy((data: MatrixLivekitMember[]) => { + expect(data.length).toEqual(2); + expectObservable(data[0].membership$).toBe("a", { + a: bobMembership, + }); + expectObservable(data[0].connection$).toBe("a", { + // The real connection should be from transportA as per the membership + a: connectionA, + }); + expectObservable(data[0].participant$).toBe("a", { + // No participant as Bob is not publishing on his membership transport + a: null, + }); + return true; + }), + }, + ); }); - - // let lastMatrixLkItems: MatrixLivekitMember[] = []; - // matrixLivekitMerger.matrixLivekitMember$.subscribe((items) => { - // lastMatrixLkItems = items; - // }); - - // vi.mocked(bobMembership).getTransport = vi - // .fn() - // .mockReturnValue(connectionA.transport); - - // fakeMemberships$.next([bobMembership]); - - // const lkMap = new ConnectionManagerData(); - // lkMap.add(connectionA, []); - // lkMap.add(connectionB, [ - // mockRemoteParticipant({ identity: bobParticipantId }) - // ]); - - // fakeManagerData$.next(lkMap); - - // const items = lastMatrixLkItems; - // expect(items).toHaveLength(1); - // const item = items[0]; - - // // Assert the expected membership - // expect(item.membership.userId).toEqual(bobMembership.userId); - // expect(item.membership.deviceId).toEqual(bobMembership.deviceId); - - // expect(item.participant).not.toBeDefined(); - - // // The transport info should come from the membership transports and not only from the publishing connection - // expect(item.connection?.transport?.livekit_service_url).toEqual( - // bobMembership.transports[0]?.livekit_service_url - // ); - // expect(item.connection?.transport?.livekit_alias).toEqual( - // bobMembership.transports[0]?.livekit_alias - // ); }); }); From 93659931caf309c4b434f198aed7a5bccd321537 Mon Sep 17 00:00:00 2001 From: Valere Date: Mon, 10 Nov 2025 11:20:20 +0100 Subject: [PATCH 39/75] fixup: update integration tests --- .../remoteMembers/integration.test.ts | 103 +++++++++++------- 1 file changed, 63 insertions(+), 40 deletions(-) diff --git a/src/state/CallViewModel/remoteMembers/integration.test.ts b/src/state/CallViewModel/remoteMembers/integration.test.ts index 1d616700..d72505da 100644 --- a/src/state/CallViewModel/remoteMembers/integration.test.ts +++ b/src/state/CallViewModel/remoteMembers/integration.test.ts @@ -12,7 +12,6 @@ import EventEmitter from "events"; import fetchMock from "fetch-mock"; import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { type Room as MatrixRoom, type RoomMember } from "matrix-js-sdk"; -import { logger } from "matrix-js-sdk/lib/logger"; import { type Epoch, @@ -150,14 +149,22 @@ test("bob, carl, then bob joining no tracks yet", () => { const items = e.value; expect(items.length).toBe(1); const item = items[0]!; - expect(item.membership).toStrictEqual(bobMembership); - expect( - areLivekitTransportsEqual( - item.connection!.transport, - bobMembership.transports[0]! as LivekitTransport, - ), - ).toBe(true); - expect(item.participant).toBeUndefined(); + expectObservable(item.membership$).toBe("a", { + a: bobMembership, + }); + expectObservable(item.connection$).toBe("a", { + a: expect.toSatisfy((co) => { + expect( + areLivekitTransportsEqual( + co.transport, + bobMembership.transports[0]! as LivekitTransport, + ), + ); + }), + }); + expectObservable(item.participant$).toBe("a", { + a: null, + }); return true; }), b: expect.toSatisfy((e: Epoch) => { @@ -166,51 +173,67 @@ test("bob, carl, then bob joining no tracks yet", () => { { const item = items[0]!; - expect(item.membership).toStrictEqual(bobMembership); - expect(item.participant).toBeUndefined(); + expectObservable(item.membership$).toBe("a", { + a: bobMembership, + }); + expectObservable(item.participant$).toBe("a", { + a: null, + }); } { const item = items[1]!; - expect(item.membership).toStrictEqual(carlMembership); - expect(item.participantId).toStrictEqual( - `${carlMembership.userId}:${carlMembership.deviceId}`, - ); - expect( - areLivekitTransportsEqual( - item.connection!.transport, - carlMembership.transports[0]! as LivekitTransport, - ), - ).toBe(true); - expect(item.participant).toBeUndefined(); + + expectObservable(item.membership$).toBe("a", { + a: carlMembership, + }); + expectObservable(item.participant$).toBe("a", { + a: null, + }); + expectObservable(item.connection$).toBe("a", { + a: expect.toSatisfy((connection) => { + expect( + areLivekitTransportsEqual( + connection.transport, + carlMembership.transports[0]! as LivekitTransport, + ), + ).toBe(true); + return true; + }), + }); } return true; }), c: expect.toSatisfy((e: Epoch) => { const items = e.value; - logger.info(`E Items length: ${items.length}`); expect(items.length).toBe(3); - { - expect(items[0]!.membership).toStrictEqual(bobMembership); - } - { - expect(items[1]!.membership).toStrictEqual(carlMembership); - } + expectObservable(items[0].membership$).toBe("a", { + a: bobMembership, + }); + expectObservable(items[1].membership$).toBe("b", { + a: carlMembership, + }); { const item = items[2]!; - expect(item.membership).toStrictEqual(daveMembership); - expect(item.participantId).toStrictEqual( - `${daveMembership.userId}:${daveMembership.deviceId}`, - ); - expect( - areLivekitTransportsEqual( - item.connection!.transport, - daveMembership.transports[0]! as LivekitTransport, - ), - ).toBe(true); - expect(item.participant).toBeUndefined(); + expectObservable(item.membership$).toBe("a", { + a: daveMembership, + }); + expectObservable(item.connection$).toBe("a", { + a: expect.toSatisfy((connection) => { + expect( + areLivekitTransportsEqual( + connection.transport, + daveMembership.transports[0]! as LivekitTransport, + ), + ).toBe(true); + return true; + }), + }); + expectObservable(item.participant$).toBe("a", { + a: null, + }); } return true; }), From 93c4dc5beb742473d4bb6b720ebfc3cc07474775 Mon Sep 17 00:00:00 2001 From: Timo K Date: Mon, 10 Nov 2025 15:55:01 +0100 Subject: [PATCH 40/75] make it run --- src/state/CallViewModel/CallViewModel.ts | 52 ++++++++++++------- .../localMember/LocalTransport.ts | 7 +-- .../CallViewModel/remoteMembers/Connection.ts | 14 +++-- .../remoteMembers/ConnectionManager.ts | 2 +- .../remoteMembers/MatrixLivekitMembers.ts | 12 +++-- src/state/ObservableScope.test.ts | 27 +++++++++- 6 files changed, 80 insertions(+), 34 deletions(-) diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index f26c4b3b..b508ff80 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -268,19 +268,15 @@ export class CallViewModel { }); // ------------------------------------------------------------------------ - // CallNotificationLifecycle - // consider inlining these!!! - private sentCallNotification$ = createSentCallNotification$( - this.scope, - this.matrixRTCSession, - ); - private receivedDecline$ = createReceivedDecline$(this.matrixRoom); private callLifecycle = createCallNotificationLifecycle$({ scope: this.scope, memberships$: this.memberships$, - sentCallNotification$: this.sentCallNotification$, - receivedDecline$: this.receivedDecline$, + sentCallNotification$: createSentCallNotification$( + this.scope, + this.matrixRTCSession, + ), + receivedDecline$: createReceivedDecline$(this.matrixRoom), options: this.options, localUser: { userId: this.userId, deviceId: this.deviceId }, }); @@ -331,24 +327,44 @@ export class CallViewModel { public readonly audioParticipants$ = this.scope.behavior( this.matrixLivekitMembers$.pipe( + switchMap((membersWithEpoch) => { + const members = membersWithEpoch.value; + const a$ = combineLatest( + members.map((member) => + combineLatest([member.connection$, member.participant$]).pipe( + map(([connection, participant]) => { + // do not render audio for local participant + if (!connection || !participant || participant.isLocal) + return null; + const livekitRoom = connection.livekitRoom; + const url = connection.transport.livekit_service_url; + + return { url, livekitRoom, participant: participant.identity }; + }), + ), + ), + ); + return a$; + }), map((members) => - members.value.reduce((acc, curr) => { - const url = curr.connection?.transport.livekit_service_url; - const livekitRoom = curr.connection?.livekitRoom; - const participant = curr.participant?.identity; + members.reduce((acc, curr) => { + if (!curr) return acc; - if (!url || !livekitRoom || !participant) return acc; - - const existing = acc.find((item) => item.url === url); + const existing = acc.find((item) => item.url === curr.url); if (existing) { - existing.participants.push(participant); + existing.participants.push(curr.participant); } else { - acc.push({ livekitRoom, participants: [participant], url }); + acc.push({ + livekitRoom: curr.livekitRoom, + participants: [curr.participant], + url: curr.url, + }); } return acc; }, []), ), ), + [], ); public readonly handsRaised$ = this.scope.behavior( diff --git a/src/state/CallViewModel/localMember/LocalTransport.ts b/src/state/CallViewModel/localMember/LocalTransport.ts index b1fd71e9..94c89deb 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.ts @@ -16,14 +16,9 @@ import { type MatrixClient } from "matrix-js-sdk"; import { combineLatest, distinctUntilChanged, first, from, map } from "rxjs"; import { logger } from "matrix-js-sdk/lib/logger"; import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery"; -import { deepCompare } from "matrix-js-sdk/lib/utils"; import { type Behavior } from "../../Behavior.ts"; -import { - Epoch, - mapEpoch, - type ObservableScope, -} from "../../ObservableScope.ts"; +import { type Epoch, type ObservableScope } from "../../ObservableScope.ts"; import { Config } from "../../../config/Config.ts"; import { MatrixRTCTransportMissingError } from "../../../utils/errors.ts"; import { getSFUConfigWithOpenID } from "../../../livekit/openIDSFU.ts"; diff --git a/src/state/CallViewModel/remoteMembers/Connection.ts b/src/state/CallViewModel/remoteMembers/Connection.ts index cae45d4a..60251541 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.ts @@ -18,7 +18,7 @@ import { RoomEvent, } from "livekit-client"; import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; -import { BehaviorSubject, type Observable } from "rxjs"; +import { BehaviorSubject, map, type Observable } from "rxjs"; import { type Logger } from "matrix-js-sdk/lib/logger"; import { @@ -184,7 +184,7 @@ export class Connection { * It filters the participants to only those that are associated with a membership that claims to publish on this connection. */ - public readonly participantsWithTrack$: Behavior; + public readonly participants$: Behavior; /** * The media transport to connect to. @@ -211,13 +211,19 @@ export class Connection { this.transport = transport; this.client = client; - this.participantsWithTrack$ = scope.behavior( + this.participants$ = scope.behavior( + // only tracks remote participants connectedParticipantsObserver(this.livekitRoom, { additionalRoomEvents: [ RoomEvent.TrackPublished, RoomEvent.TrackUnpublished, ], - }), + }).pipe( + map((participants) => [ + this.livekitRoom.localParticipant, + ...participants, + ]), + ), [], ); diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts index ce984aec..73ca3d16 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts @@ -181,7 +181,7 @@ export function createConnectionManager$({ // Map the connections to list of {connection, participants}[] const listOfConnectionsWithPublishingParticipants = connections.value.map((connection) => { - return connection.participantsWithTrack$.pipe( + return connection.participants$.pipe( map((participants) => ({ connection, participants, diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts index 764862f2..fbfd0563 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts @@ -105,6 +105,9 @@ export function createMatrixLivekitMembers$({ new Epoch([membershipsWithTransports, managerData] as const, epoch), ), generateItemsWithEpoch( + // Generator function. + // creates an array of `{key, data}[]` + // Each change in the keys (new key, missing key) will result in a call to the factory function. function* ([membershipsWithTransports, managerData]) { for (const { membership, transport } of membershipsWithTransports) { // TODO! cannot use membership.membershipID yet, Currently its hardcoded by the jwt service to @@ -125,8 +128,11 @@ export function createMatrixLivekitMembers$({ }; } }, + // Each update where the key of the generator array do not change will result in updates to the `data$` observable in the factory. (scope, data$, participantId, userId) => { const member = matrixRoom.getMember(userId); + // will only get called once per `participantId, userId` pair. + // updates to data$ and as a result to displayName$ and mxcAvatarUrl$ are more frequent. return { participantId, userId, @@ -134,11 +140,9 @@ export function createMatrixLivekitMembers$({ displayName$: scope.behavior( displaynameMap$.pipe( map((displayNames) => { - const name = displayNames.get(userId); - if (name === undefined) { + const name = displayNames.get(userId) ?? ""; + if (name === "") logger.warn(`No display name for user ${userId}`); - return ""; - } return name; }), ), diff --git a/src/state/ObservableScope.test.ts b/src/state/ObservableScope.test.ts index 4b0f3b4f..b41f47c2 100644 --- a/src/state/ObservableScope.test.ts +++ b/src/state/ObservableScope.test.ts @@ -6,7 +6,8 @@ Please see LICENSE in the repository root for full details. */ import { describe, expect, it } from "vitest"; -import { BehaviorSubject, timer } from "rxjs"; +import { BehaviorSubject, combineLatest, timer } from "rxjs"; +import { logger } from "matrix-js-sdk/lib/logger"; import { Epoch, @@ -72,4 +73,28 @@ describe("Epoch", () => { scope.behavior(a$, undefined); }); + + it("diamonds emits in a predictable order", () => { + const sb$ = new BehaviorSubject("initial"); + const root$ = sb$.pipe(trackEpoch()); + const derivedA$ = root$.pipe(mapEpoch((e) => e + "-A")); + const derivedB$ = root$.pipe(mapEpoch((e) => e + "-B")); + combineLatest([root$, derivedB$, derivedA$]).subscribe( + ([root, derivedA, derivedB]) => { + logger.log( + "combined" + + root.epoch + + root.value + + "\n" + + derivedA.epoch + + derivedA.value + + "\n" + + derivedB.epoch + + derivedB.value, + ); + }, + ); + sb$.next("updated"); + sb$.next("ANOTERUPDATE"); + }); }); From 85f659bcc902657dc920b3f8e034433ba42959c4 Mon Sep 17 00:00:00 2001 From: Timo K Date: Tue, 11 Nov 2025 15:51:48 +0100 Subject: [PATCH 41/75] Introduce MatrixMemberMetadata and use it to simplify username and avatar computation This removes member from the tiles entirely! --- src/rtcSessionHelpers.ts | 160 ------------------ src/state/Behavior.ts | 4 - src/state/CallViewModel/CallViewModel.ts | 49 +++--- .../localMember/LocalMembership.ts | 75 +++++++- .../CallViewModel/localMember/Publisher.ts | 6 + .../remoteMembers/MatrixLivekitMembers.ts | 52 +----- .../remoteMembers/MatrixMemberMetadata.ts | 148 ++++++++++++++++ .../remoteMembers/displayname.ts | 76 --------- src/utils/displayname.ts | 8 +- 9 files changed, 256 insertions(+), 322 deletions(-) delete mode 100644 src/rtcSessionHelpers.ts create mode 100644 src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.ts delete mode 100644 src/state/CallViewModel/remoteMembers/displayname.ts diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts deleted file mode 100644 index a53418f7..00000000 --- a/src/rtcSessionHelpers.ts +++ /dev/null @@ -1,160 +0,0 @@ -/* -Copyright 2023, 2024 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - type MatrixRTCSession, - isLivekitTransportConfig, - type LivekitTransportConfig, - type LivekitTransport, -} from "matrix-js-sdk/lib/matrixrtc"; -import { logger } from "matrix-js-sdk/lib/logger"; -import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery"; -import { type MatrixClient } from "matrix-js-sdk"; - -import { PosthogAnalytics } from "./analytics/PosthogAnalytics"; -import { Config } from "./config/Config"; -import { ElementWidgetActions, widget } from "./widget"; -import { MatrixRTCTransportMissingError } from "./utils/errors"; -import { getUrlParams } from "./UrlParams"; -import { getSFUConfigWithOpenID } from "./livekit/openIDSFU.ts"; -import { MatrixRTCMode } from "./settings/settings.ts"; - -const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci"; - -async function makeTransportInternal( - client: MatrixClient, - roomId: string, -): Promise { - logger.log("Searching for a preferred transport"); - //TODO refactor this to use the jwt service returned alias. - const livekitAlias = roomId; - - // TODO-MULTI-SFU: Either remove this dev tool or make it more official - const urlFromStorage = - localStorage.getItem("robin-matrixrtc-auth") ?? - localStorage.getItem("timo-focus-url"); - if (urlFromStorage !== null) { - const transportFromStorage: LivekitTransport = { - type: "livekit", - livekit_service_url: urlFromStorage, - livekit_alias: livekitAlias, - }; - logger.log( - "Using LiveKit transport from local storage: ", - transportFromStorage, - ); - return transportFromStorage; - } - - // Prioritize the .well-known/matrix/client, if available, over the configured SFU - const domain = client.getDomain(); - if (domain) { - // we use AutoDiscovery instead of relying on the MatrixClient having already - // been fully configured and started - const wellKnownFoci = (await AutoDiscovery.getRawClientConfig(domain))?.[ - FOCI_WK_KEY - ]; - if (Array.isArray(wellKnownFoci)) { - const transport: LivekitTransportConfig | undefined = wellKnownFoci.find( - (f) => f && isLivekitTransportConfig(f), - ); - if (transport !== undefined) { - logger.log("Using LiveKit transport from .well-known: ", transport); - return { ...transport, livekit_alias: livekitAlias }; - } - } - } - - const urlFromConf = Config.get().livekit?.livekit_service_url; - if (urlFromConf) { - const transportFromConf: LivekitTransport = { - type: "livekit", - livekit_service_url: urlFromConf, - livekit_alias: livekitAlias, - }; - logger.log("Using LiveKit transport from config: ", transportFromConf); - return transportFromConf; - } - - throw new MatrixRTCTransportMissingError(domain ?? ""); -} - -export async function makeTransport( - client: MatrixClient, - roomId: string, -): Promise { - const transport = await makeTransportInternal(client, roomId); - // this will call the jwt/sfu/get endpoint to pre create the livekit room. - await getSFUConfigWithOpenID( - client, - transport.livekit_service_url, - transport.livekit_alias, - ); - return transport; -} - -export interface EnterRTCSessionOptions { - encryptMedia: boolean; - matrixRTCMode: MatrixRTCMode; -} - -/** - * TODO! document this function properly - * @param rtcSession - * @param transport - * @param options - */ -export async function enterRTCSession( - rtcSession: MatrixRTCSession, - transport: LivekitTransport, - { encryptMedia, matrixRTCMode }: EnterRTCSessionOptions, -): Promise { - PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date()); - PosthogAnalytics.instance.eventCallStarted.track(rtcSession.room.roomId); - - // This must be called before we start trying to join the call, as we need to - // have started tracking by the time calls start getting created. - // groupCallOTelMembership?.onJoinCall(); - - const { features, matrix_rtc_session: matrixRtcSessionConfig } = Config.get(); - const useDeviceSessionMemberEvents = - features?.feature_use_device_session_member_events; - const { sendNotificationType: notificationType, callIntent } = getUrlParams(); - const multiSFU = matrixRTCMode !== MatrixRTCMode.Legacy; - // Multi-sfu does not need a preferred foci list. just the focus that is actually used. - rtcSession.joinRoomSession( - multiSFU ? [] : [transport], - multiSFU ? transport : undefined, - { - notificationType, - callIntent, - manageMediaKeys: encryptMedia, - ...(useDeviceSessionMemberEvents !== undefined && { - useLegacyMemberEvents: !useDeviceSessionMemberEvents, - }), - delayedLeaveEventRestartMs: - matrixRtcSessionConfig?.delayed_leave_event_restart_ms, - delayedLeaveEventDelayMs: - matrixRtcSessionConfig?.delayed_leave_event_delay_ms, - delayedLeaveEventRestartLocalTimeoutMs: - matrixRtcSessionConfig?.delayed_leave_event_restart_local_timeout_ms, - networkErrorRetryMs: matrixRtcSessionConfig?.network_error_retry_ms, - makeKeyDelay: matrixRtcSessionConfig?.wait_for_key_rotation_ms, - membershipEventExpiryMs: - matrixRtcSessionConfig?.membership_event_expiry_ms, - useExperimentalToDeviceTransport: true, - unstableSendStickyEvents: matrixRTCMode === MatrixRTCMode.Matrix_2_0, - }, - ); - if (widget) { - try { - await widget.api.transport.send(ElementWidgetActions.JoinCall, {}); - } catch (e) { - logger.error("Failed to send join action", e); - } - } -} diff --git a/src/state/Behavior.ts b/src/state/Behavior.ts index 71b18a55..3c88dc00 100644 --- a/src/state/Behavior.ts +++ b/src/state/Behavior.ts @@ -18,10 +18,6 @@ import { BehaviorSubject } from "rxjs"; */ export type Behavior = Omit, "next" | "observers">; -export type BehaviorWithEpoch = Behavior & { - pipeEpoch(): Behavior<{ value: T; epoch: number }>; -}; - /** * Creates a Behavior which never changes in value. */ diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index b508ff80..76eeaeac 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -109,7 +109,10 @@ import { createReceivedDecline$, createSentCallNotification$, } from "./CallNotificationLifecycle.ts"; -import { createRoomMembers$ } from "./remoteMembers/displayname.ts"; +import { + createMatrixMemberMetadata$, + createRoomMembers$, +} from "./remoteMembers/MatrixMemberMetadata.ts"; const logger = rootLogger.getChild("[CallViewModel]"); //TODO @@ -240,7 +243,6 @@ export class CallViewModel { membershipsWithTransport$: this.membershipsAndTransports.membershipsWithTransport$, connectionManager: this.connectionManager, - matrixRoom: this.matrixRoom, }); private connectOptions$ = this.scope.behavior( @@ -280,11 +282,9 @@ export class CallViewModel { options: this.options, localUser: { userId: this.userId, deviceId: this.deviceId }, }); - + public autoLeave$ = this.callLifecycle.autoLeave$; // ------------------------------------------------------------------------ - // ROOM MEMBER tracking TODO - // eslint-disable-next-line @typescript-eslint/no-unused-vars - private roomMembers$ = createRoomMembers$(this.scope, this.matrixRoom); + /** * If there is a configuration error with the call (e.g. misconfigured E2EE). * This is a fatal error that prevents the call from being created/joined. @@ -305,14 +305,6 @@ export class CallViewModel { "user" | "timeout" | "decline" | "allOthersLeft" >(); - /** - * Whether we are joined to the call. This reflects our local state rather - * than whether all connections are truly up and running. - */ - // DISCUSS ? lets think why we need joined and how to do it better - // eslint-disable-next-line @typescript-eslint/no-unused-vars - private readonly joined$ = this.localMembership.connected$; - /** * Whether various media/event sources should pretend to be disconnected from * all network input, even if their connection still technically works. @@ -385,6 +377,14 @@ export class CallViewModel { ), ); + private roomMembers$ = createRoomMembers$(this.scope, this.matrixRoom); + + private matrixMemberMetadataStore = createMatrixMemberMetadata$( + this.scope, + this.scope.behavior(this.memberships$.pipe(map((mems) => mems.value))), + this.roomMembers$, + ); + /** * List of user media (camera feeds) that we want tiles for. */ @@ -400,20 +400,10 @@ export class CallViewModel { userId, participant$, connection$, - displayName$, - mxcAvatarUrl$, } of matrixLivekitMembers) for (let dup = 0; dup < 1 + duplicateTiles; dup++) yield { - keys: [ - dup, - participantId, - userId, - participant$, - connection$, - displayName$, - mxcAvatarUrl$, - ], + keys: [dup, participantId, userId, participant$, connection$], data: undefined, }; }, @@ -425,8 +415,6 @@ export class CallViewModel { userId, participant$, connection$, - displayName$, - mxcAvatarUrl$, ) => { const livekitRoom$ = scope.behavior( connection$.pipe(map((c) => c?.livekitRoom)), @@ -434,6 +422,11 @@ export class CallViewModel { const focusUrl$ = scope.behavior( connection$.pipe(map((c) => c?.transport.livekit_service_url)), ); + const displayName$ = scope.behavior( + this.matrixMemberMetadataStore + .createDisplayNameBehavior$(userId) + .pipe(map((name) => name ?? userId)), + ); return new UserMedia( scope, @@ -446,7 +439,7 @@ export class CallViewModel { this.mediaDevices, this.pretendToBeDisconnected$, displayName$, - mxcAvatarUrl$, + this.matrixMemberMetadataStore.createAvatarUrlBehavior$(userId), this.handsRaised$.pipe(map((v) => v[participantId]?.time ?? null)), this.reactions$.pipe(map((v) => v[participantId] ?? undefined)), ); diff --git a/src/state/CallViewModel/localMember/LocalMembership.ts b/src/state/CallViewModel/localMember/LocalMembership.ts index c6b8b170..df4d3b6b 100644 --- a/src/state/CallViewModel/localMember/LocalMembership.ts +++ b/src/state/CallViewModel/localMember/LocalMembership.ts @@ -43,14 +43,17 @@ import { type MuteStates } from "../../MuteStates"; import { type ProcessorState } from "../../../livekit/TrackProcessorContext"; import { type MediaDevices } from "../../MediaDevices"; import { and$ } from "../../../utils/observable"; -import { - enterRTCSession, - type EnterRTCSessionOptions, -} from "../../../rtcSessionHelpers"; import { type ElementCallError } from "../../../utils/errors"; -import { ElementWidgetActions, type WidgetHelpers } from "../../../widget"; +import { + ElementWidgetActions, + widget, + type WidgetHelpers, +} from "../../../widget"; import { areLivekitTransportsEqual } from "../remoteMembers/MatrixLivekitMembers"; import { getUrlParams } from "../../../UrlParams.ts"; +import { PosthogAnalytics } from "../../../analytics/PosthogAnalytics.ts"; +import { MatrixRTCMode } from "../../../settings/settings.ts"; +import { Config } from "../../../config/Config.ts"; export enum LivekitState { Uninitialized = "uninitialized", @@ -535,3 +538,65 @@ export function observeSharingScreen$(p: Participant): Observable { ParticipantEvent.LocalTrackUnpublished, ).pipe(map((p) => p.isScreenShareEnabled)); } + +interface EnterRTCSessionOptions { + encryptMedia: boolean; + matrixRTCMode: MatrixRTCMode; +} + +/** + * TODO! document this function properly + * @param rtcSession + * @param transport + * @param options + */ +async function enterRTCSession( + rtcSession: MatrixRTCSession, + transport: LivekitTransport, + { encryptMedia, matrixRTCMode }: EnterRTCSessionOptions, +): Promise { + PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date()); + PosthogAnalytics.instance.eventCallStarted.track(rtcSession.room.roomId); + + // This must be called before we start trying to join the call, as we need to + // have started tracking by the time calls start getting created. + // groupCallOTelMembership?.onJoinCall(); + + const { features, matrix_rtc_session: matrixRtcSessionConfig } = Config.get(); + const useDeviceSessionMemberEvents = + features?.feature_use_device_session_member_events; + const { sendNotificationType: notificationType, callIntent } = getUrlParams(); + const multiSFU = matrixRTCMode !== MatrixRTCMode.Legacy; + // Multi-sfu does not need a preferred foci list. just the focus that is actually used. + rtcSession.joinRoomSession( + multiSFU ? [] : [transport], + multiSFU ? transport : undefined, + { + notificationType, + callIntent, + manageMediaKeys: encryptMedia, + ...(useDeviceSessionMemberEvents !== undefined && { + useLegacyMemberEvents: !useDeviceSessionMemberEvents, + }), + delayedLeaveEventRestartMs: + matrixRtcSessionConfig?.delayed_leave_event_restart_ms, + delayedLeaveEventDelayMs: + matrixRtcSessionConfig?.delayed_leave_event_delay_ms, + delayedLeaveEventRestartLocalTimeoutMs: + matrixRtcSessionConfig?.delayed_leave_event_restart_local_timeout_ms, + networkErrorRetryMs: matrixRtcSessionConfig?.network_error_retry_ms, + makeKeyDelay: matrixRtcSessionConfig?.wait_for_key_rotation_ms, + membershipEventExpiryMs: + matrixRtcSessionConfig?.membership_event_expiry_ms, + useExperimentalToDeviceTransport: true, + unstableSendStickyEvents: matrixRTCMode === MatrixRTCMode.Matrix_2_0, + }, + ); + if (widget) { + try { + await widget.api.transport.send(ElementWidgetActions.JoinCall, {}); + } catch (e) { + logger.error("Failed to send join action", e); + } + } +} diff --git a/src/state/CallViewModel/localMember/Publisher.ts b/src/state/CallViewModel/localMember/Publisher.ts index c10201bf..1c436397 100644 --- a/src/state/CallViewModel/localMember/Publisher.ts +++ b/src/state/CallViewModel/localMember/Publisher.ts @@ -74,6 +74,12 @@ export class Publisher { this.observeMediaDevices(scope, devices, controlledAudioDevices); this.workaroundRestartAudioInputTrackChrome(devices, scope); + this.scope.onEnd(() => { + this.logger?.info( + "[PublishConnection] Scope ended -> stop publishing all tracks", + ); + void this.stopPublishing(); + }); } /** diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts index fbfd0563..326eb0f6 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts @@ -13,19 +13,17 @@ import { type LivekitTransport, type CallMembership, } from "matrix-js-sdk/lib/matrixrtc"; -import { combineLatest, filter, fromEvent, map, startWith } from "rxjs"; -// eslint-disable-next-line rxjs/no-internal -import { type NodeStyleEventEmitter } from "rxjs/internal/observable/fromEvent"; -import { RoomStateEvent, type Room as MatrixRoom } from "matrix-js-sdk"; -import { logger } from "matrix-js-sdk/lib/logger"; +import { combineLatest, filter, map } from "rxjs"; +import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { type Behavior } from "../../Behavior"; import { type IConnectionManager } from "./ConnectionManager"; import { Epoch, type ObservableScope } from "../../ObservableScope"; -import { memberDisplaynames$ } from "./displayname"; import { type Connection } from "./Connection"; import { generateItemsWithEpoch } from "../../../utils/observable"; +const logger = rootLogger.getChild("MatrixLivekitMembers"); + /** * Represents a Matrix call member and their associated LiveKit participation. * `livekitParticipant` can be undefined if the member is not yet connected to the livekit room @@ -39,8 +37,6 @@ export interface MatrixLivekitMember { LocalLivekitParticipant | RemoteLivekitParticipant | null >; connection$: Behavior; - displayName$: Behavior; - mxcAvatarUrl$: Behavior; } interface Props { @@ -49,16 +45,7 @@ interface Props { Epoch<{ membership: CallMembership; transport?: LivekitTransport }[]> >; connectionManager: IConnectionManager; - // TODO this is too much information for that class, - // apparently needed to get a room member to later get the Avatar - // => Extract an AvatarService instead? - // Better with just `getMember` - matrixRoom: Pick & NodeStyleEventEmitter; - // roomMember$: Behavior>; } -// Alternative structure idea: -// const livekitMatrixMember$ = (callMemberships$,connectionManager,scope): Observable => { - /** * Combines MatrixRTC and Livekit worlds. * @@ -73,22 +60,11 @@ export function createMatrixLivekitMembers$({ scope, membershipsWithTransport$, connectionManager, - matrixRoom, }: Props): Behavior> { /** * Stream of all the call members and their associated livekit data (if available). */ - const displaynameMap$ = memberDisplaynames$( - scope, - matrixRoom, - scope.behavior( - membershipsWithTransport$.pipe( - map((ms) => ms.value.map((m) => m.membership)), - ), - ), - ); - return scope.behavior( combineLatest([ membershipsWithTransport$, @@ -130,29 +106,15 @@ export function createMatrixLivekitMembers$({ }, // Each update where the key of the generator array do not change will result in updates to the `data$` observable in the factory. (scope, data$, participantId, userId) => { - const member = matrixRoom.getMember(userId); + logger.debug( + `Updating data$ for participantId: ${participantId}, userId: ${userId}`, + ); // will only get called once per `participantId, userId` pair. // updates to data$ and as a result to displayName$ and mxcAvatarUrl$ are more frequent. return { participantId, userId, ...scope.splitBehavior(data$), - displayName$: scope.behavior( - displaynameMap$.pipe( - map((displayNames) => { - const name = displayNames.get(userId) ?? ""; - if (name === "") - logger.warn(`No display name for user ${userId}`); - return name; - }), - ), - ), - mxcAvatarUrl$: scope.behavior( - fromEvent(matrixRoom, RoomStateEvent.Members).pipe( - startWith(undefined), - map(() => member?.getMxcAvatarUrl()), - ), - ), }; }, ), diff --git a/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.ts b/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.ts new file mode 100644 index 00000000..ad603708 --- /dev/null +++ b/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.ts @@ -0,0 +1,148 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type RoomMember, RoomStateEvent } from "matrix-js-sdk"; +import { combineLatest, fromEvent, map } from "rxjs"; +import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; +import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; +import { type Room as MatrixRoom } from "matrix-js-sdk/lib/matrix"; +// eslint-disable-next-line rxjs/no-internal + +import { type ObservableScope } from "../../ObservableScope"; +import { + calculateDisplayName, + shouldDisambiguate, +} from "../../../utils/displayname"; +import { type Behavior } from "../../Behavior"; + +const logger = rootLogger.getChild("[MatrixMemberMetadata]"); + +export type RoomMemberMap = Map< + string, + Pick +>; +export function roomToMembersMap(matrixRoom: MatrixRoom): RoomMemberMap { + return matrixRoom.getMembers().reduce((acc, member) => { + acc.set(member.userId, { + userId: member.userId, + getMxcAvatarUrl: member.getMxcAvatarUrl.bind(member), + rawDisplayName: member.rawDisplayName, + }); + return acc; + }, new Map()); +} + +export function createRoomMembers$( + scope: ObservableScope, + matrixRoom: MatrixRoom, +): Behavior { + return scope.behavior( + fromEvent(matrixRoom, RoomStateEvent.Members).pipe( + map(() => roomToMembersMap(matrixRoom)), + ), + roomToMembersMap(matrixRoom), + ); +} +/** + * Displayname for each member of the call. This will disambiguate + * any displayname that clashes with another member. Only members + * joined to the call are considered here. + * + * @returns Map uses the Matrix user ID as the key. + */ +// don't do this work more times than we need to. This is achieved by converting to a behavior: +export const memberDisplaynames$ = ( + scope: ObservableScope, + memberships$: Behavior[]>, + roomMembers$: Behavior, +): Behavior> => { + // This map tracks userIds that at some point needed disambiguation. + // This is a memory leak bound to the number of participants. + // A call application will always increase the memory if there have been more members in a call. + // Its capped by room member participants. + const shouldDisambiguateTrackerMap = new Set(); + return scope.behavior( + combineLatest([ + // Handle call membership changes + memberships$, + // Additionally handle display name changes (implicitly reacting to them) + roomMembers$, + // TODO: do we need: pauseWhen(this.pretendToBeDisconnected$), + ]).pipe( + map(([memberships, roomMembers]) => { + const displaynameMap = new Map(); + // We only consider RTC members for disambiguation as they are the only visible members. + for (const rtcMember of memberships) { + const member = roomMembers.get(rtcMember.userId); + if (!member) { + logger.error(`Could not find member for user ${rtcMember.userId}`); + continue; + } + const disambiguateComputed = shouldDisambiguate( + member, + memberships, + roomMembers, + ); + + const disambiguate = + shouldDisambiguateTrackerMap.has(rtcMember.userId) || + disambiguateComputed; + if (disambiguate) shouldDisambiguateTrackerMap.add(rtcMember.userId); + displaynameMap.set( + rtcMember.userId, + calculateDisplayName(member, disambiguate), + ); + } + return displaynameMap; + }), + ), + ); +}; + +export const createMatrixMemberMetadata$ = ( + scope: ObservableScope, + memberships$: Behavior[]>, + roomMembers$: Behavior, +): { + createDisplayNameBehavior$: (userId: string) => Behavior; + createAvatarUrlBehavior$: (userId: string) => Behavior; + displaynameMap$: Behavior>; + avatarMap$: Behavior>; +} => { + const displaynameMap$ = memberDisplaynames$( + scope, + memberships$, + roomMembers$, + ); + const avatarMap$ = scope.behavior( + roomMembers$.pipe( + map((roomMembers) => + Array.from(roomMembers.keys()).reduce((acc, key) => { + acc.set(key, roomMembers.get(key)?.getMxcAvatarUrl()); + return acc; + }, new Map()), + ), + ), + ); + return { + createDisplayNameBehavior$: (userId: string) => + scope.behavior( + displaynameMap$.pipe( + map((displaynameMap) => displaynameMap.get(userId)), + ), + ), + createAvatarUrlBehavior$: (userId: string) => + scope.behavior( + roomMembers$.pipe( + map((roomMembers) => roomMembers.get(userId)?.getMxcAvatarUrl()), + ), + ), + // mostly for testing purposes + displaynameMap$, + avatarMap$, + }; +}; diff --git a/src/state/CallViewModel/remoteMembers/displayname.ts b/src/state/CallViewModel/remoteMembers/displayname.ts deleted file mode 100644 index 901f3613..00000000 --- a/src/state/CallViewModel/remoteMembers/displayname.ts +++ /dev/null @@ -1,76 +0,0 @@ -/* -Copyright 2025 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { type RoomMember, RoomStateEvent } from "matrix-js-sdk"; -import { combineLatest, fromEvent, map, startWith } from "rxjs"; -import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; -import { logger } from "matrix-js-sdk/lib/logger"; -import { type Room as MatrixRoom } from "matrix-js-sdk/lib/matrix"; -// eslint-disable-next-line rxjs/no-internal -import { type NodeStyleEventEmitter } from "rxjs/internal/observable/fromEvent"; - -import { type ObservableScope } from "../../ObservableScope"; -import { - calculateDisplayName, - shouldDisambiguate, -} from "../../../utils/displayname"; -import { type Behavior } from "../../Behavior"; - -export function createRoomMembers$( - scope: ObservableScope, - matrixRoom: MatrixRoom, -): Behavior[]> { - return scope.behavior( - fromEvent(matrixRoom, RoomStateEvent.Members).pipe( - map(() => matrixRoom.getMembers()), - ), - [], - ); -} -/** - * Displayname for each member of the call. This will disambiguate - * any displayname that clashes with another member. Only members - * joined to the call are considered here. - * - * @returns Map uses the Matrix user ID as the key. - */ -// don't do this work more times than we need to. This is achieved by converting to a behavior: -export const memberDisplaynames$ = ( - scope: ObservableScope, - matrixRoom: Pick & NodeStyleEventEmitter, - // roomMember$: Behavior>; - memberships$: Behavior, -): Behavior> => - scope.behavior( - combineLatest([ - // Handle call membership changes - memberships$, - // Additionally handle display name changes (implicitly reacting to them) - fromEvent(matrixRoom, RoomStateEvent.Members).pipe(startWith(null)), - // TODO: do we need: pauseWhen(this.pretendToBeDisconnected$), - ]).pipe( - map(([memberships, _displayNames]) => { - const displaynameMap = new Map(); - const room = matrixRoom; - - // We only consider RTC members for disambiguation as they are the only visible members. - for (const rtcMember of memberships) { - const member = room.getMember(rtcMember.userId); - if (member === null) { - logger.error(`Could not find member for user ${rtcMember.userId}`); - continue; - } - const disambiguate = shouldDisambiguate(member, memberships, room); - displaynameMap.set( - rtcMember.userId, - calculateDisplayName(member, disambiguate), - ); - } - return displaynameMap; - }), - ), - ); diff --git a/src/utils/displayname.ts b/src/utils/displayname.ts index 1e141255..5ab5de9b 100644 --- a/src/utils/displayname.ts +++ b/src/utils/displayname.ts @@ -10,7 +10,7 @@ import { removeHiddenChars as removeHiddenCharsUncached, } from "matrix-js-sdk/lib/utils"; -import type { Room } from "matrix-js-sdk"; +import type { RoomMember } from "matrix-js-sdk"; import type { CallMembership } from "matrix-js-sdk/lib/matrixrtc"; // Calling removeHiddenChars() can be slow on Safari, so we cache the results. @@ -40,8 +40,8 @@ function removeHiddenChars(str: string): string { // Borrowed from https://github.com/matrix-org/matrix-js-sdk/blob/f10deb5ef2e8f061ff005af0476034382ea128ca/src/models/room-member.ts#L409 export function shouldDisambiguate( member: { rawDisplayName?: string; userId: string }, - memberships: CallMembership[], - room: Pick, + memberships: Pick[], + roomMembers: Map>, ): boolean { const { rawDisplayName: displayName, userId } = member; if (!displayName || displayName === userId) return false; @@ -65,7 +65,7 @@ export function shouldDisambiguate( // displayname, after hidden character removal. return ( memberships - .map((m) => m.userId && room.getMember(m.userId)) + .map((m) => m.userId && roomMembers.get(m.userId)) // NOTE: We *should* have a room member for everyone. .filter((m) => !!m) .filter((m) => m.userId !== userId) From 8671d3fd675ed855bf2679e6e8929719dcd3887d Mon Sep 17 00:00:00 2001 From: Timo K Date: Tue, 11 Nov 2025 15:52:35 +0100 Subject: [PATCH 42/75] Very bit test overhaul. All displayname tests are now done in the Metadata file. and not in the CallViewModel anymore. --- src/room/GroupCallView.test.tsx | 15 +- src/state/CallViewModel/CallViewModel.test.ts | 124 +--- .../localMember/LocalMembership.test.ts} | 25 +- .../remoteMembers/Connection.test.ts | 259 ++++---- .../MatrixLivekitMembers.test.ts | 20 +- .../MatrixMemberMetadata.test.ts | 611 ++++++++++++++++++ .../remoteMembers/displayname.test.ts | 286 -------- .../remoteMembers/integration.test.ts | 29 +- src/state/CallViewModelWidget.test.ts | 4 +- src/tile/MediaView.test.tsx | 9 +- src/utils/displayname-integration.test.ts | 4 +- src/utils/displayname.test.ts | 47 +- src/utils/test-fixtures.ts | 3 - 13 files changed, 806 insertions(+), 630 deletions(-) rename src/{rtcSessionHelpers.test.ts => state/CallViewModel/localMember/LocalMembership.test.ts} (86%) create mode 100644 src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.test.ts delete mode 100644 src/state/CallViewModel/remoteMembers/displayname.test.ts diff --git a/src/room/GroupCallView.test.tsx b/src/room/GroupCallView.test.tsx index ad884865..1181bde7 100644 --- a/src/room/GroupCallView.test.tsx +++ b/src/room/GroupCallView.test.tsx @@ -77,13 +77,13 @@ const leaveRTCSession = vi.hoisted(() => ), ); -vi.mock("../rtcSessionHelpers", async (importOriginal) => { - // TODO: perhaps there is a more elegant way to manage the type import here? - // eslint-disable-next-line @typescript-eslint/consistent-type-imports - const orig = await importOriginal(); - // TODO: leaveRTCSession no longer exists! Tests need adapting. - return { ...orig, enterRTCSession, leaveRTCSession }; -}); +// vi.mock("../rtcSessionHelpers", async (importOriginal) => { +// // TODO: perhaps there is a more elegant way to manage the type import here? +// // eslint-disable-next-line @typescript-eslint/consistent-type-imports +// const orig = await importOriginal(); +// // TODO: leaveRTCSession no longer exists! Tests need adapting. +// return { ...orig, enterRTCSession, leaveRTCSession }; +// }); let playSound: MockedFunction< NonNullable>["playSound"] @@ -266,6 +266,7 @@ test.skip("GroupCallView leaves the session when an error occurs", async () => { test.skip("GroupCallView shows errors that occur during joining", async () => { const user = userEvent.setup(); + // This should not mock this error that deep. it should only mock the CallViewModel. enterRTCSession.mockRejectedValue(new MatrixRTCTransportMissingError("")); onTestFinished(() => { enterRTCSession.mockReset(); diff --git a/src/state/CallViewModel/CallViewModel.test.ts b/src/state/CallViewModel/CallViewModel.test.ts index 13693dc1..d54b6279 100644 --- a/src/state/CallViewModel/CallViewModel.test.ts +++ b/src/state/CallViewModel/CallViewModel.test.ts @@ -71,8 +71,6 @@ import type { RaisedHandInfo, ReactionInfo } from "../../reactions/index.ts"; import { alice, aliceDoppelganger, - aliceDoppelgangerId, - aliceDoppelgangerRtcMember, aliceId, aliceParticipant, aliceRtcMember, @@ -80,11 +78,7 @@ import { bobId, bobRtcMember, bobZeroWidthSpace, - bobZeroWidthSpaceId, - bobZeroWidthSpaceRtcMember, daveRTL, - daveRTLId, - daveRTLRtcMember, local, localId, localRtcMember, @@ -128,7 +122,7 @@ const yesNo = { const daveRtcMember = mockRtcMembership("@dave:example.org", "DDDD"); const carol = local; -const carolId = localId; + const dave = mockMatrixRoomMember(daveRtcMember, { rawDisplayName: "Dave" }); const daveId = `${dave.userId}:${daveRtcMember.deviceId}`; @@ -437,7 +431,7 @@ test.skip("test missing RTC config error", async () => { }, new BehaviorSubject({} as Record), new BehaviorSubject({} as Record), - of({ processor: undefined, supported: false }), + constant({ processor: undefined, supported: false }), ); const failPromise = Promise.withResolvers(); @@ -1073,120 +1067,6 @@ it("should show at least one tile per MatrixRTCSession", () => { }); }); -test("should disambiguate users with the same displayname", () => { - withTestScheduler(({ behavior, expectObservable }) => { - const scenarioInputMarbles = "abcde"; - const expectedLayoutMarbles = "abcde"; - - withCallViewModel( - { - rtcMembers$: behavior(scenarioInputMarbles, { - a: [localRtcMember], - b: [localRtcMember, aliceRtcMember], - c: [localRtcMember, aliceRtcMember, aliceDoppelgangerRtcMember], - d: [ - localRtcMember, - aliceRtcMember, - aliceDoppelgangerRtcMember, - bobRtcMember, - ], - e: [localRtcMember, aliceDoppelgangerRtcMember, bobRtcMember], - }), - }, - (vm) => { - expectObservable(vm.memberDisplaynames$).toBe(expectedLayoutMarbles, { - // Carol has no displayname - So userId is used. - a: new Map([[carolId, carol.userId]]), - b: new Map([ - [carolId, carol.userId], - [aliceId, alice.rawDisplayName], - ]), - // The second alice joins. - c: new Map([ - [carolId, carol.userId], - [aliceId, "Alice (@alice:example.org)"], - [aliceDoppelgangerId, "Alice (@alice2:example.org)"], - ]), - // Bob also joins - d: new Map([ - [carolId, carol.userId], - [aliceId, "Alice (@alice:example.org)"], - [aliceDoppelgangerId, "Alice (@alice2:example.org)"], - [bobId, bob.rawDisplayName], - ]), - // Alice leaves, and the displayname should reset. - e: new Map([ - [carolId, carol.userId], - [aliceDoppelgangerId, "Alice"], - [bobId, bob.rawDisplayName], - ]), - }); - }, - ); - }); -}); - -test("should disambiguate users with invisible characters", () => { - withTestScheduler(({ behavior, expectObservable }) => { - const scenarioInputMarbles = "ab"; - const expectedLayoutMarbles = "ab"; - - withCallViewModel( - { - rtcMembers$: behavior(scenarioInputMarbles, { - a: [localRtcMember], - b: [localRtcMember, bobRtcMember, bobZeroWidthSpaceRtcMember], - }), - }, - (vm) => { - expectObservable(vm.memberDisplaynames$).toBe(expectedLayoutMarbles, { - // Carol has no displayname - So userId is used. - a: new Map([[carolId, carol.userId]]), - // Both Bobs join, and should handle zero width hacks. - b: new Map([ - [carolId, carol.userId], - [bobId, `Bob (${bob.userId})`], - [ - bobZeroWidthSpaceId, - `${bobZeroWidthSpace.rawDisplayName} (${bobZeroWidthSpace.userId})`, - ], - ]), - }); - }, - ); - }); -}); - -test("should strip RTL characters from displayname", () => { - withTestScheduler(({ behavior, expectObservable }) => { - const scenarioInputMarbles = "ab"; - const expectedLayoutMarbles = "ab"; - - withCallViewModel( - { - rtcMembers$: behavior(scenarioInputMarbles, { - a: [localRtcMember], - b: [localRtcMember, daveRtcMember, daveRTLRtcMember], - }), - }, - (vm) => { - expectObservable(vm.memberDisplaynames$).toBe(expectedLayoutMarbles, { - // Carol has no displayname - So userId is used. - a: new Map([[carolId, carol.userId]]), - // Both Dave's join. Since after stripping - b: new Map([ - [carolId, carol.userId], - // Not disambiguated - [daveId, "Dave"], - // This one is, since it's using RTL. - [daveRTLId, `evaD (${daveRTL.userId})`], - ]), - }); - }, - ); - }); -}); - it("should rank raised hands above video feeds and below speakers and presenters", () => { withTestScheduler(({ schedule, expectObservable }) => { // There should always be one tile for each MatrixRTCSession diff --git a/src/rtcSessionHelpers.test.ts b/src/state/CallViewModel/localMember/LocalMembership.test.ts similarity index 86% rename from src/rtcSessionHelpers.test.ts rename to src/state/CallViewModel/localMember/LocalMembership.test.ts index a2b49390..763946d4 100644 --- a/src/rtcSessionHelpers.test.ts +++ b/src/state/CallViewModel/localMember/LocalMembership.test.ts @@ -5,14 +5,31 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; +import { + type LivekitTransport, + type MatrixRTCSession, +} from "matrix-js-sdk/lib/matrixrtc"; import { expect, test, vi } from "vitest"; import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery"; import EventEmitter from "events"; -import { enterRTCSession } from "../src/rtcSessionHelpers"; -import { mockConfig } from "./utils/test"; -import { MatrixRTCMode } from "./settings/settings"; +import { MatrixRTCMode } from "../../../settings/settings"; +import { mockConfig } from "../../../utils/test"; +import * as LocalMembership from "./LocalMembership"; + +// Read private function so we do not have to make it public +const enterRTCSession = ( + LocalMembership as unknown as { + enterRTCSession: ( + rtcSession: MatrixRTCSession, + transport: LivekitTransport, + { + encryptMedia, + matrixRTCMode, + }: { encryptMedia: boolean; matrixRTCMode: MatrixRTCMode }, + ) => Promise; + } +).enterRTCSession; const MATRIX_RTC_MODE = MatrixRTCMode.Legacy; const getUrlParams = vi.hoisted(() => vi.fn(() => ({}))); diff --git a/src/state/CallViewModel/remoteMembers/Connection.test.ts b/src/state/CallViewModel/remoteMembers/Connection.test.ts index a3a42928..01a73301 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.test.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.test.ts @@ -10,18 +10,16 @@ import { describe, expect, it, - type Mock, type MockedObject, onTestFinished, vi, } from "vitest"; -import { BehaviorSubject, of } from "rxjs"; +import { BehaviorSubject } from "rxjs"; import { type LocalParticipant, type RemoteParticipant, type Room as LivekitRoom, RoomEvent, - type RoomOptions, ConnectionState as LivekitConnectionState, } from "livekit-client"; import fetchMock from "fetch-mock"; @@ -41,10 +39,6 @@ import { import { ObservableScope } from "../../ObservableScope.ts"; import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts"; import { FailToGetOpenIdToken } from "../../../utils/errors.ts"; -import { mockMediaDevices, mockMuteStates } from "../../../utils/test.ts"; -import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx"; -import { type MuteStates } from "../../MuteStates.ts"; - let testScope: ObservableScope; let client: MockedObject; @@ -395,20 +389,12 @@ describe("Publishing participants observations", () => { const bobIsAPublisher = Promise.withResolvers(); const danIsAPublisher = Promise.withResolvers(); const observedPublishers: PublishingParticipant[][] = []; - const s = connection.allLivekitParticipants$.subscribe((publishers) => { + const s = connection.participants$.subscribe((publishers) => { observedPublishers.push(publishers); - if ( - publishers.some( - (p) => p.participant?.identity === "@bob:example.org:DEV111", - ) - ) { + if (publishers.some((p) => p.identity === "@bob:example.org:DEV111")) { bobIsAPublisher.resolve(); } - if ( - publishers.some( - (p) => p.participant?.identity === "@dan:example.org:DEV333", - ) - ) { + if (publishers.some((p) => p.identity === "@dan:example.org:DEV333")) { danIsAPublisher.resolve(); } }); @@ -466,9 +452,7 @@ describe("Publishing participants observations", () => { await bobIsAPublisher.promise; const publishers = observedPublishers.pop(); expect(publishers?.length).toEqual(1); - expect(publishers?.[0].participant?.identity).toEqual( - "@bob:example.org:DEV111", - ); + expect(publishers?.[0].identity).toEqual("@bob:example.org:DEV111"); // Now let's make dan join the rtc memberships rtcMemberships.push({ @@ -482,14 +466,10 @@ describe("Publishing participants observations", () => { const twoPublishers = observedPublishers.pop(); expect(twoPublishers?.length).toEqual(2); expect( - twoPublishers?.some( - (p) => p.participant?.identity === "@bob:example.org:DEV111", - ), + twoPublishers?.some((p) => p.identity === "@bob:example.org:DEV111"), ).toBeTruthy(); expect( - twoPublishers?.some( - (p) => p.participant?.identity === "@dan:example.org:DEV333", - ), + twoPublishers?.some((p) => p.identity === "@dan:example.org:DEV333"), ).toBeTruthy(); // Now let's make bob leave the livekit room @@ -504,26 +484,27 @@ describe("Publishing participants observations", () => { fakeRemoteLivekitParticipant("@bob:example.org:DEV111"), ); - const updatedPublishers = observedPublishers.pop(); - // Bob is not connected to the room but he is still in the rtc memberships declaring that - // he is using that focus to publish, so he should still appear as a publisher - expect(updatedPublishers?.length).toEqual(2); - const pp = updatedPublishers?.find( - (p) => p.membership.userId == "@bob:example.org", - ); - expect(pp).toBeDefined(); - expect(pp!.participant).not.toBeDefined(); - expect( - updatedPublishers?.some( - (p) => p.participant?.identity === "@dan:example.org:DEV333", - ), - ).toBeTruthy(); - // Now if bob is not in the rtc memberships, he should disappear - const noBob = rtcMemberships.filter( - ({ membership }) => membership.userId !== "@bob:example.org", - ); - fakeMembershipsFocusMap$.next(noBob); - expect(observedPublishers.pop()?.length).toEqual(1); + // TODO: evaluate this test. It looks like this is not the task of the Connection anymore. Valere? + // const updatedPublishers = observedPublishers.pop(); + // // Bob is not connected to the room but he is still in the rtc memberships declaring that + // // he is using that focus to publish, so he should still appear as a publisher + // expect(updatedPublishers?.length).toEqual(2); + // const pp = updatedPublishers?.find((p) => + // p.identity.startsWith("@bob:example.org"), + // ); + // expect(pp).toBeDefined(); + // expect(pp!).not.toBeDefined(); + // expect( + // updatedPublishers?.some( + // (p) => p.participant?.identity === "@dan:example.org:DEV333", + // ), + // ).toBeTruthy(); + // // Now if bob is not in the rtc memberships, he should disappear + // const noBob = rtcMemberships.filter( + // ({ membership }) => membership.userId !== "@bob:example.org", + // ); + // fakeMembershipsFocusMap$.next(noBob); + // expect(observedPublishers.pop()?.length).toEqual(1); }); it("should be scoped to parent scope", (): void => { @@ -532,7 +513,7 @@ describe("Publishing participants observations", () => { const connection = setupRemoteConnection(); let observedPublishers: PublishingParticipant[][] = []; - const s = connection.allLivekitParticipants$.subscribe((publishers) => { + const s = connection.participants$.subscribe((publishers) => { observedPublishers.push(publishers); }); onTestFinished(() => s.unsubscribe()); @@ -566,9 +547,7 @@ describe("Publishing participants observations", () => { // We should have bob has a publisher now const publishers = observedPublishers.pop(); expect(publishers?.length).toEqual(1); - expect(publishers?.[0].participant?.identity).toEqual( - "@bob:example.org:DEV111", - ); + expect(publishers?.[0]?.identity).toEqual("@bob:example.org:DEV111"); // end the parent scope testScope.end(); @@ -590,108 +569,112 @@ describe("Publishing participants observations", () => { }); }); -describe("PublishConnection", () => { - // let fakeBlurProcessor: ProcessorWrapper; - let roomFactoryMock: Mock<() => LivekitRoom>; - let muteStates: MockedObject; +// +// NOT USED ANYMORE ? +// +// This setup look like sth for the Publisher. Not a connection. - function setUpPublishConnection(): void { - setupTest(); +// describe("PublishConnection", () => { +// // let fakeBlurProcessor: ProcessorWrapper; +// let roomFactoryMock: Mock<() => LivekitRoom>; +// let muteStates: MockedObject; - roomFactoryMock = vi.fn().mockReturnValue(fakeLivekitRoom); +// function setUpPublishConnection(): void { +// setupTest(); - muteStates = mockMuteStates(); +// roomFactoryMock = vi.fn().mockReturnValue(fakeLivekitRoom); - // fakeBlurProcessor = vi.mocked>({ - // name: "BackgroundBlur", - // restart: vi.fn().mockResolvedValue(undefined), - // setOptions: vi.fn().mockResolvedValue(undefined), - // getOptions: vi.fn().mockReturnValue({ strength: 0.5 }), - // isRunning: vi.fn().mockReturnValue(false) - // }); - } +// muteStates = mockMuteStates(); - describe("Livekit room creation", () => { - function createSetup(): void { - setUpPublishConnection(); +// // fakeBlurProcessor = vi.mocked>({ +// // name: "BackgroundBlur", +// // restart: vi.fn().mockResolvedValue(undefined), +// // setOptions: vi.fn().mockResolvedValue(undefined), +// // getOptions: vi.fn().mockReturnValue({ strength: 0.5 }), +// // isRunning: vi.fn().mockReturnValue(false) +// // }); +// } - const fakeTrackProcessorSubject$ = new BehaviorSubject({ - supported: true, - processor: undefined, - }); +// describe("Livekit room creation", () => { +// function createSetup(): void { +// setUpPublishConnection(); - const opts: ConnectionOpts = { - client: client, - transport: livekitFocus, - remoteTransports$: fakeMembershipsFocusMap$, - scope: testScope, - livekitRoomFactory: roomFactoryMock, - }; +// const fakeTrackProcessorSubject$ = new BehaviorSubject({ +// supported: true, +// processor: undefined, +// }); - const audioInput = { - available$: of(new Map([["mic1", { id: "mic1" }]])), - selected$: new BehaviorSubject({ id: "mic1" }), - select(): void {}, - }; +// const opts: ConnectionOpts = { +// client: client, +// transport: livekitFocus, +// scope: testScope, +// livekitRoomFactory: roomFactoryMock, +// }; - const videoInput = { - available$: of(new Map([["cam1", { id: "cam1" }]])), - selected$: new BehaviorSubject({ id: "cam1" }), - select(): void {}, - }; +// const audioInput = { +// available$: of(new Map([["mic1", { id: "mic1" }]])), +// selected$: new BehaviorSubject({ id: "mic1" }), +// select(): void {}, +// }; - const audioOutput = { - available$: of(new Map([["speaker", { id: "speaker" }]])), - selected$: new BehaviorSubject({ id: "speaker" }), - select(): void {}, - }; +// const videoInput = { +// available$: of(new Map([["cam1", { id: "cam1" }]])), +// selected$: new BehaviorSubject({ id: "cam1" }), +// select(): void {}, +// }; - // TODO understand what is wrong with our mocking that requires ts-expect-error - const fakeDevices = mockMediaDevices({ - // @ts-expect-error Mocking only - audioInput, - // @ts-expect-error Mocking only - videoInput, - // @ts-expect-error Mocking only - audioOutput, - }); +// const audioOutput = { +// available$: of(new Map([["speaker", { id: "speaker" }]])), +// selected$: new BehaviorSubject({ id: "speaker" }), +// select(): void {}, +// }; - new PublishConnection( - opts, - fakeDevices, - muteStates, - undefined, - fakeTrackProcessorSubject$, - ); - } +// // TODO understand what is wrong with our mocking that requires ts-expect-error +// const fakeDevices = mockMediaDevices({ +// // @ts-expect-error Mocking only +// audioInput, +// // @ts-expect-error Mocking only +// videoInput, +// // @ts-expect-error Mocking only +// audioOutput, +// }); - it("should create room with proper initial audio and video settings", () => { - createSetup(); +// new Connection( +// opts, +// fakeDevices, +// muteStates, +// undefined, +// fakeTrackProcessorSubject$, +// ); +// } - expect(roomFactoryMock).toHaveBeenCalled(); +// it("should create room with proper initial audio and video settings", () => { +// createSetup(); - const lastCallArgs = - roomFactoryMock.mock.calls[roomFactoryMock.mock.calls.length - 1]; +// expect(roomFactoryMock).toHaveBeenCalled(); - const roomOptions = lastCallArgs.pop() as unknown as RoomOptions; - expect(roomOptions).toBeDefined(); +// const lastCallArgs = +// roomFactoryMock.mock.calls[roomFactoryMock.mock.calls.length - 1]; - expect(roomOptions!.videoCaptureDefaults?.deviceId).toEqual("cam1"); - expect(roomOptions!.audioCaptureDefaults?.deviceId).toEqual("mic1"); - expect(roomOptions!.audioOutput?.deviceId).toEqual("speaker"); - }); +// const roomOptions = lastCallArgs.pop() as unknown as RoomOptions; +// expect(roomOptions).toBeDefined(); - it("respect controlledAudioDevices", () => { - // TODO: Refactor the code to make it testable. - // The UrlParams module is a singleton has a cache and is very hard to test. - // This breaks other tests as well if not handled properly. - // vi.mock(import("./../UrlParams"), () => { - // return { - // getUrlParams: vi.fn().mockReturnValue({ - // controlledAudioDevices: true - // }) - // }; - // }); - }); - }); -}); +// expect(roomOptions!.videoCaptureDefaults?.deviceId).toEqual("cam1"); +// expect(roomOptions!.audioCaptureDefaults?.deviceId).toEqual("mic1"); +// expect(roomOptions!.audioOutput?.deviceId).toEqual("speaker"); +// }); + +// it("respect controlledAudioDevices", () => { +// // TODO: Refactor the code to make it testable. +// // The UrlParams module is a singleton has a cache and is very hard to test. +// // This breaks other tests as well if not handled properly. +// // vi.mock(import("./../UrlParams"), () => { +// // return { +// // getUrlParams: vi.fn().mockReturnValue({ +// // controlledAudioDevices: true +// // }) +// // }; +// // }); +// }); +// }); +// }); diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts index ccf93a30..e675f723 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts @@ -5,12 +5,11 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { describe, test, vi, expect, beforeEach, afterEach } from "vitest"; +import { describe, test, expect, beforeEach, afterEach } from "vitest"; import { type CallMembership, type LivekitTransport, } from "matrix-js-sdk/lib/matrixrtc"; -import { type Room as MatrixRoom, type RoomMember } from "matrix-js-sdk"; import { getParticipantId } from "matrix-js-sdk/lib/matrixrtc/utils"; import { combineLatest, map, type Observable } from "rxjs"; @@ -34,7 +33,6 @@ import { import { type Connection } from "./Connection.ts"; let testScope: ObservableScope; -let mockMatrixRoom: MatrixRoom; const transportA: LivekitTransport = { type: "livekit", @@ -61,17 +59,6 @@ const carlMembership = mockCallMembership( beforeEach(() => { testScope = new ObservableScope(); - mockMatrixRoom = vi.mocked({ - getMember: vi.fn().mockImplementation((userId: string) => { - return { - userId, - rawDisplayName: userId.replace("@", "").replace(":example.org", ""), - getMxcAvatarUrl: vi.fn().mockReturnValue(null), - } as unknown as RoomMember; - }), - addEventListener: vi.fn(), - removeEventListener: vi.fn(), - } as unknown as MatrixRoom); }); afterEach(() => { @@ -110,7 +97,6 @@ test("should signal participant not yet connected to livekit", () => { connectionManager: { connectionManagerData$: connectionManagerData$, } as unknown as IConnectionManager, - matrixRoom: mockMatrixRoom, }); expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", { @@ -191,7 +177,6 @@ test("should signal participant on a connection that is publishing", () => { connectionManager: { connectionManagerData$: connectionManagerData$, } as unknown as IConnectionManager, - matrixRoom: mockMatrixRoom, }); expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", { @@ -243,7 +228,6 @@ test("should signal participant on a connection that is not publishing", () => { connectionManager: { connectionManagerData$: connectionManagerData$, } as unknown as IConnectionManager, - matrixRoom: mockMatrixRoom, }); expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", { @@ -307,7 +291,6 @@ describe("Publication edge case", () => { connectionManager: { connectionManagerData$: connectionManagerData$, } as unknown as IConnectionManager, - matrixRoom: mockMatrixRoom, }); expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe( @@ -374,7 +357,6 @@ describe("Publication edge case", () => { connectionManager: { connectionManagerData$: connectionManagerData$, } as unknown as IConnectionManager, - matrixRoom: mockMatrixRoom, }); expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe( diff --git a/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.test.ts b/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.test.ts new file mode 100644 index 00000000..bc4d329c --- /dev/null +++ b/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.test.ts @@ -0,0 +1,611 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { afterEach, beforeEach, describe, vi } from "vitest"; +import { + type MatrixEvent, + type RoomMember, + type RoomState, + RoomStateEvent, +} from "matrix-js-sdk"; +import EventEmitter from "events"; +import { it } from "vitest"; + +import { ObservableScope } from "../../ObservableScope.ts"; +import type { Room as MatrixRoom } from "matrix-js-sdk/lib/models/room"; +import { + mockCallMembership, + mockMatrixRoomMember, + withTestScheduler, +} from "../../../utils/test.ts"; +import { + createMatrixMemberMetadata$, + createRoomMembers$, +} from "./MatrixMemberMetadata.ts"; +let testScope: ObservableScope; +let mockMatrixRoom: MatrixRoom; + +describe("MatrixMemberMetadata", () => { + /* + * To be populated in the test setup. + * Maps userId to a partial/mock RoomMember object. + */ + let fakeMembersMap: Map>; + + beforeEach(() => { + testScope = new ObservableScope(); + fakeMembersMap = new Map>(); + + const roomEmitter = new EventEmitter(); + mockMatrixRoom = { + on: roomEmitter.on.bind(roomEmitter), + off: roomEmitter.off.bind(roomEmitter), + emit: roomEmitter.emit.bind(roomEmitter), + // addListener: roomEmitter.addListener.bind(roomEmitter), + // removeListener: roomEmitter.removeListener.bind(roomEmitter), + getMember: vi.fn().mockImplementation((userId: string) => { + const member = fakeMembersMap.get(userId); + if (member) { + return member as RoomMember; + } + return null; + }), + getMembers: vi.fn().mockImplementation(() => { + const members = Array.from(fakeMembersMap.values()); + return members; + }), + } as unknown as MatrixRoom; + }); + + function fakeMemberWith(data: Partial): void { + const userId = data.userId || "@alice:example.com"; + const member: Partial = { + userId: userId, + rawDisplayName: data.rawDisplayName ?? userId, + getMxcAvatarUrl: + data.getMxcAvatarUrl || + vi.fn().mockImplementation(() => { + return `mxc://example.com/${userId}`; + }), + ...data, + } as unknown as RoomMember; + fakeMembersMap.set(userId, member); + } + + afterEach(() => { + fakeMembersMap.clear(); + }); + + describe("displayname", () => { + function updateDisplayName( + userId: `@${string}:${string}`, + newDisplayName: string, + ): void { + const member = fakeMembersMap.get(userId); + if (member) { + member.rawDisplayName = newDisplayName; + // Emit the event to notify listeners + mockMatrixRoom.emit( + RoomStateEvent.Members, + {} as unknown as MatrixEvent, + {} as unknown as RoomState, + member as RoomMember, + ); + } else { + throw new Error(`No member found with userId: ${userId}`); + } + } + + // TODO this is a regression, now there the own user is not always in the map. Ask Timo if fine + it("should show our own user if present in rtc session and room", () => { + withTestScheduler(({ behavior, expectObservable }) => { + fakeMemberWith({ + userId: "@local:example.com", + rawDisplayName: "it's a me", + }); + const memberships$ = behavior("a", { + a: [mockCallMembership("@local:example.com", "DEVICE1")], + }); + const metadataStore = createMatrixMemberMetadata$( + testScope, + memberships$, + createRoomMembers$(testScope, mockMatrixRoom), + ); + const dn$ = + metadataStore.createDisplayNameBehavior$("@local:example.com"); + + expectObservable(dn$).toBe("a", { + a: "it's a me", + }); + expectObservable(metadataStore.displaynameMap$).toBe("a", { + a: new Map([["@local:example.com", "it's a me"]]), + }); + }); + }); + + function setUpBasicRoom(): void { + fakeMemberWith({ + userId: "@local:example.com", + rawDisplayName: "it's a me", + }); + fakeMemberWith({ userId: "@alice:example.com", rawDisplayName: "Alice" }); + fakeMemberWith({ userId: "@bob:example.com", rawDisplayName: "Bob" }); + fakeMemberWith({ userId: "@carl:example.com", rawDisplayName: "Carl" }); + fakeMemberWith({ userId: "@evil:example.com", rawDisplayName: "Carl" }); + fakeMemberWith({ userId: "@bob:foo.bar", rawDisplayName: "Bob" }); + fakeMemberWith({ userId: "@no-name:foo.bar" }); + } + + it("should get displayName for users", () => { + setUpBasicRoom(); + + withTestScheduler(({ behavior, expectObservable }) => { + const memberships$ = behavior("a", { + a: [ + mockCallMembership("@alice:example.com", "DEVICE1"), + mockCallMembership("@bob:example.com", "DEVICE1"), + ], + }); + const metadataStore = createMatrixMemberMetadata$( + testScope, + memberships$, + createRoomMembers$(testScope, mockMatrixRoom), + ); + const aliceDispName$ = + metadataStore.createDisplayNameBehavior$("@alice:example.com"); + + expectObservable(aliceDispName$).toBe("a", { + a: "Alice", + }); + + expectObservable(metadataStore.displaynameMap$).toBe("a", { + a: new Map([ + ["@alice:example.com", "Alice"], + ["@bob:example.com", "Bob"], + ]), + }); + }); + }); + + it("should use userId if no display name", () => { + withTestScheduler(({ behavior, expectObservable }) => { + setUpBasicRoom(); + + const memberships$ = behavior("a", { + a: [mockCallMembership("@no-name:foo.bar", "D000")], + }); + const metadataStore = createMatrixMemberMetadata$( + testScope, + memberships$, + createRoomMembers$(testScope, mockMatrixRoom), + ); + + expectObservable(metadataStore.displaynameMap$).toBe("a", { + a: new Map([ + ["@no-name:foo.bar", "@no-name:foo.bar"], + ]), + }); + }); + }); + + it("should disambiguate users with same display name", () => { + withTestScheduler(({ behavior, expectObservable }) => { + setUpBasicRoom(); + + const memberships$ = behavior("a", { + a: [ + mockCallMembership("@bob:example.com", "DEVICE1"), + mockCallMembership("@bob:example.com", "DEVICE2"), + mockCallMembership("@bob:foo.bar", "BOB000"), + mockCallMembership("@carl:example.com", "C000"), + mockCallMembership("@evil:example.com", "E000"), + ], + }); + + const metadataStore = createMatrixMemberMetadata$( + testScope, + memberships$, + createRoomMembers$(testScope, mockMatrixRoom), + ); + + expectObservable(metadataStore.displaynameMap$).toBe("a", { + a: new Map([ + // ["@local:example.com", "it's a me"], + ["@bob:example.com", "Bob (@bob:example.com)"], + ["@bob:example.com", "Bob (@bob:example.com)"], + ["@bob:foo.bar", "Bob (@bob:foo.bar)"], + ["@carl:example.com", "Carl (@carl:example.com)"], + ["@evil:example.com", "Carl (@evil:example.com)"], + ]), + }); + }); + }); + + it("should start to disambiguate reactivly when needed", () => { + withTestScheduler(({ behavior, expectObservable }) => { + setUpBasicRoom(); + + const memberships$ = behavior("ab", { + a: [mockCallMembership("@bob:example.com", "DEVICE1")], + b: [ + mockCallMembership("@bob:example.com", "DEVICE1"), + mockCallMembership("@bob:foo.bar", "BOB000"), + ], + }); + + const metadataStore = createMatrixMemberMetadata$( + testScope, + memberships$, + createRoomMembers$(testScope, mockMatrixRoom), + ); + + expectObservable(metadataStore.displaynameMap$).toBe("ab", { + a: new Map([["@bob:example.com", "Bob"]]), + b: new Map([ + ["@bob:example.com", "Bob (@bob:example.com)"], + ["@bob:foo.bar", "Bob (@bob:foo.bar)"], + ]), + }); + }); + }); + + it("should keep disambiguated name when other leave", () => { + withTestScheduler(({ behavior, expectObservable }) => { + setUpBasicRoom(); + + const memberships$ = behavior("ab", { + a: [ + mockCallMembership("@bob:example.com", "DEVICE1"), + mockCallMembership("@bob:foo.bar", "BOB000"), + ], + b: [mockCallMembership("@bob:example.com", "DEVICE1")], + }); + + const metadataStore = createMatrixMemberMetadata$( + testScope, + memberships$, + createRoomMembers$(testScope, mockMatrixRoom), + ); + + expectObservable(metadataStore.displaynameMap$).toBe("ab", { + a: new Map([ + ["@bob:example.com", "Bob (@bob:example.com)"], + ["@bob:foo.bar", "Bob (@bob:foo.bar)"], + ]), + b: new Map([ + ["@bob:example.com", "Bob (@bob:example.com)"], + ]), + }); + }); + }); + + it("should disambiguate on name change", () => { + withTestScheduler(({ behavior, schedule, expectObservable }) => { + setUpBasicRoom(); + + const memberships$ = behavior("a", { + a: [ + mockCallMembership("@bob:example.com", "B000"), + mockCallMembership("@carl:example.com", "C000"), + ], + }); + const metadataStore = createMatrixMemberMetadata$( + testScope, + memberships$, + createRoomMembers$(testScope, mockMatrixRoom), + ); + + schedule("-a", { + a: () => { + updateDisplayName("@carl:example.com", "Bob"); + }, + }); + + expectObservable(metadataStore.displaynameMap$).toBe("ab", { + a: new Map([ + ["@bob:example.com", "Bob"], + ["@carl:example.com", "Carl"], + ]), + b: new Map([ + ["@bob:example.com", "Bob (@bob:example.com)"], + ["@carl:example.com", "Bob (@carl:example.com)"], + ]), + }); + }); + }); + + it("should track individual member id with createDisplayNameBehavior", () => { + withTestScheduler(({ behavior, schedule, expectObservable }) => { + setUpBasicRoom(); + const BOB = "@bob:example.com"; + const CARL = "@carl:example.com"; + // for this test we build a mock environment that does all possible changes: + // - memberships join/leave + // - room join/leave + // - disambiguate + const memberships$ = behavior("ab-d", { + a: [mockCallMembership(CARL, "C000")], + b: [ + mockCallMembership(CARL, "C000"), + // bob joins + mockCallMembership(BOB, "B000"), + ], + // c carl gets renamed to BOB + d: [ + // carl leaves + mockCallMembership(BOB, "B000"), + ], + }); + schedule("--a-", { + a: () => { + // carl renames + updateDisplayName(CARL, "Bob"); + }, + }); + + const metadataStore = createMatrixMemberMetadata$( + testScope, + memberships$, + createRoomMembers$(testScope, mockMatrixRoom), + ); + + const bob$ = metadataStore.createDisplayNameBehavior$(BOB); + const carl$ = metadataStore.createDisplayNameBehavior$(CARL); + + expectObservable(bob$).toBe("abc-", { + a: undefined, + b: "Bob", + c: "Bob (@bob:example.com)", + // bob stays disambiguate even though carl left + // d: "Bob (@bob:example.com)", + }); + + expectObservable(carl$).toBe("a-cd", { + a: "Carl", + // b: "Carl", + // carl gets renamed and disambiguate + c: "Bob (@carl:example.com)", + d: undefined, + }); + }); + }); + + it("should disambiguate users with invisible characters", () => { + withTestScheduler(({ behavior, expectObservable }) => { + const bobRtcMember = mockCallMembership("@bob:example.org", "BBBB"); + const bobZeroWidthSpaceRtcMember = mockCallMembership( + "@bob2:example.org", + "BBBB", + ); + const bob = mockMatrixRoomMember(bobRtcMember, { + rawDisplayName: "Bob", + }); + const bobZeroWidthSpace = mockMatrixRoomMember( + bobZeroWidthSpaceRtcMember, + { + rawDisplayName: "Bo\u200bb", + }, + ); + fakeMemberWith(bob); + fakeMemberWith(bobZeroWidthSpace); + fakeMemberWith({ userId: "@carol:example.org" }); + const memberships$ = behavior("ab", { + a: [mockCallMembership("@carol:example.org", "1111"), bobRtcMember], + b: [ + mockCallMembership("@carol:example.org", "1111"), + bobRtcMember, + bobZeroWidthSpaceRtcMember, + ], + }); + + const metadataStore = createMatrixMemberMetadata$( + testScope, + memberships$, + createRoomMembers$(testScope, mockMatrixRoom), + ); + + const bob$ = + metadataStore.createDisplayNameBehavior$("@bob:example.org"); + const bob2$ = + metadataStore.createDisplayNameBehavior$("@bob2:example.org"); + const carol$ = + metadataStore.createDisplayNameBehavior$("@carol:example.org"); + expectObservable(bob$).toBe("ab", { + a: "Bob", + b: "Bob (@bob:example.org)", + }); + expectObservable(bob2$).toBe("ab", { + a: undefined, + b: "Bo\u200bb (@bob2:example.org)", + }); + expectObservable(carol$).toBe("a-", { + a: "@carol:example.org", + }); + + expectObservable(metadataStore.displaynameMap$).toBe("ab", { + // Carol has no displayname - So userId is used. + a: new Map([ + ["@carol:example.org", "@carol:example.org"], + ["@bob:example.org", "Bob"], + ]), + // Other Bob joins, and should handle zero width hacks. + b: new Map([ + ["@carol:example.org", "@carol:example.org"], + [bobRtcMember.userId, `Bob (@bob:example.org)`], + [ + bobZeroWidthSpace.userId, + `${bobZeroWidthSpace.rawDisplayName} (${bobZeroWidthSpace.userId})`, + ], + ]), + }); + }); + }); + + it("should strip RTL characters from displayname", () => { + withTestScheduler(({ behavior, expectObservable }) => { + const daveRtcMember = mockCallMembership("@dave:example.org", "DDDD"); + const daveRTLRtcMember = mockCallMembership( + "@dave2:example.org", + "DDDD", + ); + const dave = mockMatrixRoomMember(daveRtcMember, { + rawDisplayName: "Dave", + }); + const daveRTL = mockMatrixRoomMember(daveRTLRtcMember, { + rawDisplayName: "\u202eevaD", + }); + + fakeMemberWith({ userId: "@carol:example.org" }); + fakeMemberWith(daveRTL); + fakeMemberWith(dave); + const memberships$ = behavior("ab", { + a: [mockCallMembership("@carol:example.org", "DDDD")], + b: [ + mockCallMembership("@carol:example.org", "DDDD"), + daveRtcMember, + daveRTLRtcMember, + ], + }); + + const metadataStore = createMatrixMemberMetadata$( + testScope, + memberships$, + createRoomMembers$(testScope, mockMatrixRoom), + ); + + expectObservable(metadataStore.displaynameMap$).toBe("ab", { + // Carol has no displayname - So userId is used. + a: new Map([["@carol:example.org", "@carol:example.org"]]), + // Both Dave's join. Since after stripping + b: new Map([ + ["@carol:example.org", "@carol:example.org"], + // Not disambiguated + ["@dave:example.org", "Dave"], + // This one is, since it's using RTL. + ["@dave2:example.org", "evaD (@dave2:example.org)"], + ]), + }); + }); + }); + }); + + describe("avatarUrl", () => { + function updateAvatarUrl( + userId: `@${string}:${string}`, + avatarUrl: string, + ): void { + const member = fakeMembersMap.get(userId); + if (member) { + member.getMxcAvatarUrl = vi.fn().mockReturnValue(avatarUrl); + // Emit the event to notify listeners + mockMatrixRoom.emit( + RoomStateEvent.Members, + {} as unknown as MatrixEvent, + {} as unknown as RoomState, + member as RoomMember, + ); + } else { + throw new Error(`No member found with userId: ${userId}`); + } + } + + // TODO this is a regression, now there the own user is not always in the map. Ask Timo if fine + it("should use avatar url from room members", () => { + withTestScheduler(({ behavior, expectObservable }) => { + fakeMemberWith({ + userId: "@local:example.com", + }); + fakeMemberWith({ + userId: "@alice:example.com", + getMxcAvatarUrl: vi.fn().mockReturnValue("mxc://custom.url/avatar"), + }); + const memberships$ = behavior("a", { + a: [ + mockCallMembership("@local:example.com", "DEVICE1"), + mockCallMembership("@alice:example.com", "DEVICE1"), + ], + }); + const metadataStore = createMatrixMemberMetadata$( + testScope, + memberships$, + createRoomMembers$(testScope, mockMatrixRoom), + ); + const local$ = + metadataStore.createAvatarUrlBehavior$("@local:example.com"); + + const alice$ = + metadataStore.createAvatarUrlBehavior$("@alice:example.com"); + + expectObservable(local$).toBe("a", { + a: "mxc://example.com/@local:example.com", + }); + expectObservable(alice$).toBe("a", { + a: "mxc://custom.url/avatar", + }); + expectObservable(metadataStore.avatarMap$).toBe("a", { + a: new Map([ + ["@local:example.com", "mxc://example.com/@local:example.com"], + ["@alice:example.com", "mxc://custom.url/avatar"], + ]), + }); + }); + }); + + it("should update on avatar change and user join/leave", () => { + withTestScheduler(({ behavior, schedule, expectObservable }) => { + fakeMemberWith({ userId: "@carl:example.com" }); + fakeMemberWith({ userId: "@bob:example.com" }); + const memberships$ = behavior("ab-d", { + a: [mockCallMembership("@bob:example.com", "B000")], + b: [ + mockCallMembership("@bob:example.com", "B000"), + mockCallMembership("@carl:example.com", "C000"), + ], + d: [mockCallMembership("@carl:example.com", "C000")], + }); + + const metadataStore = createMatrixMemberMetadata$( + testScope, + memberships$, + createRoomMembers$(testScope, mockMatrixRoom), + ); + + schedule("--c-", { + c: () => { + updateAvatarUrl( + "@carl:example.com", + "mxc://updated.me/updatedAvatar", + ); + }, + }); + + const bob$ = metadataStore.createAvatarUrlBehavior$("@bob:example.com"); + const carl$ = + metadataStore.createAvatarUrlBehavior$("@carl:example.com"); + expectObservable(bob$).toBe("a---", { + a: "mxc://example.com/@bob:example.com", + }); + expectObservable(carl$).toBe("a-c-", { + a: "mxc://example.com/@carl:example.com", + + c: "mxc://updated.me/updatedAvatar", + }); + expectObservable(metadataStore.avatarMap$).toBe("a-c-", { + a: new Map([ + ["@bob:example.com", "mxc://example.com/@bob:example.com"], + ["@carl:example.com", "mxc://example.com/@carl:example.com"], + ]), + // expect an update once we update the avatar URL + c: new Map([ + ["@bob:example.com", "mxc://example.com/@bob:example.com"], + ["@carl:example.com", "mxc://updated.me/updatedAvatar"], + ]), + }); + }); + }); + }); +}); diff --git a/src/state/CallViewModel/remoteMembers/displayname.test.ts b/src/state/CallViewModel/remoteMembers/displayname.test.ts deleted file mode 100644 index 60a29a18..00000000 --- a/src/state/CallViewModel/remoteMembers/displayname.test.ts +++ /dev/null @@ -1,286 +0,0 @@ -/* -Copyright 2025 Element Creations Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { afterEach, beforeEach, test, vi } from "vitest"; -import { - type MatrixEvent, - type RoomMember, - type RoomState, - RoomStateEvent, -} from "matrix-js-sdk"; -import EventEmitter from "events"; - -import { ObservableScope } from "../../ObservableScope.ts"; -import type { Room as MatrixRoom } from "matrix-js-sdk/lib/models/room"; -import { mockCallMembership, withTestScheduler } from "../../../utils/test.ts"; -import { memberDisplaynames$ } from "./displayname.ts"; - -let testScope: ObservableScope; -let mockMatrixRoom: MatrixRoom; - -/* - * To be populated in the test setup. - * Maps userId to a partial/mock RoomMember object. - */ -let fakeMembersMap: Map>; - -beforeEach(() => { - testScope = new ObservableScope(); - fakeMembersMap = new Map>(); - - const roomEmitter = new EventEmitter(); - mockMatrixRoom = { - on: roomEmitter.on.bind(roomEmitter), - off: roomEmitter.off.bind(roomEmitter), - emit: roomEmitter.emit.bind(roomEmitter), - // addListener: roomEmitter.addListener.bind(roomEmitter), - // removeListener: roomEmitter.removeListener.bind(roomEmitter), - getMember: vi.fn().mockImplementation((userId: string) => { - const member = fakeMembersMap.get(userId); - if (member) { - return member as RoomMember; - } - return null; - }), - } as unknown as MatrixRoom; -}); - -function fakeMemberWith(data: Partial): void { - const userId = data.userId || "@alice:example.com"; - const member: Partial = { - userId: userId, - rawDisplayName: data.rawDisplayName ?? userId, - ...data, - } as unknown as RoomMember; - fakeMembersMap.set(userId, member); - // return member as RoomMember; -} - -function updateDisplayName( - userId: `@${string}:${string}`, - newDisplayName: string, -): void { - const member = fakeMembersMap.get(userId); - if (member) { - member.rawDisplayName = newDisplayName; - // Emit the event to notify listeners - mockMatrixRoom.emit( - RoomStateEvent.Members, - {} as unknown as MatrixEvent, - {} as unknown as RoomState, - member as RoomMember, - ); - } else { - throw new Error(`No member found with userId: ${userId}`); - } -} - -afterEach(() => { - fakeMembersMap.clear(); -}); - -// TODO this is a regression, now there the own user is not always in the map. Ask Timo if fine -test.skip("should always have our own user", () => { - withTestScheduler(({ behavior, expectObservable }) => { - const dn$ = memberDisplaynames$( - testScope, - mockMatrixRoom, - behavior("a", { - a: [], - }), - ); - - expectObservable(dn$).toBe("a", { - a: new Map([ - ["@local:example.com", "@local:example.com"], - ]), - }); - }); -}); - -function setUpBasicRoom(): void { - fakeMemberWith({ userId: "@local:example.com", rawDisplayName: "it's a me" }); - fakeMemberWith({ userId: "@alice:example.com", rawDisplayName: "Alice" }); - fakeMemberWith({ userId: "@bob:example.com", rawDisplayName: "Bob" }); - fakeMemberWith({ userId: "@carl:example.com", rawDisplayName: "Carl" }); - fakeMemberWith({ userId: "@evil:example.com", rawDisplayName: "Carl" }); - fakeMemberWith({ userId: "@bob:foo.bar", rawDisplayName: "Bob" }); - fakeMemberWith({ userId: "@no-name:foo.bar" }); -} - -test("should get displayName for users", () => { - setUpBasicRoom(); - - withTestScheduler(({ behavior, expectObservable }) => { - const dn$ = memberDisplaynames$( - testScope, - mockMatrixRoom, - behavior("a", { - a: [ - mockCallMembership("@alice:example.com", "DEVICE1"), - mockCallMembership("@bob:example.com", "DEVICE1"), - ], - }), - ); - - expectObservable(dn$).toBe("a", { - a: new Map([ - // ["@local:example.com", "it's a me"], - ["@alice:example.com", "Alice"], - ["@bob:example.com", "Bob"], - ]), - }); - }); -}); - -test("should use userId if no display name", () => { - withTestScheduler(({ behavior, expectObservable }) => { - setUpBasicRoom(); - - const dn$ = memberDisplaynames$( - testScope, - mockMatrixRoom, - behavior("a", { - a: [mockCallMembership("@no-name:foo.bar", "D000")], - }), - ); - - expectObservable(dn$).toBe("a", { - a: new Map([ - // ["@local:example.com", "it's a me"], - ["@no-name:foo.bar", "@no-name:foo.bar"], - ]), - }); - }); -}); - -test("should disambiguate users with same display name", () => { - withTestScheduler(({ behavior, expectObservable }) => { - setUpBasicRoom(); - - const dn$ = memberDisplaynames$( - testScope, - mockMatrixRoom, - behavior("a", { - a: [ - mockCallMembership("@bob:example.com", "DEVICE1"), - mockCallMembership("@bob:example.com", "DEVICE2"), - mockCallMembership("@bob:foo.bar", "BOB000"), - mockCallMembership("@carl:example.com", "C000"), - mockCallMembership("@evil:example.com", "E000"), - ], - }), - ); - - expectObservable(dn$).toBe("a", { - a: new Map([ - // ["@local:example.com", "it's a me"], - ["@bob:example.com", "Bob (@bob:example.com)"], - ["@bob:example.com", "Bob (@bob:example.com)"], - ["@bob:foo.bar", "Bob (@bob:foo.bar)"], - ["@carl:example.com", "Carl (@carl:example.com)"], - ["@evil:example.com", "Carl (@evil:example.com)"], - ]), - }); - }); -}); - -test("should disambiguate when needed", () => { - withTestScheduler(({ behavior, expectObservable }) => { - setUpBasicRoom(); - - const dn$ = memberDisplaynames$( - testScope, - mockMatrixRoom, - behavior("ab", { - a: [mockCallMembership("@bob:example.com", "DEVICE1")], - b: [ - mockCallMembership("@bob:example.com", "DEVICE1"), - mockCallMembership("@bob:foo.bar", "BOB000"), - ], - }), - ); - - expectObservable(dn$).toBe("ab", { - a: new Map([ - // ["@local:example.com", "it's a me"], - ["@bob:example.com", "Bob"], - ]), - b: new Map([ - // ["@local:example.com", "it's a me"], - ["@bob:example.com", "Bob (@bob:example.com)"], - ["@bob:foo.bar", "Bob (@bob:foo.bar)"], - ]), - }); - }); -}); - -test.skip("should keep disambiguated name when other leave", () => { - withTestScheduler(({ behavior, expectObservable }) => { - setUpBasicRoom(); - - const dn$ = memberDisplaynames$( - testScope, - mockMatrixRoom, - behavior("ab", { - a: [ - mockCallMembership("@bob:example.com", "DEVICE1"), - mockCallMembership("@bob:foo.bar", "BOB000"), - ], - b: [mockCallMembership("@bob:example.com", "DEVICE1")], - }), - ); - - expectObservable(dn$).toBe("ab", { - a: new Map([ - // ["@local:example.com", "it's a me"], - ["@bob:example.com", "Bob (@bob:example.com)"], - ["@bob:foo.bar", "Bob (@bob:foo.bar)"], - ]), - b: new Map([ - // ["@local:example.com", "it's a me"], - ["@bob:example.com", "Bob (@bob:example.com)"], - ]), - }); - }); -}); - -test("should disambiguate on name change", () => { - withTestScheduler(({ behavior, schedule, expectObservable }) => { - setUpBasicRoom(); - - const dn$ = memberDisplaynames$( - testScope, - mockMatrixRoom, - behavior("a", { - a: [ - mockCallMembership("@bob:example.com", "B000"), - mockCallMembership("@carl:example.com", "C000"), - ], - }), - ); - - schedule("-a", { - a: () => { - updateDisplayName("@carl:example.com", "Bob"); - }, - }); - - expectObservable(dn$).toBe("ab", { - a: new Map([ - // ["@local:example.com", "it's a me"], - ["@bob:example.com", "Bob"], - ["@carl:example.com", "Carl"], - ]), - b: new Map([ - // ["@local:example.com", "it's a me"], - ["@bob:example.com", "Bob (@bob:example.com)"], - ["@carl:example.com", "Bob (@carl:example.com)"], - ]), - }); - }); -}); diff --git a/src/state/CallViewModel/remoteMembers/integration.test.ts b/src/state/CallViewModel/remoteMembers/integration.test.ts index d72505da..6115694d 100644 --- a/src/state/CallViewModel/remoteMembers/integration.test.ts +++ b/src/state/CallViewModel/remoteMembers/integration.test.ts @@ -11,7 +11,6 @@ import { type Room as LivekitRoom } from "livekit-client"; import EventEmitter from "events"; import fetchMock from "fetch-mock"; import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; -import { type Room as MatrixRoom, type RoomMember } from "matrix-js-sdk"; import { type Epoch, @@ -40,7 +39,6 @@ let testScope: ObservableScope; let ecConnectionFactory: ECConnectionFactory; let mockClient: OpenIDClientParts; let lkRoomFactory: () => LivekitRoom; -let mockMatrixRoom: MatrixRoom; const createdMockLivekitRooms: Map = new Map(); @@ -90,18 +88,6 @@ beforeEach(() => { }, }; }); - - mockMatrixRoom = vi.mocked({ - getMember: vi.fn().mockImplementation((userId: string) => { - return { - userId, - rawDisplayName: userId.replace("@", "").replace(":example.org", ""), - getMxcAvatarUrl: vi.fn().mockReturnValue(null), - } as unknown as RoomMember; - }), - addEventListener: vi.fn(), - removeEventListener: vi.fn(), - } as unknown as MatrixRoom); }); afterEach(() => { @@ -141,7 +127,6 @@ test("bob, carl, then bob joining no tracks yet", () => { membershipsWithTransport$: membershipsAndTransports.membershipsWithTransport$, connectionManager, - matrixRoom: mockMatrixRoom, }); expectObservable(matrixLivekitItems$).toBe(vMarble, { @@ -153,14 +138,12 @@ test("bob, carl, then bob joining no tracks yet", () => { a: bobMembership, }); expectObservable(item.connection$).toBe("a", { - a: expect.toSatisfy((co) => { - expect( - areLivekitTransportsEqual( - co.transport, - bobMembership.transports[0]! as LivekitTransport, - ), - ); - }), + a: expect.toSatisfy((co) => + areLivekitTransportsEqual( + co.transport, + bobMembership.transports[0]! as LivekitTransport, + ), + ), }); expectObservable(item.participant$).toBe("a", { a: null, diff --git a/src/state/CallViewModelWidget.test.ts b/src/state/CallViewModelWidget.test.ts index 045e2472..35a451f9 100644 --- a/src/state/CallViewModelWidget.test.ts +++ b/src/state/CallViewModelWidget.test.ts @@ -9,11 +9,11 @@ import { test, vi, expect } from "vitest"; import EventEmitter from "events"; import { constant } from "./Behavior.ts"; -import { withCallViewModel } from "./CallViewModel.test.ts"; import { aliceParticipant, localRtcMember } from "../utils/test-fixtures.ts"; import { ElementWidgetActions, widget } from "../widget.ts"; import { E2eeType } from "../e2ee/e2eeType.ts"; -import { type CallViewModel } from "./CallViewModel.ts"; +import { withCallViewModel } from "./CallViewModel/CallViewModel.test.ts"; +import { type CallViewModel } from "./CallViewModel/CallViewModel.ts"; vi.mock("../widget", () => ({ ElementWidgetActions: { diff --git a/src/tile/MediaView.test.tsx b/src/tile/MediaView.test.tsx index c26a4d5f..c8ffbefd 100644 --- a/src/tile/MediaView.test.tsx +++ b/src/tile/MediaView.test.tsx @@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { describe, expect, it, test, vi } from "vitest"; +import { describe, expect, it, test } from "vitest"; import { render, screen } from "@testing-library/react"; import { axe } from "vitest-axe"; import { TooltipProvider } from "@vector-im/compound-web"; @@ -16,7 +16,6 @@ import { import { LocalTrackPublication, Track } from "livekit-client"; import { TrackInfo } from "@livekit/protocol"; import { type ComponentProps } from "react"; -import { type RoomMember } from "matrix-js-sdk"; import { MediaView } from "./MediaView"; import { EncryptionStatus } from "../state/MediaViewModel"; @@ -46,10 +45,8 @@ describe("MediaView", () => { mirror: false, unencryptedWarning: false, video: trackReference, - member: vi.mocked({ - userId: "@alice:example.com", - getMxcAvatarUrl: vi.fn().mockReturnValue(undefined), - } as unknown as RoomMember), + userId: "@alice:example.com", + mxcAvatarUrl: undefined, localParticipant: false, focusable: true, }; diff --git a/src/utils/displayname-integration.test.ts b/src/utils/displayname-integration.test.ts index 5ba42e70..12d65176 100644 --- a/src/utils/displayname-integration.test.ts +++ b/src/utils/displayname-integration.test.ts @@ -6,10 +6,10 @@ Please see LICENSE in the repository root for full details. */ import { afterEach, beforeAll, describe, expect, test, vi } from "vitest"; +import { type RoomMember } from "matrix-js-sdk"; import { shouldDisambiguate } from "./displayname"; import { alice } from "./test-fixtures"; -import { mockMatrixRoom } from "./test"; // Ideally these tests would be in ./displayname.test.ts but I can't figure out how to // just spy on the removeHiddenChars() function without impacting the other tests. @@ -29,7 +29,7 @@ describe("shouldDisambiguate", () => { }); test("should only call removeHiddenChars once for a single displayname", () => { - const room = mockMatrixRoom({}); + const room: Map> = new Map([]); shouldDisambiguate(alice, [], room); expect(jsUtils.removeHiddenChars).toHaveBeenCalledTimes(1); for (let i = 0; i < 10; i++) { diff --git a/src/utils/displayname.test.ts b/src/utils/displayname.test.ts index f28a3e55..442b928a 100644 --- a/src/utils/displayname.test.ts +++ b/src/utils/displayname.test.ts @@ -20,62 +20,70 @@ import { daveRTL, } from "./test-fixtures"; import { mockMatrixRoom } from "./test"; +import { roomToMembersMap } from "../state/CallViewModel/remoteMembers/MatrixMemberMetadata"; describe("shouldDisambiguate", () => { test("should not disambiguate a solo member", () => { - const room = mockMatrixRoom({}); - expect(shouldDisambiguate(alice, [], room)).toEqual(false); + const room = mockMatrixRoom({ + getMembers: () => [], + }); + expect(shouldDisambiguate(alice, [], roomToMembersMap(room))).toEqual( + false, + ); }); test("should not disambiguate a member with an empty displayname", () => { const room = mockMatrixRoom({ - getMember: (u) => - [alice, aliceDoppelganger].find((m) => m.userId === u) ?? null, + getMembers: () => [alice, aliceDoppelganger], }); expect( shouldDisambiguate( { rawDisplayName: "", userId: alice.userId }, [aliceRtcMember, aliceDoppelgangerRtcMember], - room, + roomToMembersMap(room), ), ).toEqual(false); }); test("should disambiguate a member with RTL characters", () => { - const room = mockMatrixRoom({}); - expect(shouldDisambiguate(daveRTL, [], room)).toEqual(true); + const room = mockMatrixRoom({ getMembers: () => [] }); + expect(shouldDisambiguate(daveRTL, [], roomToMembersMap(room))).toEqual( + true, + ); }); test("should disambiguate a member with a matching displayname", () => { const room = mockMatrixRoom({ - getMember: (u) => - [alice, aliceDoppelganger].find((m) => m.userId === u) ?? null, + getMembers: () => [alice, aliceDoppelganger], }); expect( shouldDisambiguate( alice, [aliceRtcMember, aliceDoppelgangerRtcMember], - room, + roomToMembersMap(room), ), ).toEqual(true); expect( shouldDisambiguate( aliceDoppelganger, [aliceRtcMember, aliceDoppelgangerRtcMember], - room, + roomToMembersMap(room), ), ).toEqual(true); }); test("should disambiguate a member with a matching displayname with hidden spaces", () => { const room = mockMatrixRoom({ - getMember: (u) => - [bob, bobZeroWidthSpace].find((m) => m.userId === u) ?? null, + getMembers: () => [bob, bobZeroWidthSpace], }); expect( - shouldDisambiguate(bob, [bobRtcMember, bobZeroWidthSpaceRtcMember], room), + shouldDisambiguate( + bob, + [bobRtcMember, bobZeroWidthSpaceRtcMember], + roomToMembersMap(room), + ), ).toEqual(true); expect( shouldDisambiguate( bobZeroWidthSpace, [bobRtcMember, bobZeroWidthSpaceRtcMember], - room, + roomToMembersMap(room), ), ).toEqual(true); }); @@ -83,11 +91,14 @@ describe("shouldDisambiguate", () => { "should disambiguate a member with a displayname containing a mxid-like string '%s'", (rawDisplayName) => { const room = mockMatrixRoom({ - getMember: (u) => - [alice, aliceDoppelganger].find((m) => m.userId === u) ?? null, + getMembers: () => [alice, aliceDoppelganger], }); expect( - shouldDisambiguate({ rawDisplayName, userId: alice.userId }, [], room), + shouldDisambiguate( + { rawDisplayName, userId: alice.userId }, + [], + roomToMembersMap(room), + ), ).toEqual(true); }, ); diff --git a/src/utils/test-fixtures.ts b/src/utils/test-fixtures.ts index 9d93267e..e3824ac9 100644 --- a/src/utils/test-fixtures.ts +++ b/src/utils/test-fixtures.ts @@ -37,7 +37,6 @@ export const aliceDoppelganger = mockMatrixRoomMember( rawDisplayName: "Alice", }, ); -export const aliceDoppelgangerId = `${aliceDoppelganger.userId}:${aliceDoppelgangerRtcMember.deviceId}`; export const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB"); export const bob = mockMatrixRoomMember(bobRtcMember, { @@ -55,10 +54,8 @@ export const bobZeroWidthSpace = mockMatrixRoomMember( rawDisplayName: "Bo\u200bb", }, ); -export const bobZeroWidthSpaceId = `${bobZeroWidthSpace.userId}:${bobZeroWidthSpaceRtcMember.deviceId}`; export const daveRTLRtcMember = mockRtcMembership("@dave2:example.org", "DDDD"); export const daveRTL = mockMatrixRoomMember(daveRTLRtcMember, { rawDisplayName: "\u202eevaD", }); -export const daveRTLId = `${daveRTL.userId}:${daveRTLRtcMember.deviceId}`; From 9f4d954cfab4f1a0b41e59faa02f1e8e3b2b2f79 Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 12 Nov 2025 12:09:31 +0100 Subject: [PATCH 43/75] The source of the local participant is the createLocalMembership$ and not the MatrixLivekitMembers! Co-authored-by: Valere --- locales/en/app.json | 12 +-- src/state/CallViewModel/CallViewModel.ts | 85 +++++++++++++++++-- .../localMember/LocalMembership.ts | 15 +++- .../remoteMembers/Connection.test.ts | 28 +++--- .../CallViewModel/remoteMembers/Connection.ts | 20 +++-- .../remoteMembers/ConnectionManager.ts | 6 +- .../remoteMembers/MatrixLivekitMembers.ts | 9 +- 7 files changed, 130 insertions(+), 45 deletions(-) diff --git a/locales/en/app.json b/locales/en/app.json index 2c6801bc..2b9eae5a 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -74,16 +74,16 @@ "matrix_id": "Matrix ID: {{id}}", "matrixRTCMode": { "Comptibility": { - "label": "Compatibility: state events & multi SFU", - "description": "Compatible with homeservers that do not support sticky events (but all other EC clients are v0.17.0 or later)" + "description": "Compatible with homeservers that do not support sticky events (but all other EC clients are v0.17.0 or later)", + "label": "Compatibility: state events & multi SFU" }, "Legacy": { - "label": "Legacy: state events & oldest membership SFU", - "description": "Compatible with old versions of EC that do not support multi SFU" + "description": "Compatible with old versions of EC that do not support multi SFU", + "label": "Legacy: state events & oldest membership SFU" }, "Matrix_2_0": { - "label": "Matrix 2.0: sticky events & multi SFU", - "description": "Compatible only with homservers supporting sticky events and all EC clients v0.17.0 or later" + "description": "Compatible only with homservers supporting sticky events and all EC clients v0.17.0 or later", + "label": "Matrix 2.0: sticky events & multi SFU" } }, "mute_all_audio": "Mute all audio (participants, reactions, join sounds)", diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 76eeaeac..916d8e93 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -103,7 +103,10 @@ import { } from "../SessionBehaviors.ts"; import { ECConnectionFactory } from "./remoteMembers/ConnectionFactory.ts"; import { createConnectionManager$ } from "./remoteMembers/ConnectionManager.ts"; -import { createMatrixLivekitMembers$ } from "./remoteMembers/MatrixLivekitMembers.ts"; +import { + createMatrixLivekitMembers$, + type MatrixLivekitMember, +} from "./remoteMembers/MatrixLivekitMembers.ts"; import { createCallNotificationLifecycle$, createReceivedDecline$, @@ -269,6 +272,38 @@ export class CallViewModel { options: this.connectOptions$, }); + private localRtcMembership$ = this.scope.behavior( + this.memberships$.pipe( + map( + (memberships) => + memberships.value.find( + (membership) => + membership.userId === this.userId && + membership.deviceId === this.deviceId, + ) ?? null, + ), + ), + ); + private localMatrixLivekitMemberUninitialized = { + membership$: this.localRtcMembership$, + participant$: this.localMembership.participant$, + connection$: this.localMembership.connection$, + userId: this.userId, + }; + + private localMatrixLivekitMember$: Behavior = + this.scope.behavior( + this.localRtcMembership$.pipe( + switchMap((membership) => { + if (!membership) return of(null); + return of( + // casting is save here since we know that localRtcMembership$ is !== null since we reached this case. + this.localMatrixLivekitMemberUninitialized as MatrixLivekitMember, + ); + }), + ), + ); + // ------------------------------------------------------------------------ private callLifecycle = createCallNotificationLifecycle$({ @@ -283,6 +318,7 @@ export class CallViewModel { localUser: { userId: this.userId, deviceId: this.deviceId }, }); public autoLeave$ = this.callLifecycle.autoLeave$; + // ------------------------------------------------------------------------ /** @@ -377,12 +413,10 @@ export class CallViewModel { ), ); - private roomMembers$ = createRoomMembers$(this.scope, this.matrixRoom); - private matrixMemberMetadataStore = createMatrixMemberMetadata$( this.scope, this.scope.behavior(this.memberships$.pipe(map((mems) => mems.value))), - this.roomMembers$, + createRoomMembers$(this.scope, this.matrixRoom), ); /** @@ -390,22 +424,55 @@ export class CallViewModel { */ // TODO this also needs the local participant to be added. private readonly userMedia$ = this.scope.behavior( - combineLatest([this.matrixLivekitMembers$, duplicateTiles.value$]).pipe( + combineLatest([ + this.localMatrixLivekitMember$, + this.matrixLivekitMembers$, + duplicateTiles.value$, + ]).pipe( // Generate a collection of MediaItems from the list of expected (whether // present or missing) LiveKit participants. generateItems( - function* ([{ value: matrixLivekitMembers }, duplicateTiles]) { + function* ([ + localMatrixLivekitMember, + { value: matrixLivekitMembers }, + duplicateTiles, + ]) { + // add local member if available + if (localMatrixLivekitMember) { + const { + userId, + participant$, + connection$, + // membership$, + } = localMatrixLivekitMember; + const participantId = participant$.value?.identity; // should be membership$.value.membershipID which is not optional + // const participantId = membership$.value.membershipID; + if (participantId) { + for (let dup = 0; dup < 1 + duplicateTiles; dup++) { + yield { + keys: [dup, participantId, userId, participant$, connection$], + data: undefined, + }; + } + } + } + // add remote members that are available for (const { - participantId, userId, participant$, connection$, - } of matrixLivekitMembers) - for (let dup = 0; dup < 1 + duplicateTiles; dup++) + // membership$ + } of matrixLivekitMembers) { + const participantId = participant$.value?.identity; + // const participantId = membership$.value?.identity; + if (!participantId) continue; + for (let dup = 0; dup < 1 + duplicateTiles; dup++) { yield { keys: [dup, participantId, userId, participant$, connection$], data: undefined, }; + } + } }, ( scope, diff --git a/src/state/CallViewModel/localMember/LocalMembership.ts b/src/state/CallViewModel/localMember/LocalMembership.ts index df4d3b6b..86112f4e 100644 --- a/src/state/CallViewModel/localMember/LocalMembership.ts +++ b/src/state/CallViewModel/localMember/LocalMembership.ts @@ -10,6 +10,7 @@ import { type E2EEOptions, type Participant, ParticipantEvent, + type LocalParticipant, } from "livekit-client"; import { observeParticipantEvents } from "@livekit/components-core"; import { @@ -54,6 +55,7 @@ import { getUrlParams } from "../../../UrlParams.ts"; import { PosthogAnalytics } from "../../../analytics/PosthogAnalytics.ts"; import { MatrixRTCMode } from "../../../settings/settings.ts"; import { Config } from "../../../config/Config.ts"; +import { type Connection } from "../remoteMembers/Connection.ts"; export enum LivekitState { Uninitialized = "uninitialized", @@ -82,8 +84,8 @@ type LocalMemberMatrixState = | { state: MatrixState.Disconnected }; export interface LocalMemberConnectionState { - livekit$: BehaviorSubject; - matrix$: BehaviorSubject; + livekit$: Behavior; + matrix$: Behavior; } /* @@ -145,7 +147,8 @@ export const createLocalMembership$ = ({ // Use null here since behavior cannot be initialised with undefined. sharingScreen$: Behavior; toggleScreenSharing: (() => void) | null; - + participant$: Behavior; + connection$: Behavior; // deprecated fields /** @deprecated use state instead*/ homeserverConnected$: Behavior; @@ -317,6 +320,7 @@ export const createLocalMembership$ = ({ state.livekit$.next({ state: LivekitState.Error, error }); }); }); + combineLatest([localTransport$, connectRequested$]).subscribe( ([transport, connectRequested]) => { if ( @@ -515,6 +519,9 @@ export const createLocalMembership$ = ({ alternativeScreenshareToggle, ); + const participant$ = scope.behavior( + connection$.pipe(map((c) => c?.livekitRoom.localParticipant ?? null)), + ); return { startTracks, requestConnect, @@ -526,6 +533,8 @@ export const createLocalMembership$ = ({ configError$, sharingScreen$, toggleScreenSharing, + participant$, + connection$, }; }; diff --git a/src/state/CallViewModel/remoteMembers/Connection.test.ts b/src/state/CallViewModel/remoteMembers/Connection.test.ts index 01a73301..eb9ae89d 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.test.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.test.ts @@ -389,15 +389,17 @@ describe("Publishing participants observations", () => { const bobIsAPublisher = Promise.withResolvers(); const danIsAPublisher = Promise.withResolvers(); const observedPublishers: PublishingParticipant[][] = []; - const s = connection.participants$.subscribe((publishers) => { - observedPublishers.push(publishers); - if (publishers.some((p) => p.identity === "@bob:example.org:DEV111")) { - bobIsAPublisher.resolve(); - } - if (publishers.some((p) => p.identity === "@dan:example.org:DEV333")) { - danIsAPublisher.resolve(); - } - }); + const s = connection.remoteParticipantsWithTracks$.subscribe( + (publishers) => { + observedPublishers.push(publishers); + if (publishers.some((p) => p.identity === "@bob:example.org:DEV111")) { + bobIsAPublisher.resolve(); + } + if (publishers.some((p) => p.identity === "@dan:example.org:DEV333")) { + danIsAPublisher.resolve(); + } + }, + ); onTestFinished(() => s.unsubscribe()); // The publishingParticipants$ observable is derived from the current members of the // livekitRoom and the rtc membership in order to publish the members that are publishing @@ -513,9 +515,11 @@ describe("Publishing participants observations", () => { const connection = setupRemoteConnection(); let observedPublishers: PublishingParticipant[][] = []; - const s = connection.participants$.subscribe((publishers) => { - observedPublishers.push(publishers); - }); + const s = connection.remoteParticipantsWithTracks$.subscribe( + (publishers) => { + observedPublishers.push(publishers); + }, + ); onTestFinished(() => s.unsubscribe()); let participants: RemoteParticipant[] = [ diff --git a/src/state/CallViewModel/remoteMembers/Connection.ts b/src/state/CallViewModel/remoteMembers/Connection.ts index 60251541..fa66183a 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.ts @@ -179,12 +179,13 @@ export class Connection { } /** - * An observable of the participants that are publishing on this connection. + * An observable of the participants that are publishing on this connection. (Excluding our local participant) * This is derived from `participantsIncludingSubscribers$` and `remoteTransports$`. * It filters the participants to only those that are associated with a membership that claims to publish on this connection. */ - - public readonly participants$: Behavior; + public readonly remoteParticipantsWithTracks$: Behavior< + PublishingParticipant[] + >; /** * The media transport to connect to. @@ -211,7 +212,9 @@ export class Connection { this.transport = transport; this.client = client; - this.participants$ = scope.behavior( + // REMOTE participants with track!!! + // this.remoteParticipantsWithTracks$ + this.remoteParticipantsWithTracks$ = scope.behavior( // only tracks remote participants connectedParticipantsObserver(this.livekitRoom, { additionalRoomEvents: [ @@ -219,10 +222,11 @@ export class Connection { RoomEvent.TrackUnpublished, ], }).pipe( - map((participants) => [ - this.livekitRoom.localParticipant, - ...participants, - ]), + map((participants) => { + return participants.filter( + (participant) => participant.getTrackPublications().length > 0, + ); + }), ), [], ); diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts index 73ca3d16..32d42d75 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts @@ -55,8 +55,8 @@ export class ConnectionManagerData { public getConnectionForTransport( transport: LivekitTransport, - ): Connection | undefined { - return this.store.get(this.getKey(transport))?.[0]; + ): Connection | null { + return this.store.get(this.getKey(transport))?.[0] ?? null; } public getParticipantForTransport( @@ -181,7 +181,7 @@ export function createConnectionManager$({ // Map the connections to list of {connection, participants}[] const listOfConnectionsWithPublishingParticipants = connections.value.map((connection) => { - return connection.participants$.pipe( + return connection.remoteParticipantsWithTracks$.pipe( map((participants) => ({ connection, participants, diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts index 326eb0f6..3b31cd33 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts @@ -30,13 +30,14 @@ const logger = rootLogger.getChild("MatrixLivekitMembers"); * or if it has no livekit transport at all. */ export interface MatrixLivekitMember { - participantId: string; - userId: string; membership$: Behavior; participant$: Behavior< LocalLivekitParticipant | RemoteLivekitParticipant | null >; - connection$: Behavior; + connection$: Behavior; + // participantId: string; We do not want a participantId here since it will be generated by the jwt + // TODO decide if we can also drop the userId. Its in the matrix membership anyways. + userId: string; } interface Props { @@ -96,7 +97,7 @@ export function createMatrixLivekitMembers$({ participants.find((p) => p.identity == participantId) ?? null; const connection = transport ? managerData.getConnectionForTransport(transport) - : undefined; + : null; yield { keys: [participantId, membership.userId], From 8d421899a649ba28bcf6dec286938d829ae501c5 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 12 Nov 2025 10:16:04 -0500 Subject: [PATCH 44/75] Fix formatting of doc comment --- src/state/ObservableScope.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/state/ObservableScope.ts b/src/state/ObservableScope.ts index ae46a242..27f501c7 100644 --- a/src/state/ObservableScope.ts +++ b/src/state/ObservableScope.ts @@ -156,8 +156,8 @@ export class ObservableScope { * Splits a Behavior of objects with static properties into an object with * Behavior properties. * - * For example, splitting a Behavior<{ name: string, age: number }> results in - * an object of type { name$: Behavior age$: Behavior }. + * For example, splitting a `Behavior<{ name: string; age: number }>` results + * in an object of type `{ name$: Behavior; age$: Behavior }`. */ public splitBehavior( input$: Behavior, From 27b76b4b1dd296080fe780c03eb6f616b60c788b Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 12 Nov 2025 14:28:26 -0500 Subject: [PATCH 45/75] Remove unused Async type --- src/state/Async.ts | 59 ---------------------------------------------- 1 file changed, 59 deletions(-) delete mode 100644 src/state/Async.ts diff --git a/src/state/Async.ts b/src/state/Async.ts deleted file mode 100644 index e95eaa39..00000000 --- a/src/state/Async.ts +++ /dev/null @@ -1,59 +0,0 @@ -/* -Copyright 2025 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { catchError, from, map, type Observable, of, startWith } from "rxjs"; - -/** - * Data that may need to be loaded asynchronously. - * - * This type is for when you need to represent the current state of an operation - * involving Promises as **immutable data**. See the async$ function below. - */ -export type Async = - | { state: "loading" } - | { state: "error"; value: Error } - | { state: "ready"; value: A }; - -export const loading: Async = { state: "loading" }; -export function error(value: Error): Async { - return { state: "error", value }; -} - -export function ready(value: A): Async { - return { state: "ready", value }; -} - -/** - * Turn a Promise into an Observable async value. The Observable will have the - * value "loading" while the Promise is pending, "ready" when the Promise - * resolves, and "error" when the Promise rejects. - */ -export function async$(promise: Promise): Observable> { - return from(promise).pipe( - map(ready), - startWith(loading), - catchError((e: unknown) => - of(error((e as Error) ?? new Error("Unknown error"))), - ), - ); -} - -/** - * If the async value is ready, apply the given function to the inner value. - */ -export function mapAsync( - async: Async, - project: (value: A) => B, -): Async { - return async.state === "ready" ? ready(project(async.value)) : async; -} - -export function unwrapAsync(fallback: A): (async: Async) => A { - return (async: Async) => { - return async.state === "ready" ? async.value : fallback; - }; -} From a62d8368a1ecaa514f5143f4fb0ccb010b3bfc91 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 12 Nov 2025 15:02:19 -0500 Subject: [PATCH 46/75] Fix and simplify screen sharing --- src/room/InCallView.tsx | 3 +- src/state/CallViewModel/CallViewModel.ts | 3 +- .../localMember/LocalMembership.ts | 85 ++++++------------- 3 files changed, 28 insertions(+), 63 deletions(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 7f469460..c506e96b 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -734,8 +734,7 @@ export const InCallView: FC = ({ Behavior; requestDisconnect: () => Observable | null; connectionState: LocalMemberConnectionState; - // Use null here since behavior cannot be initialised with undefined. - sharingScreen$: Behavior; + sharingScreen$: Behavior; + /** + * Callback to toggle screen sharing. If null, screen sharing is not possible. + */ toggleScreenSharing: (() => void) | null; participant$: Behavior; connection$: Behavior; @@ -453,72 +452,40 @@ export const createLocalMembership$ = ({ }); /** - * Returns undefined if scrennSharing is not yet ready. + * Whether the user is currently sharing their screen. */ const sharingScreen$ = scope.behavior( connection$.pipe( - switchMap((c) => { - if (!c) return of(null); - if (c.state$.value.state === "ConnectedToLkRoom") - return observeSharingScreen$(c.livekitRoom.localParticipant); - return of(false); - }), + switchMap((c) => + c === null + ? of(false) + : observeSharingScreen$(c.livekitRoom.localParticipant), + ), ), - null, ); const toggleScreenSharing = "getDisplayMedia" in (navigator.mediaDevices ?? {}) && !getUrlParams().hideScreensharing ? (): void => - // If a connection is ready... - void connection$ - .pipe( - // I dont see why we need this. isnt the check later on superseeding it? - takeWhile( - (c) => c !== null && c.state$.value.state !== "FailedToStart", - ), - switchMap((c) => - c?.state$.value.state === "ConnectedToLkRoom" ? of(c) : NEVER, - ), - take(1), - scope.bind(), - ) - // ...toggle screen sharing. - .subscribe( - (c) => - void c.livekitRoom.localParticipant - .setScreenShareEnabled(!sharingScreen$.value, { - audio: true, - selfBrowserSurface: "include", - surfaceSwitching: "include", - systemAudio: "include", - }) - .catch(logger.error), - ) + // If a connection is ready, toggle screen sharing. + // We deliberately do nothing in the case of a null connection because + // it looks nice for the call control buttons to all become available + // at once upon joining the call, rather than introducing a disabled + // state. The user can just click again. + // We also allow screen sharing to be toggled even if the connection + // is still initializing or publishing tracks, because there's no + // technical reason to disallow this. LiveKit will publish if it can. + void connection$.value?.livekitRoom.localParticipant + .setScreenShareEnabled(!sharingScreen$.value, { + audio: true, + selfBrowserSurface: "include", + surfaceSwitching: "include", + systemAudio: "include", + }) + .catch(logger.error) : null; - // we do not need all the auto waiting since we can just check via sharingScreen$.value !== undefined - let alternativeScreenshareToggle: (() => void) | null = null; - if ( - "getDisplayMedia" in (navigator.mediaDevices ?? {}) && - !getUrlParams().hideScreensharing - ) { - alternativeScreenshareToggle = (): void => - void connection$.value?.livekitRoom.localParticipant - .setScreenShareEnabled(!sharingScreen$.value, { - audio: true, - selfBrowserSurface: "include", - surfaceSwitching: "include", - systemAudio: "include", - }) - .catch(logger.error); - } - logger.log( - "alternativeScreenshareToggle so that it is used", - alternativeScreenshareToggle, - ); - const participant$ = scope.behavior( connection$.pipe(map((c) => c?.livekitRoom.localParticipant ?? null)), ); From c7f50b53f5db78ab5b74c8bf73ef0a9c82862da3 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 12 Nov 2025 15:41:41 -0500 Subject: [PATCH 47/75] Fix decryption errors The code had regressed to a state where it was attempting to use one encryption worker for all LiveKit rooms, which does not currently work. --- src/state/CallViewModel/CallViewModel.ts | 17 +++-------------- .../localMember/LocalMembership.ts | 4 ---- .../CallViewModel/localMember/Publisher.ts | 4 +--- .../remoteMembers/ConnectionFactory.ts | 11 +++++++++-- 4 files changed, 13 insertions(+), 23 deletions(-) diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 06970b93..aa7f32be 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -8,12 +8,10 @@ Please see LICENSE in the repository root for full details. import { type BaseKeyProvider, type ConnectionState, - type E2EEOptions, ExternalE2EEKeyProvider, type Room as LivekitRoom, type RoomOptions, } from "livekit-client"; -import E2EEWorker from "livekit-client/e2ee-worker?worker"; import { type Room as MatrixRoom } from "matrix-js-sdk"; import { combineLatest, @@ -179,19 +177,11 @@ export class CallViewModel { private readonly userId = this.matrixRoom.client.getUserId()!; private readonly deviceId = this.matrixRoom.client.getDeviceId()!; - private readonly livekitE2EEKeyProvider = getE2eeKeyProvider( + private readonly livekitKeyProvider = getE2eeKeyProvider( this.options.encryptionSystem, this.matrixRTCSession, ); - private readonly e2eeLivekitOptions: E2EEOptions | undefined = this - .livekitE2EEKeyProvider - ? { - keyProvider: this.livekitE2EEKeyProvider, - worker: new E2EEWorker(), - } - : undefined; - private memberships$ = createMemberships$(this.scope, this.matrixRTCSession); private membershipsAndTransports = membershipsAndTransports$( @@ -215,7 +205,7 @@ export class CallViewModel { this.matrixRoom.client, this.mediaDevices, this.trackProcessorState$, - this.e2eeLivekitOptions, + this.livekitKeyProvider, getUrlParams().controlledAudioDevices, ); @@ -251,7 +241,7 @@ export class CallViewModel { private connectOptions$ = this.scope.behavior( matrixRTCMode.value$.pipe( map((mode) => ({ - encryptMedia: this.e2eeLivekitOptions !== undefined, + encryptMedia: this.livekitKeyProvider !== undefined, // TODO. This might need to get called again on each cahnge of matrixRTCMode... matrixRTCMode: mode, })), @@ -266,7 +256,6 @@ export class CallViewModel { matrixRTCSession: this.matrixRTCSession, matrixRoom: this.matrixRoom, localTransport$: this.localTransport$, - e2eeLivekitOptions: this.e2eeLivekitOptions, trackProcessorState$: this.trackProcessorState$, widget, options: this.connectOptions$, diff --git a/src/state/CallViewModel/localMember/LocalMembership.ts b/src/state/CallViewModel/localMember/LocalMembership.ts index d4408f27..1d517643 100644 --- a/src/state/CallViewModel/localMember/LocalMembership.ts +++ b/src/state/CallViewModel/localMember/LocalMembership.ts @@ -7,7 +7,6 @@ Please see LICENSE in the repository root for full details. import { type LocalTrack, - type E2EEOptions, type Participant, ParticipantEvent, type LocalParticipant, @@ -105,7 +104,6 @@ interface Props { matrixRTCSession: MatrixRTCSession; matrixRoom: MatrixRoom; localTransport$: Behavior; - e2eeLivekitOptions: E2EEOptions | undefined; trackProcessorState$: Behavior; widget: WidgetHelpers | null; } @@ -132,7 +130,6 @@ export const createLocalMembership$ = ({ matrixRTCSession, localTransport$, matrixRoom, - e2eeLivekitOptions, trackProcessorState$, widget, }: Props): { @@ -252,7 +249,6 @@ export const createLocalMembership$ = ({ connection, mediaDevices, muteStates, - e2eeLivekitOptions, trackProcessorState$, ), ); diff --git a/src/state/CallViewModel/localMember/Publisher.ts b/src/state/CallViewModel/localMember/Publisher.ts index 1c436397..ff4afbd6 100644 --- a/src/state/CallViewModel/localMember/Publisher.ts +++ b/src/state/CallViewModel/localMember/Publisher.ts @@ -5,7 +5,6 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ import { - type E2EEOptions, LocalVideoTrack, type Room as LivekitRoom, Track, @@ -55,7 +54,6 @@ export class Publisher { private connection: Connection, devices: MediaDevices, private readonly muteStates: MuteStates, - e2eeLivekitOptions: E2EEOptions | undefined, trackerProcessorState$: Behavior, private logger?: Logger, ) { @@ -64,7 +62,7 @@ export class Publisher { const room = connection.livekitRoom; - room.setE2EEEnabled(e2eeLivekitOptions !== undefined)?.catch((e: Error) => { + room.setE2EEEnabled(room.options.e2ee !== undefined)?.catch((e: Error) => { this.logger?.error("Failed to set E2EE enabled on room", e); }); diff --git a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts index a9e2b8fb..9f448cd9 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts @@ -10,8 +10,10 @@ import { type E2EEOptions, Room as LivekitRoom, type RoomOptions, + type BaseKeyProvider, } from "livekit-client"; import { type Logger } from "matrix-js-sdk/lib/logger"; +import E2EEWorker from "livekit-client/e2ee-worker?worker"; import { type ObservableScope } from "../../ObservableScope.ts"; import { Connection } from "./Connection.ts"; @@ -46,7 +48,7 @@ export class ECConnectionFactory implements ConnectionFactory { private client: OpenIDClientParts, private devices: MediaDevices, private processorState$: Behavior, - private e2eeLivekitOptions: E2EEOptions | undefined, + livekitKeyProvider: BaseKeyProvider | undefined, private controlledAudioDevices: boolean, livekitRoomFactory?: () => LivekitRoom, ) { @@ -55,7 +57,12 @@ export class ECConnectionFactory implements ConnectionFactory { generateRoomOption( this.devices, this.processorState$.value, - this.e2eeLivekitOptions, + livekitKeyProvider && { + keyProvider: livekitKeyProvider, + // It's important that every room use a separate E2EE worker. + // They get confused if given streams from multiple rooms. + worker: new E2EEWorker(), + }, this.controlledAudioDevices, ), ); From 0115242a2b5dcc976d6dbb71b57f94efc7d99c68 Mon Sep 17 00:00:00 2001 From: Timo K Date: Thu, 13 Nov 2025 11:35:37 +0100 Subject: [PATCH 48/75] tests first batch --- src/room/InCallView.test.tsx | 7 +- src/room/InCallView.tsx | 14 +- .../__snapshots__/InCallView.test.tsx.snap | 52 +------ src/state/CallViewModel/CallViewModel.test.ts | 5 +- .../remoteMembers/Connection.test.ts | 146 ++++-------------- .../CallViewModel/remoteMembers/Connection.ts | 3 +- .../remoteMembers/ConnectionManager.test.ts | 7 +- src/state/ObservableScope.test.ts | 32 ++-- src/state/ObservableScope.ts | 25 ++- src/utils/test-viewmodel.ts | 5 + 10 files changed, 98 insertions(+), 198 deletions(-) diff --git a/src/room/InCallView.test.tsx b/src/room/InCallView.test.tsx index d388ebc3..a137074b 100644 --- a/src/room/InCallView.test.tsx +++ b/src/room/InCallView.test.tsx @@ -43,9 +43,6 @@ import { LivekitRoomAudioRenderer } from "../livekit/MatrixAudioRenderer"; import { MediaDevicesContext } from "../MediaDevicesContext"; import { HeaderStyle } from "../UrlParams"; -// vi.hoisted(() => { -// localStorage = {} as unknown as Storage; -// }); vi.hoisted( () => (global.ImageData = class MockImageData { @@ -109,6 +106,7 @@ function createInCallView(): RenderResult & { getUserId: () => localRtcMember.userId, getDeviceId: () => localRtcMember.deviceId, getRoom: (rId) => (rId === roomId ? room : null), + getDomain: () => "example.com", } as Partial as MatrixClient; const room = mockMatrixRoom({ relations: { @@ -119,7 +117,8 @@ function createInCallView(): RenderResult & { } as unknown as RelationsContainer, client, roomId, - getMember: (userId) => roomMembers.get(userId) ?? null, + // getMember: (userId) => roomMembers.get(userId) ?? null, + getMembers: () => Array.from(roomMembers.values()), getMxcAvatarUrl: () => null, hasEncryptionStateEvent: vi.fn().mockReturnValue(true), getCanonicalAlias: () => null, diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index c506e96b..1474fb81 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -118,16 +118,16 @@ export interface ActiveCallProps } export const ActiveCall: FC = (props) => { - const mediaDevices = useMediaDevices(); const [vm, setVm] = useState(null); - const { autoLeaveWhenOthersLeft, waitForCallPickup, sendNotificationType } = - useUrlParams(); - + const urlParams = useUrlParams(); + const mediaDevices = useMediaDevices(); const trackProcessorState$ = useTrackProcessorObservable$(); useEffect(() => { const scope = new ObservableScope(); const reactionsReader = new ReactionsReader(scope, props.rtcSession); + const { autoLeaveWhenOthersLeft, waitForCallPickup, sendNotificationType } = + urlParams; const vm = new CallViewModel( scope, props.rtcSession, @@ -152,13 +152,11 @@ export const ActiveCall: FC = (props) => { }, [ props.rtcSession, props.matrixRoom, - mediaDevices, props.muteStates, props.e2eeSystem, - autoLeaveWhenOthersLeft, - sendNotificationType, - waitForCallPickup, props.onLeft, + urlParams, + mediaDevices, trackProcessorState$, ]); diff --git a/src/room/__snapshots__/InCallView.test.tsx.snap b/src/room/__snapshots__/InCallView.test.tsx.snap index 661d13ad..a1b0d226 100644 --- a/src/room/__snapshots__/InCallView.test.tsx.snap +++ b/src/room/__snapshots__/InCallView.test.tsx.snap @@ -49,66 +49,26 @@ exports[`InCallView > rendering > renders 1`] = `