From 8e6eb70e5b60032c33ad03bb085b353cd4cbdcc7 Mon Sep 17 00:00:00 2001 From: Valere Date: Mon, 13 Oct 2025 13:52:01 +0200 Subject: [PATCH 01/22] refactor: use `EnterRTCSessionOptions` instead of unnamed bools --- src/rtcSessionHelpers.test.ts | 20 +++++++++++++++----- src/rtcSessionHelpers.ts | 34 ++++++++++++++++++++++++++++++---- src/state/CallViewModel.ts | 10 ++++++---- 3 files changed, 51 insertions(+), 13 deletions(-) diff --git a/src/rtcSessionHelpers.test.ts b/src/rtcSessionHelpers.test.ts index 258d2f9a..e7f204ec 100644 --- a/src/rtcSessionHelpers.test.ts +++ b/src/rtcSessionHelpers.test.ts @@ -15,6 +15,7 @@ import { mockConfig } from "./utils/test"; import { ElementWidgetActions, widget } from "./widget"; import { ErrorCode } from "./utils/errors.ts"; +const USE_MUTI_SFU = false; const getUrlParams = vi.hoisted(() => vi.fn(() => ({}))); vi.mock("./UrlParams", () => ({ getUrlParams })); @@ -93,7 +94,10 @@ test("It joins the correct Session", async () => { livekit_service_url: "http://my-well-known-service-url.com", type: "livekit", }, - true, + { + encryptMedia: true, + useMultiSfu: USE_MUTI_SFU, + } ); expect(mockedSession.joinRoomSession).toHaveBeenLastCalledWith( @@ -123,12 +127,12 @@ test("It joins the correct Session", async () => { focus_selection: "oldest_membership", type: "livekit", }, - { + expect.objectContaining({ manageMediaKeys: false, useLegacyMemberEvents: false, useNewMembershipManager: true, useExperimentalToDeviceTransport: false, - }, + }), ); }); @@ -197,7 +201,10 @@ test("It fails with configuration error if no live kit url config is set in fall livekit_service_url: "http://my-well-known-service-url.com", type: "livekit", }, - true, + { + encryptMedia: true, + useMultiSfu: USE_MUTI_SFU, + } ), ).rejects.toThrowError( expect.objectContaining({ code: ErrorCode.MISSING_MATRIX_RTC_TRANSPORT }), @@ -240,6 +247,9 @@ test("It should not fail with configuration error if homeserver config has livek livekit_service_url: "http://my-well-known-service-url.com", type: "livekit", }, - true, + { + encryptMedia: true, + useMultiSfu: USE_MUTI_SFU, + } ); }); diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index 3cdd82e7..1bb9f11e 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -97,14 +97,40 @@ export async function makeTransport( return transport; } +export interface EnterRTCSessionOptions { + encryptMedia: boolean; + // TODO: remove this flag, the new membership manager is stable enough + useNewMembershipManager?: boolean; + // TODO: remove this flag, to-device transport is stable enough now + useExperimentalToDeviceTransport?: boolean; + /** EXPERIMENTAL: If true, will use the multi-sfu codepath where each member connects to its SFU instead of everyone connecting to an elected on. */ + useMultiSfu?: boolean; +} + +/** + * TODO! document this function properly + * @param rtcSession + * @param transport + * @param options + */ export async function enterRTCSession( rtcSession: MatrixRTCSession, transport: LivekitTransport, - encryptMedia: boolean, - useNewMembershipManager = true, - useExperimentalToDeviceTransport = false, - useMultiSfu = true, + options: EnterRTCSessionOptions = { + encryptMedia: true, + useNewMembershipManager: true, + useExperimentalToDeviceTransport: false, + useMultiSfu: true, + }, ): Promise { + + const { + encryptMedia, + useNewMembershipManager = true, + useExperimentalToDeviceTransport = false, + useMultiSfu = true, + } = options; + PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date()); PosthogAnalytics.instance.eventCallStarted.track(rtcSession.room.roomId); diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index b2f6a464..82fd7cab 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -2016,10 +2016,12 @@ export class CallViewModel extends ViewModel { await enterRTCSession( this.matrixRTCSession, localTransport.value, - this.options.encryptionSystem.kind !== E2eeType.NONE, - true, - true, - multiSfu.value$.value, + { + encryptMedia: this.options.encryptionSystem.kind !== E2eeType.NONE, + useExperimentalToDeviceTransport: true, + useNewMembershipManager: true, + useMultiSfu: multiSfu.value$.value + } ); } catch (e) { logger.error("Error entering RTC session", e); From 8823be67c57392b1bc832acc690ba8efc12a8bbf Mon Sep 17 00:00:00 2001 From: Valere Date: Mon, 13 Oct 2025 15:43:12 +0200 Subject: [PATCH 02/22] refactor extract inner classes to their own files --- src/state/CallViewModel.ts | 167 +++++-------------------------------- src/state/ScreenShare.ts | 54 ++++++++++++ src/state/UserMedia.ts | 117 ++++++++++++++++++++++++++ 3 files changed, 190 insertions(+), 148 deletions(-) create mode 100644 src/state/ScreenShare.ts create mode 100644 src/state/UserMedia.ts diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 82fd7cab..dff8ae56 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -5,39 +5,32 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { observeParticipantEvents } from "@livekit/components-core"; import { - ConnectionState, type BaseKeyProvider, + ConnectionState, type E2EEOptions, ExternalE2EEKeyProvider, - type Room as LivekitRoom, type LocalParticipant, - ParticipantEvent, RemoteParticipant, - type Participant, + type Room as LivekitRoom, } from "livekit-client"; import E2EEWorker from "livekit-client/e2ee-worker?worker"; import { ClientEvent, + type EventTimelineSetHandlerMap, + EventType, + type Room as MatrixRoom, + RoomEvent, type RoomMember, RoomStateEvent, SyncState, - type Room as MatrixRoom, - type EventTimelineSetHandlerMap, - EventType, - RoomEvent, } from "matrix-js-sdk"; import { deepCompare } from "matrix-js-sdk/lib/utils"; import { - BehaviorSubject, - EMPTY, - NEVER, - type Observable, - Subject, combineLatest, concat, distinctUntilChanged, + EMPTY, endWith, filter, from, @@ -45,6 +38,8 @@ import { ignoreElements, map, merge, + NEVER, + type Observable, of, pairwise, race, @@ -53,6 +48,7 @@ import { skip, skipWhile, startWith, + Subject, switchAll, switchMap, switchScan, @@ -90,7 +86,6 @@ import { finalizeValue, pauseWhen, } from "../utils/observable"; -import { ObservableScope } from "./ObservableScope"; import { duplicateTiles, multiSfu, @@ -114,11 +109,10 @@ import { type ReactionInfo, type ReactionOption, } from "../reactions"; -import { observeSpeaker$ } from "./observeSpeaker"; import { shallowEquals } from "../utils/array"; import { calculateDisplayName, shouldDisambiguate } from "../utils/displayname"; import { type MediaDevices } from "./MediaDevices"; -import { constant, type Behavior } from "./Behavior"; +import { type Behavior, constant } from "./Behavior"; import { enterRTCSession, getLivekitAlias, @@ -137,6 +131,8 @@ import { type ProcessorState } from "../livekit/TrackProcessorContext"; import { ElementWidgetActions, widget } from "../widget"; import { PublishConnection } from "./PublishConnection.ts"; import { type Async, async$, mapAsync, ready } from "./Async"; +import { sharingScreen$, UserMedia } from "./UserMedia.ts"; +import { ScreenShare } from "./ScreenShare.ts"; export interface CallViewModelOptions { encryptionSystem: EncryptionSystem; @@ -297,117 +293,6 @@ interface LayoutScanState { tiles: TileStore; } -class UserMedia { - private readonly scope = new ObservableScope(); - public readonly vm: UserMediaViewModel; - private readonly participant$: BehaviorSubject< - LocalParticipant | RemoteParticipant | undefined - >; - - public readonly speaker$: Behavior; - public readonly presenter$: Behavior; - public constructor( - public readonly id: string, - member: RoomMember, - participant: LocalParticipant | RemoteParticipant | undefined, - encryptionSystem: EncryptionSystem, - livekitRoom: LivekitRoom, - mediaDevices: MediaDevices, - pretendToBeDisconnected$: Behavior, - displayname$: Observable, - handRaised$: Observable, - reaction$: Observable, - ) { - this.participant$ = new BehaviorSubject(participant); - - if (participant?.isLocal) { - this.vm = new LocalUserMediaViewModel( - this.id, - member, - this.participant$ as Behavior, - encryptionSystem, - livekitRoom, - mediaDevices, - this.scope.behavior(displayname$), - this.scope.behavior(handRaised$), - this.scope.behavior(reaction$), - ); - } else { - this.vm = new RemoteUserMediaViewModel( - id, - member, - this.participant$.asObservable() as Observable< - RemoteParticipant | undefined - >, - encryptionSystem, - livekitRoom, - pretendToBeDisconnected$, - this.scope.behavior(displayname$), - this.scope.behavior(handRaised$), - this.scope.behavior(reaction$), - ); - } - - this.speaker$ = this.scope.behavior(observeSpeaker$(this.vm.speaking$)); - - this.presenter$ = this.scope.behavior( - this.participant$.pipe( - switchMap((p) => (p === undefined ? of(false) : sharingScreen$(p))), - ), - ); - } - - public updateParticipant( - newParticipant: LocalParticipant | RemoteParticipant | undefined, - ): void { - if (this.participant$.value !== newParticipant) { - // Update the BehaviourSubject in the UserMedia. - this.participant$.next(newParticipant); - } - } - - public destroy(): void { - this.scope.end(); - this.vm.destroy(); - } -} - -class ScreenShare { - private readonly scope = new ObservableScope(); - public readonly vm: ScreenShareViewModel; - private readonly participant$: BehaviorSubject< - LocalParticipant | RemoteParticipant - >; - - public constructor( - id: string, - member: RoomMember, - participant: LocalParticipant | RemoteParticipant, - encryptionSystem: EncryptionSystem, - livekitRoom: LivekitRoom, - pretendToBeDisconnected$: Behavior, - displayName$: Observable, - ) { - this.participant$ = new BehaviorSubject(participant); - - this.vm = new ScreenShareViewModel( - id, - member, - this.participant$.asObservable(), - encryptionSystem, - livekitRoom, - pretendToBeDisconnected$, - this.scope.behavior(displayName$), - participant.isLocal, - ); - } - - public destroy(): void { - this.scope.end(); - this.vm.destroy(); - } -} - type MediaItem = UserMedia | ScreenShare; function getRoomMemberFromRtcMember( @@ -432,16 +317,6 @@ function getRoomMemberFromRtcMember( return { id, member }; } -function sharingScreen$(p: Participant): Observable { - return observeParticipantEvents( - p, - ParticipantEvent.TrackPublished, - ParticipantEvent.TrackUnpublished, - ParticipantEvent.LocalTrackPublished, - ParticipantEvent.LocalTrackUnpublished, - ).pipe(map((p) => p.isScreenShareEnabled)); -} - export class CallViewModel extends ViewModel { private readonly urlParams = getUrlParams(); @@ -2013,16 +1888,12 @@ export class CallViewModel extends ViewModel { this.scope.reconcile(this.localTransport$, async (localTransport) => { if (localTransport?.state === "ready") { try { - await enterRTCSession( - this.matrixRTCSession, - localTransport.value, - { - encryptMedia: this.options.encryptionSystem.kind !== E2eeType.NONE, - useExperimentalToDeviceTransport: true, - useNewMembershipManager: true, - useMultiSfu: multiSfu.value$.value - } - ); + await enterRTCSession(this.matrixRTCSession, localTransport.value, { + encryptMedia: this.options.encryptionSystem.kind !== E2eeType.NONE, + useExperimentalToDeviceTransport: true, + useNewMembershipManager: true, + useMultiSfu: multiSfu.value$.value, + }); } catch (e) { logger.error("Error entering RTC session", e); } diff --git a/src/state/ScreenShare.ts b/src/state/ScreenShare.ts new file mode 100644 index 00000000..6da18df3 --- /dev/null +++ b/src/state/ScreenShare.ts @@ -0,0 +1,54 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ +import { ObservableScope } from "./ObservableScope.ts"; +import { ScreenShareViewModel } from "./MediaViewModel.ts"; +import { BehaviorSubject, type Observable } from "rxjs"; +import { + LocalParticipant, + RemoteParticipant, + type Room as LivekitRoom, +} from "livekit-client"; +import type { RoomMember } from "matrix-js-sdk"; +import type { EncryptionSystem } from "../e2ee/sharedKeyManagement.ts"; +import type { Behavior } from "./Behavior.ts"; + +// TODO Document this +export class ScreenShare { + private readonly scope = new ObservableScope(); + public readonly vm: ScreenShareViewModel; + private readonly participant$: BehaviorSubject< + LocalParticipant | RemoteParticipant + >; + + public constructor( + id: string, + member: RoomMember, + participant: LocalParticipant | RemoteParticipant, + encryptionSystem: EncryptionSystem, + livekitRoom: LivekitRoom, + pretendToBeDisconnected$: Behavior, + displayName$: Observable, + ) { + this.participant$ = new BehaviorSubject(participant); + + this.vm = new ScreenShareViewModel( + id, + member, + this.participant$.asObservable(), + encryptionSystem, + livekitRoom, + pretendToBeDisconnected$, + this.scope.behavior(displayName$), + participant.isLocal, + ); + } + + public destroy(): void { + this.scope.end(); + this.vm.destroy(); + } +} diff --git a/src/state/UserMedia.ts b/src/state/UserMedia.ts new file mode 100644 index 00000000..be44b998 --- /dev/null +++ b/src/state/UserMedia.ts @@ -0,0 +1,117 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { ObservableScope } from "./ObservableScope.ts"; +import { + LocalUserMediaViewModel, + RemoteUserMediaViewModel, + UserMediaViewModel, +} from "./MediaViewModel.ts"; +import { BehaviorSubject, map, type Observable, of, switchMap } from "rxjs"; +import { + LocalParticipant, + Participant, + ParticipantEvent, + RemoteParticipant, + type Room as LivekitRoom, +} from "livekit-client"; +import type { Behavior } from "./Behavior.ts"; +import type { RoomMember } from "matrix-js-sdk"; +import type { EncryptionSystem } from "../e2ee/sharedKeyManagement.ts"; +import type { MediaDevices } from "./MediaDevices.ts"; +import type { ReactionOption } from "../reactions"; +import { observeSpeaker$ } from "./observeSpeaker.ts"; +import { observeParticipantEvents } from "@livekit/components-core"; + +/** + * TODO Document this + */ +export class UserMedia { + private readonly scope = new ObservableScope(); + public readonly vm: UserMediaViewModel; + private readonly participant$: BehaviorSubject< + LocalParticipant | RemoteParticipant | undefined + >; + + public readonly speaker$: Behavior; + public readonly presenter$: Behavior; + + public constructor( + public readonly id: string, + member: RoomMember, + participant: LocalParticipant | RemoteParticipant | undefined, + encryptionSystem: EncryptionSystem, + livekitRoom: LivekitRoom, + mediaDevices: MediaDevices, + pretendToBeDisconnected$: Behavior, + displayname$: Observable, + handRaised$: Observable, + reaction$: Observable, + ) { + this.participant$ = new BehaviorSubject(participant); + + if (participant?.isLocal) { + this.vm = new LocalUserMediaViewModel( + this.id, + member, + this.participant$ as Behavior, + encryptionSystem, + livekitRoom, + mediaDevices, + this.scope.behavior(displayname$), + this.scope.behavior(handRaised$), + this.scope.behavior(reaction$), + ); + } else { + this.vm = new RemoteUserMediaViewModel( + id, + member, + this.participant$.asObservable() as Observable< + RemoteParticipant | undefined + >, + encryptionSystem, + livekitRoom, + pretendToBeDisconnected$, + this.scope.behavior(displayname$), + this.scope.behavior(handRaised$), + this.scope.behavior(reaction$), + ); + } + + this.speaker$ = this.scope.behavior(observeSpeaker$(this.vm.speaking$)); + + this.presenter$ = this.scope.behavior( + this.participant$.pipe( + switchMap((p) => (p === undefined ? of(false) : sharingScreen$(p))), + ), + ); + } + + public updateParticipant( + newParticipant: LocalParticipant | RemoteParticipant | undefined, + ): void { + if (this.participant$.value !== newParticipant) { + // Update the BehaviourSubject in the UserMedia. + this.participant$.next(newParticipant); + } + } + + public destroy(): void { + this.scope.end(); + this.vm.destroy(); + } +} + +export function sharingScreen$(p: Participant): Observable { + return observeParticipantEvents( + p, + ParticipantEvent.TrackPublished, + ParticipantEvent.TrackUnpublished, + ParticipantEvent.LocalTrackPublished, + ParticipantEvent.LocalTrackUnpublished, + ).pipe(map((p) => p.isScreenShareEnabled)); +} From 0e1b61a5e879a3586350a8490d381c0592b04e4b Mon Sep 17 00:00:00 2001 From: Valere Date: Mon, 13 Oct 2025 16:24:55 +0200 Subject: [PATCH 03/22] refactor: Split out all exports of CallViewModel to their own file --- src/grid/GridLayout.tsx | 2 +- src/grid/OneOnOneLayout.tsx | 2 +- src/grid/SpotlightExpandedLayout.tsx | 2 +- src/grid/SpotlightLandscapeLayout.tsx | 2 +- src/grid/SpotlightPortraitLayout.tsx | 2 +- src/room/InCallView.tsx | 7 +- src/state/CallViewModel.test.ts | 7 +- src/state/CallViewModel.ts | 150 ++++++-------------------- src/state/GridLikeLayout.ts | 2 +- src/state/OneOnOneLayout.ts | 2 +- src/state/PipLayout.ts | 2 +- src/state/SpotlightExpandedLayout.ts | 2 +- src/state/layout-types.ts | 102 ++++++++++++++++++ 13 files changed, 146 insertions(+), 138 deletions(-) create mode 100644 src/state/layout-types.ts diff --git a/src/grid/GridLayout.tsx b/src/grid/GridLayout.tsx index 51fbc6ea..cf46e8b4 100644 --- a/src/grid/GridLayout.tsx +++ b/src/grid/GridLayout.tsx @@ -14,7 +14,7 @@ import { import { distinctUntilChanged } from "rxjs"; import { useObservableEagerState } from "observable-hooks"; -import { type GridLayout as GridLayoutModel } from "../state/CallViewModel"; +import { type GridLayout as GridLayoutModel } from "../state/layout-types.ts"; import styles from "./GridLayout.module.css"; import { useInitial } from "../useInitial"; import { type CallLayout, arrangeTiles } from "./CallLayout"; diff --git a/src/grid/OneOnOneLayout.tsx b/src/grid/OneOnOneLayout.tsx index 675e4d0a..6c5ae69f 100644 --- a/src/grid/OneOnOneLayout.tsx +++ b/src/grid/OneOnOneLayout.tsx @@ -9,7 +9,7 @@ import { type ReactNode, useCallback, useMemo } from "react"; import { useObservableEagerState } from "observable-hooks"; import classNames from "classnames"; -import { type OneOnOneLayout as OneOnOneLayoutModel } from "../state/CallViewModel"; +import { type OneOnOneLayout as OneOnOneLayoutModel } from "../state/layout-types.ts"; import { type CallLayout, arrangeTiles } from "./CallLayout"; import styles from "./OneOnOneLayout.module.css"; import { type DragCallback, useUpdateLayout } from "./Grid"; diff --git a/src/grid/SpotlightExpandedLayout.tsx b/src/grid/SpotlightExpandedLayout.tsx index 9dd2a109..ac47f0d4 100644 --- a/src/grid/SpotlightExpandedLayout.tsx +++ b/src/grid/SpotlightExpandedLayout.tsx @@ -7,7 +7,7 @@ Please see LICENSE in the repository root for full details. import { type ReactNode, useCallback } from "react"; -import { type SpotlightExpandedLayout as SpotlightExpandedLayoutModel } from "../state/CallViewModel"; +import { type SpotlightExpandedLayout as SpotlightExpandedLayoutModel } from "../state/layout-types.ts"; import { type CallLayout } from "./CallLayout"; import { type DragCallback, useUpdateLayout } from "./Grid"; import styles from "./SpotlightExpandedLayout.module.css"; diff --git a/src/grid/SpotlightLandscapeLayout.tsx b/src/grid/SpotlightLandscapeLayout.tsx index 96343296..d87be1f1 100644 --- a/src/grid/SpotlightLandscapeLayout.tsx +++ b/src/grid/SpotlightLandscapeLayout.tsx @@ -10,7 +10,7 @@ import { useObservableEagerState } from "observable-hooks"; import classNames from "classnames"; import { type CallLayout } from "./CallLayout"; -import { type SpotlightLandscapeLayout as SpotlightLandscapeLayoutModel } from "../state/CallViewModel"; +import { type SpotlightLandscapeLayout as SpotlightLandscapeLayoutModel } from "../state/layout-types.ts"; import styles from "./SpotlightLandscapeLayout.module.css"; import { useUpdateLayout, useVisibleTiles } from "./Grid"; diff --git a/src/grid/SpotlightPortraitLayout.tsx b/src/grid/SpotlightPortraitLayout.tsx index ad11ed11..a6d1241c 100644 --- a/src/grid/SpotlightPortraitLayout.tsx +++ b/src/grid/SpotlightPortraitLayout.tsx @@ -10,7 +10,7 @@ import { useObservableEagerState } from "observable-hooks"; import classNames from "classnames"; import { type CallLayout, arrangeTiles } from "./CallLayout"; -import { type SpotlightPortraitLayout as SpotlightPortraitLayoutModel } from "../state/CallViewModel"; +import { type SpotlightPortraitLayout as SpotlightPortraitLayoutModel } from "../state/layout-types.ts"; import styles from "./SpotlightPortraitLayout.module.css"; import { useUpdateLayout, useVisibleTiles } from "./Grid"; import { useBehavior } from "../useBehavior"; diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index fd631bae..3a27e250 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -59,11 +59,7 @@ import { type MuteStates } from "../state/MuteStates"; import { type MatrixInfo } from "./VideoPreview"; import { InviteButton } from "../button/InviteButton"; import { LayoutToggle } from "./LayoutToggle"; -import { - CallViewModel, - type GridMode, - type Layout, -} from "../state/CallViewModel"; +import { CallViewModel, GridMode } from "../state/CallViewModel"; import { Grid, type TileProps } from "../grid/Grid"; import { useInitial } from "../useInitial"; import { SpotlightTile } from "../tile/SpotlightTile"; @@ -113,6 +109,7 @@ import { useAudioContext } from "../useAudioContext"; import ringtoneMp3 from "../sound/ringtone.mp3?url"; import ringtoneOgg from "../sound/ringtone.ogg?url"; import { useTrackProcessorObservable$ } from "../livekit/TrackProcessorContext.tsx"; +import { Layout } from "../state/layout-types.ts"; const maxTapDurationMs = 400; diff --git a/src/state/CallViewModel.test.ts b/src/state/CallViewModel.test.ts index 055720c8..f03b648d 100644 --- a/src/state/CallViewModel.test.ts +++ b/src/state/CallViewModel.test.ts @@ -46,11 +46,8 @@ import { } from "matrix-js-sdk/lib/matrixrtc"; import { deepCompare } from "matrix-js-sdk/lib/utils"; -import { - CallViewModel, - type CallViewModelOptions, - type Layout, -} from "./CallViewModel"; +import { CallViewModel, type CallViewModelOptions } from "./CallViewModel"; +import { type Layout } from "./layout-types"; import { mockLocalParticipant, mockMatrixRoom, diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index dff8ae56..3cabf697 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -94,10 +94,6 @@ import { } from "../settings/settings"; import { isFirefox } from "../Platform"; import { setPipEnabled$ } from "../controls"; -import { - type GridTileViewModel, - type SpotlightTileViewModel, -} from "./TileViewModel"; import { TileStore } from "./TileStore"; import { gridLikeLayout } from "./GridLikeLayout"; import { spotlightExpandedLayout } from "./SpotlightExpandedLayout"; @@ -133,6 +129,15 @@ import { PublishConnection } from "./PublishConnection.ts"; import { type Async, async$, mapAsync, ready } from "./Async"; import { sharingScreen$, UserMedia } from "./UserMedia.ts"; import { ScreenShare } from "./ScreenShare.ts"; +import { + GridLayoutMedia, + Layout, + LayoutMedia, + OneOnOneLayoutMedia, + SpotlightExpandedLayoutMedia, + SpotlightLandscapeLayoutMedia, + SpotlightPortraitLayoutMedia, +} from "./layout-types.ts"; export interface CallViewModelOptions { encryptionSystem: EncryptionSystem; @@ -157,99 +162,6 @@ const smallMobileCallThreshold = 3; // with the interface const showFooterMs = 4000; -export interface GridLayoutMedia { - type: "grid"; - spotlight?: MediaViewModel[]; - grid: UserMediaViewModel[]; -} - -export interface SpotlightLandscapeLayoutMedia { - type: "spotlight-landscape"; - spotlight: MediaViewModel[]; - grid: UserMediaViewModel[]; -} - -export interface SpotlightPortraitLayoutMedia { - type: "spotlight-portrait"; - spotlight: MediaViewModel[]; - grid: UserMediaViewModel[]; -} - -export interface SpotlightExpandedLayoutMedia { - type: "spotlight-expanded"; - spotlight: MediaViewModel[]; - pip?: UserMediaViewModel; -} - -export interface OneOnOneLayoutMedia { - type: "one-on-one"; - local: UserMediaViewModel; - remote: UserMediaViewModel; -} - -export interface PipLayoutMedia { - type: "pip"; - spotlight: MediaViewModel[]; -} - -export type LayoutMedia = - | GridLayoutMedia - | SpotlightLandscapeLayoutMedia - | SpotlightPortraitLayoutMedia - | SpotlightExpandedLayoutMedia - | OneOnOneLayoutMedia - | PipLayoutMedia; - -export interface GridLayout { - type: "grid"; - spotlight?: SpotlightTileViewModel; - grid: GridTileViewModel[]; - setVisibleTiles: (value: number) => void; -} - -export interface SpotlightLandscapeLayout { - type: "spotlight-landscape"; - spotlight: SpotlightTileViewModel; - grid: GridTileViewModel[]; - setVisibleTiles: (value: number) => void; -} - -export interface SpotlightPortraitLayout { - type: "spotlight-portrait"; - spotlight: SpotlightTileViewModel; - grid: GridTileViewModel[]; - setVisibleTiles: (value: number) => void; -} - -export interface SpotlightExpandedLayout { - type: "spotlight-expanded"; - spotlight: SpotlightTileViewModel; - pip?: GridTileViewModel; -} - -export interface OneOnOneLayout { - type: "one-on-one"; - local: GridTileViewModel; - remote: GridTileViewModel; -} - -export interface PipLayout { - type: "pip"; - spotlight: SpotlightTileViewModel; -} - -/** - * A layout defining the media tiles present on screen and their visual - * arrangement. - */ -export type Layout = - | GridLayout - | SpotlightLandscapeLayout - | SpotlightPortraitLayout - | SpotlightExpandedLayout - | OneOnOneLayout - | PipLayout; - export type GridMode = "grid" | "spotlight"; export type WindowMode = "normal" | "narrow" | "flat" | "pip"; @@ -295,28 +207,6 @@ interface LayoutScanState { type MediaItem = UserMedia | ScreenShare; -function getRoomMemberFromRtcMember( - rtcMember: CallMembership, - room: MatrixRoom, -): { id: string; member: RoomMember | undefined } { - // WARN! This is not exactly the sender but the user defined in the state key. - // This will be available once we change to the new "member as object" format in the MatrixRTC object. - let id = rtcMember.sender + ":" + rtcMember.deviceId; - - if (!rtcMember.sender) { - return { id, member: undefined }; - } - if ( - rtcMember.sender === room.client.getUserId() && - rtcMember.deviceId === room.client.getDeviceId() - ) { - id = "local"; - } - - const member = room.getMember(rtcMember.sender) ?? undefined; - return { id, member }; -} - export class CallViewModel extends ViewModel { private readonly urlParams = getUrlParams(); @@ -2004,3 +1894,25 @@ function getE2eeKeyProvider( return keyProvider; } } + +function getRoomMemberFromRtcMember( + rtcMember: CallMembership, + room: MatrixRoom, +): { id: string; member: RoomMember | undefined } { + // WARN! This is not exactly the sender but the user defined in the state key. + // This will be available once we change to the new "member as object" format in the MatrixRTC object. + let id = rtcMember.sender + ":" + rtcMember.deviceId; + + if (!rtcMember.sender) { + return { id, member: undefined }; + } + if ( + rtcMember.sender === room.client.getUserId() && + rtcMember.deviceId === room.client.getDeviceId() + ) { + id = "local"; + } + + const member = room.getMember(rtcMember.sender) ?? undefined; + return { id, member }; +} diff --git a/src/state/GridLikeLayout.ts b/src/state/GridLikeLayout.ts index 0740f26c..0d130834 100644 --- a/src/state/GridLikeLayout.ts +++ b/src/state/GridLikeLayout.ts @@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type Layout, type LayoutMedia } from "./CallViewModel"; +import { type Layout, type LayoutMedia } from "./layout-types.ts"; import { type TileStore } from "./TileStore"; export type GridLikeLayoutType = diff --git a/src/state/OneOnOneLayout.ts b/src/state/OneOnOneLayout.ts index b8c7b8fb..10268945 100644 --- a/src/state/OneOnOneLayout.ts +++ b/src/state/OneOnOneLayout.ts @@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type OneOnOneLayout, type OneOnOneLayoutMedia } from "./CallViewModel"; +import { type OneOnOneLayout, type OneOnOneLayoutMedia } from "./layout-types"; import { type TileStore } from "./TileStore"; /** diff --git a/src/state/PipLayout.ts b/src/state/PipLayout.ts index ab066410..56e9aeb2 100644 --- a/src/state/PipLayout.ts +++ b/src/state/PipLayout.ts @@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type PipLayout, type PipLayoutMedia } from "./CallViewModel"; +import { type PipLayout, type PipLayoutMedia } from "./layout-types.ts"; import { type TileStore } from "./TileStore"; /** diff --git a/src/state/SpotlightExpandedLayout.ts b/src/state/SpotlightExpandedLayout.ts index 4baba0a1..8ccc49dd 100644 --- a/src/state/SpotlightExpandedLayout.ts +++ b/src/state/SpotlightExpandedLayout.ts @@ -8,7 +8,7 @@ Please see LICENSE in the repository root for full details. import { type SpotlightExpandedLayout, type SpotlightExpandedLayoutMedia, -} from "./CallViewModel"; +} from "./layout-types"; import { type TileStore } from "./TileStore"; /** diff --git a/src/state/layout-types.ts b/src/state/layout-types.ts new file mode 100644 index 00000000..f28ada46 --- /dev/null +++ b/src/state/layout-types.ts @@ -0,0 +1,102 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { GridTileViewModel, SpotlightTileViewModel } from "./TileViewModel.ts"; +import { MediaViewModel, UserMediaViewModel } from "./MediaViewModel.ts"; + +export interface GridLayoutMedia { + type: "grid"; + spotlight?: MediaViewModel[]; + grid: UserMediaViewModel[]; +} + +export interface SpotlightLandscapeLayoutMedia { + type: "spotlight-landscape"; + spotlight: MediaViewModel[]; + grid: UserMediaViewModel[]; +} + +export interface SpotlightPortraitLayoutMedia { + type: "spotlight-portrait"; + spotlight: MediaViewModel[]; + grid: UserMediaViewModel[]; +} + +export interface SpotlightExpandedLayoutMedia { + type: "spotlight-expanded"; + spotlight: MediaViewModel[]; + pip?: UserMediaViewModel; +} + +export interface OneOnOneLayoutMedia { + type: "one-on-one"; + local: UserMediaViewModel; + remote: UserMediaViewModel; +} + +export interface PipLayoutMedia { + type: "pip"; + spotlight: MediaViewModel[]; +} + +export type LayoutMedia = + | GridLayoutMedia + | SpotlightLandscapeLayoutMedia + | SpotlightPortraitLayoutMedia + | SpotlightExpandedLayoutMedia + | OneOnOneLayoutMedia + | PipLayoutMedia; + +export interface GridLayout { + type: "grid"; + spotlight?: SpotlightTileViewModel; + grid: GridTileViewModel[]; + setVisibleTiles: (value: number) => void; +} + +export interface SpotlightLandscapeLayout { + type: "spotlight-landscape"; + spotlight: SpotlightTileViewModel; + grid: GridTileViewModel[]; + setVisibleTiles: (value: number) => void; +} + +export interface SpotlightPortraitLayout { + type: "spotlight-portrait"; + spotlight: SpotlightTileViewModel; + grid: GridTileViewModel[]; + setVisibleTiles: (value: number) => void; +} + +export interface SpotlightExpandedLayout { + type: "spotlight-expanded"; + spotlight: SpotlightTileViewModel; + pip?: GridTileViewModel; +} + +export interface OneOnOneLayout { + type: "one-on-one"; + local: GridTileViewModel; + remote: GridTileViewModel; +} + +export interface PipLayout { + type: "pip"; + spotlight: SpotlightTileViewModel; +} + +/** + * A layout defining the media tiles present on screen and their visual + * arrangement. + */ +export type Layout = + | GridLayout + | SpotlightLandscapeLayout + | SpotlightPortraitLayout + | SpotlightExpandedLayout + | OneOnOneLayout + | PipLayout; From a5aba928dd985e27362d48dfe92d218f28e05b0b Mon Sep 17 00:00:00 2001 From: Valere Date: Mon, 13 Oct 2025 16:39:14 +0200 Subject: [PATCH 04/22] dependency: depends on js-sdk develop --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 29b774d5..411d98a2 100644 --- a/package.json +++ b/package.json @@ -109,7 +109,7 @@ "livekit-client": "^2.13.0", "lodash-es": "^4.17.21", "loglevel": "^1.9.1", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=toger5/use-relation-based-CallMembership-create-ts", + "matrix-js-sdk": "matrix-org/matrix-js-sdk#develop", "matrix-widget-api": "^1.13.0", "normalize.css": "^8.0.1", "observable-hooks": "^4.2.3", diff --git a/yarn.lock b/yarn.lock index 044bf4af..fb6e7dcc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7545,7 +7545,7 @@ __metadata: livekit-client: "npm:^2.13.0" lodash-es: "npm:^4.17.21" loglevel: "npm:^1.9.1" - matrix-js-sdk: "github:matrix-org/matrix-js-sdk#head=toger5/use-relation-based-CallMembership-create-ts" + matrix-js-sdk: "matrix-org/matrix-js-sdk#develop" matrix-widget-api: "npm:^1.13.0" normalize.css: "npm:^8.0.1" observable-hooks: "npm:^4.2.3" @@ -10335,9 +10335,9 @@ __metadata: languageName: node linkType: hard -"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=toger5/use-relation-based-CallMembership-create-ts": +"matrix-js-sdk@matrix-org/matrix-js-sdk#develop": version: 38.4.0 - resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=4608506288c6beaa252982d224e996e23e51f681" + resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=fd949fe486038099ee111a72b57ce711e85bc352" dependencies: "@babel/runtime": "npm:^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm": "npm:^15.3.0" @@ -10353,7 +10353,7 @@ __metadata: sdp-transform: "npm:^2.14.1" unhomoglyph: "npm:^1.0.6" uuid: "npm:13" - checksum: 10c0/2e896d6a92cb3bbb47c120a39dd1a0030b4bf02289cb914f6c848b564208f421ada605e8efb68f6d9d55a0d2e3f86698b6076cb029e9bab2bac0f70f7250dd17 + checksum: 10c0/d334811074726482b58089fef6c9e98a462fbc1d91c63798307648bdc1349d2e154aa31f391690e2c9cd90eee61a3be9fe8872873e7f828b529d9268d2a25b78 languageName: node linkType: hard From a9db9c8b59303e91810c5bb77431290785845c1a Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 14 Oct 2025 10:46:57 +0200 Subject: [PATCH 05/22] ErrorHandling: publish connection error handling --- src/home/useGroupCallRooms.ts | 4 +- src/room/InCallView.tsx | 8 +++- src/rtcSessionHelpers.test.ts | 61 ++----------------------------- src/rtcSessionHelpers.ts | 1 - src/state/CallViewModel.test.ts | 63 +++++++++++++++++++++++++++++++- src/state/CallViewModel.ts | 65 +++++++++++++++++++++++++-------- src/state/Connection.ts | 33 ++++++++++++++++- src/state/PublishConnection.ts | 3 ++ src/state/ScreenShare.ts | 9 +++-- src/state/UserMedia.ts | 21 ++++++----- src/state/layout-types.ts | 10 ++++- src/utils/test.ts | 2 +- 12 files changed, 183 insertions(+), 97 deletions(-) diff --git a/src/home/useGroupCallRooms.ts b/src/home/useGroupCallRooms.ts index 149af4b0..ad69b864 100644 --- a/src/home/useGroupCallRooms.ts +++ b/src/home/useGroupCallRooms.ts @@ -141,8 +141,8 @@ export function useGroupCallRooms(client: MatrixClient): GroupCallRoom[] { .filter(roomIsJoinable); const sortedRooms = sortRooms(client, rooms); Promise.all( - sortedRooms.map(async (room) => { - const session = await client.matrixRTC.getRoomSession(room); + sortedRooms.map((room) => { + const session = client.matrixRTC.getRoomSession(room); return { roomAlias: room.getCanonicalAlias() ?? undefined, roomName: room.name, diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 3a27e250..6b07dacc 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -59,7 +59,7 @@ import { type MuteStates } from "../state/MuteStates"; import { type MatrixInfo } from "./VideoPreview"; import { InviteButton } from "../button/InviteButton"; import { LayoutToggle } from "./LayoutToggle"; -import { CallViewModel, GridMode } from "../state/CallViewModel"; +import { CallViewModel, type GridMode } from "../state/CallViewModel"; import { Grid, type TileProps } from "../grid/Grid"; import { useInitial } from "../useInitial"; import { SpotlightTile } from "../tile/SpotlightTile"; @@ -109,7 +109,7 @@ import { useAudioContext } from "../useAudioContext"; import ringtoneMp3 from "../sound/ringtone.mp3?url"; import ringtoneOgg from "../sound/ringtone.ogg?url"; import { useTrackProcessorObservable$ } from "../livekit/TrackProcessorContext.tsx"; -import { Layout } from "../state/layout-types.ts"; +import { type Layout } from "../state/layout-types.ts"; const maxTapDurationMs = 400; @@ -297,6 +297,10 @@ export const InCallView: FC = ({ const audioOutputSwitcher = useBehavior(vm.audioOutputSwitcher$); const sharingScreen = useBehavior(vm.sharingScreen$); + const fatalCallError = useBehavior(vm.configError$); + // Stop the rendering and throw for the error boundary + if (fatalCallError) throw fatalCallError; + // We need to set the proper timings on the animation based upon the sound length. const ringDuration = pickupPhaseAudio?.soundDuration["waiting"] ?? 1; useEffect((): (() => void) => { diff --git a/src/rtcSessionHelpers.test.ts b/src/rtcSessionHelpers.test.ts index e7f204ec..cf73dc1f 100644 --- a/src/rtcSessionHelpers.test.ts +++ b/src/rtcSessionHelpers.test.ts @@ -13,7 +13,6 @@ import EventEmitter from "events"; import { enterRTCSession, leaveRTCSession } from "../src/rtcSessionHelpers"; import { mockConfig } from "./utils/test"; import { ElementWidgetActions, widget } from "./widget"; -import { ErrorCode } from "./utils/errors.ts"; const USE_MUTI_SFU = false; const getUrlParams = vi.hoisted(() => vi.fn(() => ({}))); @@ -97,38 +96,20 @@ test("It joins the correct Session", async () => { { encryptMedia: true, useMultiSfu: USE_MUTI_SFU, - } + }, ); expect(mockedSession.joinRoomSession).toHaveBeenLastCalledWith( [ - { - livekit_alias: "my-oldest-member-service-alias", - livekit_service_url: "http://my-oldest-member-service-url.com", - type: "livekit", - }, { livekit_alias: "roomId", livekit_service_url: "http://my-well-known-service-url.com", type: "livekit", }, - { - livekit_alias: "roomId", - livekit_service_url: "http://my-well-known-service-url2.com", - type: "livekit", - }, - { - livekit_alias: "roomId", - livekit_service_url: "http://my-default-service-url.com", - type: "livekit", - }, ], - { - focus_selection: "oldest_membership", - type: "livekit", - }, + undefined, expect.objectContaining({ - manageMediaKeys: false, + manageMediaKeys: true, useLegacyMemberEvents: false, useNewMembershipManager: true, useExperimentalToDeviceTransport: false, @@ -177,40 +158,6 @@ test("leaveRTCSession doesn't close the widget when returning to lobby", async ( await testLeaveRTCSession("user", false); }); -test("It fails with configuration error if no live kit url config is set in fallback", async () => { - mockConfig({}); - vi.spyOn(AutoDiscovery, "getRawClientConfig").mockResolvedValue({}); - - const mockedSession = vi.mocked({ - room: { - roomId: "roomId", - client: { - getDomain: vi.fn().mockReturnValue("example.org"), - }, - }, - memberships: [], - getFocusInUse: vi.fn(), - joinRoomSession: vi.fn(), - }) as unknown as MatrixRTCSession; - - await expect( - enterRTCSession( - mockedSession, - { - livekit_alias: "roomId", - livekit_service_url: "http://my-well-known-service-url.com", - type: "livekit", - }, - { - encryptMedia: true, - useMultiSfu: USE_MUTI_SFU, - } - ), - ).rejects.toThrowError( - expect.objectContaining({ code: ErrorCode.MISSING_MATRIX_RTC_TRANSPORT }), - ); -}); - test("It should not fail with configuration error if homeserver config has livekit url but not fallback", async () => { mockConfig({}); vi.spyOn(AutoDiscovery, "getRawClientConfig").mockResolvedValue({ @@ -250,6 +197,6 @@ test("It should not fail with configuration error if homeserver config has livek { encryptMedia: true, useMultiSfu: USE_MUTI_SFU, - } + }, ); }); diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index 1bb9f11e..c5052339 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -123,7 +123,6 @@ export async function enterRTCSession( useMultiSfu: true, }, ): Promise { - const { encryptMedia, useNewMembershipManager = true, diff --git a/src/state/CallViewModel.test.ts b/src/state/CallViewModel.test.ts index f03b648d..9fa619f1 100644 --- a/src/state/CallViewModel.test.ts +++ b/src/state/CallViewModel.test.ts @@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { test, vi, onTestFinished, it, describe } from "vitest"; +import { test, vi, onTestFinished, it, describe, expect } from "vitest"; import EventEmitter from "events"; import { BehaviorSubject, @@ -45,6 +45,7 @@ import { MatrixRTCSessionEvent, } from "matrix-js-sdk/lib/matrixrtc"; import { deepCompare } from "matrix-js-sdk/lib/utils"; +import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery"; import { CallViewModel, type CallViewModelOptions } from "./CallViewModel"; import { type Layout } from "./layout-types"; @@ -58,6 +59,7 @@ import { MockRTCSession, mockMediaDevices, mockMuteStates, + mockConfig, } from "../utils/test"; import { ECAddonConnectionState, @@ -92,6 +94,10 @@ import { MediaDevices } from "./MediaDevices"; import { getValue } from "../utils/observable"; import { type Behavior, constant } from "./Behavior"; import type { ProcessorState } from "../livekit/TrackProcessorContext.tsx"; +import { + type ElementCallError, + MatrixRTCTransportMissingError, +} from "../utils/errors.ts"; const getUrlParams = vi.hoisted(() => vi.fn(() => ({}))); vi.mock("../UrlParams", () => ({ getUrlParams })); @@ -365,6 +371,61 @@ function withCallViewModel( continuation(vm, rtcSession, { raisedHands$: raisedHands$ }, setSyncState); } +test("test missing RTC config error", async () => { + const rtcMemberships$ = new BehaviorSubject([]); + const emitter = new EventEmitter(); + const client = vi.mocked({ + on: emitter.on.bind(emitter), + off: emitter.off.bind(emitter), + getSyncState: vi.fn().mockReturnValue(SyncState.Syncing), + getUserId: vi.fn().mockReturnValue("@user:localhost"), + getUser: vi.fn().mockReturnValue(null), + getDeviceId: vi.fn().mockReturnValue("DEVICE"), + credentials: { + userId: "@user:localhost", + }, + getCrypto: vi.fn().mockReturnValue(undefined), + getDomain: vi.fn().mockReturnValue("example.org"), + } as unknown as MatrixClient); + + const matrixRoom = mockMatrixRoom({ + roomId: "!myRoomId:example.com", + client, + getMember: vi.fn().mockReturnValue(undefined), + }); + + const fakeRtcSession = new MockRTCSession(matrixRoom).withMemberships( + rtcMemberships$, + ); + + mockConfig({}); + vi.spyOn(AutoDiscovery, "getRawClientConfig").mockResolvedValue({}); + + const callVM = new CallViewModel( + fakeRtcSession.asMockedSession(), + matrixRoom, + mockMediaDevices({}), + mockMuteStates(), + { + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + autoLeaveWhenOthersLeft: false, + }, + new BehaviorSubject({} as Record), + new BehaviorSubject({} as Record), + of({ processor: undefined, supported: false }), + ); + + const failPromise = Promise.withResolvers(); + callVM.configError$.subscribe((error) => { + if (error) { + failPromise.resolve(error); + } + }); + + const error = await failPromise.promise; + expect(error).toBeInstanceOf(MatrixRTCTransportMissingError); +}); + test("participants are retained during a focus switch", () => { withTestScheduler(({ behavior, expectObservable }) => { // Participants disappear on frame 2 and come back on frame 3 diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 3cabf697..3cdba405 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -27,6 +27,7 @@ import { } from "matrix-js-sdk"; import { deepCompare } from "matrix-js-sdk/lib/utils"; import { + BehaviorSubject, combineLatest, concat, distinctUntilChanged, @@ -76,7 +77,7 @@ import { ViewModel } from "./ViewModel"; import { LocalUserMediaViewModel, type MediaViewModel, - RemoteUserMediaViewModel, + type RemoteUserMediaViewModel, ScreenShareViewModel, type UserMediaViewModel, } from "./MediaViewModel"; @@ -130,14 +131,15 @@ import { type Async, async$, mapAsync, ready } from "./Async"; import { sharingScreen$, UserMedia } from "./UserMedia.ts"; import { ScreenShare } from "./ScreenShare.ts"; import { - GridLayoutMedia, - Layout, - LayoutMedia, - OneOnOneLayoutMedia, - SpotlightExpandedLayoutMedia, - SpotlightLandscapeLayoutMedia, - SpotlightPortraitLayoutMedia, + type GridLayoutMedia, + type Layout, + type LayoutMedia, + type OneOnOneLayoutMedia, + type SpotlightExpandedLayoutMedia, + type SpotlightLandscapeLayoutMedia, + type SpotlightPortraitLayoutMedia, } from "./layout-types.ts"; +import { ElementCallError, UnknownCallError } from "../utils/errors.ts"; export interface CallViewModelOptions { encryptionSystem: EncryptionSystem; @@ -224,6 +226,19 @@ export class CallViewModel extends ViewModel { } : undefined; + private readonly _configError$ = new BehaviorSubject( + null, + ); + + /** + * If there is a configuration error with the call (e.g. misconfigured E2EE). + * This is a fatal error that prevents the call from being created/joined. + * Should render a blocking error screen. + */ + public get configError$(): Behavior { + return this._configError$; + } + private readonly join$ = new Subject(); public join(): void { @@ -273,7 +288,7 @@ export class CallViewModel extends ViewModel { * The transport that we would personally prefer to publish on (if not for the * transport preferences of others, perhaps). */ - private readonly preferredTransport = makeTransport(this.matrixRTCSession); + private readonly preferredTransport$: Observable>; /** * Lists the transports used by ourselves, plus all other MatrixRTC session @@ -287,11 +302,7 @@ export class CallViewModel extends ViewModel { switchMap((joined) => joined ? combineLatest( - [ - async$(this.preferredTransport), - this.memberships$, - multiSfu.value$, - ], + [this.preferredTransport$, this.memberships$, multiSfu.value$], (preferred, memberships, multiSfu) => { const oldestMembership = this.matrixRTCSession.getOldestMembership(); @@ -313,6 +324,13 @@ export class CallViewModel extends ViewModel { local = ready(selection); } } + if (local.state === "error") { + this._configError$.next( + local.value instanceof ElementCallError + ? local.value + : new UnknownCallError(local.value), + ); + } return { local, remote }; }, ) @@ -1743,6 +1761,10 @@ export class CallViewModel extends ViewModel { ) { super(); + this.preferredTransport$ = async$( + makeTransport(this.matrixRTCSession), + ).pipe(this.scope.bind()); + // Start and stop local and remote connections as needed this.connectionInstructions$ .pipe(this.scope.bind()) @@ -1765,11 +1787,21 @@ export class CallViewModel extends ViewModel { logger.info( `Connected to ${c.localTransport.livekit_service_url}`, ), - (e) => + (e) => { + // We only want to report fatal errors `_configError$` for the publish connection. + // If there is an error with another connection, it will not terminate the call and will be displayed + // on eacn tile. + if ( + c instanceof PublishConnection && + e instanceof ElementCallError + ) { + this._configError$.next(e); + } logger.error( `Failed to start connection to ${c.localTransport.livekit_service_url}`, e, - ), + ); + }, ); } }); @@ -1778,6 +1810,7 @@ export class CallViewModel extends ViewModel { this.scope.reconcile(this.localTransport$, async (localTransport) => { if (localTransport?.state === "ready") { try { + this._configError$.next(null); await enterRTCSession(this.matrixRTCSession, localTransport.value, { encryptMedia: this.options.encryptionSystem.kind !== E2eeType.NONE, useExperimentalToDeviceTransport: true, diff --git a/src/state/Connection.ts b/src/state/Connection.ts index b7864677..bdb77ed8 100644 --- a/src/state/Connection.ts +++ b/src/state/Connection.ts @@ -10,6 +10,7 @@ import { connectionStateObserver, } from "@livekit/components-core"; import { + ConnectionError, type ConnectionState, type E2EEOptions, Room as LivekitRoom, @@ -29,6 +30,10 @@ import { import { type Behavior } from "./Behavior"; import { type ObservableScope } from "./ObservableScope"; import { defaultLiveKitOptions } from "../livekit/options"; +import { + InsufficientCapacityError, + SFURoomCreationRestrictedError, +} from "../utils/errors.ts"; export interface ConnectionOpts { /** The focus server to connect to. */ @@ -88,6 +93,9 @@ export class Connection { * 1. Request an OpenId token `request_token` (allows matrix users to verify their identity with a third-party service.) * 2. Use this token to request the SFU config to the MatrixRtc authentication service. * 3. Connect to the configured LiveKit room. + * + * @throws {InsufficientCapacityError} if the LiveKit server indicates that it has insufficient capacity to accept the connection. + * @throws {SFURoomCreationRestrictedError} if the LiveKit server indicates that the room does not exist and cannot be created. */ public async start(): Promise { this.stopped = false; @@ -105,7 +113,30 @@ export class Connection { state: "ConnectingToLkRoom", focus: this.localTransport, }); - await this.livekitRoom.connect(url, jwt); + try { + await this.livekitRoom.connect(url, jwt); + } catch (e) { + // LiveKit uses 503 to indicate that the server has hit its track limits. + // https://github.com/livekit/livekit/blob/fcb05e97c5a31812ecf0ca6f7efa57c485cea9fb/pkg/service/rtcservice.go#L171 + // It also errors with a status code of 200 (yes, really) for room + // participant limits. + // LiveKit Cloud uses 429 for connection limits. + // Either way, all these errors can be explained as "insufficient capacity". + if (e instanceof ConnectionError) { + if (e.status === 503 || e.status === 200 || e.status === 429) { + throw new InsufficientCapacityError(); + } + if (e.status === 404) { + // error msg is "Could not establish signal connection: requested room does not exist" + // The room does not exist. There are two different modes of operation for the SFU: + // - the room is created on the fly when connecting (livekit `auto_create` option) + // - Only authorized users can create rooms, so the room must exist before connecting (done by the auth jwt service) + // In the first case there will not be a 404, so we are in the second case. + throw new SFURoomCreationRestrictedError(); + } + } + throw e; + } // If we were stopped while connecting, don't proceed to update state. if (this.stopped) return; diff --git a/src/state/PublishConnection.ts b/src/state/PublishConnection.ts index 9a219483..12ee84cc 100644 --- a/src/state/PublishConnection.ts +++ b/src/state/PublishConnection.ts @@ -94,6 +94,9 @@ export class PublishConnection extends Connection { * 2. Use this token to request the SFU config to the MatrixRtc authentication service. * 3. Connect to the configured LiveKit room. * 4. Create local audio and video tracks based on the current mute states and publish them to the room. + * + * @throws {InsufficientCapacityError} if the LiveKit server indicates that it has insufficient capacity to accept the connection. + * @throws {SFURoomCreationRestrictedError} if the LiveKit server indicates that the room does not exist and cannot be created. */ public async start(): Promise { this.stopped = false; diff --git a/src/state/ScreenShare.ts b/src/state/ScreenShare.ts index 6da18df3..06f61d41 100644 --- a/src/state/ScreenShare.ts +++ b/src/state/ScreenShare.ts @@ -4,14 +4,15 @@ Copyright 2025 New Vector Ltd. SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { ObservableScope } from "./ObservableScope.ts"; -import { ScreenShareViewModel } from "./MediaViewModel.ts"; import { BehaviorSubject, type Observable } from "rxjs"; import { - LocalParticipant, - RemoteParticipant, + type LocalParticipant, + type RemoteParticipant, type Room as LivekitRoom, } from "livekit-client"; + +import { ObservableScope } from "./ObservableScope.ts"; +import { ScreenShareViewModel } from "./MediaViewModel.ts"; import type { RoomMember } from "matrix-js-sdk"; import type { EncryptionSystem } from "../e2ee/sharedKeyManagement.ts"; import type { Behavior } from "./Behavior.ts"; diff --git a/src/state/UserMedia.ts b/src/state/UserMedia.ts index be44b998..5309bc24 100644 --- a/src/state/UserMedia.ts +++ b/src/state/UserMedia.ts @@ -5,27 +5,28 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ +import { BehaviorSubject, map, type Observable, of, switchMap } from "rxjs"; +import { + type LocalParticipant, + type Participant, + ParticipantEvent, + type RemoteParticipant, + type Room as LivekitRoom, +} from "livekit-client"; +import { observeParticipantEvents } from "@livekit/components-core"; + import { ObservableScope } from "./ObservableScope.ts"; import { LocalUserMediaViewModel, RemoteUserMediaViewModel, - UserMediaViewModel, + type UserMediaViewModel, } from "./MediaViewModel.ts"; -import { BehaviorSubject, map, type Observable, of, switchMap } from "rxjs"; -import { - LocalParticipant, - Participant, - ParticipantEvent, - RemoteParticipant, - type Room as LivekitRoom, -} from "livekit-client"; import type { Behavior } from "./Behavior.ts"; import type { RoomMember } from "matrix-js-sdk"; import type { EncryptionSystem } from "../e2ee/sharedKeyManagement.ts"; import type { MediaDevices } from "./MediaDevices.ts"; import type { ReactionOption } from "../reactions"; import { observeSpeaker$ } from "./observeSpeaker.ts"; -import { observeParticipantEvents } from "@livekit/components-core"; /** * TODO Document this diff --git a/src/state/layout-types.ts b/src/state/layout-types.ts index f28ada46..3796715c 100644 --- a/src/state/layout-types.ts +++ b/src/state/layout-types.ts @@ -5,8 +5,14 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { GridTileViewModel, SpotlightTileViewModel } from "./TileViewModel.ts"; -import { MediaViewModel, UserMediaViewModel } from "./MediaViewModel.ts"; +import { + type GridTileViewModel, + type SpotlightTileViewModel, +} from "./TileViewModel.ts"; +import { + type MediaViewModel, + type UserMediaViewModel, +} from "./MediaViewModel.ts"; export interface GridLayoutMedia { type: "grid"; diff --git a/src/utils/test.ts b/src/utils/test.ts index 2da8ed31..a3f26933 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -196,7 +196,7 @@ export function mockRtcMembership( content: data, }); - const cms = new CallMembership(event); + const cms = new CallMembership(event, data); vi.mocked(cms).getTransport = vi.fn().mockReturnValue(fociPreferred[0]); return cms; } From 60332dc2db6fc624d49c401c66f989543420a7a3 Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 14 Oct 2025 12:16:24 +0200 Subject: [PATCH 06/22] fix js-sdk dependency format --- package.json | 2 +- yarn.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 411d98a2..e02952a6 100644 --- a/package.json +++ b/package.json @@ -109,7 +109,7 @@ "livekit-client": "^2.13.0", "lodash-es": "^4.17.21", "loglevel": "^1.9.1", - "matrix-js-sdk": "matrix-org/matrix-js-sdk#develop", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=develop", "matrix-widget-api": "^1.13.0", "normalize.css": "^8.0.1", "observable-hooks": "^4.2.3", diff --git a/yarn.lock b/yarn.lock index fb6e7dcc..8c0ce84b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7545,7 +7545,7 @@ __metadata: livekit-client: "npm:^2.13.0" lodash-es: "npm:^4.17.21" loglevel: "npm:^1.9.1" - matrix-js-sdk: "matrix-org/matrix-js-sdk#develop" + matrix-js-sdk: "github:matrix-org/matrix-js-sdk#head=develop" matrix-widget-api: "npm:^1.13.0" normalize.css: "npm:^8.0.1" observable-hooks: "npm:^4.2.3" @@ -10335,7 +10335,7 @@ __metadata: languageName: node linkType: hard -"matrix-js-sdk@matrix-org/matrix-js-sdk#develop": +"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=develop": version: 38.4.0 resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=fd949fe486038099ee111a72b57ce711e85bc352" dependencies: From 58d60b35fd34bb4d00eedf8420ea1e85f1a65aa9 Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 14 Oct 2025 12:25:31 +0200 Subject: [PATCH 07/22] fix CI failing with Invalid value "iife" for option "worker.format" UMD and IIFE output formats are not supported for code-splitting builds. see https://github.com/vitejs/vite/issues/18585 --- vite.config.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vite.config.ts b/vite.config.ts index cfc80279..a0bb9de5 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -97,6 +97,9 @@ export default ({ cert: fs.readFileSync("./backend/dev_tls_m.localhost.crt"), }, }, + worker: { + format: "es", + }, build: { minify: mode === "production" ? true : false, sourcemap: true, From 93d763f58f6f76889c097e5464fef616779f76ea Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 14 Oct 2025 14:06:54 +0200 Subject: [PATCH 08/22] devtool: quick display of focus URL in stats tile --- src/RTCConnectionStats.tsx | 25 ++++++++++++++++++++++++- src/state/CallViewModel.ts | 8 +++++++- src/state/MediaViewModel.ts | 9 +++++++++ src/state/ScreenShare.ts | 2 ++ src/state/UserMedia.ts | 3 +++ src/tile/GridTile.tsx | 1 + src/tile/MediaView.tsx | 4 ++++ src/tile/SpotlightTile.tsx | 2 +- src/utils/test.ts | 2 ++ 9 files changed, 53 insertions(+), 3 deletions(-) diff --git a/src/RTCConnectionStats.tsx b/src/RTCConnectionStats.tsx index dcd8d019..d51089cf 100644 --- a/src/RTCConnectionStats.tsx +++ b/src/RTCConnectionStats.tsx @@ -19,10 +19,26 @@ import mediaViewStyles from "../src/tile/MediaView.module.css"; interface Props { audio?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; video?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; + focusUrl?: string; } +const extractDomain = (url: string): string => { + try { + const parsedUrl = new URL(url); + return parsedUrl.hostname; // Returns "kdk.cpm" + } catch (error) { + console.error("Invalid URL:", error); + return url; + } +}; + // This is only used in developer mode for debugging purposes, so we don't need full localization -export const RTCConnectionStats: FC = ({ audio, video, ...rest }) => { +export const RTCConnectionStats: FC = ({ + audio, + video, + focusUrl, + ...rest +}) => { const [showModal, setShowModal] = useState(false); const [modalContents, setModalContents] = useState< "video" | "audio" | "none" @@ -55,6 +71,13 @@ export const RTCConnectionStats: FC = ({ audio, video, ...rest }) => { + {focusUrl && ( +
+ +  {extractDomain(focusUrl)} + +
+ )} {audio && (