From 5f3db07b762f0c50a68c18dc9c3da709eb47c1aa Mon Sep 17 00:00:00 2001 From: "Timo K." Date: Tue, 13 Oct 2026 18:49:39 +0200 Subject: [PATCH 01/20] use the matrixRtcSdk --- component/index.tsx | 2 +- sdk-target-based-on-call-view-model/README.md | 7 +- sdk-target-based-on-call-view-model/helper.ts | 26 +- .../index.html | 2 +- sdk-target-based-on-call-view-model/main.ts | 301 ++--- src/App.tsx | 2 +- src/button/ReactionToggleButton.test.tsx | 3 +- src/config/ConfigOptions.ts | 20 +- src/e2ee/e2eeType.ts | 8 +- src/e2ee/matrixKeyProvider.test.ts | 72 -- src/e2ee/matrixKeyProvider.ts | 75 -- src/e2ee/sharedKeyManagement.ts | 7 +- src/livekit/MatrixAudioRenderer.test.tsx | 342 ------ src/livekit/MatrixAudioRenderer.tsx | 216 ---- src/livekit/options.test.ts | 213 ---- src/livekit/options.ts | 152 --- src/room/CallEventAudioRenderer.test.tsx | 1 - src/room/CallView.test.tsx | 2 +- src/room/InCallView.test.tsx | 123 +- src/room/InCallView.tsx | 201 +++- src/room/KnockLobbyView.test.tsx | 4 +- src/room/LobbyView.test.tsx | 4 +- src/room/LobbyView.tsx | 2 +- src/room/ReactionAudioRenderer.test.tsx | 1 - src/room/ReactionsOverlay.test.tsx | 4 +- .../__snapshots__/InCallView.test.tsx.snap | 3 + src/settings/DeveloperSettingsTab.test.tsx | 102 +- src/settings/DeveloperSettingsTab.tsx | 100 +- src/settings/SettingsModal.tsx | 15 +- .../DeveloperSettingsTab.test.tsx.snap | 60 +- src/state/Behavior.ts | 25 +- .../CallNotificationLifecycle.test.ts | 13 +- .../CallNotificationLifecycle.ts | 29 +- src/state/CallViewModel/CallViewModel.test.ts | 140 +-- src/state/CallViewModel/CallViewModel.ts | 706 +++-------- .../CallViewModel/CallViewModelTestUtils.ts | 242 ++-- .../localMember/HomeserverConnected.test.ts | 336 ------ .../localMember/HomeserverConnected.ts | 136 --- .../localMember/LocalMember.test.ts | 920 --------------- .../CallViewModel/localMember/LocalMember.ts | 1031 ----------------- .../localMember/LocalTransport.test.ts | 126 -- .../localMember/LocalTransport.ts | 41 - .../localMember/Publisher.test.ts | 402 ------- .../CallViewModel/localMember/Publisher.ts | 445 ------- .../RtcTransportAutoDiscovery.test.ts | 153 --- .../localMember/RtcTransportAutoDiscovery.ts | 103 -- .../remoteMembers/Connection.test.ts | 749 ------------ .../CallViewModel/remoteMembers/Connection.ts | 417 ------- .../remoteMembers/ConnectionFactory.ts | 201 ---- .../remoteMembers/ConnectionMap.test.ts | 453 -------- .../remoteMembers/ECConnectionFactory.test.ts | 287 ----- .../MatrixLivekitMembers.test.ts | 400 ------- .../remoteMembers/MatrixLivekitMembers.ts | 162 --- .../MatrixMemberMetadata.test.ts | 632 ---------- .../remoteMembers/MatrixMemberMetadata.ts | 160 --- .../remoteMembers/integration.test.ts | 223 ---- src/state/CallViewModelWidget.test.ts | 3 - src/state/ObservableScope.test.ts | 293 ----- src/state/ObservableScope.ts | 330 +----- src/state/SessionBehaviors.test.ts | 100 -- src/state/SessionBehaviors.ts | 106 -- src/state/media/LocalScreenShareViewModel.ts | 8 +- src/state/media/LocalUserMediaViewModel.ts | 108 +- src/state/media/MediaViewModel.test.ts | 242 ++-- src/state/media/MemberMediaViewModel.ts | 281 ++--- src/state/media/RemoteScreenShareViewModel.ts | 32 +- src/state/media/RemoteUserMediaViewModel.ts | 39 +- src/state/media/ScreenShareViewModel.ts | 7 +- src/state/media/UserMediaViewModel.ts | 92 +- src/state/media/WrappedUserMediaViewModel.ts | 130 +-- src/state/media/observeRtpStreamStats.ts | 74 -- src/state/observeTrackReference.ts | 28 - src/tile/GridTile.test.tsx | 18 +- src/tile/MediaView.test.tsx | 29 +- src/tile/MediaView.tsx | 26 +- src/tile/SpotlightTile.test.tsx | 11 +- src/tile/SpotlightTile.tsx | 4 +- .../BlurBackgroundTransformer.ts | 0 src/tracks/MemberAudioRenderer.tsx | 177 +++ .../TrackProcessorContext.test.ts | 0 .../TrackProcessorContext.tsx | 0 src/utils/displayname-integration.test.ts | 40 - src/utils/displayname.test.ts | 133 --- src/utils/displayname.ts | 114 -- src/utils/errors.ts | 45 + src/utils/matrix.ts | 35 - src/utils/observable.test.ts | 104 -- src/utils/observable.ts | 293 +---- src/utils/test-fixtures.ts | 21 - src/utils/test-viewmodel.ts | 64 +- src/utils/test.ts | 333 +++--- vite-component-dev.config.ts | 4 +- ...-target-based-on-call-view-model.config.ts | 17 +- vitest.config.ts | 2 + 94 files changed, 1664 insertions(+), 12281 deletions(-) delete mode 100644 src/e2ee/matrixKeyProvider.test.ts delete mode 100644 src/e2ee/matrixKeyProvider.ts delete mode 100644 src/livekit/MatrixAudioRenderer.test.tsx delete mode 100644 src/livekit/MatrixAudioRenderer.tsx delete mode 100644 src/livekit/options.test.ts delete mode 100644 src/livekit/options.ts delete mode 100644 src/state/CallViewModel/localMember/HomeserverConnected.test.ts delete mode 100644 src/state/CallViewModel/localMember/HomeserverConnected.ts delete mode 100644 src/state/CallViewModel/localMember/LocalMember.test.ts delete mode 100644 src/state/CallViewModel/localMember/LocalMember.ts delete mode 100644 src/state/CallViewModel/localMember/LocalTransport.test.ts delete mode 100644 src/state/CallViewModel/localMember/LocalTransport.ts delete mode 100644 src/state/CallViewModel/localMember/Publisher.test.ts delete mode 100644 src/state/CallViewModel/localMember/Publisher.ts delete mode 100644 src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.test.ts delete mode 100644 src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.ts delete mode 100644 src/state/CallViewModel/remoteMembers/Connection.test.ts delete mode 100644 src/state/CallViewModel/remoteMembers/Connection.ts delete mode 100644 src/state/CallViewModel/remoteMembers/ConnectionFactory.ts delete mode 100644 src/state/CallViewModel/remoteMembers/ConnectionMap.test.ts delete mode 100644 src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts delete mode 100644 src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts delete mode 100644 src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts delete mode 100644 src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.test.ts delete mode 100644 src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.ts delete mode 100644 src/state/CallViewModel/remoteMembers/integration.test.ts delete mode 100644 src/state/ObservableScope.test.ts delete mode 100644 src/state/SessionBehaviors.test.ts delete mode 100644 src/state/SessionBehaviors.ts delete mode 100644 src/state/media/observeRtpStreamStats.ts delete mode 100644 src/state/observeTrackReference.ts rename src/{livekit => tracks}/BlurBackgroundTransformer.ts (100%) create mode 100644 src/tracks/MemberAudioRenderer.tsx rename src/{livekit => tracks}/TrackProcessorContext.test.ts (100%) rename src/{livekit => tracks}/TrackProcessorContext.tsx (100%) delete mode 100644 src/utils/displayname-integration.test.ts delete mode 100644 src/utils/displayname.test.ts delete mode 100644 src/utils/displayname.ts delete mode 100644 src/utils/observable.test.ts diff --git a/component/index.tsx b/component/index.tsx index b3f7b6209..4a4b4d8cf 100644 --- a/component/index.tsx +++ b/component/index.tsx @@ -74,7 +74,7 @@ import { import { MediaDevicesContext } from "../src/MediaDevicesContext"; import { MediaDevices } from "../src/state/MediaDevices"; import { ObservableScope } from "../src/state/ObservableScope"; -import { ProcessorProvider } from "../src/livekit/TrackProcessorContext"; +import { ProcessorProvider } from "../src/tracks/TrackProcessorContext"; import { Config } from "../src/config/Config"; import { type ConfigOptions } from "../src/config/ConfigOptions"; import { i18n } from "../src/utils/i18n"; diff --git a/sdk-target-based-on-call-view-model/README.md b/sdk-target-based-on-call-view-model/README.md index 1715bda8e..1c7e6ee23 100644 --- a/sdk-target-based-on-call-view-model/README.md +++ b/sdk-target-based-on-call-view-model/README.md @@ -1,8 +1,9 @@ -# SDK mode (EXPERIMENTAL, superseded) +# SDK mode (LEGACY) > Superseded by the `sdk/` package (`@element-hq/matrixrtc-sdk`) and its harness in -> `sdk/dev`; see `sdk-plan.md`. This folder still builds, with its own script, until -> that package can do what this does. +> `sdk/dev`; see `sdk/SdkArchitecture.md`. This bundle is in production use, so it +> keeps building as a thin layer over that package, with its interface unchanged, +> until its consumers have moved to the package directly. It will then be removed. EC can be build in sdk mode. This will result in a compiled js file that can be imported in very simple webapps. diff --git a/sdk-target-based-on-call-view-model/helper.ts b/sdk-target-based-on-call-view-model/helper.ts index 9f7e02a7d..9ff8c53d6 100644 --- a/sdk-target-based-on-call-view-model/helper.ts +++ b/sdk-target-based-on-call-view-model/helper.ts @@ -6,14 +6,12 @@ Please see LICENSE in the repository root for full details. */ /** - * This file contains helper functions and types for the MatrixRTC SDK. + * This file contains helper functions and types for the legacy SDK bundle. */ import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; -import { scan } from "rxjs"; import { type WidgetHelpers } from "../src/widget"; -import { type LivekitRoomItem } from "../src/state/CallViewModel/CallViewModel"; export const tryMakeSticky = (widget: WidgetHelpers): void => { const logger = rootLogger.getChild("[MatrixRTCSdk]"); @@ -27,25 +25,5 @@ export const tryMakeSticky = (widget: WidgetHelpers): void => { logger.error("failed to make sticky MatrixRTCSdk", error); }); }; +/** The data channel topic this bundle has always used, so that old and new builds talk. */ export const TEXT_LK_TOPIC = "matrixRTC"; -/** - * simple helper operator to combine the last emitted and the current emitted value of a rxjs observable - * - * I think there should be a builtin for this but i did not find it... - */ -export const currentAndPrev = scan< - LivekitRoomItem[], - { - prev: LivekitRoomItem[]; - current: LivekitRoomItem[]; - } ->( - ({ current: lastCurrentVal }, items) => ({ - prev: lastCurrentVal, - current: items, - }), - { - prev: [], - current: [], - }, -); diff --git a/sdk-target-based-on-call-view-model/index.html b/sdk-target-based-on-call-view-model/index.html index 8883b9a3c..4cf8a4cc5 100644 --- a/sdk-target-based-on-call-view-model/index.html +++ b/sdk-target-based-on-call-view-model/index.html @@ -30,7 +30,7 @@ div.appendChild(child); }); - sdk.members$.subscribe((memberObjects) => { + sdk.remoteMembers$.subscribe((memberObjects) => { const div = document.getElementById("members"); div.innerHTML = "

Members:

"; diff --git a/sdk-target-based-on-call-view-model/main.ts b/sdk-target-based-on-call-view-model/main.ts index 0a4233dc0..26dad98ad 100644 --- a/sdk-target-based-on-call-view-model/main.ts +++ b/sdk-target-based-on-call-view-model/main.ts @@ -6,62 +6,74 @@ Please see LICENSE in the repository root for full details. */ /** - * EXPERIMENTAL + * LEGACY, in production use * - * This file is the entrypoint for the sdk build of element call: - * `pnpm build:sdk-target-based-on-call-view-model` - * use in widgets. - * It exposes the `createMatrixRTCSdk` which creates the `MatrixRTCSdk` interface (see below) that - * can be used to join a rtc session and exchange realtime data. - * It takes care of all the tricky bits: + * This file is the entrypoint of the legacy SDK bundle, + * `pnpm build:sdk-target-based-on-call-view-model`, for use in widgets. It + * exposes `createMatrixRTCSdk`, which creates the `MatrixRTCSdk` interface + * (see below) to join an RTC session and exchange realtime data. + * + * It is a thin layer over the `@element-hq/matrixrtc-sdk` package and Element + * Call's call view model, which take care of all the tricky bits: * - sending delayed events * - finding the right sfu * - handling the media stream * - sending join/leave state or sticky events - * - setting up encryption and scharing keys + * - setting up encryption and sharing keys + * + * The interface is kept as it was for its existing consumers. New consumers + * use the `@element-hq/matrixrtc-sdk` package directly; this bundle goes away + * once the existing ones have moved. */ import { combineLatest, + filter, map, type Observable, of, - shareReplay, - Subject, switchMap, - tap, } from "rxjs"; +import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; +import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { - type CallMembership, - MatrixRTCSessionEvent, - MatrixRTCSessionManager, -} from "matrix-js-sdk/lib/matrixrtc"; -import { - type Room as LivekitRoom, - type TextStreamReader, - type LocalParticipant, - type RemoteParticipant, -} from "livekit-client"; + type Behavior, + constant, + createMatrixRTCClient, + E2eeType, + ObservableScope, + type RTCMember, +} from "@element-hq/matrixrtc-sdk"; -// TODO how can this get fixed? to just be part of `livekit-client` -// Can this be done in the tsconfig.json -import { type TextStreamInfo } from "../node_modules/livekit-client/dist/src/room/types"; -import { type Behavior, constant } from "../src/state/Behavior"; import { callViewModelOptionsFromParams, createCallViewModel$, } from "../src/state/CallViewModel/CallViewModel"; -import { ObservableScope } from "../src/state/ObservableScope"; import { getUrlParams } from "../src/UrlParams"; import { MuteStates } from "../src/state/MuteStates"; import { MediaDevices } from "../src/state/MediaDevices"; -import { E2eeType } from "../src/e2ee/e2eeType"; -import { currentAndPrev, TEXT_LK_TOPIC, tryMakeSticky } from "./helper"; -import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; +import { TEXT_LK_TOPIC, tryMakeSticky } from "./helper"; import { initializeWidget } from "../src/widget"; -import { type Connection } from "../src/state/CallViewModel/remoteMembers/Connection"; import { createWidgetHostBridge } from "../src/HostBridge"; import { observeElementSize$ } from "../src/utils/elementSize"; +import { + matrixRTCClientOptions, + selectedDeviceId$, +} from "../src/room/InCallView"; + +/** + * A member as this bundle has always presented it. The media transport is + * the SDK's business now, so the transport objects are gone: `connection` and + * `participant` are always null and only kept so that existing code keeps + * destructuring. `membership` is what consumers read. + */ +interface MatrixRTCSdkMember { + /** @deprecated Always null: the SDK owns the transport connection. */ + connection: null; + membership: CallMembership; + /** @deprecated Always null: the SDK owns the transport participant. */ + participant: null; +} interface MatrixRTCSdk { /** @@ -81,21 +93,11 @@ interface MatrixRTCSdk { /** * flattened list of remote members */ - remoteMembers$: Behavior< - { - connection: Connection | null; - membership: CallMembership; - participant: LocalParticipant | RemoteParticipant | null; - }[] - >; + remoteMembers$: Behavior; /** * flattened local member */ - localMember$: Behavior<{ - connection: Connection | null; - membership: CallMembership; - participant: LocalParticipant | null; - } | null>; + localMember$: Behavior; /** Use the LocalMemberConnectionState returned from `join` for a more detailed connection state */ connected$: Behavior; sendData?: (data: unknown) => Promise; @@ -124,12 +126,6 @@ export async function createMatrixRTCSdk( const room = client.getRoom(roomId); if (room === null) throw Error("could not get room from client"); - // rtc session - const slot = { application, id }; - const rtcSessionManager = new MatrixRTCSessionManager(logger, client, slot); - rtcSessionManager.start(); - const rtcSession = rtcSessionManager.getRoomSession(room); - // media devices const mediaDevices = new MediaDevices(scope, { controlledAudioDevices, @@ -142,134 +138,63 @@ export async function createMatrixRTCSdk( hostBridge, ); - // call view model + // rtc client: the session, the transport and the media, as in the app + const encryptionSystem = { kind: E2eeType.PER_PARTICIPANT } as const; + const rtcClient = createMatrixRTCClient( + scope, + client, + room, + { + microphoneEnabled: muteStates.audio.enabled$.value, + cameraEnabled: muteStates.video.enabled$.value, + audioInputDeviceId$: selectedDeviceId$(scope, mediaDevices.audioInput), + videoInputDeviceId$: selectedDeviceId$(scope, mediaDevices.videoInput), + audioOutputDeviceId$: controlledAudioDevices + ? constant(undefined) + : selectedDeviceId$(scope, mediaDevices.audioOutput), + videoProcessor$: constant(undefined), + }, + { + ...matrixRTCClientOptions(urlParams, encryptionSystem), + application, + slot: id, + }, + ); + + // call view model: the host bridge's hang-up handling and the leave flow const callViewModel = createCallViewModel$( scope, - rtcSession, + rtcClient, room, mediaDevices, muteStates, { ...callViewModelOptionsFromParams(urlParams), - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + encryptionSystem, hostBridge, // The SDK owns its page, so the body is the space it has windowSize$: scope.behavior(observeElementSize$(document.body)), }, of({}), of({}), - constant({ supported: false, processor: undefined }), ); logger.info("CallViewModelCreated"); - // create data listener - const data$ = new Subject<{ rtcBackendIdentity: string; data: string }>(); - - const lkTextStreamHandlerFunction = async ( - reader: TextStreamReader, - participantInfo: { identity: string }, - livekitRoom: LivekitRoom, - ): Promise => { - const info = reader.info; - logger.info( - `Received text stream from ${participantInfo.identity}\n` + - ` Topic: ${info.topic}\n` + - ` Timestamp: ${info.timestamp}\n` + - ` ID: ${info.id}\n` + - ` Size: ${info.size}`, // Optional, only available if the stream was sent with `sendText` - ); - - const participants = callViewModel.livekitRoomItems$.value.find( - (i) => i.livekitRoom === livekitRoom, - )?.participants; - if (participants && participants.includes(participantInfo.identity)) { - const text = await reader.readAll(); - logger.info(`Received text: ${text}`); - data$.next({ rtcBackendIdentity: participantInfo.identity, data: text }); - } else { - logger.warn( - "Received text from unknown participant", - participantInfo.identity, - ); - } - }; - - const livekitRoomItemsSub = callViewModel.livekitRoomItems$ - .pipe( - tap((beforecurrentAndPrev) => { - logger.info( - `LiveKit room items updated: ${beforecurrentAndPrev.length}`, - beforecurrentAndPrev, - ); - }), - currentAndPrev, - tap((aftercurrentAndPrev) => { - logger.info( - `LiveKit room items updated: ${aftercurrentAndPrev.current.length}, ${aftercurrentAndPrev.prev.length}`, - aftercurrentAndPrev, - ); - }), - ) - .subscribe({ - next: ({ prev, current }) => { - const prevRooms = prev.map((i) => i.livekitRoom); - const currentRooms = current.map((i) => i.livekitRoom); - const addedRooms = currentRooms.filter((r) => !prevRooms.includes(r)); - const removedRooms = prevRooms.filter((r) => !currentRooms.includes(r)); - addedRooms.forEach((r) => { - logger.info(`Registering text stream handler for room `); - r.registerTextStreamHandler( - TEXT_LK_TOPIC, - (reader, participantInfo) => - void lkTextStreamHandlerFunction(reader, participantInfo, r), - ); - }); - removedRooms.forEach((r) => { - logger.info(`Unregistering text stream handler for room `); - r.unregisterTextStreamHandler(TEXT_LK_TOPIC); - }); - }, - complete: () => { - logger.info("Livekit room items subscription completed"); - for (const item of callViewModel.livekitRoomItems$.value) { - logger.info("unregistering room item from room", item.transport); - item.livekitRoom.unregisterTextStreamHandler(TEXT_LK_TOPIC); - } - }, - }); - - // create sendData function - const sendFn: Behavior<(data: string) => Promise> = - scope.behavior( - callViewModel.localMatrixLivekitMember$.pipe( - switchMap((m) => { - if (!m) - return of((data: string): never => { - throw Error("local membership not yet ready."); - }); - return m.participant.value$.pipe( - map((p) => { - if (p === null) { - return (data: string): never => { - throw Error("local participant not yet ready to send data."); - }; - } else { - return async (data: string): Promise => - p.sendText(data, { topic: TEXT_LK_TOPIC }); - } - }), - ); - }), - ), - ); + // Data arrives from attested members only; the SDK has matched the sender + const data$ = rtcClient.data$.pipe( + filter(({ topic }) => topic === TEXT_LK_TOPIC), + map(({ member, text }) => { + logger.info(`Received text from ${member.id}: ${text}`); + return { rtcBackendIdentity: member.id, data: text }; + }), + ); const sendData = async (data: unknown): Promise => { const dataString = JSON.stringify(data); logger.info("try sending: ", dataString); try { - await Promise.resolve(); - const info = await sendFn.value(dataString); - logger.info(`Sent text with stream ID: ${info.id}`); + await rtcClient.sendData(TEXT_LK_TOPIC, dataString); + logger.info("sent text"); } catch (e) { logger.error("failed sending: ", dataString, e); } @@ -288,13 +213,12 @@ export async function createMatrixRTCSdk( // after hangup gets called const leaveSubs = callViewModel.leave$.subscribe(() => { const scheduleWidgetCloseOnLeave = async (): Promise => { - const leaveResolver = Promise.withResolvers(); logger.info("waiting for RTC leave"); - rtcSession.on(MatrixRTCSessionEvent.JoinStateChanged, (isJoined) => { - logger.info("received RTC join update: ", isJoined); - if (!isJoined) leaveResolver.resolve(); + await new Promise((resolve) => { + rtcClient.status$ + .pipe(filter((status) => status === "disconnected")) + .subscribe(() => resolve()); }); - await leaveResolver.promise; logger.info("send Unstick"); await hostBridge .setAlwaysOnScreen(false) @@ -315,6 +239,15 @@ export async function createMatrixRTCSdk( logger.info("createMatrixRTCSdk done"); + const flatten = (member: RTCMember): Observable => + member.membership$.pipe( + map((membership) => ({ + connection: null, + membership, + participant: null, + })), + ); + return { join: (): void => { // first lets try making the widget sticky @@ -326,54 +259,22 @@ export async function createMatrixRTCSdk( }, stop: (): void => { leaveSubs.unsubscribe(); - livekitRoomItemsSub.unsubscribe(); scope.end(); }, data$, localMember$: scope.behavior( - callViewModel.localMatrixLivekitMember$.pipe( - tap((member) => - logger.info("localMatrixLivekitMember$ next: ", member), - ), - switchMap((member) => { - if (member === null) return of(null); - return combineLatest([ - member.connection$, - member.membership$, - member.participant.value$, - ]).pipe( - map(([connection, membership, participant]) => ({ - connection, - membership, - participant, - })), - ); - }), - tap((member) => logger.info("localMember$ next: ", member)), + rtcClient.localMember$.pipe( + switchMap((member) => (member === null ? of(null) : flatten(member))), ), ), connected$: callViewModel.connected$, remoteMembers$: scope.behavior( - callViewModel.remoteMatrixLivekitMembers$.pipe( - switchMap((members) => { - const listOfMemberObservables = members.map((member) => - combineLatest([ - member.connection$, - member.membership$, - member.participant.value$, - ]).pipe( - map(([connection, membership, participant]) => ({ - connection, - membership, - participant, - })), - // using shareReplay instead of a Behavior here because the behavior would need - // a tricky scope.end() setup. - shareReplay({ bufferSize: 1, refCount: true }), - ), - ); - return combineLatest(listOfMemberObservables); - }), + rtcClient.remoteMembers$.pipe( + switchMap((members) => + members.length === 0 + ? of([]) + : combineLatest(members.map((member) => flatten(member))), + ), ), [], ), diff --git a/src/App.tsx b/src/App.tsx index c07553afc..a92d56b59 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -35,7 +35,7 @@ import { ErrorPage, LoadingPage } from "./FullScreenView"; import { Initializer } from "./initializer"; import { type WidgetHelpers } from "./widget"; import { useTheme } from "./useTheme"; -import { ProcessorProvider } from "./livekit/TrackProcessorContext"; +import { ProcessorProvider } from "./tracks/TrackProcessorContext"; import { type AppViewModel } from "./state/AppViewModel"; import { MediaDevicesContext } from "./MediaDevicesContext"; import { diff --git a/src/button/ReactionToggleButton.test.tsx b/src/button/ReactionToggleButton.test.tsx index 5c8d375cb..54201aa9b 100644 --- a/src/button/ReactionToggleButton.test.tsx +++ b/src/button/ReactionToggleButton.test.tsx @@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details. */ import { act, render } from "@testing-library/react"; -import { expect, test, vi } from "vitest"; +import { expect, test } from "vitest"; import { TooltipProvider } from "@vector-im/compound-web"; import { userEvent } from "@testing-library/user-event"; import { type ReactNode } from "react"; @@ -20,7 +20,6 @@ import { type MockRTCSession } from "../utils/test"; import { ReactionsSenderProvider } from "../reactions/useReactionsSender"; import { initializeWidget } from "../widget"; initializeWidget(); -vi.mock("livekit-client/e2ee-worker?worker"); const localIdent = `${localRtcMember.userId}:${localRtcMember.deviceId}`; diff --git a/src/config/ConfigOptions.ts b/src/config/ConfigOptions.ts index 5f9910662..65aa04da2 100644 --- a/src/config/ConfigOptions.ts +++ b/src/config/ConfigOptions.ts @@ -6,23 +6,9 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -/** - * The MatrixRTC mode determines how Element Call interacts with the - * MatrixRTC backend and other participants. Selectable via the Developer - * Settings, or pinned for a deployment via `matrix_rtc_mode` in config.json. - */ -export enum MatrixRTCMode { - /** Multi-SFU transport, legacy JWT endpoint, state events. */ - Compatibility = "compatibility", - /** - * Multi-SFU transport with: - * - sticky events - * - hashed RTC backend identity - * - the new endpoint for the jwt token on the local membership (remote memberships will always try the new jwt endpoint first -> then the legacy one) - * - use the hashed identity for the local membership - */ - Matrix_2_0 = "matrix_2_0", -} +import { MatrixRTCMode } from "@element-hq/matrixrtc-sdk"; + +export { MatrixRTCMode }; export interface DelayedLeaveTimings { /** diff --git a/src/e2ee/e2eeType.ts b/src/e2ee/e2eeType.ts index 79e26281f..ce843355b 100644 --- a/src/e2ee/e2eeType.ts +++ b/src/e2ee/e2eeType.ts @@ -1,12 +1,8 @@ /* -Copyright 2023, 2024 New Vector Ltd. +Copyright 2026 Element Creations Ltd. SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -export enum E2eeType { - NONE = 0, - PER_PARTICIPANT = 1, - SHARED_KEY = 2, -} +export { E2eeType } from "@element-hq/matrixrtc-sdk"; diff --git a/src/e2ee/matrixKeyProvider.test.ts b/src/e2ee/matrixKeyProvider.test.ts deleted file mode 100644 index f46218e79..000000000 --- a/src/e2ee/matrixKeyProvider.test.ts +++ /dev/null @@ -1,72 +0,0 @@ -/* -Copyright 2024 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { describe, expect, test, vi } from "vitest"; -import { - type MatrixRTCSession, - MatrixRTCSessionEvent, -} from "matrix-js-sdk/lib/matrixrtc"; -import { KeyProviderEvent } from "livekit-client"; - -import { MatrixKeyProvider } from "./matrixKeyProvider"; - -function mockRTCSession(): MatrixRTCSession { - return { - on: vi.fn(), - off: vi.fn(), - reemitEncryptionKeys: vi.fn(), - } as unknown as MatrixRTCSession; -} - -describe("matrixKeyProvider", () => { - test("initializes", () => { - const keyProvider = new MatrixKeyProvider(); - expect(keyProvider).toBeTruthy(); - }); - - test("listens for key requests and emits existing keys", () => { - const keyProvider = new MatrixKeyProvider(); - - const session = mockRTCSession(); - - keyProvider.setRTCSession(session); - - expect(session.on).toHaveBeenCalledWith( - MatrixRTCSessionEvent.EncryptionKeyChanged, - expect.any(Function), - ); - expect(session.off).not.toHaveBeenCalled(); - }); - - test("stops listening when session changes", () => { - const keyProvider = new MatrixKeyProvider(); - - const session1 = mockRTCSession(); - const session2 = mockRTCSession(); - - keyProvider.setRTCSession(session1); - expect(session1.off).not.toHaveBeenCalled(); - - keyProvider.setRTCSession(session2); - expect(session1.off).toHaveBeenCalledWith( - MatrixRTCSessionEvent.EncryptionKeyChanged, - expect.any(Function), - ); - }); - - test("emits existing keys", () => { - const keyProvider = new MatrixKeyProvider(); - const setKeyListener = vi.fn(); - keyProvider.on(KeyProviderEvent.SetKey, setKeyListener); - - const session = mockRTCSession(); - - keyProvider.setRTCSession(session); - - expect(session.reemitEncryptionKeys).toHaveBeenCalled(); - }); -}); diff --git a/src/e2ee/matrixKeyProvider.ts b/src/e2ee/matrixKeyProvider.ts deleted file mode 100644 index f1a66a28a..000000000 --- a/src/e2ee/matrixKeyProvider.ts +++ /dev/null @@ -1,75 +0,0 @@ -/* -Copyright 2023, 2024 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { BaseKeyProvider } from "livekit-client"; -import { - type MatrixRTCSession, - MatrixRTCSessionEvent, -} from "matrix-js-sdk/lib/matrixrtc"; -import { logger as rootLogger, type Logger } from "matrix-js-sdk/lib/logger"; -import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; - -export class MatrixKeyProvider extends BaseKeyProvider { - private rtcSession?: MatrixRTCSession; - private logger: Logger; - public constructor() { - super({ ratchetWindowSize: 10, keyringSize: 256 }); - this.logger = rootLogger.getChild("[MatrixKeyProvider]"); - } - - public setRTCSession(rtcSession: MatrixRTCSession): void { - if (this.rtcSession) { - this.rtcSession.off( - MatrixRTCSessionEvent.EncryptionKeyChanged, - this.onEncryptionKeyChanged, - ); - } - - this.rtcSession = rtcSession; - - this.rtcSession.on( - MatrixRTCSessionEvent.EncryptionKeyChanged, - this.onEncryptionKeyChanged, - ); - - // The new session could be aware of keys of which the old session wasn't, - // so emit key changed events - this.rtcSession.reemitEncryptionKeys(); - } - - private onEncryptionKeyChanged = ( - encryptionKey: Uint8Array, - encryptionKeyIndex: number, - membershipParts: CallMembershipIdentityParts, - rtcBackendIdentity: string, - ): void => { - crypto.subtle - .importKey("raw", encryptionKey, "HKDF", false, [ - "deriveBits", - "deriveKey", - ]) - .then( - (keyMaterial) => { - this.onSetEncryptionKey( - keyMaterial, - rtcBackendIdentity, - encryptionKeyIndex, - ); - - this.logger.debug( - `Sent new key to livekit room=${this.rtcSession?.room.roomId} participantId=${rtcBackendIdentity} (before hash: ${membershipParts.userId}:${membershipParts.deviceId}) encryptionKeyIndex=${encryptionKeyIndex}`, - ); - }, - (e) => { - this.logger.error( - `Failed to create key material from buffer for livekit room=${this.rtcSession?.room.roomId} participantId before hash=${membershipParts.userId}:${membershipParts.deviceId} encryptionKeyIndex=${encryptionKeyIndex}`, - e, - ); - }, - ); - }; -} diff --git a/src/e2ee/sharedKeyManagement.ts b/src/e2ee/sharedKeyManagement.ts index b29ede319..fb329a274 100644 --- a/src/e2ee/sharedKeyManagement.ts +++ b/src/e2ee/sharedKeyManagement.ts @@ -13,7 +13,7 @@ import { useLocalStorage, } from "../useLocalStorage"; import { getUrlParams, useUrlParams } from "../UrlParams"; -import { E2eeType } from "./e2eeType"; +import { E2eeType, type EncryptionSystem } from "@element-hq/matrixrtc-sdk"; import { useClient } from "../ClientContext"; /** @@ -82,10 +82,7 @@ export function getKeyForRoom(roomId: string): string | null { return keyForRoom(roomId, paramsRoomId, password); } -export type Unencrypted = { kind: E2eeType.NONE }; -export type SharedSecret = { kind: E2eeType.SHARED_KEY; secret: string }; -export type PerParticipantE2EE = { kind: E2eeType.PER_PARTICIPANT }; -export type EncryptionSystem = Unencrypted | SharedSecret | PerParticipantE2EE; +export { type EncryptionSystem } from "@element-hq/matrixrtc-sdk"; export function useRoomEncryptionSystem(roomId: string): EncryptionSystem { const { client } = useClient(); diff --git a/src/livekit/MatrixAudioRenderer.test.tsx b/src/livekit/MatrixAudioRenderer.test.tsx deleted file mode 100644 index ab252a0e7..000000000 --- a/src/livekit/MatrixAudioRenderer.test.tsx +++ /dev/null @@ -1,342 +0,0 @@ -/* -Copyright 2023, 2024 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { afterEach, beforeEach, expect, it, vi } from "vitest"; -import { render, type RenderResult } from "@testing-library/react"; -import { - getTrackReferenceId, - type TrackReference, -} from "@livekit/components-core"; -import { - type Participant, - type RemoteAudioTrack, - type Room, - Track, -} from "livekit-client"; -import { type ReactNode } from "react"; -import { useTracks } from "@livekit/components-react"; - -import { testAudioContext } from "../useAudioContext.test"; -import * as MediaDevicesContext from "../MediaDevicesContext"; -import { LivekitRoomAudioRenderer } from "./MatrixAudioRenderer"; -import { - exampleTransportLocator, - mockLocalParticipant, - mockMediaDevices, - mockRemoteParticipant, - mockTrack, -} from "../utils/test"; -import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; -import { initializeWidget } from "../widget"; -initializeWidget(); -export const TestAudioContextConstructor = vi.fn( - class { - public constructor() { - return testAudioContext; - } - }, -); - -const MediaDevicesProvider = MediaDevicesContext.MediaDevicesContext.Provider; - -beforeEach(() => { - vi.stubGlobal("AudioContext", TestAudioContextConstructor); -}); - -afterEach(() => { - vi.unstubAllGlobals(); - vi.clearAllMocks(); -}); - -vi.mock("@livekit/components-react", async (importOriginal) => { - return { - ...(await importOriginal()), - AudioTrack: (props: { trackRef: TrackReference }): ReactNode => { - return ( - - ); - }, - useTracks: vi.fn(), - }; -}); - -let tracks: TrackReference[] = []; - -/** - * Render the test component with given rtc members and livekit participant identities. - * - * It is possible to have rtc members that are not in livekit (e.g. not yet joined) and vice versa. - * - * @param rtcMembers - Array of active rtc members with userId and deviceId. - * @param livekitParticipantIdentities - Array of livekit participant (that are publishing). - * @param explicitTracks - Array of tracks available in livekit, if not provided, one audio track per livekitParticipantIdentities will be created. - * */ - -function renderTestComponent( - rtcMembers: { userId: string; deviceId: string }[], - livekitParticipantIdentities: string[], - explicitTracks?: { - participantId: string; - kind: Track.Kind; - source: Track.Source; - }[], -): RenderResult { - const liveKitParticipants = livekitParticipantIdentities.map((identity) => - mockRemoteParticipant({ identity }), - ); - const participants = rtcMembers.flatMap(({ userId, deviceId }) => { - const p = liveKitParticipants.find( - (p) => p.identity === `${userId}:${deviceId}`, - ); - return p === undefined ? [] : [p]; - }); - const livekitRoom = { - remoteParticipants: new Map( - liveKitParticipants.map((p) => [p.identity, p]), - ), - } as unknown as Room; - - if ((explicitTracks?.length ?? 0) > 0) { - tracks = explicitTracks!.map(({ participantId, source, kind }) => { - const participant = - liveKitParticipants.find((p) => p.identity === participantId) ?? - mockRemoteParticipant({ identity: participantId }); - return mockTrack(participant, kind, source); - }); - } else { - tracks = participants.map((p) => mockTrack(p)); - } - - vi.mocked(useTracks).mockReturnValue(tracks); - return render( - - p.identity)} - livekitRoom={livekitRoom} - transport={exampleTransportLocator} - /> - , - ); -} - -it("should render for member", () => { - const { container, queryAllByTestId } = renderTestComponent( - [{ userId: "@alice", deviceId: "DEV0" }], - ["@alice:DEV0"], - ); - expect(container).toBeTruthy(); - expect(queryAllByTestId("audio")).toHaveLength(1); -}); - -function spyOnWarn(): ReturnType { - const warn = vi.fn(); - vi.spyOn(rootLogger, "getChild").mockReturnValue({ - warn, - } as unknown as typeof rootLogger); - return warn; -} - -it("should not render or warn for the local participant", () => { - const warn = spyOnWarn(); - const local = mockLocalParticipant({ identity: "@alice:DEV0" }); - vi.mocked(useTracks).mockReturnValue([mockTrack(local)]); - const { queryAllByTestId } = render( - - - , - ); - expect(queryAllByTestId("audio")).toHaveLength(0); - expect(warn).not.toHaveBeenCalled(); -}); - -it("should warn only once per unexpected participant", () => { - const warn = spyOnWarn(); - const { rerender } = renderTestComponent( - [{ userId: "@bob", deviceId: "DEV0" }], - ["@alice:DEV0"], - [ - { - participantId: "@alice:DEV0", - kind: Track.Kind.Audio, - source: Track.Source.Microphone, - }, - ], - ); - expect(warn).toHaveBeenCalledTimes(1); - rerender( - - - , - ); - expect(warn).toHaveBeenCalledTimes(1); -}); - -it("should not render without member", () => { - const { container, queryAllByTestId } = renderTestComponent( - [{ userId: "@bob", deviceId: "DEV0" }], - ["@alice:DEV0"], - ); - expect(container).toBeTruthy(); - expect(queryAllByTestId("audio")).toHaveLength(0); -}); - -const TEST_CASES: { - name: string; - rtcUsers: { userId: string; deviceId: string }[]; - livekitParticipantIdentities: string[]; - explicitTracks?: { - participantId: string; - kind: Track.Kind; - source: Track.Source; - }[]; - expectedAudioTracks: number; -}[] = [ - { - name: "single user single device", - rtcUsers: [ - { userId: "@alice", deviceId: "DEV0" }, - { userId: "@alice", deviceId: "DEV1" }, - { userId: "@bob", deviceId: "DEV0" }, - ], - livekitParticipantIdentities: ["@alice:DEV0", "@bob:DEV0", "@alice:DEV1"], - expectedAudioTracks: 3, - }, - // Charlie is a rtc member but not in livekit - { - name: "Charlie is rtc member but not in livekit", - rtcUsers: [ - { userId: "@alice", deviceId: "DEV0" }, - { userId: "@bob", deviceId: "DEV0" }, - { userId: "@charlie", deviceId: "DEV0" }, - ], - livekitParticipantIdentities: ["@alice:DEV0", "@bob:DEV0"], - expectedAudioTracks: 2, - }, - // Charlie is in livekit but not rtc member - { - name: "Charlie is in livekit but not rtc member", - rtcUsers: [ - { userId: "@alice", deviceId: "DEV0" }, - { userId: "@bob", deviceId: "DEV0" }, - ], - livekitParticipantIdentities: ["@alice:DEV0", "@bob:DEV0", "@charlie:DEV0"], - expectedAudioTracks: 2, - }, - { - name: "no audio track, only video track", - rtcUsers: [{ userId: "@alice", deviceId: "DEV0" }], - livekitParticipantIdentities: ["@alice:DEV0"], - explicitTracks: [ - { - participantId: "@alice:DEV0", - kind: Track.Kind.Video, - source: Track.Source.Camera, - }, - ], - expectedAudioTracks: 0, - }, - { - name: "Audio track from unknown source", - rtcUsers: [{ userId: "@alice", deviceId: "DEV0" }], - livekitParticipantIdentities: ["@alice:DEV0"], - explicitTracks: [ - { - participantId: "@alice:DEV0", - kind: Track.Kind.Audio, - source: Track.Source.Unknown, - }, - ], - expectedAudioTracks: 1, - }, - { - name: "Audio track from other device", - rtcUsers: [{ userId: "@alice", deviceId: "DEV0" }], - livekitParticipantIdentities: ["@alice:DEV0"], - explicitTracks: [ - { - participantId: "@alice:DEV1", - kind: Track.Kind.Audio, - source: Track.Source.Microphone, - }, - ], - expectedAudioTracks: 0, - }, - { - name: "two audio tracks, microphone and screenshare", - rtcUsers: [{ userId: "@alice", deviceId: "DEV0" }], - livekitParticipantIdentities: ["@alice:DEV0"], - explicitTracks: [ - { - participantId: "@alice:DEV0", - kind: Track.Kind.Audio, - source: Track.Source.Microphone, - }, - { - participantId: "@alice:DEV0", - kind: Track.Kind.Audio, - source: Track.Source.ScreenShareAudio, - }, - ], - expectedAudioTracks: 2, - }, -]; - -it.each(TEST_CASES)( - `should render sound test cases $name`, - ({ - rtcUsers, - livekitParticipantIdentities, - explicitTracks, - expectedAudioTracks, - }) => { - const { queryAllByTestId } = renderTestComponent( - rtcUsers, - livekitParticipantIdentities, - explicitTracks, - ); - expect(queryAllByTestId("audio")).toHaveLength(expectedAudioTracks); - }, -); - -it("should not setup audioContext gain and pan if there is no need to.", () => { - renderTestComponent([{ userId: "@bob", deviceId: "DEV0" }], ["@bob:DEV0"]); - const audioTrack = tracks[0].publication.track! as RemoteAudioTrack; - - expect(audioTrack.setAudioContext).toHaveBeenCalledTimes(1); - expect(audioTrack.setAudioContext).toHaveBeenCalledWith(undefined); - expect(audioTrack.setWebAudioPlugins).toHaveBeenCalledTimes(1); - expect(audioTrack.setWebAudioPlugins).toHaveBeenCalledWith([]); - - expect(testAudioContext.gain.gain.value).toEqual(1); - expect(testAudioContext.pan.pan.value).toEqual(0); -}); - -it("should setup audioContext gain and pan", () => { - vi.spyOn(MediaDevicesContext, "useEarpieceAudioConfig").mockReturnValue({ - pan: 1, - volume: 0.1, - }); - - renderTestComponent([{ userId: "@bob", deviceId: "DEV0" }], ["@bob:DEV0"]); - - const audioTrack = tracks[0].publication.track! as RemoteAudioTrack; - expect(audioTrack.setAudioContext).toHaveBeenCalled(); - expect(audioTrack.setWebAudioPlugins).toHaveBeenCalled(); - - expect(testAudioContext.gain.gain.value).toEqual(0.1); - expect(testAudioContext.pan.pan.value).toEqual(1); -}); diff --git a/src/livekit/MatrixAudioRenderer.tsx b/src/livekit/MatrixAudioRenderer.tsx deleted file mode 100644 index 4ca1d3bae..000000000 --- a/src/livekit/MatrixAudioRenderer.tsx +++ /dev/null @@ -1,216 +0,0 @@ -/* -Copyright 2025 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { getTrackReferenceId } from "@livekit/components-core"; -import { type Room as LivekitRoom } from "livekit-client"; -import { type RemoteAudioTrack, Track } from "livekit-client"; -import { useEffect, useMemo, useRef, useState, type ReactNode } from "react"; -import { - useTracks, - AudioTrack, - type AudioTrackProps, -} from "@livekit/components-react"; -import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; - -import { useEarpieceAudioConfig } from "../MediaDevicesContext"; -import { useReactiveState } from "../useReactiveState"; -import * as controls from "../controls"; -import { type TransportLocator } from "./auth"; - -export interface MatrixAudioRendererProps { - transport: TransportLocator; - livekitRoom: LivekitRoom; - /** - * The list of participant identities to render audio for. - * This list needs to be composed based on the matrixRTC members so that we do not play audio from users - * that are not expected to be in the rtc session (local user is excluded). - */ - validIdentities: string[]; - /** - * If set to `true`, mutes all audio tracks rendered by the component. - * @remarks - * If set to `true`, the server will stop sending audio track data to the client. - */ - muted?: boolean; -} - -/** - * Takes care of handling remote participants’ audio tracks and makes sure that microphones and screen share are audible. - * - * It also takes care of the earpiece audio configuration for iOS devices. - * This is done by using the WebAudio API to create a stereo pan effect that mimics the earpiece audio. - * @example - * ```tsx - * - * - * - * ``` - * @public - */ -export function LivekitRoomAudioRenderer({ - transport, - livekitRoom, - validIdentities, - muted, -}: MatrixAudioRendererProps): ReactNode { - const logger = rootLogger.getChild("[MatrixAudioRenderer]"); - // Identities we have already warned about, so that re-renders (which happen - // on every active speaker update) don't repeat the warning. - const warnedIdentities = useRef(new Set()); - const tracks = useTracks( - [ - Track.Source.Microphone, - Track.Source.ScreenShareAudio, - Track.Source.Unknown, - ], - { - updateOnlyOn: [], - onlySubscribed: true, - room: livekitRoom, - }, - ) - // Only keep remote audio tracks (we never render our own audio) - .filter( - (ref) => - ref.publication.kind === Track.Kind.Audio && !ref.participant.isLocal, - ) - // Only keep tracks from participants that are in the validIdentities list - .filter((ref) => { - const { identity } = ref.participant; - const isValid = validIdentities.includes(identity); - if (!isValid && !warnedIdentities.current.has(identity)) { - warnedIdentities.current.add(identity); - // Log that there is an invalid identity, that means that someone is publishing audio that is not expected to be in the call. - logger.warn( - `Audio track ${identity} from ${JSON.stringify(transport)} has no matching matrix call member`, - `current members: ${validIdentities.join()}`, - `track will not get rendered`, - ); - } else if (isValid) { - warnedIdentities.current.delete(identity); - } - return isValid; - }); - - // This component is also (in addition to the "only play audio for connected members" logic above) - // responsible for mimicking earpiece audio on iPhones. - // The Safari audio devices enumeration does not expose an earpiece audio device. - // We alternatively use the audioContext pan node to only use one of the stereo channels. - - // This component does get additionally complicated because of a Safari bug. - // (see: https://bugs.webkit.org/show_bug.cgi?id=251532 - // and the related issues: https://bugs.webkit.org/show_bug.cgi?id=237878 - // and https://bugs.webkit.org/show_bug.cgi?id=231105) - // - // AudioContext gets stopped if the webview gets moved into the background. - // Once the phone is in standby audio playback will stop. - // So we can only use the pan trick only works is the phone is not in standby. - // If earpiece mode is not used we do not use audioContext to allow standby playback. - // shouldUseAudioContext is set to false if stereoPan === 0 to allow standby bluetooth playback. - - const { pan: stereoPan, volume: volumeFactor } = useEarpieceAudioConfig(); - const shouldUseAudioContext = stereoPan !== 0; - - // initialize the potentially used audio context. - const [audioContext, setAudioContext] = useState( - undefined, - ); - useEffect(() => { - const ctx = new AudioContext(); - setAudioContext(ctx); - return (): void => { - void ctx.close(); - }; - }, []); - const audioNodes = useMemo( - () => ({ - gain: audioContext?.createGain(), - pan: audioContext?.createStereoPanner(), - }), - [audioContext], - ); - - // Simple effects to update the gain and pan node based on the props - useEffect(() => { - if (audioNodes.pan) audioNodes.pan.pan.value = stereoPan; - }, [audioNodes.pan, stereoPan]); - useEffect(() => { - if (audioNodes.gain) audioNodes.gain.gain.value = volumeFactor; - }, [audioNodes.gain, volumeFactor]); - - return ( - // We add all audio elements into one
for the browser developer tool experience/tidyness. -
- {tracks.map((trackRef) => ( - - ))} -
- ); -} - -interface StereoPanAudioTrackProps { - muted?: boolean; - audioContext?: AudioContext; - audioNodes: { - gain?: GainNode; - pan?: StereoPannerNode; - }; -} - -/** - * This wraps `livekit.AudioTrack` to allow adding audio nodes to a track. - * It main purpose is to remount the AudioTrack component when switching from - * audioContext to normal audio playback. - * As of now the AudioTrack component does not support adding audio nodes while being mounted. - * @param props The component props - * @param props.trackRef The track reference - * @param props.muted If the track should be muted - * @param props.audioContext The audio context to use - * @param props.audioNodes The audio nodes to use - * @returns - */ -function AudioTrackWithAudioNodes({ - trackRef, - muted, - audioContext, - audioNodes, - ...props -}: StereoPanAudioTrackProps & - AudioTrackProps & - React.RefAttributes): ReactNode { - // This is used to unmount/remount the AudioTrack component. - // Mounting needs to happen after the audioContext is set. - // (adding the audio context when already mounted did not work outside strict mode) - const [trackReady, setTrackReady] = useReactiveState( - () => false, - // We only want the track to reset once both (audioNodes and audioContext) are set. - // for unsetting the audioContext its enough if one of the two is undefined. - [audioContext && audioNodes], - ); - - useEffect(() => { - if (!trackRef || trackReady) return; - const track = trackRef.publication.track as RemoteAudioTrack; - const useContext = audioContext && audioNodes.gain && audioNodes.pan; - track.setAudioContext(useContext ? audioContext : undefined); - track.setWebAudioPlugins( - useContext ? [audioNodes.gain!, audioNodes.pan!] : [], - ); - setTrackReady(true); - controls.setPlaybackStarted(); - }, [audioContext, audioNodes, setTrackReady, trackReady, trackRef]); - - return ( - trackReady && - ); -} diff --git a/src/livekit/options.test.ts b/src/livekit/options.test.ts deleted file mode 100644 index 03134f0ce..000000000 --- a/src/livekit/options.test.ts +++ /dev/null @@ -1,213 +0,0 @@ -/* -Copyright 2026 Element Creations Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { describe, expect, it, vi } from "vitest"; -import { VideoPresets, type VideoPreset } from "livekit-client"; - -import { buildLiveKitOptions, getLiveKitOptions } from "./options"; -import { Config } from "../config/Config"; - -vi.mock("../config/Config", () => ({ - Config: { - get: vi.fn(), - }, -})); - -describe("buildLiveKitOptions", () => { - it("returns sensible defaults with no config", () => { - const opts = buildLiveKitOptions(); - expect(opts.adaptiveStream).toBe(true); - expect(opts.dynacast).toBe(true); - expect(opts.videoCaptureDefaults?.resolution).toEqual( - VideoPresets.h720.resolution, - ); - expect(opts.publishDefaults?.videoCodec).toBe("vp8"); - expect(opts.publishDefaults?.videoEncoding).toEqual({ - maxBitrate: 1_700_000, - maxFramerate: 30, - }); - expect(opts.publishDefaults?.screenShareEncoding).toEqual({ - maxBitrate: 5_000_000, - maxFramerate: 30, - }); - expect(opts.publishDefaults?.videoSimulcastLayers).toEqual([ - VideoPresets.h180, - VideoPresets.h360, - ]); - }); - - it("applies video codec from config", () => { - const opts = buildLiveKitOptions({ video_codec: "vp9" }); - expect(opts.publishDefaults?.videoCodec).toBe("vp9"); - }); - - it("applies video resolution and encoding from config", () => { - const baseVideoConfig = { - max_resolution: 1080, - max_bitrate: 3_000_000, - max_framerate: 60, - }; - const opts1080 = buildLiveKitOptions({ - video: baseVideoConfig, - }); - const opts1440 = buildLiveKitOptions({ - video: { ...baseVideoConfig, max_resolution: 1440 }, - }); - const opts2160 = buildLiveKitOptions({ - video: { ...baseVideoConfig, max_resolution: 2160 }, - }); - expect(opts1080.videoCaptureDefaults?.resolution).toEqual( - VideoPresets.h1080.resolution, - ); - expect(opts1440.videoCaptureDefaults?.resolution).toEqual( - VideoPresets.h1440.resolution, - ); - expect(opts2160.videoCaptureDefaults?.resolution).toEqual( - VideoPresets.h2160.resolution, - ); - expect(opts1080.publishDefaults?.videoEncoding).toEqual({ - maxBitrate: 3_000_000, - maxFramerate: 60, - }); - }); - - it("applies screen share encoding from config", () => { - const opts = buildLiveKitOptions({ - screen_share: { - max_bitrate: 8_000_000, - max_framerate: 15, - }, - }); - expect(opts.publishDefaults?.screenShareEncoding).toEqual({ - maxBitrate: 8_000_000, - maxFramerate: 15, - }); - }); - - it("uses DEFAULT_CONFIG defaults when only resolution is set", () => { - const opts = buildLiveKitOptions({ - screen_share: { - max_resolution: 720, - }, - }); - // Bitrate and framerate fall back to DEFAULT_CONFIG, not the preset - expect(opts.publishDefaults?.screenShareEncoding).toEqual({ - maxBitrate: 5_000_000, - maxFramerate: 30, - }); - }); - - it("maps low resolutions to the closest preset, rounding up", () => { - const expectations: [number, VideoPreset][] = [ - [180, VideoPresets.h180], - [360, VideoPresets.h360], - [480, VideoPresets.h540], - [540, VideoPresets.h540], - [720, VideoPresets.h720], - ]; - for (const [height, preset] of expectations) { - const opts = buildLiveKitOptions({ video: { max_resolution: height } }); - expect(opts.videoCaptureDefaults?.resolution).toEqual(preset.resolution); - } - }); - - it("screen share layers fall back to max_framerate, then 30", () => { - const fromMax = buildLiveKitOptions({ - screen_share: { - max_framerate: 15, - simulcast_layers: [{ height: 540, bitrate: 1_000_000 }], - }, - }); - expect( - fromMax.publishDefaults?.screenShareSimulcastLayers?.[0], - ).toMatchObject({ encoding: { maxFramerate: 15 } }); - - const fromDefault = buildLiveKitOptions({ - screen_share: { - simulcast_layers: [{ height: 540, bitrate: 1_000_000 }], - }, - }); - expect( - fromDefault.publishDefaults?.screenShareSimulcastLayers?.[0], - ).toMatchObject({ encoding: { maxFramerate: 30 } }); - }); - - it("applies custom video simulcast layers", () => { - const opts = buildLiveKitOptions({ - video: { - simulcast_layers: [ - { height: 180, bitrate: 100_000 }, - { height: 360, bitrate: 300_000 }, - { height: 540, bitrate: 600_000 }, - ], - max_framerate: 24, - }, - }); - const layers = opts.publishDefaults?.videoSimulcastLayers; - expect(layers).toHaveLength(3); - expect(layers?.[0]).toMatchObject({ - width: 320, - height: 180, - encoding: { maxBitrate: 100_000, maxFramerate: 24 }, - }); - expect(layers?.[2]).toMatchObject({ - width: 960, - height: 540, - encoding: { maxBitrate: 600_000, maxFramerate: 24 }, - }); - }); - - it("applies custom screen share simulcast layers", () => { - const opts = buildLiveKitOptions({ - screen_share: { - simulcast_layers: [{ height: 540, bitrate: 1_000_000, framerate: 5 }], - }, - }); - const layers = opts.publishDefaults?.screenShareSimulcastLayers; - expect(layers).toHaveLength(1); - expect(layers?.[0]).toMatchObject({ - width: 960, - height: 540, - encoding: { maxBitrate: 1_000_000, maxFramerate: 5 }, - }); - }); - - it("does not include screenShareSimulcastLayers when not configured", () => { - const opts = buildLiveKitOptions(); - expect(opts.publishDefaults?.screenShareSimulcastLayers).toBeUndefined(); - }); - - it("backupCodec always uses stock VP8 720p encoding", () => { - const opts = buildLiveKitOptions({ - video_codec: "av1", - video: { max_bitrate: 10_000_000, max_framerate: 60 }, - }); - const backup = opts.publishDefaults?.backupCodec as { - codec: string; - encoding: { maxBitrate: number; maxFramerate: number }; - }; - expect(backup.codec).toBe("vp8"); - expect(backup.encoding).toEqual(VideoPresets.h720.encoding); - }); -}); - -describe("getLiveKitOptions", () => { - it("reads from Config singleton", () => { - vi.mocked(Config.get).mockReturnValue({ - media_quality: { video_codec: "h264" }, - } as ReturnType); - const opts = getLiveKitOptions(); - expect(opts.publishDefaults?.videoCodec).toBe("h264"); - }); - - it("throws when Config is not initialized", () => { - vi.mocked(Config.get).mockImplementation(() => { - throw new Error("Config not initialized"); - }); - expect(() => getLiveKitOptions()).toThrow("Config not initialized"); - }); -}); diff --git a/src/livekit/options.ts b/src/livekit/options.ts deleted file mode 100644 index a4972f002..000000000 --- a/src/livekit/options.ts +++ /dev/null @@ -1,152 +0,0 @@ -/* -Copyright 2023, 2024 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - AudioPresets, - DefaultReconnectPolicy, - type RoomOptions, - type TrackPublishDefaults, - type VideoPreset, - VideoPresets, - VideoPreset as VideoPresetClass, -} from "livekit-client"; - -import { Config } from "../config/Config"; -import { DEFAULT_CONFIG, type ConfigOptions } from "../config/ConfigOptions"; - -/** - * Find the closest matching VideoPreset for a given height. - */ -function videoPresetForHeight(height: number): VideoPreset { - if (height <= 180) return VideoPresets.h180; - if (height <= 360) return VideoPresets.h360; - if (height <= 540) return VideoPresets.h540; - if (height <= 720) return VideoPresets.h720; - if (height <= 1080) return VideoPresets.h1080; - if (height <= 1440) return VideoPresets.h1440; - return VideoPresets.h2160; -} - -/** - * Build LiveKit publish options from config, falling back to sensible defaults. - */ -function buildPublishOptions( - mediaQuality: ConfigOptions["media_quality"], -): TrackPublishDefaults { - const defaults = DEFAULT_CONFIG.media_quality; - const videoConf = mediaQuality?.video; - const screenConf = mediaQuality?.screen_share; - const codec = mediaQuality?.video_codec ?? defaults.video_codec; - - // Camera video encoding - const videoEncoding = { - maxBitrate: videoConf?.max_bitrate ?? defaults.video.max_bitrate, - maxFramerate: videoConf?.max_framerate ?? defaults.video.max_framerate, - }; - - // Camera simulcast layers - let videoSimulcastLayers: VideoPreset[]; - if (videoConf?.simulcast_layers) { - videoSimulcastLayers = videoConf.simulcast_layers.map( - (layer) => - new VideoPresetClass( - Math.round((layer.height * 16) / 9), - layer.height, - layer.bitrate, - videoConf?.max_framerate ?? defaults.video.max_framerate, - ), - ); - } else { - videoSimulcastLayers = [VideoPresets.h180, VideoPresets.h360]; - } - - // Screen share encoding - const screenShareEncoding = { - maxBitrate: screenConf?.max_bitrate ?? defaults.screen_share.max_bitrate, - maxFramerate: - screenConf?.max_framerate ?? defaults.screen_share.max_framerate, - }; - - // Screen share simulcast layers - let screenShareSimulcastLayers: VideoPreset[] | undefined; - if (screenConf?.simulcast_layers) { - screenShareSimulcastLayers = screenConf.simulcast_layers.map( - (layer) => - new VideoPresetClass( - Math.round((layer.height * 16) / 9), - layer.height, - layer.bitrate, - layer.framerate ?? screenConf?.max_framerate ?? 30, - ), - ); - } - - return { - audioPreset: AudioPresets.music, - dtx: true, - // disable red because the livekit server strips out red packets for clients - // that don't support it (firefox) but of course that doesn't work with e2ee. - red: false, - forceStereo: false, - simulcast: true, - videoSimulcastLayers: videoSimulcastLayers as VideoPreset[], - screenShareEncoding, - ...(screenShareSimulcastLayers && { - screenShareSimulcastLayers: screenShareSimulcastLayers as VideoPreset[], - }), - stopMicTrackOnMute: false, - videoCodec: codec, - videoEncoding, - backupCodec: { - codec: "vp8", - encoding: VideoPresets.h720.encoding, - }, - } as TrackPublishDefaults; -} - -/** - * Build LiveKit RoomOptions from config. - * Call this after Config.init() has resolved. - */ -export function buildLiveKitOptions( - mediaQuality?: ConfigOptions["media_quality"], -): RoomOptions { - const videoHeight = - mediaQuality?.video?.max_resolution ?? - DEFAULT_CONFIG.media_quality.video.max_resolution; - const basePreset = videoPresetForHeight(videoHeight); - - return { - // automatically manage subscribed video quality - adaptiveStream: true, - - // optimize publishing bandwidth and CPU for published tracks - dynacast: true, - - // capture settings - videoCaptureDefaults: { - resolution: basePreset.resolution, - }, - - // publish settings - publishDefaults: buildPublishOptions(mediaQuality), - - // default LiveKit options that seem to be sane - stopLocalTrackOnUnpublish: true, - reconnectPolicy: new DefaultReconnectPolicy(), - disconnectOnPageLeave: true, - webAudioMix: false, - }; -} - -/** - * Get LiveKit options, reading from the loaded Config singleton. - * Requires Config.init() to have resolved first. - */ -export function getLiveKitOptions(): RoomOptions { - return buildLiveKitOptions(Config.get().media_quality); -} diff --git a/src/room/CallEventAudioRenderer.test.tsx b/src/room/CallEventAudioRenderer.test.tsx index 1d8fd33d2..87d350fa8 100644 --- a/src/room/CallEventAudioRenderer.test.tsx +++ b/src/room/CallEventAudioRenderer.test.tsx @@ -41,7 +41,6 @@ import { import { MAX_PARTICIPANT_COUNT_FOR_SOUND } from "../state/CallViewModel/CallViewModel"; import { initializeWidget } from "../widget"; initializeWidget(); -vitest.mock("livekit-client/e2ee-worker?worker"); vitest.mock("../useAudioContext"); vitest.mock("../soundUtils"); vitest.mock("../rtcSessionHelpers", async (importOriginal) => ({ diff --git a/src/room/CallView.test.tsx b/src/room/CallView.test.tsx index 8ac920b84..2c2208c56 100644 --- a/src/room/CallView.test.tsx +++ b/src/room/CallView.test.tsx @@ -61,7 +61,7 @@ import { } from "../HostBridge"; import { type JoinCallData } from "../widget"; import { MatrixRTCTransportMissingError } from "../utils/errors"; -import { ProcessorProvider } from "../livekit/TrackProcessorContext"; +import { ProcessorProvider } from "../tracks/TrackProcessorContext"; import { MediaDevicesContext } from "../MediaDevicesContext"; import { constant } from "../state/Behavior"; diff --git a/src/room/InCallView.test.tsx b/src/room/InCallView.test.tsx index 357bc186e..7f2111615 100644 --- a/src/room/InCallView.test.tsx +++ b/src/room/InCallView.test.tsx @@ -15,22 +15,20 @@ import { vi, } from "vitest"; import { act, render, type RenderResult } from "@testing-library/react"; -import { type LocalParticipant } from "livekit-client"; -import { BehaviorSubject, of } from "rxjs"; +import { BehaviorSubject } from "rxjs"; import { BrowserRouter } from "react-router-dom"; import { TooltipProvider } from "@vector-im/compound-web"; -import { RoomContext, useLocalParticipant } from "@livekit/components-react"; import userEvent from "@testing-library/user-event"; import { ActiveCall, InCallView } from "./InCallView"; import { - mockLivekitRoom, - mockLocalParticipant, mockMediaDevices, mockMuteStates, - mockRemoteParticipant, mockRtcMembership, type MockRTCSession, + mockMatrixRTCClient, + mockMemberMedia, + mockRTCMember, } from "../utils/test"; import { E2eeType } from "../e2ee/e2eeType"; import { @@ -41,15 +39,17 @@ import { type CallViewModel, type CallViewModelOptions, } from "../state/CallViewModel/CallViewModel"; -import { alice, local } from "../utils/test-fixtures"; +import { alice, local, aliceRtcMember } from "../utils/test-fixtures"; import { ReactionsSenderProvider } from "../reactions/useReactionsSender"; import { useRoomEncryptionSystem } from "../e2ee/sharedKeyManagement"; -import { LivekitRoomAudioRenderer } from "../livekit/MatrixAudioRenderer"; +import { MemberAudioRenderer } from "../tracks/MemberAudioRenderer"; +import type * as MatrixRTCSdk from "@element-hq/matrixrtc-sdk"; +import { constant, type ObservableScope } from "@element-hq/matrixrtc-sdk"; import { MediaDevicesContext } from "../MediaDevicesContext"; import { type MediaDevices as ECMediaDevices } from "../state/MediaDevices"; import { AppBar } from "../AppBar"; import { type MatrixInfo } from "./VideoPreview"; -import { ProcessorProvider } from "../livekit/TrackProcessorContext"; +import { ProcessorProvider } from "../tracks/TrackProcessorContext"; import { initializeWidget } from "../widget"; import { RootElementProvider } from "../RootElementContext"; @@ -68,21 +68,19 @@ vi.mock("../soundUtils"); vi.mock("../useAudioContext"); vi.mock("../tile/GridTile"); vi.mock("../tile/SpotlightTile"); -vi.mock("@livekit/components-react"); -vi.mock("livekit-client/e2ee-worker?worker"); vi.mock("../e2ee/sharedKeyManagement"); -vi.mock("../livekit/MatrixAudioRenderer"); +vi.mock("../tracks/MemberAudioRenderer"); +// ActiveCall builds the client itself; the view is what these tests are about +const createMatrixRTCClientMock = vi.hoisted(() => vi.fn()); +vi.mock("@element-hq/matrixrtc-sdk", async (importOriginal) => ({ + ...(await importOriginal()), + createMatrixRTCClient: createMatrixRTCClientMock, +})); vi.mock("react-use-measure", () => ({ default: (): [() => void, object] => [(): void => {}, {}], })); const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC"); -const localParticipant = mockLocalParticipant({ - identity: "@local:example.org:AAAAAA", -}); -const remoteParticipant = mockRemoteParticipant({ - identity: "@alice:example.org:AAAAAA", -}); const matrixInfo = { userId: "", @@ -100,21 +98,26 @@ let useRoomEncryptionSystemMock: MockedFunction; beforeEach(() => { vi.clearAllMocks(); - // MatrixAudioRenderer is tested separately. - ( - LivekitRoomAudioRenderer as MockedFunction - ).mockImplementation((_props) => { - return
mocked: MatrixAudioRenderer
; - }); - ( - useLocalParticipant as MockedFunction - ).mockImplementation( - () => - ({ - isScreenShareEnabled: false, - localParticipant: localRtcMember as unknown as LocalParticipant, - }) as unknown as ReturnType, + createMatrixRTCClientMock.mockImplementation((scope: ObservableScope) => + mockMatrixRTCClient(scope, { + localMember$: constant( + mockRTCMember(true, { + membership: localRtcMember, + media$: constant(mockMemberMedia({ local: true })), + }), + ), + remoteMembers$: constant([ + mockRTCMember(false, { + membership: aliceRtcMember, + media$: constant(mockMemberMedia()), + }), + ]), + }), ); + // MemberAudioRenderer is tested separately. + ( + MemberAudioRenderer as MockedFunction + ).mockImplementation(() =>
mocked: MemberAudioRenderer
); useRoomEncryptionSystemMock = useRoomEncryptionSystem as typeof useRoomEncryptionSystemMock; useRoomEncryptionSystemMock.mockReturnValue({ kind: E2eeType.NONE }); @@ -131,14 +134,6 @@ function createInCallView(args: CreateInCallViewArgs = {}): RenderResult & { } { const mediaDevices = args.mediaDevices ?? mockMediaDevices({}); const muteState = mockMuteStates(); - const livekitRoom = mockLivekitRoom( - { - localParticipant, - }, - { - remoteParticipants$: of([remoteParticipant]), - }, - ); const { vm, footerVm, developerSettingsVm, rtcSession } = getBasicCallViewModelEnvironment( [local, alice], @@ -174,9 +169,7 @@ function createInCallView(args: CreateInCallViewArgs = {}): RenderResult & { vm={vm} rtcSession={rtcSession.asMockedSession()} > - - {content} - + {content} , @@ -244,18 +237,16 @@ describe("ActiveCall", () => { - - {}} - /> - + {}} + /> @@ -294,18 +285,16 @@ describe("ActiveCall", () => { - - {}} - /> - + {}} + /> diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index bc0d68c4e..761f93e71 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -46,7 +46,10 @@ import { } from "../state/CallViewModel/CallViewModel.ts"; import { Grid, type TileProps } from "../grid/Grid"; import { SpotlightTile } from "../tile/SpotlightTile"; -import { type EncryptionSystem } from "../e2ee/sharedKeyManagement"; +import { + type EncryptionSystem, + type MatrixRTCMode, +} from "@element-hq/matrixrtc-sdk"; import { E2eeType } from "../e2ee/e2eeType"; import { makeGridLayout } from "../grid/GridLayout"; import { type CallLayoutOutputs } from "../grid/CallLayout"; @@ -63,9 +66,36 @@ import { import { ReactionsAudioRenderer } from "./ReactionAudioRenderer"; import { ReactionsOverlay } from "./ReactionsOverlay"; import { CallEventAudioRenderer } from "./CallEventAudioRenderer"; -import { matrixRTCMode as matrixRTCModeSetting } from "../settings/settings"; +import { + createMatrixRTCClient, + type MatrixRTCClientOptions, + constant, + type Behavior, +} from "@element-hq/matrixrtc-sdk"; +import { map } from "rxjs"; +import { + advancedCamera, + advancedScreenShare, + autoGainControlSetting, + cameraBitrate, + cameraCodec, + cameraFramerate, + cameraResolution, + customLivekitUrl, + echoCancellationSetting, + matrixRTCMode as matrixRTCModeSetting, + noiseSuppressionSetting, + parseResolution, + screenShareBitrate, + screenShareCodec, + screenShareFramerate, + screenShareResolution, +} from "../settings/settings"; +import { Config } from "../config/Config"; +import { type MediaDevice } from "../state/MediaDevices"; +import { createSentCallNotification$ } from "../state/CallViewModel/CallNotificationLifecycle.ts"; import { ReactionsReader } from "../reactions/ReactionsReader"; -import { LivekitRoomAudioRenderer } from "../livekit/MatrixAudioRenderer.tsx"; +import { MemberAudioRenderer } from "../tracks/MemberAudioRenderer.tsx"; import { muteAllAudio$ } from "../state/MuteAllAudioModel.ts"; import { useMediaDevices } from "../MediaDevicesContext.ts"; import { EarpieceOverlay } from "./EarpieceOverlay.tsx"; @@ -76,10 +106,9 @@ import { } from "../AppBar.tsx"; import { useBehavior } from "../useBehavior.ts"; import { useValueBehavior } from "../useValueBehavior.ts"; -import { constant } from "../state/Behavior.ts"; import { Toast } from "../Toast.tsx"; import overlayStyles from "../Overlay.module.css"; -import { useTrackProcessorState$ } from "../livekit/TrackProcessorContext.tsx"; +import { useTrackProcessorState$ } from "../tracks/TrackProcessorContext.tsx"; import { type Layout } from "../state/layout-types.ts"; import { ObservableScope } from "../state/ObservableScope.ts"; import { CallFooter, type FooterSnapshot } from "../components/CallFooter.tsx"; @@ -100,7 +129,7 @@ declare module "react" { export interface ActiveCallProps extends Omit< InCallViewProps, - "vm" | "livekitRoom" | "connState" | "footerVm" | "developerSettingsVm" + "vm" | "footerVm" | "developerSettingsVm" > { e2eeSystem: EncryptionSystem; // TODO refactor those reasons into an enum @@ -131,9 +160,28 @@ export const ActiveCall: FC = (props) => { const { autoLeaveWhenOthersLeft, waitForCallPickup, sendNotificationType } = urlParams; + const rtcClient = createMatrixRTCClient( + scope, + props.client, + props.matrixRoom, + { + microphoneEnabled: props.muteStates.audio.enabled$.value, + cameraEnabled: props.muteStates.video.enabled$.value, + audioInputDeviceId$: selectedDeviceId$(scope, mediaDevices.audioInput), + videoInputDeviceId$: selectedDeviceId$(scope, mediaDevices.videoInput), + // A host that routes audio itself leaves the browser's output alone + audioOutputDeviceId$: urlParams.controlledAudioDevices + ? constant(undefined) + : selectedDeviceId$(scope, mediaDevices.audioOutput), + videoProcessor$: scope.behavior( + trackProcessorState$.pipe(map((state) => state.processor)), + ), + }, + matrixRTCClientOptions(urlParams, props.e2eeSystem), + ); const vm = createCallViewModel$( scope, - props.rtcSession, + rtcClient, props.matrixRoom, mediaDevices, props.muteStates, @@ -143,14 +191,14 @@ export const ActiveCall: FC = (props) => { hostBridge, autoLeaveWhenOthersLeft, waitForCallPickup: waitForCallPickup && sendNotificationType === "ring", - // We merely sample the current mode here, so the user would need to - // manually rejoin to switch to a different one. - matrixRTCMode: matrixRTCModeSetting.value$.value, + sentCallNotification$: createSentCallNotification$( + scope, + props.rtcSession, + ), windowSize$: scope.behavior(observeElementSize$(rootElement)), }, reactionsReader.raisedHands$, reactionsReader.reactions$, - trackProcessorState$, ); // TODO move this somewhere else once we use the callViewModel in the lobby as well! vm.join(); @@ -250,16 +298,6 @@ export const InCallView: FC = ({ const { sendReaction, toggleRaisedHand } = useReactionsSender(); useWakeLock(); - // TODO-MULTI-SFU This is unused now?? - // const connectionState = useBehavior(vm.livekitConnectionState$); - - // annoyingly we don't get the disconnection reason this way, - // only by listening for the emitted event - // This needs to be done differential. with the vm connection state we start with Disconnected. - // TODO-MULTI-SFU decide how to handle this properly - // @BillCarsonFr - // if (connectionState === ConnectionState.Disconnected) - // throw new ConnectionLostError(); const containerRef1 = useRef(null); const [containerRef2, bounds] = useMeasure(); @@ -296,7 +334,7 @@ export const InCallView: FC = ({ ); const ringingVm = useBehavior(vm.ringingVm$); - const audioParticipants = useBehavior(vm.livekitRoomItems$); + const remoteMembers = useBehavior(vm.remoteMembers$); const participantCount = useBehavior(vm.participantCount$); const reconnecting = useBehavior(vm.reconnecting$); const screenShareError = useBehavior(vm.screenShareError$); @@ -646,7 +684,11 @@ export const InCallView: FC = ({ const footer = footerVm !== null && ( ); - const allConnections = useBehavior(vm.allConnections$); + const connectedTransports = useBehavior(vm.connectedTransports$); + const localMember = useBehavior(vm.localMember$); + const localTransport = useBehavior( + localMember?.transport$ ?? constant(undefined), + ); return ( // The pointer handler here exists to control the visibility of the footer, @@ -665,15 +707,7 @@ export const InCallView: FC = ({ onPointerOut={onPointerOut} > {header} - {audioParticipants.map(({ livekitRoom, transport, participants }) => ( - - ))} + {renderContent()} @@ -694,22 +728,99 @@ export const InCallView: FC = ({ tab={settingsTab} onTabChange={setSettingsTab} developerSettingsVm={developerSettingsVm} - livekitRooms={allConnections.map( - ({ livekitRoom: room, livekitAlias, transport, role }) => ({ - room, - livekitAlias, - role, - url: - // It should be relatively clear to the user which kind of URL - // this is, based on the scheme (ws:// vs. http://) - "url" in transport - ? transport.url - : transport.livekit_service_url, - }), - )} + transports={connectedTransports.map((transport) => ({ + transport, + local: transport === localTransport, + }))} /> )}
); }; + +/** The id of the device the user picked, for the client to capture or play on. */ +export function selectedDeviceId$( + scope: ObservableScope, + device: MediaDevice, +): Behavior { + return scope.behavior(device.selected$.pipe(map((d) => d?.id))); +} + +/** + * What the client needs from Element Call's configuration, settings and URL: + * the SDK reads none of them itself. Shared with the legacy SDK bundle so that + * a widget built from it behaves like the app. + */ +export function matrixRTCClientOptions( + urlParams: ReturnType, + encryptionSystem: EncryptionSystem, +): MatrixRTCClientOptions { + const config = Config.get(); + const session = config.matrix_rtc_session; + const cameraSettings = advancedCamera.getValue() + ? { + resolution: { + ...parseResolution(cameraResolution.getValue()), + frameRate: cameraFramerate.getValue(), + }, + maxBitrate: cameraBitrate.getValue(), + maxFramerate: cameraFramerate.getValue(), + codec: cameraCodec.getValue(), + } + : undefined; + const screenConfig = config.media_quality?.screen_share; + const screenShareSettings = advancedScreenShare.getValue() + ? { + resolution: { + ...parseResolution(screenShareResolution.getValue()), + frameRate: screenShareFramerate.getValue(), + }, + maxBitrate: screenShareBitrate.getValue(), + maxFramerate: screenShareFramerate.getValue(), + codec: screenShareCodec.getValue(), + } + : screenConfig?.max_resolution + ? { + resolution: { + width: Math.round((screenConfig.max_resolution * 16) / 9), + height: screenConfig.max_resolution, + frameRate: screenConfig.max_framerate ?? 30, + }, + } + : undefined; + return { + encryptionSystem, + // matrix_rtc_mode in config.json overrides the user's Developer Settings + // choice. We merely sample the current mode here, so the user would need + // to manually rejoin to switch to a different one. + matrixRTCMode: + (config.matrix_rtc_mode as MatrixRTCMode | undefined) ?? + matrixRTCModeSetting.value$.value, + sendNotificationType: urlParams.sendNotificationType, + applicationData: urlParams.callIntent + ? { "m.call.intent": urlParams.callIntent } + : undefined, + timings: { + syncDisconnectGracePeriodMs: config.sync_disconnect_grace_period_ms, + networkErrorRetryMs: session.network_error_retry_ms, + waitForKeyRotationMs: session.wait_for_key_rotation_ms, + membershipEventExpiryMs: session.membership_event_expiry_ms, + keyRotationParticipantLimit: session.key_rotation_participant_limit, + delayedLeave: session.delayed_leave, + delegatedDelayedLeave: session.delegated_delayed_leave, + }, + mediaQuality: config.media_quality, + capture: { + audio: { + echoCancellation: echoCancellationSetting.getValue(), + noiseSuppression: noiseSuppressionSetting.getValue(), + autoGainControl: autoGainControlSetting.getValue(), + }, + camera: cameraSettings, + screenShare: screenShareSettings, + }, + transportUrl: customLivekitUrl.value$.value ?? undefined, + fallbackTransportUrl: config.livekit?.livekit_service_url, + }; +} diff --git a/src/room/KnockLobbyView.test.tsx b/src/room/KnockLobbyView.test.tsx index a513952b7..fda9cee06 100644 --- a/src/room/KnockLobbyView.test.tsx +++ b/src/room/KnockLobbyView.test.tsx @@ -14,14 +14,14 @@ import { type MatrixClient, type RoomSummary } from "matrix-js-sdk"; import { KnockLobbyView } from "./KnockLobbyView"; import { LeaveToHomeProvider } from "../LeaveToHomeContext"; import { MediaDevicesContext } from "../MediaDevicesContext"; -import { type ProcessorState } from "../livekit/TrackProcessorContext"; +import { type ProcessorState } from "../tracks/TrackProcessorContext"; import { mockMediaDevices } from "../utils/test"; vi.mock("@livekit/components-react", () => ({ usePreviewTracks: (): unknown[] => [], })); -vi.mock("../livekit/TrackProcessorContext", () => ({ +vi.mock("../tracks/TrackProcessorContext", () => ({ useTrackProcessor: (): ProcessorState => ({ supported: false, processor: undefined, diff --git a/src/room/LobbyView.test.tsx b/src/room/LobbyView.test.tsx index aedd9fdd5..fbe4248e7 100644 --- a/src/room/LobbyView.test.tsx +++ b/src/room/LobbyView.test.tsx @@ -28,7 +28,7 @@ import { } from "../utils/test"; import { type MediaDevices } from "../state/MediaDevices"; import { MediaDevicesContext } from "../MediaDevicesContext"; -import { type ProcessorState } from "../livekit/TrackProcessorContext"; +import { type ProcessorState } from "../tracks/TrackProcessorContext"; import { type EncryptionSystem } from "../e2ee/sharedKeyManagement"; import lobbyStyles from "./LobbyView.module.css"; import headerStyles from "../Header.module.css"; @@ -41,7 +41,7 @@ vi.mock("@livekit/components-react", () => ({ usePreviewTracks: (): unknown[] => [], })); -vi.mock("../livekit/TrackProcessorContext", () => ({ +vi.mock("../tracks/TrackProcessorContext", () => ({ useTrackProcessor: (): ProcessorState => ({ supported: false, processor: undefined, diff --git a/src/room/LobbyView.tsx b/src/room/LobbyView.tsx index 5814e4316..13ba55b74 100644 --- a/src/room/LobbyView.tsx +++ b/src/room/LobbyView.tsx @@ -42,7 +42,7 @@ import { useInitial } from "../useInitial"; import { useTrackProcessor, useTrackProcessorSync, -} from "../livekit/TrackProcessorContext"; +} from "../tracks/TrackProcessorContext"; import { getValue } from "../utils/observable"; import { useBehavior } from "../useBehavior"; import { CallFooter, type FooterSnapshot } from "../components/CallFooter"; diff --git a/src/room/ReactionAudioRenderer.test.tsx b/src/room/ReactionAudioRenderer.test.tsx index 0f847572c..246faa16f 100644 --- a/src/room/ReactionAudioRenderer.test.tsx +++ b/src/room/ReactionAudioRenderer.test.tsx @@ -48,7 +48,6 @@ function TestComponent({ vm }: { vm: CallViewModel }): ReactNode { ); } -vitest.mock("livekit-client/e2ee-worker?worker"); vitest.mock("../useAudioContext"); vitest.mock("../soundUtils"); diff --git a/src/room/ReactionsOverlay.test.tsx b/src/room/ReactionsOverlay.test.tsx index 727fbb582..61a940196 100644 --- a/src/room/ReactionsOverlay.test.tsx +++ b/src/room/ReactionsOverlay.test.tsx @@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details. */ import { render } from "@testing-library/react"; -import { expect, test, afterEach, vi } from "vitest"; +import { expect, test, afterEach } from "vitest"; import { act } from "react"; import { showReactions } from "../settings/settings"; @@ -22,8 +22,6 @@ import { getBasicCallViewModelEnvironment } from "../utils/test-viewmodel"; import { initializeWidget } from "../widget"; initializeWidget(); -vi.mock("livekit-client/e2ee-worker?worker"); - afterEach(() => { showReactions.setValue(showReactions.defaultValue); }); diff --git a/src/room/__snapshots__/InCallView.test.tsx.snap b/src/room/__snapshots__/InCallView.test.tsx.snap index fd0ff4661..16e3eff3a 100644 --- a/src/room/__snapshots__/InCallView.test.tsx.snap +++ b/src/room/__snapshots__/InCallView.test.tsx.snap @@ -84,6 +84,9 @@ exports[`InCallView > rendering > renders 1`] = ` class="_nav_e4b327 _rightNav_e4b327" /> +
+ mocked: MemberAudioRenderer +
diff --git a/src/settings/DeveloperSettingsTab.test.tsx b/src/settings/DeveloperSettingsTab.test.tsx index 82764d8ec..44aaa9155 100644 --- a/src/settings/DeveloperSettingsTab.test.tsx +++ b/src/settings/DeveloperSettingsTab.test.tsx @@ -11,10 +11,10 @@ import userEvent, { type UserEvent } from "@testing-library/user-event"; import { TooltipProvider } from "@vector-im/compound-web"; import type { MatrixClient } from "matrix-js-sdk"; -import type { Room as LivekitRoom } from "livekit-client"; import { DeveloperSettingsTab, type DeveloperSettingsSnapshot, + type TransportInfo, } from "./DeveloperSettingsTab"; import { outOfCallDeveloperSettingsTabViewModel } from "./DeveloperSettingsTabViewModel"; import { createStaticViewModel } from "../state/ViewModel"; @@ -23,9 +23,13 @@ import { enableExtendedLivekitLogs as enableExtendedLivekitLogsSetting, matrixRTCMode as matrixRTCModeSetting, } from "./settings"; -import { MatrixRTCMode } from "../config/ConfigOptions"; -import { exampleTransport, mockConfig } from "../utils/test"; -import { getSFUConfig } from "../livekit/auth"; +import { + authenticateWithTransport, + constant, + MatrixRTCMode, +} from "@element-hq/matrixrtc-sdk"; +import type * as MatrixRTCSdk from "@element-hq/matrixrtc-sdk"; +import { mockConfig } from "../utils/test"; // Mock url params hook to avoid environment-dependent snapshot churn. vi.mock("../UrlParams", () => ({ @@ -36,32 +40,30 @@ vi.mock("../UrlParams", () => ({ })); // IMPORTANT: mock the same specifier used by DeveloperSettingsTab -vi.mock("../livekit/auth", () => ({ - getSFUConfig: vi.fn().mockResolvedValue({ +vi.mock("@element-hq/matrixrtc-sdk", async (importOriginal) => ({ + ...(await importOriginal()), + authenticateWithTransport: vi.fn().mockResolvedValue({ url: "mock-url", jwt: "mock-jwt", }), })); -// Provide a minimal mock of a Livekit Room structure used by the component. -function createMockLivekitRoom( - wsUrl: string, - serverInfo: object, - metadata: string, -): { isLocal: boolean; url: string; room: LivekitRoom; livekitAlias: string } { - const mockRoom = { - serverInfo, - metadata, - engine: { client: { ws: { url: wsUrl } } }, - localParticipant: { identity: "localParticipantIdentity" }, - remoteParticipants: new Map(), - } as unknown as LivekitRoom; - +/** A transport the panel shows, as the client would hand it out. */ +function mockTransport( + url: string, + local: boolean, + resolved?: { url: string; roomAlias: string; identity: string }, +): TransportInfo { return { - isLocal: true, - url: wsUrl, - room: mockRoom, - livekitAlias: "TestAlias", + local, + transport: { + type: "livekit", + id: url, + raw: { type: "livekit", livekit_service_url: url }, + resolved$: constant( + resolved && { type: "livekit", token: "secret", ...resolved }, + ), + }, }; } @@ -96,36 +98,20 @@ describe("DeveloperSettingsTab", () => { it("renders and matches snapshot", async () => { const client = createMockMatrixClient(); - const livekitRooms: { - room: LivekitRoom; - url: string; - isLocal?: boolean; - livekitAlias: string; - }[] = [ - createMockLivekitRoom( - "wss://local-sfu.example.org", - { region: "local", version: "1.2.3" }, - "local-metadata", - ), - { - isLocal: false, - livekitAlias: "TestAlias2", - url: "wss://remote-sfu.example.org", - room: { - localParticipant: { identity: "localParticipantIdentity" }, - remoteParticipants: new Map(), - serverInfo: { region: "remote", version: "4.5.6" }, - metadata: "remote-metadata", - engine: { client: { ws: { url: "wss://remote-sfu.example.org" } } }, - } as unknown as LivekitRoom, - }, + const transports = [ + mockTransport("https://local-sfu.example.org/jwt", true, { + url: "wss://local-sfu.example.org", + roomAlias: "TestAlias", + identity: "localParticipantIdentity", + }), + mockTransport("https://remote-sfu.example.org/jwt", false), ]; const { container } = render( , @@ -169,7 +155,7 @@ describe("DeveloperSettingsTab", () => { const saveButton = screen.getByRole("button", { name: "Save" }); await user.click(saveButton); - expect(getSFUConfig).not.toHaveBeenCalled(); + expect(authenticateWithTransport).not.toHaveBeenCalled(); expect(customTransportSetting.getValue()).toBe(null); }); @@ -192,7 +178,7 @@ describe("DeveloperSettingsTab", () => { const saveButton = screen.getByRole("button", { name: "Save" }); await user.click(saveButton); - expect(getSFUConfig).not.toHaveBeenCalled(); + expect(authenticateWithTransport).not.toHaveBeenCalled(); expect(customTransportSetting.getValue()).toBe(null); }); @@ -218,7 +204,7 @@ describe("DeveloperSettingsTab", () => { name: "Reset overwrite", }); await user.click(cancelButton); - expect(getSFUConfig).not.toHaveBeenCalled(); + expect(authenticateWithTransport).not.toHaveBeenCalled(); expect(customTransportSetting.getValue()).toBe(null); }); @@ -245,11 +231,11 @@ describe("DeveloperSettingsTab", () => { const saveButton = screen.getByRole("button", { name: "Save" }); await user.click(saveButton); - expect(getSFUConfig).toHaveBeenCalledWith( - expect.objectContaining({ - transport: { type: "livekit", url: "wss://example.livekit.valid" }, - roomId: "#testRoom", - }), + expect(authenticateWithTransport).toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + "wss://example.livekit.valid", + "#testRoom", ); expect(customTransportSetting.getValue()).toEqual({ @@ -279,8 +265,8 @@ describe("DeveloperSettingsTab", () => { }); const saveButton = screen.getByRole("button", { name: "Save" }); - (getSFUConfig as Mock).mockImplementation(() => { - throw new Error("Failed to get SFU config"); + (authenticateWithTransport as Mock).mockImplementation(() => { + throw new Error("Invalid URL"); }); await user.click(saveButton); expect( diff --git a/src/settings/DeveloperSettingsTab.tsx b/src/settings/DeveloperSettingsTab.tsx index e31282e81..0642db575 100644 --- a/src/settings/DeveloperSettingsTab.tsx +++ b/src/settings/DeveloperSettingsTab.tsx @@ -11,7 +11,6 @@ import { type ReactNode, useCallback, useEffect, - useMemo, useId, useState, } from "react"; @@ -32,7 +31,6 @@ import { RadioControl, Separator, } from "@vector-im/compound-web"; -import { type Room as LivekitRoom } from "livekit-client"; import { FieldRow, InputField } from "../input/Input"; import { Config } from "../config/Config"; @@ -67,6 +65,10 @@ import styles from "./DeveloperSettingsTab.module.css"; import settingsStyles from "./SettingsModal.module.css"; import { Slider } from "../Slider"; import { useUrlParams } from "../UrlParams"; +import { + authenticateWithTransport, + type TransportMetadata, +} from "@element-hq/matrixrtc-sdk"; import { useBehavior } from "../useBehavior"; import { type ViewModel } from "../state/ViewModel.ts"; import { getSFUConfig } from "../livekit/auth/index.ts"; @@ -75,6 +77,13 @@ import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; /** * The state of MatrixRTC's media key rotation. */ +/** One connected transport, as the panel shows it. */ +export interface TransportInfo { + transport: TransportMetadata; + /** Whether this is the transport the local member publishes on. */ + local: boolean; +} + export interface KeyRotationInfo { /** Whether the call is large enough that MatrixRTC has stopped rotating the media key. */ suppressed: boolean; @@ -106,19 +115,15 @@ const KeyRotationStatus: FC<{ info: KeyRotationInfo }> = ({ info }) => ( interface Props { client: MatrixClient; roomId?: string; - livekitRooms?: { - room: LivekitRoom; - url: string; - isLocal?: boolean; - livekitAlias?: string; - }[]; + /** The transports the client is connected to, while in a call. */ + transports?: TransportInfo[]; env: ImportMetaEnv; vm: ViewModel; } export const DeveloperSettingsTab: FC = ({ client, - livekitRooms, + transports, roomId, env, vm, @@ -185,17 +190,6 @@ export const DeveloperSettingsTab: FC = ({ const urlParams = useUrlParams(); - const localSfuUrl = useMemo((): URL | null => { - const localRoom = livekitRooms?.find((r) => r.isLocal)?.room; - if (localRoom?.engine.client.ws?.url) { - // strip the URL params - const url = new URL(localRoom.engine.client.ws.url); - url.search = ""; - return url; - } - return null; - }, [livekitRooms]); - const MediaQualitySettings: React.FC<{ id: string; header: string; @@ -521,9 +515,7 @@ export const DeveloperSettingsTab: FC = ({ if (userId === null || deviceId === null) { throw new Error("Invalid user or device ID"); } - const transport = JSON.parse(customTransportTextBuffer); - // Validate the transport by attempting to get a token for it - await getSFUConfig({ + await authenticateWithTransport( client, membership: { userId, deviceId, memberId: "" }, transport, @@ -601,39 +593,8 @@ export const DeveloperSettingsTab: FC = ({ - {livekitRooms?.map((livekitRoom) => ( -
-

- {t("developer_mode.livekit_sfu", { - url: livekitRoom.url || "unknown", - })} -

-

LivekitAlias: {livekitRoom.livekitAlias}

-

connectionState (wont hot reload): {livekitRoom.room.state}

- {livekitRoom.isLocal &&

ws-url: {localSfuUrl?.href}

} -

- {t("developer_mode.livekit_server_info")}( - {livekitRoom.isLocal ? "local" : "remote"}) -

- 
-            {livekitRoom.room.serverInfo
-              ? JSON.stringify(livekitRoom.room.serverInfo, null, 2)
-              : "undefined"}
-            {livekitRoom.room.metadata}
-
-

Local Participant

- 
-            {livekitRoom.room.localParticipant.identity}
-
-

Remote Participants

-
    - {Array.from(livekitRoom.room.remoteParticipants.keys()).map( - (id) => ( -
  • {id}
    • - ), - )} -
-
+ {transports?.map(({ transport, local }) => ( + ))} = ({ ); }; -/** - * The text shown in the custom transport field for a given setting value. - */ -function transportText(transport: UnstableLivekitTransport | null): string { - return transport === null ? "" : JSON.stringify(transport); -} +const TransportBox: FC = ({ transport, local }) => { + const { t } = useTranslation(); + const resolved = useBehavior(transport.resolved$); + return ( +
+

{t("developer_mode.livekit_sfu", { url: transport.id })}

+

+ {t("developer_mode.livekit_server_info")}({local ? "local" : "remote"}) +

+ 
+        {resolved === undefined
+          ? "not connected"
+          : JSON.stringify(
+              { ...resolved, token: resolved.token && "" },
+              null,
+              2,
+            )}
+
+
+ ); +}; diff --git a/src/settings/SettingsModal.tsx b/src/settings/SettingsModal.tsx index 733068acc..7c1e79271 100644 --- a/src/settings/SettingsModal.tsx +++ b/src/settings/SettingsModal.tsx @@ -9,7 +9,6 @@ import { type FC, type ReactNode, useEffect, useState } from "react"; import { useTranslation } from "react-i18next"; import { type MatrixClient } from "matrix-js-sdk"; import { Button, Root as Form, Separator } from "@vector-im/compound-web"; -import { type Room as LivekitRoom } from "livekit-client"; import { Modal } from "../Modal"; import styles from "./SettingsModal.module.css"; @@ -28,10 +27,11 @@ import { import { PreferencesSettingsTab } from "./PreferencesSettingsTab"; import { Slider } from "../Slider"; import { DeviceSelection } from "./DeviceSelection"; -import { useTrackProcessor } from "../livekit/TrackProcessorContext"; +import { useTrackProcessor } from "../tracks/TrackProcessorContext"; import { DeveloperSettingsTab, type DeveloperSettingsSnapshot, + type TransportInfo, } from "./DeveloperSettingsTab"; import { FieldRow, InputField } from "../input/Input"; import { useSubmitRageshake } from "./submit-rageshake"; @@ -56,11 +56,8 @@ interface Props { onTabChange: (tab: SettingsTab) => void; client: MatrixClient; roomId?: string; - livekitRooms?: { - room: LivekitRoom; - url: string; - role: "publisher" | "subscriber"; - }[]; + /** The transports the client is connected to, while in a call. */ + transports?: TransportInfo[]; /** Only available while in a call. Used by the developer tab. */ developerSettingsVm?: ViewModel; } @@ -74,7 +71,7 @@ export const SettingsModal: FC = ({ onTabChange, client, roomId, - livekitRooms, + transports, developerSettingsVm, }) => { const { t } = useTranslation(); @@ -227,7 +224,7 @@ export const SettingsModal: FC = ({ diff --git a/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap b/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap index 54d1ea478..a2e8671c1 100644 --- a/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap +++ b/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap @@ -361,19 +361,8 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = ` class="_livekit_room_box_2ddec4" >

- LiveKit SFU: wss://local-sfu.example.org + LiveKit SFU: https://local-sfu.example.org/jwt

-

- LivekitAlias: - TestAlias -

-

- connectionState (wont hot reload): -

-

- ws-url: - wss://local-sfu.example.org/ -

LiveKit Server Info ( @@ -384,37 +373,20 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = ` class="_pre_2ddec4" > { - "region": "local", - "version": "1.2.3" + "type": "livekit", + "token": "<redacted>", + "url": "wss://local-sfu.example.org", + "roomAlias": "TestAlias", + "identity": "localParticipantIdentity" } - local-metadata -

- Local Participant -

- 
-      localParticipantIdentity
-
-

- Remote Participants -

-

- LiveKit SFU: wss://remote-sfu.example.org + LiveKit SFU: https://remote-sfu.example.org/jwt

-

- LivekitAlias: - TestAlias2 -

-

- connectionState (wont hot reload): -

LiveKit Server Info ( @@ -424,24 +396,8 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = ` 

-      {
-  "region": "remote",
-  "version": "4.5.6"
-}
-      remote-metadata
+      not connected
-

- Local Participant -

- 
-      localParticipantIdentity
-
-

- Remote Participants -

-
= Omit< - BehaviorSubject, - "next" | "observers" | "error" ->; - -/** - * Creates a Behavior which never changes in value. - */ -export function constant(value: T): Behavior { - return new BehaviorSubject(value); -} +export { type Behavior, constant } from "@element-hq/matrixrtc-sdk"; diff --git a/src/state/CallViewModel/CallNotificationLifecycle.test.ts b/src/state/CallViewModel/CallNotificationLifecycle.test.ts index e5281cf65..a624f5fef 100644 --- a/src/state/CallViewModel/CallNotificationLifecycle.test.ts +++ b/src/state/CallViewModel/CallNotificationLifecycle.test.ts @@ -27,7 +27,6 @@ import { createCallNotificationLifecycle$, type RingAttempt, } from "./CallNotificationLifecycle"; -import { Epoch, trackEpoch } from "../ObservableScope"; import { constant } from "../Behavior"; function mockRingEvent( @@ -44,8 +43,8 @@ function mockRingEvent( } const defaultProps = { - memberships$: constant(new Epoch([])), - matrixRoomMembers$: constant(new Map([[alice.userId, alice]])), + memberUserIds$: constant([]), + roomMemberUserIds$: constant([alice.userId]), receivedDecline$: NEVER, options: { waitForCallPickup: true, @@ -122,7 +121,7 @@ test("ring attempt times out after nobody joins", () => { scope, ...defaultProps, // No one ever joins (only local user) - memberships$: constant(new Epoch([])), + memberUserIds$: constant([]), sentCallNotification$: hot("-a", { a: mockRingEvent("$notif1", 30), }), @@ -140,8 +139,8 @@ test("ring attempt is accepted once recipient joins", () => { const { ringAttempts$ } = createCallNotificationLifecycle$({ scope, ...defaultProps, - memberships$: scope.behavior( - behavior("a-b", { a: [], b: [aliceRtcMember] }).pipe(trackEpoch()), + memberUserIds$: scope.behavior( + behavior("a-b", { a: [], b: [aliceRtcMember.userId] }), ), sentCallNotification$: hot("-a", { a: mockRingEvent("$notif1", 30), @@ -160,7 +159,7 @@ test("ring attempt is immediately accepted if recipient is already joined", () = const { ringAttempts$ } = createCallNotificationLifecycle$({ scope, ...defaultProps, - memberships$: constant(new Epoch([aliceRtcMember])), + memberUserIds$: constant([aliceRtcMember.userId]), sentCallNotification$: hot("-a", { a: mockRingEvent("$notif1", 30), }), diff --git a/src/state/CallViewModel/CallNotificationLifecycle.ts b/src/state/CallViewModel/CallNotificationLifecycle.ts index 59930fd64..ec165996e 100644 --- a/src/state/CallViewModel/CallNotificationLifecycle.ts +++ b/src/state/CallViewModel/CallNotificationLifecycle.ts @@ -6,7 +6,6 @@ Please see LICENSE in the repository root for full details. */ import { - type CallMembership, type IRTCNotificationContent, type MatrixRTCSession, MatrixRTCSessionEvent, @@ -35,9 +34,7 @@ import { } from "matrix-js-sdk"; import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; -import { type Behavior } from "../Behavior"; -import { type Epoch, type ObservableScope } from "../ObservableScope"; -import { type RoomMemberMap } from "./remoteMembers/MatrixMemberMetadata"; +import { type Behavior, type ObservableScope } from "@element-hq/matrixrtc-sdk"; export type AutoLeaveReason = "allOthersLeft" | "timeout" | "decline"; @@ -81,8 +78,10 @@ export function createReceivedDecline$( export interface Props { scope: ObservableScope; - memberships$: Behavior>; - matrixRoomMembers$: Behavior; + /** The user ids of the session's members, ours included. */ + memberUserIds$: Behavior; + /** The user ids of the room's members, ours included. */ + roomMemberUserIds$: Behavior; sentCallNotification$: Observable; receivedDecline$: Observable< Parameters @@ -93,8 +92,8 @@ export interface Props { export function createCallNotificationLifecycle$({ scope, - memberships$, - matrixRoomMembers$, + memberUserIds$, + roomMemberUserIds$, sentCallNotification$, receivedDecline$, options, @@ -127,7 +126,7 @@ export function createCallNotificationLifecycle$({ switchMap((notificationEvent) => { // We assume that there is only one other user in the room when ringing // TODO: Respect io.element.functional_members - const recipient = [...matrixRoomMembers$.value.keys()].find( + const recipient = roomMemberUserIds$.value.find( (userId) => userId !== localUser.userId, ); if (recipient === undefined) { @@ -140,8 +139,8 @@ export function createCallNotificationLifecycle$({ map(() => "timeout" as const), ); // Call is accepted when the recipient joins - const accept$ = memberships$.pipe( - filter((ms) => ms.value.some((m) => m.userId === recipient)), + const accept$ = memberUserIds$.pipe( + filter((userIds) => userIds.includes(recipient)), map(() => "accept" as const), ); // Call is declined when we receive a decline event @@ -171,12 +170,12 @@ export function createCallNotificationLifecycle$({ scope.share, ); - const allOthersLeft$ = memberships$.pipe( + const allOthersLeft$ = memberUserIds$.pipe( pairwise(), filter( - ([{ value: prev }, { value: current }]) => - current.every((m) => m.userId === localUser.userId) && - prev.some((m) => m.userId !== localUser.userId), + ([prev, current]) => + current.every((userId) => userId === localUser.userId) && + prev.some((userId) => userId !== localUser.userId), ), map(() => {}), ); diff --git a/src/state/CallViewModel/CallViewModel.test.ts b/src/state/CallViewModel/CallViewModel.test.ts index 1b1c38d4a..6f659ac31 100644 --- a/src/state/CallViewModel/CallViewModel.test.ts +++ b/src/state/CallViewModel/CallViewModel.test.ts @@ -8,30 +8,18 @@ Please see LICENSE in the repository root for full details. import { test, vi, onTestFinished, it, describe, expect } from "vitest"; import { - BehaviorSubject, combineLatest, debounceTime, distinctUntilChanged, map, NEVER, type Observable, - of, switchMap, } from "rxjs"; -import { SyncState } from "matrix-js-sdk"; import { - ConnectionState, - type LocalTrackPublication, - type Participant, - type RemoteParticipant, -} from "livekit-client"; -import * as ComponentsCore from "@livekit/components-core"; -import { - Status, type CallMembership, type IRTCNotificationContent, MatrixRTCSessionEvent, - type UnstableLivekitTransport, } from "matrix-js-sdk/lib/matrixrtc"; import { deepCompare } from "matrix-js-sdk/lib/utils"; @@ -42,7 +30,6 @@ import { withTestScheduler, mockRtcMembership, testScope, - exampleTransport, } from "../../utils/test.ts"; import { E2eeType } from "../../e2ee/e2eeType.ts"; import { @@ -64,6 +51,7 @@ import { getValue } from "../../utils/observable.ts"; import { type Behavior, constant } from "../Behavior.ts"; import { localParticipant, + type Participant, withCallViewModel as withCallViewModelInMode, } from "./CallViewModelTestUtils.ts"; import { MatrixRTCMode } from "../../config/ConfigOptions.ts"; @@ -80,11 +68,6 @@ vi.mock("rxjs", async (importOriginal) => ({ interval: (): Observable => NEVER, })); -vi.mock("@livekit/components-core"); -vi.mock("livekit-client/e2ee-worker?worker"); - -vi.mock("../e2ee/matrixKeyProvider"); - const getPlatform = vi.hoisted(() => vi.fn(() => "desktop")); vi.mock("../../Platform", () => ({ get platform(): string { @@ -93,15 +76,6 @@ vi.mock("../../Platform", () => ({ isFirefox: (): boolean => false, })); -vi.mock( - "../state/CallViewModel/localMember/localTransport", - async (importOriginal) => ({ - ...(await importOriginal()), - makeTransport: async (): Promise => - Promise.resolve(exampleTransport), - }), -); - const yesNo = { y: true, n: false, @@ -278,10 +252,7 @@ describe.each(modes)("CallViewModel (%s mode)", (mode) => { b: [], }), rtcMembers$: constant([localRtcMember, aliceRtcMember, bobRtcMember]), - livekitConnectionState$: behavior(connectionInputMarbles, { - c: ConnectionState.Connected, - s: ConnectionState.Connecting, - }), + connected$: behavior(connectionInputMarbles, { c: true, s: false }), }, (vm) => { expectObservable(summarizeLayout$(vm.layout$)).toBe( @@ -1388,9 +1359,9 @@ describe.each(modes)("CallViewModel (%s mode)", (mode) => { function participantJoinLeave$( behavior: ( marbles: string, - values: Record, - ) => Behavior, - ): Behavior { + values: Record, + ) => Behavior, + ): Behavior { return behavior("a-b-c-d", { a: [], // Start empty b: [aliceParticipant], // Alice joins @@ -1592,10 +1563,6 @@ describe.each(modes)("CallViewModel (%s mode)", (mode) => { it.skip("audio output changes when toggling earpiece mode", () => { withTestScheduler(({ schedule, expectObservable }) => { - vi.mocked(ComponentsCore.createMediaDeviceObserver).mockReturnValue( - of([]), - ); - const devices = new MediaDevices(testScope(), { controlledAudioDevices: true, }); @@ -1627,78 +1594,6 @@ describe.each(modes)("CallViewModel (%s mode)", (mode) => { }); }); }); - - it.skip("media tracks are paused while reconnecting to MatrixRTC", () => { - withTestScheduler(({ schedule, expectObservable }) => { - const trackRunning$ = new BehaviorSubject(true); - const originalPublications = localParticipant.trackPublications; - localParticipant.trackPublications = new Map([ - [ - "video", - { - track: new (class { - public get isUpstreamPaused(): boolean { - return !trackRunning$.value; - } - public async pauseUpstream(): Promise { - trackRunning$.next(false); - return Promise.resolve(); - } - public async resumeUpstream(): Promise { - trackRunning$.next(true); - return Promise.resolve(); - } - })(), - } as unknown as LocalTrackPublication, - ], - ]); - onTestFinished(() => { - localParticipant.trackPublications = originalPublications; - }); - - // There are three indicators that the client might be disconnected from - // MatrixRTC: whether the sync loop is connected, whether the membership is - // present in local room state, and whether the membership manager thinks - // we've hit the timeout for the delayed leave event. Let's test all - // combinations of these conditions. - const syncingMarbles = " nyny----n--y"; - const membershipStatusMarbles = " y---ny-n-yn-y"; - const probablyLeftMarbles = " n-----y-ny---n"; - const expectedReconnectingMarbles = "n-ynyny------n"; - const expectedTrackRunningMarbles = "nynynyn------y"; - - withCallViewModel( - { initialSyncState: SyncState.Reconnecting }, - (vm, rtcSession, _subjects, setSyncState) => { - schedule(syncingMarbles, { - y: () => setSyncState(SyncState.Syncing), - n: () => setSyncState(SyncState.Reconnecting), - }); - schedule(membershipStatusMarbles, { - y: () => { - rtcSession.membershipStatus = Status.Connected; - }, - }); - schedule(probablyLeftMarbles, { - y: () => { - rtcSession.probablyLeft = true; - }, - n: () => { - rtcSession.probablyLeft = false; - }, - }); - expectObservable(vm.reconnecting$).toBe( - expectedReconnectingMarbles, - yesNo, - ); - expectObservable(trackRunning$).toBe( - expectedTrackRunningMarbles, - yesNo, - ); - }, - ); - }); - }); }); describe("callViewModelOptionsFromParams", () => { @@ -1709,33 +1604,8 @@ describe("callViewModelOptionsFromParams", () => { const widgetUrl = (extra: string): string => `#?widgetId=id&parentUrl=${encodeURIComponent("http://parent")}&${extra}`; - it("carries an explicitly requested notification type", () => { - const params = computeUrlParams("", widgetUrl("sendNotificationType=ring")); - expect(callViewModelOptionsFromParams(params).sendNotificationType).toBe( - "ring", - ); - }); - - it("carries the notification type an intent implies", () => { - const params = computeUrlParams("", widgetUrl("intent=start_call_dm")); - expect(callViewModelOptionsFromParams(params).sendNotificationType).toBe( - "ring", - ); - }); - it("carries hideScreensharing", () => { const params = computeUrlParams("", widgetUrl("hideScreensharing=true")); expect(callViewModelOptionsFromParams(params).hideScreensharing).toBe(true); }); - - it("carries controlledAudioDevices and the call intent", () => { - const params = computeUrlParams( - "", - widgetUrl("controlledAudioDevices=true&intent=start_call_voice"), - ); - expect(callViewModelOptionsFromParams(params)).toMatchObject({ - controlledAudioDevices: true, - callIntent: "audio", - }); - }); }); diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 169abb98c..53de80c08 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -7,14 +7,25 @@ Please see LICENSE in the repository root for full details. */ import { - type BaseKeyProvider, - type ConnectionState, - ExternalE2EEKeyProvider, - type Room as LivekitRoom, - type RoomOptions, - type RemoteParticipant, -} from "livekit-client"; -import { type Room as MatrixRoom } from "matrix-js-sdk"; + type Behavior, + constant, + type DisconnectReason, + type EncryptionSystem, + generateItems, + type LocalRTCMember, + type MatrixRTCClient, + type ObservableScope, + pauseWhen, + type RemoteRTCMember, + type RTCMember, + type TransportMetadata, +} from "@element-hq/matrixrtc-sdk"; +import { + KnownMembership, + type Room as MatrixRoom, + type RoomMember, + RoomStateEvent, +} from "matrix-js-sdk"; import { BehaviorSubject, combineLatest, @@ -38,40 +49,15 @@ import { throttleTime, timer, takeUntil, - from, - concatWith, - catchError, - switchAll, } from "rxjs"; -import { type Logger, logger as rootLogger } from "matrix-js-sdk/lib/logger"; -import { - MembershipManagerEvent, - type UnstableLivekitTransport, - type MatrixRTCSession, - type RTCCallIntent, - type RTCNotificationType, -} from "matrix-js-sdk/lib/matrixrtc"; -import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; -import { v4 as uuidv4 } from "uuid"; -import { type IMembershipManager } from "matrix-js-sdk/lib/matrixrtc/IMembershipManager"; +import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; -import { - createToggle$, - filterBehavior, - generateItems, - mapScoped, - pauseWhen, -} from "../../utils/observable"; +import { createToggle$ } from "../../utils/observable"; import { duplicateTiles, playReactionsSound, showReactions, } from "../../settings/settings"; -import { Config } from "../../config/Config"; -import { - MatrixRTCMode, - type ResolvedDelayedLeaveTimings, -} from "../../config/ConfigOptions"; import { isFirefox, platform } from "../../Platform"; import { setPipEnabled$ } from "../../controls"; import { TileStore } from "../TileStore"; @@ -80,7 +66,6 @@ import { spotlightExpandedLayout } from "../SpotlightExpandedLayout"; import { oneOnOneDesktopLayout } from "../OneOnOneDesktopLayout"; import { oneOnOneMobileLayout } from "../OneOnOneMobileLayout"; import { pipLayout } from "../PipLayout"; -import { type EncryptionSystem } from "../../e2ee/sharedKeyManagement"; import { type RaisedHandInfo, type ReactionInfo, @@ -89,12 +74,8 @@ import { import { shallowEquals as shallowArrayEquals } from "../../utils/array"; import { shallowEquals as shallowObjectEquals } from "../../utils/object"; import { type MediaDevices } from "../MediaDevices"; -import { constant, type Behavior } from "../Behavior"; -import { E2eeType } from "../../e2ee/e2eeType"; -import { MatrixKeyProvider } from "../../e2ee/matrixKeyProvider"; import { type MuteStates } from "../MuteStates"; import { HeaderStyle, type UrlParams } from "../../UrlParams"; -import { type ProcessorState } from "../../livekit/TrackProcessorContext"; import { type HostBridge, nullHostBridge } from "../../HostBridge"; import { type Alignment, @@ -108,45 +89,18 @@ import { computeSpotlight, type OneOnOneMedia, } from "../layoutMedia.ts"; -import { ElementCallError, UnknownCallError } from "../../utils/errors.ts"; -import { Epoch, type ObservableScope } from "../ObservableScope.ts"; -import { createHomeserverConnected$ } from "./localMember/HomeserverConnected.ts"; import { - createLocalMembership$, - enterRTCSession, - TransportState, -} from "./localMember/LocalMember.ts"; -import { getLocalTransport } from "./localMember/LocalTransport.ts"; -import { - createKeyRotationSuppressed$, - createMemberships$, - membershipsAndTransports$, -} from "../SessionBehaviors.ts"; -import { - type ConnectionFactory, - ECConnectionFactory, -} from "./remoteMembers/ConnectionFactory.ts"; -import { - ConnectionMap, - createConnectionMap$, -} from "./remoteMembers/ConnectionMap.ts"; -import { - createRemoteMatrixLivekitMembers$, - type LocalMatrixLivekitMember, - type RemoteMatrixLivekitMember, -} from "./remoteMembers/MatrixLivekitMembers.ts"; + type ElementCallError, + fromMatrixRTCError, + UnknownCallError, +} from "../../utils/errors.ts"; +import { PosthogAnalytics } from "../../analytics/PosthogAnalytics.ts"; import { type AutoLeaveReason, + type CallNotificationWrapper, createCallNotificationLifecycle$, createReceivedDecline$, - createSentCallNotification$, } from "./CallNotificationLifecycle.ts"; -import { - createMatrixMemberMetadata$, - createRoomMembers$, -} from "./remoteMembers/MatrixMemberMetadata.ts"; -import { Publisher } from "./localMember/Publisher.ts"; -import { type Connection } from "./remoteMembers/Connection.ts"; import { type LayoutSwitchViewModel, createLayoutSwitchViewModel, @@ -167,13 +121,6 @@ import { import { type GridTileViewModel } from "../TileViewModel.ts"; import { type TransportLocator } from "../../livekit/auth"; -//TODO -// Larger rename -// member,membership -> rtcMember -// participant -> livekitParticipant -// matrixLivekitItem -> callMember -// js-sdk -// callMembership -> rtcMembership export interface CallViewModelOptions { encryptionSystem: EncryptionSystem; /** @@ -181,33 +128,23 @@ export interface CallViewModelOptions { * to know when the user joins or leaves. Defaults to no host. */ hostBridge?: HostBridge; - /** - * Whether the app hosting Element Call controls the audio output devices, - * rather than the browser. Defaults to false. - */ - controlledAudioDevices?: boolean; /** The style of header to show. Defaults to {@link HeaderStyle.Standard}. */ header?: HeaderStyle; /** Whether the call controls should be shown. Defaults to true. */ showControls?: boolean; /** Whether to hide the screen-sharing button. Defaults to false. */ hideScreensharing?: boolean; - /** - * Whether and what kind of notification to send when joining the call. - */ - sendNotificationType?: RTCNotificationType; - /** The kind of call being placed. */ - callIntent?: RTCCallIntent; autoLeaveWhenOthersLeft?: boolean; /** * If the call is started in a way where we want it to behave like a telephone usecase * If we sent a notification event, we want the ui to show a ringing state */ waitForCallPickup?: boolean; - /** Optional factory to create LiveKit rooms, mainly for testing purposes. */ - livekitRoomFactory?: (options?: RoomOptions) => LivekitRoom; - /** Optional behavior overriding the local connection state, mainly for testing purposes. */ - connectionState$?: Behavior; + /** + * The call notification the client sent with its join, for the ringing UI. + * Defaults to none sent. + */ + sentCallNotification$?: Observable; /** * The size of the space the call is drawn in: the page when Element Call * owns it, or the container a host mounted it in when it is a component. @@ -217,12 +154,6 @@ export interface CallViewModelOptions { * changed. */ windowSize$: Behavior<{ width: number; height: number }>; - /** Optional value overriding the local transport, for testing purposes. */ - localTransport?: TransportLocator; - /** Optional value overriding the connection factory, for testing purposes. */ - connectionFactory?: ConnectionFactory; - /** The version & compatibility mode of MatrixRTC that we should use. */ - matrixRTCMode?: MatrixRTCMode; /** Optional behavior overriding for the screensharing, for testing */ toggleScreensharing?: () => void; } @@ -242,22 +173,11 @@ export interface CallViewModelOptions { */ export function callViewModelOptionsFromParams( params: UrlParams, -): Pick< - CallViewModelOptions, - | "controlledAudioDevices" - | "header" - | "showControls" - | "hideScreensharing" - | "sendNotificationType" - | "callIntent" -> { +): Pick { return { - controlledAudioDevices: params.controlledAudioDevices, header: params.header, showControls: params.showControls, hideScreensharing: params.hideScreensharing, - sendNotificationType: params.sendNotificationType, - callIntent: params.callIntent, }; } @@ -276,12 +196,6 @@ interface LayoutScanState { tiles: TileStore; } -export type LivekitRoomItem = { - livekitRoom: LivekitRoom; - participants: string[]; - transport: TransportLocator; -}; - /** * The return of createCallViewModel$ * this interface represents the root source of data for the call view. @@ -376,12 +290,11 @@ export interface CallViewModel { * key is generated when someone joins or leaves. */ keyRotationSuppressed$: Behavior; - allConnections$: Behavior; - /** Participants sorted by livekit room so they can be used in the audio rendering */ - livekitRoomItems$: Behavior; - /** use the layout instead, this is just for the sdk export. */ - remoteMatrixLivekitMembers$: Behavior; - localMatrixLivekitMember$: Behavior; + /** The local member, once our membership has been seen in the room. */ + localMember$: Behavior; + remoteMembers$: Behavior; + /** The transports the client holds a connection to, for the developer panel. */ + connectedTransports$: Behavior; /** List of participants raising their hand */ handsRaised$: Behavior>; /** List of reactions. Keys are: membership.membershipId (currently predefined as: `${membershipEvent.userId}:${membershipEvent.deviceId}`)*/ @@ -464,14 +377,10 @@ export interface CallViewModel { switch: () => void; } | null>; - /** - * Whether the app is currently reconnecting to the LiveKit server and/or setting the matrix rtc room state. - */ + /** Connected once, and currently not: to the homeserver, the session or the media transport. */ reconnecting$: Behavior; - /** - * Shortcut for not requireing to parse and combine connectionState.matrix and connectionState.livekit - */ + /** Connected to the homeserver, the session and the media transport, all three. */ connected$: Behavior; } @@ -479,21 +388,20 @@ export interface CallViewModel { * A view model providing all the application logic needed to show the in-call * UI (may eventually be expanded to cover the lobby and feedback screens in the * future). + * + * The session itself — memberships, transports, media — is the client's; this + * adds everything that makes it a call. */ -// Throughout this class and related code we must distinguish between MatrixRTC -// state and LiveKit state. We use the common terminology of room "members", RTC -// "memberships", and LiveKit "participants". export function createCallViewModel$( scope: ObservableScope, + rtcClient: MatrixRTCClient, // A call is permanently tied to a single Matrix room - matrixRTCSession: MatrixRTCSession, matrixRoom: MatrixRoom, mediaDevices: MediaDevices, muteStates: MuteStates, options: CallViewModelOptions, handsRaisedSubject$: Observable>, reactionsSubject$: Observable>, - trackProcessorState$: Behavior, ): CallViewModel { const logger = rootLogger.getChild("[CallViewModel]"); const client = matrixRoom.client; @@ -506,277 +414,53 @@ export function createCallViewModel$( // so that callers which don't care (chiefly tests) behave as they always have. const { hostBridge = nullHostBridge, - controlledAudioDevices = false, header = HeaderStyle.Standard, showControls = true, hideScreensharing = false, - sendNotificationType, - callIntent, + sentCallNotification$ = of(null), } = options; - const livekitKeyProvider = getE2eeKeyProvider( - options.encryptionSystem, - matrixRTCSession, - logger, - ); - // matrix_rtc_mode in config.json overrides the user's Developer Settings choice. - // It is validated at config load (src/config/Config.ts) so the cast is safe. - const configMatrixRTCMode = Config.get().matrix_rtc_mode as - | MatrixRTCMode - | undefined; - const matrixRTCMode = - configMatrixRTCMode ?? options.matrixRTCMode ?? MatrixRTCMode.Compatibility; - - // Each hbar seperates a block of input variables required for the CallViewModel to function. - // The outputs of this block is written under the hbar. - // - // For mocking purposes it is recommended to only mock the functions creating those outputs. - // All other fields are just temp computations for the mentioned output. - // The class does not need anything except the values underneath the bar. - // The creations of the values under the bar are all tested independently and testing the callViewModel Should - // not test their creation. Call view model only needs: - // - memberships$ via createMemberships$ - // - localMembership via createLocalMembership$ - // - callLifecycle via createCallNotificationLifecycle$ - // - matrixMemberMetadataStore via createMatrixMemberMetadata$ - - // ------------------------------------------------------------------------ - // memberships$ - const memberships$ = createMemberships$(scope, matrixRTCSession); - - // ------------------------------------------------------------------------ - // matrixLivekitMembers$ AND localMembership - - const membershipsAndTransports = membershipsAndTransports$( - scope, - memberships$, - ); - - const ownMembershipIdentity: CallMembershipIdentityParts = { - userId, - deviceId, - // Consumed by the sticky membership manager as `member.id`, *and* stamped - // into every to-device key event by the key transport. A pre-sticky - // membership advertises `${userId}:${deviceId}` as its `membershipID` - // instead, so a uuid there names a member no peer can resolve. - memberId: - matrixRTCMode === MatrixRTCMode.Matrix_2_0 - ? uuidv4() - : `${userId}:${deviceId}`, - }; - - const localTransport$ = options.localTransport - ? constant(options.localTransport) - : from(getLocalTransport(client)); - - const connectionFactory = - options.connectionFactory ?? - new ECConnectionFactory( - client, - matrixRoom.roomId, - matrixRTCSession.slotId, - mediaDevices, - trackProcessorState$, - livekitKeyProvider, - controlledAudioDevices, - options.livekitRoomFactory, - ); - - const connectionMap$ = scope.behavior>( - localTransport$.pipe( - catchError(() => NEVER), // Swallow errors - concatWith(NEVER), // So the Observable doesn't complete prematurely - mapScoped("connectionMap$", (localTransport, scope) => - createConnectionMap$({ - scope, - connectionFactory, - localTransport, - remoteTransports$: membershipsAndTransports.transports$, - logger, - ownMembershipIdentity, - }), - ), - switchAll(), - ), - new Epoch(new ConnectionMap()), - ); - - const remoteMatrixLivekitMembers$: Behavior< - Epoch - > = createRemoteMatrixLivekitMembers$({ - scope: scope, - membershipsWithTransport$: - membershipsAndTransports.membershipsWithTransport$, - connectionMap$, - localUser: { userId, deviceId }, + // The mute switches drive the client, which reports back what the device + // allowed, so a denied permission flips the switch back + muteStates.audio.setHandler(rtcClient.setMicrophoneEnabled); + muteStates.video.setHandler(rtcClient.setCameraEnabled); + scope.onEnd(() => { + muteStates.audio.unsetHandler(); + muteStates.video.unsetHandler(); }); - const localMembership = createLocalMembership$({ - scope, - homeserverConnected: createHomeserverConnected$( - scope, - client, - matrixRTCSession, - ), - muteStates, - joinMatrixRTC: ( - transport: UnstableLivekitTransport, - delayedLeaveTimings: ResolvedDelayedLeaveTimings, - ) => { - return enterRTCSession( - matrixRTCSession, - ownMembershipIdentity, - transport, - { - encryptMedia: livekitKeyProvider !== undefined, - matrixRTCMode, - delayedLeaveTimings, - sendNotificationType, - callIntent, - }, - ); - }, - createPublisherFactory: (connection: Connection) => { - return new Publisher( - connection, - mediaDevices, - muteStates, - trackProcessorState$, - logger.getChild(`[Publisher ${JSON.stringify(connection.transport)}]`), - controlledAudioDevices, - ); - }, - connectionMap$, - client, - matrixRTCSession, - localTransport$, - roomId: matrixRoom.roomId, - hideScreensharing, - hostBridge, - baseUrl: client.baseUrl, - ownMembershipIdentity, - delayId$: scope.behavior( - ( - fromEvent( - matrixRTCSession, - MembershipManagerEvent.DelayIdChanged, - // The type of reemitted event includes the original emitted as the second arg. - ) as Observable<[string | undefined, IMembershipManager]> - ).pipe(map(([delayId]) => delayId ?? null)), - matrixRTCSession.delayId ?? null, - ), - logger: logger.getChild(`[${Date.now()}]`), - }); - - const localRtcMembership$ = scope.behavior( - memberships$.pipe( - map( - (memberships) => - memberships.value.find( - (membership) => - membership.userId === userId && membership.deviceId === deviceId, - ) ?? null, - ), - ), - ); - - const localMatrixLivekitMember$: Behavior = - scope.behavior( - localRtcMembership$.pipe( - filterBehavior((membership) => membership !== null), - map((membership$) => { - if (membership$ === null) return null; - return { - membership$, - participant: { - type: "local" as const, - value$: localMembership.participant$, - }, - connection$: localMembership.connection$, - userId, - }; - }), - ), - ); - - const matrixLivekitMembers$ = scope.behavior( + const members$ = scope.behavior( combineLatest( - [localMatrixLivekitMember$, remoteMatrixLivekitMembers$], - (local, remote) => [...(local === null ? [] : [local]), ...remote.value], + [rtcClient.localMember$, rtcClient.remoteMembers$], + (local, remote) => (local === null ? remote : [local, ...remote]), ), ); - // ------------------------------------------------------------------------ - // matrixMemberMetadataStore - - const matrixRoomMembers$ = createRoomMembers$(scope, matrixRoom); - const matrixMemberMetadataStore = createMatrixMemberMetadata$( - scope, - scope.behavior(memberships$.pipe(map((mems) => mems.value))), - matrixRoomMembers$, + const matrixRoomMembers$ = scope.behavior( + fromEvent(matrixRoom, RoomStateEvent.Members).pipe( + map(() => roomMembers(matrixRoom)), + ), + roomMembers(matrixRoom), ); - // ------------------------------------------------------------------------ - // callLifecycle - // TODO if we are in "unknown" state we need a loading rendering (or empty screen) // Otherwise it looks like we already connected and only than the ringing starts which is weird. const { ringAttempts$, autoLeave$ } = createCallNotificationLifecycle$({ scope, - memberships$, - matrixRoomMembers$, - sentCallNotification$: createSentCallNotification$(scope, matrixRTCSession), + memberUserIds$: scope.behavior( + members$.pipe(map((members) => members.map((m) => m.userId))), + ), + roomMemberUserIds$: scope.behavior( + matrixRoomMembers$.pipe(map((members) => members.map((m) => m.userId))), + ), + sentCallNotification$, receivedDecline$: createReceivedDecline$(matrixRoom), options, localUser: { userId, deviceId }, }); - const allConnections$ = scope.behavior( - connectionMap$.pipe(map((d) => d.value.getConnections())), - ); - - const livekitRoomItems$ = scope.behavior( - remoteMatrixLivekitMembers$.pipe( - // Flatten the members to a snapshot of their current connections and - // participant objects - switchMap(({ value: members }) => { - if (members.length === 0) return of([]); - return combineLatest( - members.map((m) => - combineLatest( - [m.connection$, m.participant.value$], - (connection, participant) => ({ connection, participant }), - ), - ), - ); - }), - // Keep only those with a defined connection and participant - map((members) => - members.filter( - ( - m, - ): m is { connection: Connection; participant: RemoteParticipant } => - m.connection !== null && m.participant !== null, - ), - ), - // Group members by connection - map((members) => - [...Map.groupBy(members, (m) => m.connection)].map( - ([connection, members]) => ({ - transport: { - transport: connection.transport, - serverName: connection.serverName, - }, - livekitRoom: connection.livekitRoom, - participants: members.map((m) => m.participant.identity), - }), - ), - ), - ), - ); - const handsRaised$ = scope.behavior( - handsRaisedSubject$.pipe(pauseWhen(localMembership.reconnecting$)), + handsRaisedSubject$.pipe(pauseWhen(rtcClient.reconnecting$)), ); const reactions$ = scope.behavior( @@ -789,7 +473,7 @@ export function createCallViewModel$( ]), ), ), - pauseWhen(localMembership.reconnecting$), + pauseWhen(rtcClient.reconnecting$), ), ); @@ -797,59 +481,44 @@ export function createCallViewModel$( * List of user media (camera feeds) that we want tiles for. */ const userMedia$ = scope.behavior( - combineLatest([matrixLivekitMembers$, duplicateTiles.value$]).pipe( - // Generate a collection of user media from the list of expected (whether - // present or missing) LiveKit participants. + combineLatest([members$, duplicateTiles.value$]).pipe( + // Generate a collection of user media from the list of members, whether + // their media is present or still missing. generateItems( "CallViewModel userMedia$", function* ([members, duplicateTiles]) { - for (const { - userId, - participant, - connection$, - membership$, - } of members) { - const rtcId = membership$.value.rtcBackendIdentity; - const mediaId = `${userId}:${membership$.value.deviceId}`; + for (const member of members) { + const mediaId = `${member.userId}:${member.deviceId}`; for (let dup = 0; dup < 1 + duplicateTiles; dup++) { - yield { - keys: [dup, mediaId, userId, participant, connection$, rtcId], - data: undefined, - }; + yield { keys: [dup, mediaId, member.id], data: member }; } } }, - (scope, _, dup, mediaId, userId, participant, connection$, rtcId) => - createWrappedUserMedia(scope, { + (scope, member$, dup, mediaId, _memberId) => { + const member = member$.value; + return createWrappedUserMedia(scope, { id: `${mediaId}:${dup}`, - userId, - rtcBackendIdentity: rtcId, - participant, + userId: member.userId, + rtcBackendIdentity: member.id, + ...(member.local + ? { local: true, media$: (member as LocalRTCMember).media$ } + : { local: false, media$: member.media$ }), encryptionSystem: options.encryptionSystem, - livekitRoom$: scope.behavior( - connection$.pipe(map((c) => c?.livekitRoom)), - ), - transport$: scope.behavior( - connection$.pipe(map((c) => c?.transport)), + focusUrl$: scope.behavior( + member.transport$.pipe(map((transport) => transport?.id)), ), mediaDevices, - pretendToBeDisconnected$: localMembership.reconnecting$, - displayName$: scope.behavior( - matrixMemberMetadataStore - .createDisplayNameBehavior$(scope, userId) - .pipe(map((name) => name ?? userId)), - ), - mxcAvatarUrl$: matrixMemberMetadataStore.createAvatarUrlBehavior$( - scope, - userId, - ), + pretendToBeDisconnected$: rtcClient.reconnecting$, + displayName$: member.displayName$, + mxcAvatarUrl$: member.avatarUrl$, handRaised$: scope.behavior( handsRaised$.pipe(map((v) => v[mediaId]?.time ?? null)), ), reaction$: scope.behavior( reactions$.pipe(map((v) => v[mediaId] ?? undefined)), ), - }), + }); + }, ), ), ); @@ -865,26 +534,23 @@ export function createCallViewModel$( if (pickupState !== "accept") yield { keys: [intent, recipient], data: pickupState }; }, - (scope, pickupState$, intent, userId) => - createRingingMedia({ + (scope, pickupState$, intent, userId) => { + const member$ = matrixRoomMembers$.pipe( + map((members) => members.find((m) => m.userId === userId)), + ); + return createRingingMedia({ id: `ringing:${userId}`, userId, displayName$: scope.behavior( - matrixRoomMembers$.pipe( - map( - (members) => - members.get(userId)?.rawDisplayName || userId, - ), - ), + member$.pipe(map((m) => m?.rawDisplayName || userId)), + ), + mxcAvatarUrl$: scope.behavior( + member$.pipe(map((m) => m?.getMxcAvatarUrl())), ), - mxcAvatarUrl$: - matrixMemberMetadataStore.createAvatarUrlBehavior$( - scope, - userId, - ), pickupState$, intent, - }), + }); + }, ), map(([media]) => media ?? null), ), @@ -893,9 +559,6 @@ export function createCallViewModel$( ), ); - /** - * All screen share media that we want to display. - */ const screenShares$ = scope.behavior( userMedia$.pipe( switchMap((userMedia) => @@ -927,14 +590,9 @@ export function createCallViewModel$( * - There can be multiple participants for one Matrix user if they join from * multiple devices. */ - const participantCount$ = scope.behavior( - matrixLivekitMembers$.pipe(map((ms) => ms.length)), - ); + const participantCount$ = rtcClient.memberCount$; - const keyRotationSuppressed$ = createKeyRotationSuppressed$( - scope, - matrixRTCSession, - ); + const keyRotationSuppressed$ = rtcClient.keyRotationSuppressed$; const leaveSoundEffect$ = userMedia$.pipe( pairwise(), @@ -1243,7 +901,7 @@ export function createCallViewModel$( // without a name tag!) // TODO: Only hide name tags in DMs, not group chats that just // happen to have only 2 users - members.size > 2, + members.length > 2, ), ) : of(true), @@ -1630,55 +1288,73 @@ export function createCallViewModel$( /** * Whether we are sharing our screen. */ - // reassigned here to make it publicly accessible - const sharingScreen$ = localMembership.sharingScreen$; + const sharingScreen$ = scope.behavior( + rtcClient.localMember$.pipe( + switchMap((member) => member?.sharingScreen$ ?? of(false)), + ), + ); /** - * Callback to toggle screen sharing. If null, screen sharing is not possible. + * Callback to toggle screen sharing. If null, screen sharing is not possible: + * the platform cannot capture a screen, or the host hides the button. */ - // reassigned here to make it publicly accessible const toggleScreenSharing = - options.toggleScreensharing ?? localMembership.toggleScreenSharing; + options.toggleScreensharing ?? + (hideScreensharing || !("getDisplayMedia" in (navigator.mediaDevices ?? {})) + ? null + : (): void => rtcClient.localMember$.value?.toggleScreenSharing?.()); - const errors$ = scope.behavior<{ - transportError?: ElementCallError; - matrixError?: ElementCallError; - connectionError?: ElementCallError; - publishError?: ElementCallError; - } | null>( - localMembership.localMemberState$.pipe( - map((value) => { - const returnObject: { - transportError?: ElementCallError; - matrixError?: ElementCallError; - connectionError?: ElementCallError; - publishError?: ElementCallError; - } = {}; - if (value instanceof ElementCallError) return { transportError: value }; - if (value === TransportState.Waiting) return null; - if (value.matrix instanceof ElementCallError) - returnObject.matrixError = value.matrix; - if (value.media instanceof ElementCallError) - returnObject.publishError = value.media; - else if ( - typeof value.media === "object" && - value.media.connection.state === "error" - ) - returnObject.connectionError = value.media.connection.error; - return returnObject; - }), + const screenShareError$ = scope.behavior( + rtcClient.localMember$.pipe( + switchMap((member) => member?.screenShareError$ ?? of(null)), ), - null, ); + // Tell the host and the analytics about the user's joins and leaves + const join = (): void => { + PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date()); + PosthogAnalytics.instance.eventCallStarted.track(matrixRoom.roomId); + rtcClient.join(); + hostBridge.notifyJoined().catch((e) => { + logger.error("Failed to notify the host that we joined", e); + }); + }; + const leave = (): void => { + rtcClient.leave(); + hostBridge.notifyHungUp().catch((e) => { + logger.error("Failed to notify the host that we hung up", e); + }); + }; + + let reconnectStart: { time: number; reason: DisconnectReason } | null = null; + rtcClient.disconnectReason$ + .pipe(distinctUntilChanged(), pairwise(), scope.bind()) + .subscribe(([prev, reason]) => { + if (reason !== null) { + // Only the loss of a connection that existed counts as a reconnect, + // not the startup phase + if (prev === null) reconnectStart ??= { time: Date.now(), reason }; + } else if (reconnectStart !== null) { + const reason = + reconnectStart.reason === "media" ? "livekit" : reconnectStart.reason; + PosthogAnalytics.instance.eventCallReconnecting.track( + matrixRoom.roomId, + reason, + (Date.now() - reconnectStart.time) / 1000, + ); + PosthogAnalytics.instance.eventCallEnded.cacheReconnecting(reason); + reconnectStart = null; + } + }); + return { autoLeave$: autoLeave$, ringingVm$: ringingMedia$, ringingStatusLocation: header === HeaderStyle.AppBar ? "app_bar" : "tile", leave$: leave$, hangup: (): void => userHangup$.next(), - join: localMembership.requestJoinAndPublish, - leave: localMembership.requestDisconnect, + join, + leave, toggleScreenSharing: toggleScreenSharing, sharingScreen$: sharingScreen$, @@ -1688,23 +1364,15 @@ export function createCallViewModel$( unhoverScreen: (): void => screenUnhover$.next(), fatalError$: scope.behavior( - errors$.pipe( - map((errors) => { - logger.debug("errors$ to compute any fatal errors:", errors); - return ( - errors?.transportError ?? - errors?.matrixError ?? - errors?.connectionError ?? - null - ); - }), - filter((error) => error !== null), + rtcClient.fatalError$.pipe( + map((error) => error && fromMatrixRTCError(error)), ), - null, ), - allConnections$, participantCount$: participantCount$, keyRotationSuppressed$: keyRotationSuppressed$, + localMember$: rtcClient.localMember$, + remoteMembers$: rtcClient.remoteMembers$, + connectedTransports$: rtcClient.connectedTransports$, handsRaised$: handsRaised$, reactions$: reactions$, joinSoundEffect$: joinSoundEffect$, @@ -1720,23 +1388,6 @@ export function createCallViewModel$( showLayoutSwitch$.pipe(map((show) => (show ? layoutSwitchVm : null))), ), layout$: layout$, - localMatrixLivekitMember$, - remoteMatrixLivekitMembers$: scope.behavior( - remoteMatrixLivekitMembers$.pipe( - map((members) => members.value), - tap((v) => { - const listForLogs = v - .map( - (m) => - m.membership$.value.userId + "|" + m.membership$.value.deviceId, - ) - .join(","); - logger.debug( - `matrixLivekitMembers$ updated (exported) [${listForLogs}]`, - ); - }), - ), - ), tileStoreGeneration$: tileStoreGeneration$, showSpotlightIndicators$: showSpotlightIndicators$, showSpeakingIndicators$: showSpeakingIndicators$, @@ -1750,30 +1401,17 @@ export function createCallViewModel$( overflowing$, earpieceMode$: earpieceMode$, audioOutputSwitcher$: audioOutputSwitcher$, - reconnecting$: localMembership.reconnecting$, - livekitRoomItems$, - connected$: localMembership.connected$, - screenShareError$: localMembership.screenShareError$, - dismissScreenShareError: localMembership.dismissScreenShareError, + reconnecting$: rtcClient.reconnecting$, + connected$: rtcClient.connected$, + screenShareError$, + dismissScreenShareError: (): void => + rtcClient.localMember$.value?.dismissScreenShareError(), }; } -function getE2eeKeyProvider( - e2eeSystem: EncryptionSystem, - rtcSession: MatrixRTCSession, - logger: Logger, -): BaseKeyProvider | undefined { - if (e2eeSystem.kind === E2eeType.NONE) return undefined; - - if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) { - const keyProvider = new MatrixKeyProvider(); - keyProvider.setRTCSession(rtcSession); - return keyProvider; - } else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) { - const keyProvider = new ExternalE2EEKeyProvider(); - keyProvider - .setKey(e2eeSystem.secret) - .catch((e) => logger.error("Failed to set shared key for E2EE", e)); - return keyProvider; - } +/** The members a call can be with: those in the room and those invited. */ +function roomMembers(room: MatrixRoom): RoomMember[] { + return room + .getMembersWithMembership(KnownMembership.Join) + .concat(room.getMembersWithMembership(KnownMembership.Invite)); } diff --git a/src/state/CallViewModel/CallViewModelTestUtils.ts b/src/state/CallViewModel/CallViewModelTestUtils.ts index 93c190ace..8881c27a2 100644 --- a/src/state/CallViewModel/CallViewModelTestUtils.ts +++ b/src/state/CallViewModel/CallViewModelTestUtils.ts @@ -7,39 +7,39 @@ Please see LICENSE in the repository root for full details. */ import { - ConnectionState, - type Participant, - ParticipantEvent, - type RemoteParticipant, - type Room as LivekitRoom, - type TrackPublication, -} from "livekit-client"; + type Behavior, + constant, + E2eeType, + type LocalMemberMedia, + type LocalRTCMember, + type MatrixRTCMode, + type ObservableScope, + type RemoteRTCMember, +} from "@element-hq/matrixrtc-sdk"; import { SyncState } from "matrix-js-sdk/lib/sync"; -import { BehaviorSubject, combineLatest, map, of } from "rxjs"; -import { onTestFinished, vi } from "vitest"; +import { BehaviorSubject, combineLatest, map, of, switchMap } from "rxjs"; import { ClientEvent, type RoomMember, type MatrixClient } from "matrix-js-sdk"; import EventEmitter from "events"; -import * as ComponentsCore from "@livekit/components-core"; import type { CallMembership } from "matrix-js-sdk/lib/matrixrtc"; -import { E2eeType } from "../../e2ee/e2eeType"; import { type RaisedHandInfo, type ReactionInfo } from "../../reactions"; import { type CallViewModel, createCallViewModel$, type CallViewModelOptions, } from "./CallViewModel"; +import { createSentCallNotification$ } from "./CallNotificationLifecycle"; import { - exampleTransportLocator, mockConfig, - MockConnection, - mockLivekitRoom, - mockLocalParticipant, mockMatrixRoom, mockMatrixRoomMember, + mockMatrixRTCClient, mockMediaDevices, + mockVideoTrack, + mockMemberMedia, mockMuteStates, MockRTCSession, + mockRTCMember, testScope, } from "../../utils/test"; import { @@ -52,10 +52,7 @@ import { local, localRtcMember, } from "../../utils/test-fixtures"; -import { type Behavior, constant } from "../Behavior"; -import { type ProcessorState } from "../../livekit/TrackProcessorContext"; import { type MediaDevices } from "../MediaDevices"; -import { type MatrixRTCMode } from "../../config/ConfigOptions"; mockConfig({ livekit: { livekit_service_url: "http://my-default-service-url.com" }, @@ -65,11 +62,18 @@ const carol = local; const dave = mockMatrixRoomMember(daveRTLRtcMember, { rawDisplayName: "Dave" }); +/** Whoever publishes media: a stand-in for a LiveKit participant, by identity. */ +export interface Participant { + identity: string; +} + export interface CallViewModelInputs { - remoteParticipants$: Behavior; + /** The members whose media has arrived. The local member's always has. */ + remoteParticipants$: Behavior; rtcMembers$: Behavior[]>; roomMembers: RoomMember[]; - livekitConnectionState$: Behavior; + /** Whether the local transport is connected. */ + connected$: Behavior; speaking: Map>; videoEnabled: Map>; sharingScreen: Map>; @@ -78,7 +82,7 @@ export interface CallViewModelInputs { windowSize$: Behavior<{ width: number; height: number }>; } -export const localParticipant = mockLocalParticipant({ identity: "" }); +export const localParticipant: Participant = { identity: "" }; export function withCallViewModel(mode: MatrixRTCMode) { return ( @@ -94,9 +98,7 @@ export function withCallViewModel(mode: MatrixRTCMode) { dave, daveRTL, ], - livekitConnectionState$: connectionState$ = constant( - ConnectionState.Connected, - ), + connected$ = constant(true), speaking = new Map(), videoEnabled = new Map(), sharingScreen = new Map(), @@ -146,113 +148,131 @@ export function withCallViewModel(mode: MatrixRTCMode) { const rtcSession = new MockRTCSession(room, []).withMemberships( rtcMembers$, ); - const participantsSpy = vi - .spyOn(ComponentsCore, "connectedParticipantsObserver") - .mockReturnValue(remoteParticipants$); - const mediaSpy = vi - .spyOn(ComponentsCore, "observeParticipantMedia") - .mockImplementation((p) => { - return (videoEnabled.get(p) ?? constant(false)).pipe( - map((videoEnabled) => ({ - participant: p, - isMicrophoneEnabled: false, - isCameraEnabled: videoEnabled, - isScreenShareEnabled: false, - cameraTrack: { - isMuted: !videoEnabled, - } as unknown as TrackPublication, - })), - ); - }); - const eventsSpy = vi - .spyOn(ComponentsCore, "observeParticipantEvents") - .mockImplementation((p, ...eventTypes) => { - return combineLatest([ - (eventTypes.includes(ParticipantEvent.IsSpeakingChanged) && - speaking.get(p)) || - constant(false), - (eventTypes.includes(ParticipantEvent.TrackPublished) && - sharingScreen.get(p)) || - constant(false), - ]).pipe( - map( - ([isSpeaking, isScreenShareEnabled]) => - ({ ...p, isSpeaking, isScreenShareEnabled }) as Participant, - ), - ); - }); + const scope = testScope(); - const roomEventSelectorSpy = vi - .spyOn(ComponentsCore, "roomEventSelector") - .mockImplementation((_room, _eventType) => of()); + const mediaOf = ( + scope: ObservableScope, + participant: Participant, + local: boolean, + ): LocalMemberMedia => + mockMemberMedia({ + local, + speaking$: speaking.get(participant) ?? constant(false), + screenShareEnabled$: sharingScreen.get(participant) ?? constant(false), + camera$: scope.behavior( + (videoEnabled.get(participant) ?? constant(false)).pipe( + map((enabled) => + mockVideoTrack({ source: "camera", muted$: constant(!enabled) }), + ), + ), + ), + screenShare$: scope.behavior( + (sharingScreen.get(participant) ?? constant(false)).pipe( + map((sharing) => + sharing ? mockVideoTrack({ source: "screenShare" }) : undefined, + ), + ), + ), + }); + const participantOf = ( + scope: ObservableScope, + membership: CallMembership, + ): Behavior => + scope.behavior( + remoteParticipants$.pipe( + map((participants) => + participants.find( + (p) => p.identity === membership.rtcBackendIdentity, + ), + ), + ), + ); + const memberOf = ( + scope: ObservableScope, + membership: CallMembership, + ): RemoteRTCMember => + mockRTCMember(false, { + membership, + roomMember: roomMembers.find((m) => m.userId === membership.userId), + transportUrl: "http://my-default-service-url.com", + media$: scope.behavior( + participantOf(scope, membership).pipe( + map((participant) => + participant === undefined + ? null + : mediaOf(scope, participant, false), + ), + ), + ), + }); + const remoteMembers$ = scope.behavior( + rtcMembers$.pipe( + map((memberships) => + (memberships as CallMembership[]) + .filter( + (m) => + m.userId !== localRtcMember.userId || + m.deviceId !== localRtcMember.deviceId, + ) + .map((membership) => memberOf(scope, membership)), + ), + ), + ); + const localMember$ = scope.behavior( + rtcMembers$.pipe( + map((memberships) => + (memberships as CallMembership[]).some( + (m) => + m.userId === localRtcMember.userId && + m.deviceId === localRtcMember.deviceId, + ) + ? mockRTCMember(true, { + membership: localRtcMember, + roomMember: carol, + transportUrl: "http://my-default-service-url.com", + media$: constant(mediaOf(scope, localParticipant, true)), + }) + : null, + ), + ), + ); + const rtcClient = mockMatrixRTCClient(scope, { + localMember$, + remoteMembers$, + connected$, + reconnecting$: scope.behavior( + combineLatest([connected$, of(initialSyncState)]).pipe( + switchMap(() => connected$), + map((connected) => !connected), + ), + ), + }); const muteStates = mockMuteStates(); const raisedHands$ = new BehaviorSubject>( {}, ); const reactions$ = new BehaviorSubject>({}); - const livekitRoomFactory = (): LivekitRoom => - mockLivekitRoom({ - localParticipant, - disconnect: async () => Promise.resolve(), - setE2EEEnabled: async () => Promise.resolve(), - }); - const vm = createCallViewModel$( - testScope(), - rtcSession.asMockedSession(), + scope, + rtcClient, room, mediaDevices, muteStates, { encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, autoLeaveWhenOthersLeft: false, - livekitRoomFactory, - connectionState$, + sentCallNotification$: createSentCallNotification$( + scope, + rtcSession.asMockedSession(), + ), windowSize$, - localTransport: exampleTransportLocator, - connectionFactory: { - createConnection( - scope, - role, - transport, - serverName, - ownMembershipIdentity, - logger, - ) { - return new MockConnection( - { - scope, - role, - transport, - serverName, - ownMembershipIdentity, - client: room.client, - roomId: room.roomId, - slotId: "m.call#ROOM", - livekitRoomFactory, - }, - logger, - ); - }, - }, - matrixRTCMode: mode, ...options, }, raisedHands$, reactions$, - new BehaviorSubject({ - processor: undefined, - supported: undefined, - }), ); - - onTestFinished(() => { - participantsSpy.mockRestore(); - mediaSpy.mockRestore(); - eventsSpy.mockRestore(); - roomEventSelectorSpy.mockRestore(); - }); + void mode; continuation(vm, rtcSession, { raisedHands$: raisedHands$ }, setSyncState); }; diff --git a/src/state/CallViewModel/localMember/HomeserverConnected.test.ts b/src/state/CallViewModel/localMember/HomeserverConnected.test.ts deleted file mode 100644 index 4b6bde984..000000000 --- a/src/state/CallViewModel/localMember/HomeserverConnected.test.ts +++ /dev/null @@ -1,336 +0,0 @@ -/* -Copyright 2025 Element Creations Ltd. -Copyright 2024 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { describe, it, expect, beforeEach, afterEach } from "vitest"; -import { EventEmitter } from "events"; -import { ClientEvent, SyncState } from "matrix-js-sdk"; -import { MembershipManagerEvent, Status } from "matrix-js-sdk/lib/matrixrtc"; - -import { ObservableScope } from "../../ObservableScope"; -import { createHomeserverConnected$ } from "./HomeserverConnected"; -import { testScope, withTestScheduler } from "../../../utils/test"; - -/** - * Minimal stub of a Matrix client sufficient for our tests: - ``` - createHomeserverConnected$( - scope: ObservableScope, - client: NodeStyleEventEmitter & Pick, - matrixRTCSession: NodeStyleEventEmitter & - Pick, - ) - ``` - */ -class MockMatrixClient extends EventEmitter { - private syncState: SyncState; - public constructor(initial: SyncState) { - super(); - this.syncState = initial; - } - public setSyncState(state: SyncState): void { - this.syncState = state; - // Matrix's Sync event in createHomeserverConnected$ expects [SyncState] - this.emit(ClientEvent.Sync, [state]); - } - public getSyncState(): SyncState { - return this.syncState; - } -} - -/** - * Minimal stub of MatrixRTCSession (membership manager): - ``` - createHomeserverConnected$( - scope: ObservableScope, - client: NodeStyleEventEmitter & Pick, - matrixRTCSession: NodeStyleEventEmitter & - Pick, - ) - ``` - */ -class MockMatrixRTCSession extends EventEmitter { - public membershipStatus: Status; - public probablyLeft: boolean; - - public constructor(props: { - membershipStatus: Status; - probablyLeft: boolean; - }) { - super(); - this.membershipStatus = props.membershipStatus; - this.probablyLeft = props.probablyLeft; - } - - public setMembershipStatus(status: Status): void { - this.membershipStatus = status; - this.emit(MembershipManagerEvent.StatusChanged); - } - - public setProbablyLeft(flag: boolean): void { - this.probablyLeft = flag; - this.emit(MembershipManagerEvent.ProbablyLeft); - } -} - -describe("createHomeserverConnected$", () => { - let scope: ObservableScope; - let client: MockMatrixClient; - let session: MockMatrixRTCSession; - - beforeEach(() => { - scope = new ObservableScope(); - client = new MockMatrixClient(SyncState.Error); // start disconnected - session = new MockMatrixRTCSession({ - membershipStatus: Status.Disconnected, - probablyLeft: false, - }); - }); - - afterEach(() => { - scope.end(); - }); - - // LLM generated test cases. They are a bit overkill but I improved the mocking so it is - // easy enough to read them so I think they can stay. - // Note: gracePeriodMs is set to 0 to avoid debouncing delays in tests - it("reports syncing reason when sync state is not Syncing", () => { - const hsConnected = createHomeserverConnected$(scope, client, session, 0); - expect(hsConnected.combined$.value).toEqual([false, "sync"]); - }); - - it("reports membership reason when sync is Syncing but membership is not Connected", () => { - const hsConnected = createHomeserverConnected$(scope, client, session, 0); - client.setSyncState(SyncState.Syncing); - expect(hsConnected.combined$.value).toEqual([false, "membership"]); - }); - - it("reports probablyLeft reason when membership transitions to Connected but ProbablyLeft is true", () => { - const hsConnected = createHomeserverConnected$(scope, client, session, 0); - // Make sync loop OK - client.setSyncState(SyncState.Syncing); - // Indicate probable leave before connection - session.setProbablyLeft(true); - session.setMembershipStatus(Status.Connected); - expect(hsConnected.combined$.value).toEqual([false, "probablyLeft"]); - }); - - it("becomes null (connected) only when all three conditions are satisfied", () => { - const hsConnected = createHomeserverConnected$(scope, client, session, 0); - // 1. Sync loop connected - client.setSyncState(SyncState.Syncing); - expect(hsConnected.combined$.value).toEqual([false, "membership"]); // not yet membership connected - // 2. Membership connected - session.setMembershipStatus(Status.Connected); - expect(hsConnected.combined$.value).toEqual([true, null]); // probablyLeft is false - }); - - it("returns syncing reason when sync loop leaves Syncing", () => { - const hsConnected = createHomeserverConnected$(scope, client, session, 0); - // Reach connected state - client.setSyncState(SyncState.Syncing); - session.setMembershipStatus(Status.Connected); - expect(hsConnected.combined$.value).toEqual([true, null]); - - // Sync loop error => should report syncing reason - client.setSyncState(SyncState.Error); - expect(hsConnected.combined$.value).toEqual([false, "sync"]); - }); - - it("returns membershipConnected reason when membership status becomes disconnected", () => { - const hsConnected = createHomeserverConnected$(scope, client, session, 0); - client.setSyncState(SyncState.Syncing); - session.setMembershipStatus(Status.Connected); - expect(hsConnected.combined$.value).toEqual([true, null]); - - session.setMembershipStatus(Status.Disconnected); - expect(hsConnected.combined$.value).toEqual([false, "membership"]); - }); - - it("returns certainlyConnected reason when ProbablyLeft is emitted", () => { - const hsConnected = createHomeserverConnected$(scope, client, session, 0); - client.setSyncState(SyncState.Syncing); - session.setMembershipStatus(Status.Connected); - expect(hsConnected.combined$.value).toEqual([true, null]); - - session.setProbablyLeft(true); - expect(hsConnected.combined$.value).toEqual([false, "probablyLeft"]); - }); - - it("recovers to null (connected) if ProbablyLeft becomes false again while other conditions remain true", () => { - const hsConnected = createHomeserverConnected$(scope, client, session, 0); - client.setSyncState(SyncState.Syncing); - session.setMembershipStatus(Status.Connected); - expect(hsConnected.combined$.value).toEqual([true, null]); - - session.setProbablyLeft(true); - expect(hsConnected.combined$.value).toEqual([false, "probablyLeft"]); - - // Simulate clearing the flag (in realistic scenario membership manager would update) - session.setProbablyLeft(false); - expect(hsConnected.combined$.value).toEqual([true, null]); - }); - - it("composite sequence reflects each individual failure reason", () => { - const hsConnected = createHomeserverConnected$(scope, client, session, 0); - - // Initially: sync error + membership disconnected → syncing wins (highest priority) - expect(hsConnected.combined$.value).toEqual([false, "sync"]); - - // Fix sync only → membershipConnected is now the blocker - client.setSyncState(SyncState.Syncing); - expect(hsConnected.combined$.value).toEqual([false, "membership"]); - - // Fix membership → all conditions satisfied - session.setMembershipStatus(Status.Connected); - expect(hsConnected.combined$.value).toEqual([true, null]); - - // Introduce probablyLeft → certainlyConnected - session.setProbablyLeft(true); - expect(hsConnected.combined$.value).toEqual([false, "probablyLeft"]); - - // Restore notProbablyLeft → connected again - session.setProbablyLeft(false); - expect(hsConnected.combined$.value).toEqual([true, null]); - - // Drop sync → syncing reason - client.setSyncState(SyncState.Error); - expect(hsConnected.combined$.value).toEqual([false, "sync"]); - }); -}); - -describe("createHomeserverConnected$ - combined$ reason values", () => { - let scope: ObservableScope; - let client: MockMatrixClient; - let session: MockMatrixRTCSession; - - beforeEach(() => { - scope = new ObservableScope(); - // Start with sync failing and membership disconnected - client = new MockMatrixClient(SyncState.Error); - session = new MockMatrixRTCSession({ - membershipStatus: Status.Disconnected, - probablyLeft: false, - }); - }); - - afterEach(() => { - scope.end(); - }); - - it("is [true, null] when all three conditions are satisfied", () => { - const { combined$ } = createHomeserverConnected$(scope, client, session, 0); - client.setSyncState(SyncState.Syncing); - session.setMembershipStatus(Status.Connected); - expect(combined$.value).toEqual([true, null]); - }); - - it("reports syncing when sync loop is not Syncing", () => { - const { combined$ } = createHomeserverConnected$(scope, client, session, 0); - // client starts with SyncState.Error, membership also disconnected - expect(combined$.value).toEqual([false, "sync"]); - }); - - it("reports membershipConnected when sync is fine but membership is not Connected", () => { - const { combined$ } = createHomeserverConnected$(scope, client, session, 0); - client.setSyncState(SyncState.Syncing); - // session still Status.Disconnected - expect(combined$.value).toEqual([false, "membership"]); - }); - - it("reports certainlyConnected when probablyLeft is true", () => { - const { combined$ } = createHomeserverConnected$(scope, client, session, 0); - client.setSyncState(SyncState.Syncing); - session.setMembershipStatus(Status.Connected); - session.setProbablyLeft(true); - expect(combined$.value).toEqual([false, "probablyLeft"]); - }); - - it("prioritises syncing over membershipConnected when both fail", () => { - const { combined$ } = createHomeserverConnected$(scope, client, session, 0); - // Both sync (Error) and membership (Disconnected) are failing - expect(combined$.value).toEqual([false, "sync"]); - }); - - it("updates reason as conditions change", () => { - const { combined$ } = createHomeserverConnected$(scope, client, session, 0); - // Initially: syncing fails - expect(combined$.value).toEqual([false, "sync"]); - - // Fix sync → membershipConnected is now the blocker - client.setSyncState(SyncState.Syncing); - expect(combined$.value).toEqual([false, "membership"]); - - // Fix membership → probablyLeft makes certainlyConnected fail - session.setProbablyLeft(true); - session.setMembershipStatus(Status.Connected); - expect(combined$.value).toEqual([false, "probablyLeft"]); - - // Clear probablyLeft → all conditions satisfied - session.setProbablyLeft(false); - expect(combined$.value).toEqual([true, null]); - }); -}); - -describe("createHomeserverConnected$ - Grace Period", () => { - const GRACE_PERIOD = 5; - - function marbleTest( - syncStateMarbles: string, - expectedConnectedMarbles: string, - ): void { - withTestScheduler(({ behavior, schedule, expectObservable }) => { - const syncState$ = behavior(syncStateMarbles, { - s: SyncState.Syncing, - e: SyncState.Error, - }); - const client = new MockMatrixClient(syncState$.value); - schedule(syncStateMarbles, { - s: () => client.setSyncState(SyncState.Syncing), - e: () => client.setSyncState(SyncState.Error), - }); - const session = new MockMatrixRTCSession({ - membershipStatus: Status.Connected, - probablyLeft: false, - }); - const hsConnected = createHomeserverConnected$( - testScope(), - client, - session, - GRACE_PERIOD, - ); - expectObservable(hsConnected.combined$).toBe(expectedConnectedMarbles, { - y: [true, null], - n: [false, "sync"], - }); - }); - } - - it("respects gracePeriodMs: stays true during grace period and flips false after", () => { - // - Initial state: Everything is connected - // - Sync error occurs -> should remain connected due to grace period - // - After grace period, not connected - marbleTest("se", "y-----n"); - // If the sync error takes longer to occur, it should take equally long for - // the connection state to change - marbleTest("s--e", "y-------n"); - }); - - it("recovers immediately if sync returns during grace period", () => { - // - Initial state: Connected - // - Sync error occurs - // - Sync recovers BEFORE the grace period expires - // - Connection state remains constant - marbleTest("se--s", "y"); - }); - - it("flips to true IMMEDIATELY even if a grace period was pending", () => { - // - Initial error: connection eventually flips to false - // - Back to Syncing -> Must be connected immediately (synchronously) - marbleTest("e-----s", "y----ny"); - }); -}); diff --git a/src/state/CallViewModel/localMember/HomeserverConnected.ts b/src/state/CallViewModel/localMember/HomeserverConnected.ts deleted file mode 100644 index 45e9f6cbb..000000000 --- a/src/state/CallViewModel/localMember/HomeserverConnected.ts +++ /dev/null @@ -1,136 +0,0 @@ -/* -Copyright 2025 Element Creations Ltd. -Copyright 2024 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - MembershipManagerEvent, - Status, - type MatrixRTCSession, -} from "matrix-js-sdk/lib/matrixrtc"; -import { ClientEvent, type MatrixClient, SyncState } from "matrix-js-sdk"; -import { - fromEvent, - startWith, - map, - tap, - type Observable, - distinctUntilChanged, - switchMap, - of, - delay, - combineLatest, -} from "rxjs"; -import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; - -import { Config } from "../../../config/Config"; -import { type ObservableScope } from "../../ObservableScope"; -import { type Behavior } from "../../Behavior"; -import { type NodeStyleEventEmitter } from "../../../utils/test"; - -export type HomeserverDisconnectReason = "sync" | "membership" | "probablyLeft"; - -export interface HomeserverConnected { - /** - * Emits `[true, null]` when the homeserver connection is healthy, or - * `[false, reason]` when one of the three sub-conditions fails. - */ - combined$: Behavior<[boolean, HomeserverDisconnectReason | null]>; - rtsSession$: Behavior; -} - -/** - * Behavior representing whether we consider ourselves connected to the Matrix homeserver - * for the purposes of a MatrixRTC session. - * - * `combined$` emits `null` when all conditions are satisfied, or the first failing - * reason (priority: syncing > membershipConnected > certainlyConnected): - * 1. Sync loop is not in SyncState.Syncing (after grace period) → "sync" - * 2. membershipStatus !== Status.Connected → "membership" - * 3. probablyLeft === true → "probablyLeft" - * - * @param scope - The observable scope for lifecycle management. - * @param client - The Matrix client to monitor sync state. - * @param matrixRTCSession - The RTC session to monitor membership. - * @param gracePeriodMs - Grace period in milliseconds to wait before reporting sync disconnect. - * If not provided, uses the config value (default 10000ms). - */ -export function createHomeserverConnected$( - scope: ObservableScope, - client: NodeStyleEventEmitter & Pick, - matrixRTCSession: NodeStyleEventEmitter & - Pick, - gracePeriodMs?: number, -): HomeserverConnected { - const logger = rootLogger.getChild("[HomeserverConnected]"); - // Get grace period from parameter or config (default 10000ms) - const graceMs = gracePeriodMs ?? Config.get().sync_disconnect_grace_period_ms; - - const syncing$ = ( - fromEvent(client, ClientEvent.Sync) as Observable<[SyncState]> - ).pipe( - startWith([client.getSyncState()]), - map(([state]) => state === SyncState.Syncing), - distinctUntilChanged(), - switchMap((isSyncing) => { - if (isSyncing || graceMs <= 0) { - return of(isSyncing); - } - return of(false).pipe(delay(graceMs), startWith(true)); - }), - distinctUntilChanged(), - ); - - const rtsSession$ = scope.behavior( - fromEvent(matrixRTCSession, MembershipManagerEvent.StatusChanged).pipe( - map(() => matrixRTCSession.membershipStatus ?? Status.Unknown), - ), - matrixRTCSession.membershipStatus ?? Status.Unknown, - ); - - const membershipConnected$ = rtsSession$.pipe( - map((status) => status === Status.Connected), - ); - - // This is basically notProbablyLeft$ - // - // probablyLeft is computed by a local timer that mimics the server delayed event. - // If we locally predict our server event timed out. We consider ourselves as probablyLeft - // even though we might not yet have received the delayed event leave. - // - // If that is not the case we certainly still have a valid membership on the matrix network - // independet if the sync currently works. - const certainlyConnected$ = fromEvent( - matrixRTCSession, - MembershipManagerEvent.ProbablyLeft, - ).pipe( - startWith(null), - map(() => matrixRTCSession.probablyLeft !== true), - ); - - const combined$ = scope.behavior( - combineLatest([syncing$, membershipConnected$, certainlyConnected$]).pipe( - map( - ([syncing, membership, certainly]): [ - boolean, - HomeserverDisconnectReason | null, - ] => { - if (!syncing) return [false, "sync"]; - if (!membership) return [false, "membership"]; - if (!certainly) return [false, "probablyLeft"]; - return [true, null]; - }, - ), - tap(([connected, reason]) => { - logger.info( - `Homeserver connected update: ${connected ? "connected" : reason}`, - ); - }), - ), - ); - - return { combined$, rtsSession$ }; -} diff --git a/src/state/CallViewModel/localMember/LocalMember.test.ts b/src/state/CallViewModel/localMember/LocalMember.test.ts deleted file mode 100644 index f8b4e6b6b..000000000 --- a/src/state/CallViewModel/localMember/LocalMember.test.ts +++ /dev/null @@ -1,920 +0,0 @@ -/* -Copyright 2025 Element Creations Ltd. -Copyright 2024 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - Status as RTCMemberStatus, - type UnstableLivekitTransport, - type MatrixRTCSession, -} from "matrix-js-sdk/lib/matrixrtc"; -import { - describe, - expect, - it, - vi, - beforeAll, - afterAll, - beforeEach, - afterEach, -} from "vitest"; -import { BehaviorSubject, of, Subject } from "rxjs"; -import { logger } from "matrix-js-sdk/lib/logger"; -import { - ConnectionState as LivekitConnectionState, - type LocalParticipant, - type LocalTrack, -} from "livekit-client"; -import fetchMock from "fetch-mock"; - -import { PosthogAnalytics } from "../../../analytics/PosthogAnalytics"; -import { - MatrixRTCMode, - type ResolvedDelayedLeaveTimings, -} from "../../../config/ConfigOptions"; -import { type HomeserverDisconnectReason } from "./HomeserverConnected"; -import { - flushPromises, - mockConfig, - mockLivekitRoom, - mockLocalParticipant, - mockMuteStates, - withTestScheduler, - ownMemberMock, - testScope, -} from "../../../utils/test"; -import { - TransportState, - createLocalMembership$, - enterRTCSession, - PublishState, - TrackState, - watchScreenShareToggle, -} from "./LocalMember"; -import { MatrixRTCTransportMissingError } from "../../../utils/errors"; -import { Epoch, ObservableScope } from "../../ObservableScope"; -import { constant } from "../../Behavior"; -import { ConnectionMap } from "../remoteMembers/ConnectionMap"; -import { - type ConnectionState, - type Connection, -} from "../remoteMembers/Connection"; -import { type Publisher } from "./Publisher"; -import { initializeWidget } from "../../../widget"; -import { nullHostBridge } from "../../../HostBridge"; -import * as livekitAuth from "../../../livekit/auth"; - -initializeWidget(); - -const MATRIX_RTC_MODE = MatrixRTCMode.Compatibility; -const getUrlParams = vi.hoisted(() => vi.fn(() => ({}))); -vi.mock("../../../UrlParams", () => ({ getUrlParams })); -vi.mock("@livekit/components-core", () => ({ - observeParticipantEvents: vi - .fn() - .mockReturnValue(of({ isScreenShareEnabled: false })), -})); - -describe("watchScreenShareToggle", () => { - it("reports nothing when the toggle completes", async () => { - const onError = vi.fn(); - watchScreenShareToggle(Promise.resolve(), true, logger, onError); - await flushPromises(); - expect(onError).not.toHaveBeenCalled(); - }); - - it("reports failures other than the user cancelling", async () => { - const onError = vi.fn(); - const e = new Error("NotReadableError"); - watchScreenShareToggle(Promise.reject(e), true, logger, onError); - await flushPromises(); - expect(onError).toHaveBeenCalledWith(e); - }); - - it("does not report the user cancelling the picker", async () => { - const onError = vi.fn(); - const cancelled = new DOMException("Permission denied", "NotAllowedError"); - watchScreenShareToggle(Promise.reject(cancelled), true, logger, onError); - await flushPromises(); - expect(onError).not.toHaveBeenCalled(); - }); -}); - -const timings: ResolvedDelayedLeaveTimings = { - delay_ms: 10000, - restart_ms: 4000, - restart_timeout_ms: 1000, -}; - -const delegatedTimings: ResolvedDelayedLeaveTimings = { - delay_ms: timings.delay_ms * 10, - restart_ms: timings.restart_ms! * 10, - restart_timeout_ms: timings.restart_timeout_ms! * 10, -}; - -const mockedClient = { - baseUrl: "https://matrix.example.org", - getDomain: vi.fn().mockReturnValue("example.org"), - getDeviceId: vi.fn().mockReturnValue("AAAA"), - getOpenIdToken: vi.fn().mockResolvedValue({ - access_token: "ACCCESS_TOKEN", - token_type: "Bearer", - matrix_server_name: "localhost", - expires_in: 10000, - }), - _unstable_delegateDelayedLeave: vi.fn().mockResolvedValue({}), -}; - -describe("enterRTCSession", () => { - const transport: UnstableLivekitTransport = { - livekit_service_url: "http://my-livekit-service-url.com", - type: "livekit", - }; - - const options = { - encryptMedia: true, - matrixRTCMode: MATRIX_RTC_MODE, - delayedLeaveTimings: timings, - }; - - const mockedSession = vi.mocked({ - room: { - roomId: "roomId", - client: mockedClient, - }, - memberships: [], - join: vi.fn(), - }) as unknown as MatrixRTCSession; - - beforeEach(() => - mockConfig({ - livekit: { livekit_service_url: "http://my-default-service-url.com" }, - }), - ); - - it("It joins the correct Session", () => { - enterRTCSession(mockedSession, ownMemberMock, transport, options); - - expect(mockedSession.join).toHaveBeenLastCalledWith( - { - deviceId: "DEVICE", - memberId: "@alice:example.org:DEVICE", - userId: "@alice:example.org", - }, - [transport], - expect.objectContaining({ manageMediaKeys: true }), - ); - }); - - it("passes keyRotationParticipantLimit from config to join", () => { - mockConfig({ - livekit: { livekit_service_url: "http://my-default-service-url.com" }, - matrix_rtc_session: { - network_error_retry_ms: 0, - key_rotation_participant_limit: 50, - delayed_leave: timings, - delegated_delayed_leave: timings, - }, - }); - - enterRTCSession(mockedSession, ownMemberMock, transport, options); - - expect(mockedSession.join).toHaveBeenLastCalledWith( - expect.any(Object), - expect.any(Object), - expect.objectContaining({ - keyRotationParticipantLimit: 50, - }), - ); - }); - - it("uses the specified delayed leave timings", () => { - enterRTCSession(mockedSession, ownMemberMock, transport, options); - - expect(mockedSession.join).toHaveBeenLastCalledWith( - expect.anything(), - expect.anything(), - expect.objectContaining({ - delayedLeaveEventRestartMs: timings.restart_ms, - delayedLeaveEventDelayMs: timings.delay_ms, - delayedLeaveEventRestartLocalTimeoutMs: timings.restart_timeout_ms, - }), - ); - }); -}); - -describe("LocalMembership", () => { - const defaultCreateLocalMemberValues = { - options: constant({ - encryptMedia: false, - matrixRTCMode: MatrixRTCMode.Matrix_2_0, - }), - matrixRTCSession: { - updateCallIntent: vi.fn().mockReturnValue(Promise.resolve()), - leave: vi.fn(), - } as unknown as MatrixRTCSession, - muteStates: mockMuteStates(), - trackProcessorState$: constant({ - supported: false, - processor: undefined, - }), - logger: logger, - createPublisherFactory: vi.fn().mockReturnValue({ - destroy: vi.fn(), - createAndSetupTracks: vi.fn().mockResolvedValue(undefined), - }), - joinMatrixRTC: async (): Promise => {}, - homeserverConnected: { - combined$: constant<[boolean, HomeserverDisconnectReason | null]>([ - true, - null, - ]), - rtsSession$: constant(RTCMemberStatus.Connected), - }, - roomId: "!test-room-id:example.org", - hideScreensharing: false, - hostBridge: nullHostBridge, - baseUrl: "https://matrix.example.org", - ownMembershipIdentity: ownMemberMock, - client: mockedClient, - delayId$: constant(null), - matrixRTCMode: MATRIX_RTC_MODE, - }; - - beforeEach(() => { - mockConfig({ - livekit: { livekit_service_url: "http://my-default-service-url.com" }, - matrix_rtc_session: { - network_error_retry_ms: 1000, - delayed_leave: timings, - delegated_delayed_leave: delegatedTimings, - }, - }); - fetchMock.catch(404); - }); - - afterEach(async () => { - void (await fetchMock.flush()); - fetchMock.reset(); - vi.clearAllMocks(); - }); - - it("throws error on missing RTC config error", () => { - withTestScheduler(({ scope, hot, expectObservable }) => { - const localMembership = createLocalMembership$({ - scope, - ...defaultCreateLocalMemberValues, - // we do not need any connection data since we want to fail before reaching that. - connectionMap$: constant(new Epoch(new ConnectionMap())), - localTransport$: hot( - "1ms #", - {}, - new MatrixRTCTransportMissingError("domain.com"), - ), - }); - - expectObservable(localMembership.localMemberState$).toBe("ne", { - n: TransportState.Waiting, - e: expect.toSatisfy((e) => e instanceof MatrixRTCTransportMissingError), - }); - }); - }); - - it("logs if callIntent cannot be updated", async () => { - async function reject(): Promise { - return Promise.reject(new Error("Not connected yet")); - } - const localMembership = createLocalMembership$({ - scope: testScope(), - ...defaultCreateLocalMemberValues, - matrixRTCSession: { - slotId: "m.call#ROOM", - updateCallIntent: vi.fn().mockImplementation(reject), - leave: vi.fn(), - }, - connectionMap$: constant(new Epoch(new ConnectionMap())), - localTransport$: constant(mockTransportLocator), - }); - const expextedLog = - "'not connected yet' while updating the call intent (this is expected on startup)"; - const internalLogger = vi.spyOn(localMembership.internalLoggerRef, "debug"); - - await flushPromises(); - defaultCreateLocalMemberValues.muteStates.video.setEnabled$.value?.(true); - expect(internalLogger).toHaveBeenCalledWith(expextedLog); - }); - - const mockTransport: UnstableLivekitTransport = { - type: "livekit", - url: "https://sfu.example.org", - livekit_service_url: "https://jwt.example.org", - }; - - const mockTransportLocator: livekitAuth.TransportLocator = { - transport: mockTransport, - serverName: "example.org", - }; - - const connectionTransportAConnected = { - livekitRoom: mockLivekitRoom({ - localParticipant: { - isScreenShareEnabled: false, - trackPublications: [], - } as unknown as LocalParticipant, - }), - state$: constant({ - state: "authenticated", - livekitState: LivekitConnectionState.Connected, - }), - transport: mockTransport, - serverName: "example.org", - } as Connection; - const connectionTransportAConnecting = { - ...connectionTransportAConnected, - state$: constant({ - state: "authenticated", - livekitState: LivekitConnectionState.Connecting, - }), - livekitRoom: mockLivekitRoom({}), - } as unknown as Connection; - - const delegationSpy = vi - .spyOn(livekitAuth, "delegateDelayedLeave") - .mockResolvedValue(); - afterEach(() => delegationSpy.mockClear()); - - const urlOnlyTransport: UnstableLivekitTransport = { - type: "livekit", - url: "https://sfu.example.org", - }; - - // With a `url` only we publish through the CS API, so the homeserver - // delegates. With a `livekit_service_url` we publish through the JWT - // service, so that delegates. - it.each([ - ["no", mockTransport, null, timings], - [ - "homeserver", - urlOnlyTransport, - "https://matrix.example.org/_matrix/client/unstable/io.element.msc4195/rtc/livekit/delegate_delayed_leave", - delegatedTimings, - ], - [ - "transport", - mockTransport, - "https://jwt.example.org/delegate_delayed_leave", - delegatedTimings, - ], - ])( - "joins session with %s delegation support", - async (_serviceName, transport, delegationUrl, delayedLeaveTimings) => { - const scope = testScope(); - const joinMatrixRTC = vi.fn(); - const delayId$ = new BehaviorSubject(null); - - if (delegationUrl !== null) - fetchMock.post(delegationUrl, () => ({ status: 401, body: {} })); - - const connectionMap = new ConnectionMap(); - connectionMap.add( - { ...connectionTransportAConnected, transport } as Connection, - [], - ); - const localMembership = createLocalMembership$({ - scope, - ...defaultCreateLocalMemberValues, - connectionMap$: constant(new Epoch(connectionMap)), - joinMatrixRTC, - localTransport$: constant({ transport, serverName: "example.org" }), - delayId$, - }); - - localMembership.requestJoinAndPublish(); - void (await fetchMock.flush()); - await flushPromises(); - // Joins with timings appropriate for the level of delegation support - expect(joinMatrixRTC).toHaveBeenCalledWith( - transport, - delayedLeaveTimings, - ); - - expect(delegationSpy).not.toHaveBeenCalled(); - delayId$.next("leave1"); - await flushPromises(); - if (delegationUrl === null) { - expect(delegationSpy).not.toHaveBeenCalled(); - } else { - // Delegation is supported in this test case, so go on to check that - // LocalMember actually performs delegation - const expectDelegation = (delayId: string) => - expect(delegationSpy).toHaveBeenLastCalledWith({ - client: mockedClient, - membership: ownMemberMock, - transport, - roomId: "!test-room-id:example.org", - delayId, - logger: expect.anything(), - }); - - expectDelegation("leave1"); - delayId$.next("leave2"); // Can change delegated leaves - await flushPromises(); - expectDelegation("leave2"); - } - }, - ); - - it("only starts tracks if requested", async () => { - const scope = new ObservableScope(); - - const publishers: Publisher[] = []; - - const tracks$ = new BehaviorSubject([]); - const publishing$ = constant(false); - defaultCreateLocalMemberValues.createPublisherFactory.mockImplementation( - () => { - const p = { - // It is enought to check if destroy is called. Destroy itself is tested in the publisher to make sure it does - // all the cleanup we need. - destroy: vi.fn(), - createAndSetupTracks: vi.fn().mockImplementation(async () => { - tracks$.next([{}, {}] as LocalTrack[]); - return Promise.resolve(); - }), - tracks$, - publishing$, - }; - publishers.push(p as unknown as Publisher); - return p; - }, - ); - const publisherFactory = - defaultCreateLocalMemberValues.createPublisherFactory as ReturnType< - typeof vi.fn - >; - - const connectionMap = new ConnectionMap(); - connectionMap.add(connectionTransportAConnected, []); - // connectionMap.add(connectionTransportB, []); - const localMembership = createLocalMembership$({ - scope, - ...defaultCreateLocalMemberValues, - connectionMap$: constant(new Epoch(connectionMap)), - localTransport$: constant(mockTransportLocator), - }); - await flushPromises(); - expect(publisherFactory).toHaveBeenCalledOnce(); - // expect(localMembership.tracks$.value.length).toBe(0); - expect(publishers[0].createAndSetupTracks).not.toHaveBeenCalled(); - localMembership.startTracks(); - await flushPromises(); - expect(publishers[0].createAndSetupTracks).toHaveBeenCalled(); - - scope.end(); - await flushPromises(); - // stop all tracks after ending scopes - expect(publishers[0].destroy).toHaveBeenCalled(); - // expect(publishers[0].stopTracks).toHaveBeenCalled(); - publisherFactory.mockClear(); - }); - // TODO add an integration test combining publisher and localMembership - // - it("tracks livekit state correctly", async () => { - const scope = new ObservableScope(); - const connectionMap = new ConnectionMap(); - const localTransport$ = new Subject(); - - const connectionMap$ = new BehaviorSubject(new Epoch(connectionMap)); - const publishers: Publisher[] = []; - - const publishing$ = new BehaviorSubject(false); - const createTrackResolver = Promise.withResolvers(); - const publishResolver = Promise.withResolvers(); - defaultCreateLocalMemberValues.createPublisherFactory.mockImplementation( - () => { - const p = { - // It is enought to check if destroy is called. Destroy itself is tested in the publisher to make sure it does - // all the cleanup we need. - destroy: vi.fn(), - createAndSetupTracks: vi.fn().mockImplementation(async () => { - await createTrackResolver.promise; - }), - startPublishing: vi.fn().mockImplementation(async () => { - await publishResolver.promise; - publishing$.next(true); - }), - publishing$, - }; - publishers.push(p as unknown as Publisher); - return p; - }, - ); - - const publisherFactory = - defaultCreateLocalMemberValues.createPublisherFactory as ReturnType< - typeof vi.fn - >; - - const localMembership = createLocalMembership$({ - scope, - ...defaultCreateLocalMemberValues, - connectionMap$: connectionMap$, - localTransport$, - }); - - await flushPromises(); - expect(localMembership.localMemberState$.value).toStrictEqual( - TransportState.Waiting, - ); - localTransport$.next(mockTransportLocator); - await flushPromises(); - expect(localMembership.localMemberState$.value).toStrictEqual({ - matrix: RTCMemberStatus.Connected, - media: { - connection: { state: "initializing" }, - tracks: TrackState.WaitingForUser, - }, - }); - - const connectionMap2 = new ConnectionMap(); - connectionMap2.add( - // clone because we will mutate this later. - { ...connectionTransportAConnecting } as unknown as Connection, - [], - ); - - connectionMap$.next(new Epoch(connectionMap2)); - await flushPromises(); - expect(localMembership.localMemberState$.value).toStrictEqual({ - matrix: RTCMemberStatus.Connected, - media: { - connection: { - state: "authenticated", - livekitState: LivekitConnectionState.Connecting, - }, - tracks: TrackState.WaitingForUser, - }, - }); - - ( - connectionMap2.getConnectionForTransport(mockTransportLocator)! - .state$ as BehaviorSubject - ).next({ - state: "authenticated", - livekitState: LivekitConnectionState.Connected, - }); - expect(localMembership.localMemberState$.value).toStrictEqual({ - matrix: RTCMemberStatus.Connected, - media: { - connection: { - state: "authenticated", - livekitState: LivekitConnectionState.Connected, - }, - tracks: TrackState.WaitingForUser, - }, - }); - - expect(publisherFactory).toHaveBeenCalledOnce(); - // expect(localMembership.tracks$.value.length).toBe(0); - - // ------- - localMembership.startTracks(); - // ------- - - await flushPromises(); - // expect(localMembership.localMemberState$.value).toStrictEqual({ - // matrix: RTCMemberStatus.Connected, - // media: { - // tracks: TrackState.Creating, - // connection: ConnectionState.LivekitConnected, - // }, - // }); - createTrackResolver.resolve(); - await flushPromises(); - expect( - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (localMembership.localMemberState$.value as any).media, - ).toStrictEqual(PublishState.WaitingForUser); - - // ------- - localMembership.requestJoinAndPublish(); - // ------- - - expect( - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (localMembership.localMemberState$.value as any).media, - ).toStrictEqual(PublishState.Publishing); - - publishResolver.resolve(); - await flushPromises(); - expect( - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (localMembership.localMemberState$.value as any).media, - ).toStrictEqual(PublishState.Publishing); - - expect(publishers[0].destroy).not.toHaveBeenCalled(); - - expect(localMembership.localMemberState$.isStopped).toBe(false); - scope.end(); - await flushPromises(); - // stays in connected state because it is stopped before the update to tracks update the state. - expect( - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (localMembership.localMemberState$.value as any).media, - ).toStrictEqual(PublishState.Publishing); - // stop all tracks after ending scopes - expect(publishers[0].destroy).toHaveBeenCalled(); - // expect(publishers[0].stopTracks).toHaveBeenCalled(); - }); - // TODO add tests for matrix local matrix participation. - - describe("reconnecting analytics", () => { - beforeAll(() => { - mockConfig(); - }); - - beforeEach(() => { - vi.restoreAllMocks(); - }); - - afterAll(() => { - PosthogAnalytics.resetInstance(); - }); - - it("does not fire CallReconnecting for the initial non-connected state at startup", async () => { - const scope = new ObservableScope(); - const trackSpy = vi.spyOn( - PosthogAnalytics.instance.eventCallReconnecting, - "track", - ); - - // Simulate startup where membership isn't established yet - const hsReason$ = new BehaviorSubject< - [boolean, HomeserverDisconnectReason | null] - >([false, "membership"]); - - const connectionMap = new ConnectionMap(); - connectionMap.add(connectionTransportAConnected, []); - - createLocalMembership$({ - scope, - ...defaultCreateLocalMemberValues, - homeserverConnected: { - combined$: hsReason$, - rtsSession$: constant(RTCMemberStatus.Connected), - }, - connectionMap$: constant(new Epoch(connectionMap)), - localTransport$: constant(mockTransportLocator), - }); - - await flushPromises(); - - // Membership is established — call is now connected - hsReason$.next([true, null]); - - expect(trackSpy).not.toHaveBeenCalled(); - - scope.end(); - }); - - it("fires CallReconnecting with homeserver reason and duration when reconnected", async () => { - const scope = new ObservableScope(); - const trackSpy = vi.spyOn( - PosthogAnalytics.instance.eventCallReconnecting, - "track", - ); - - const hsReason$ = new BehaviorSubject< - [boolean, HomeserverDisconnectReason | null] - >([true, null]); - - const connectionMap = new ConnectionMap(); - connectionMap.add(connectionTransportAConnected, []); - - createLocalMembership$({ - scope, - ...defaultCreateLocalMemberValues, - homeserverConnected: { - combined$: hsReason$, - rtsSession$: constant(RTCMemberStatus.Connected), - }, - connectionMap$: constant(new Epoch(connectionMap)), - localTransport$: constant(mockTransportLocator), - }); - - await flushPromises(); - - hsReason$.next([false, "sync"]); - hsReason$.next([true, null]); - - expect(trackSpy).toHaveBeenCalledWith( - defaultCreateLocalMemberValues.roomId, - "sync", - expect.any(Number), - ); - - scope.end(); - }); - - it("reports livekit reason when livekit disconnects then reconnects", async () => { - const scope = new ObservableScope(); - const trackSpy = vi.spyOn( - PosthogAnalytics.instance.eventCallReconnecting, - "track", - ); - - const connectionState$ = new BehaviorSubject({ - state: "authenticated", - livekitState: LivekitConnectionState.Connected, - }); - const mutableConnection = { - ...connectionTransportAConnected, - state$: connectionState$, - } as unknown as Connection; - - const connectionMap = new ConnectionMap(); - connectionMap.add(mutableConnection, []); - - createLocalMembership$({ - scope, - ...defaultCreateLocalMemberValues, - homeserverConnected: { - combined$: constant<[boolean, HomeserverDisconnectReason | null]>([ - true, - null, - ]), - rtsSession$: constant(RTCMemberStatus.Connected), - }, - connectionMap$: constant(new Epoch(connectionMap)), - localTransport$: constant(mockTransportLocator), - }); - - await flushPromises(); - - connectionState$.next({ - state: "authenticated", - livekitState: LivekitConnectionState.Disconnected, - }); - connectionState$.next({ - state: "authenticated", - livekitState: LivekitConnectionState.Connected, - }); - - expect(trackSpy).toHaveBeenCalledWith( - defaultCreateLocalMemberValues.roomId, - "livekit", - expect.any(Number), - ); - - scope.end(); - }); - - it("fires one event per completed reconnection cycle", async () => { - const scope = new ObservableScope(); - const trackSpy = vi.spyOn( - PosthogAnalytics.instance.eventCallReconnecting, - "track", - ); - - const hsReason$ = new BehaviorSubject< - [boolean, HomeserverDisconnectReason | null] - >([true, null]); - - const connectionMap = new ConnectionMap(); - connectionMap.add(connectionTransportAConnected, []); - - createLocalMembership$({ - scope, - ...defaultCreateLocalMemberValues, - homeserverConnected: { - combined$: hsReason$, - rtsSession$: constant(RTCMemberStatus.Connected), - }, - connectionMap$: constant(new Epoch(connectionMap)), - localTransport$: constant(mockTransportLocator), - }); - - await flushPromises(); - - hsReason$.next([false, "membership"]); - hsReason$.next([true, null]); - - hsReason$.next([false, "probablyLeft"]); - hsReason$.next([false, "sync"]); - hsReason$.next([false, "membership"]); - hsReason$.next([true, null]); - - expect(trackSpy).toHaveBeenCalledTimes(2); - expect(trackSpy).toHaveBeenNthCalledWith( - 1, - defaultCreateLocalMemberValues.roomId, - "membership", - expect.any(Number), - ); - expect(trackSpy).toHaveBeenNthCalledWith( - 2, - defaultCreateLocalMemberValues.roomId, - "probablyLeft", - expect.any(Number), - ); - - scope.end(); - }); - }); - - describe("toggleScreenSharing", () => { - let originalMediaDevices: MediaDevices | undefined; - - beforeAll(() => { - mockConfig(); - // Screen sharing is only offered when getDisplayMedia is available. - originalMediaDevices = navigator.mediaDevices; - Object.defineProperty(navigator, "mediaDevices", { - configurable: true, - value: { getDisplayMedia: vi.fn() }, - }); - }); - - afterAll(() => { - Object.defineProperty(navigator, "mediaDevices", { - configurable: true, - value: originalMediaDevices, - }); - }); - - const createMembershipWithConnection = ( - connection: Connection | null, - ): { - scope: ObservableScope; - localMembership: ReturnType; - } => { - const scope = new ObservableScope(); - const connectionMap = new ConnectionMap(); - if (connection) connectionMap.add(connection, []); - const localMembership = createLocalMembership$({ - scope, - ...defaultCreateLocalMemberValues, - connectionMap$: constant(new Epoch(connectionMap)), - localTransport$: constant(mockTransportLocator), - }); - return { scope, localMembership }; - }; - - it("surfaces a failure and clears it on dismiss", async () => { - const error = new Error("NotReadableError"); - const setScreenShareEnabled = vi.fn().mockRejectedValue(error); - const connection = { - state$: constant({ - state: "authenticated", - livekitState: LivekitConnectionState.Connected, - }), - transport: mockTransport, - serverName: "example.org", - livekitRoom: mockLivekitRoom({ - localParticipant: mockLocalParticipant({ - isScreenShareEnabled: false, - setScreenShareEnabled, - }), - }), - } as unknown as Connection; - const { scope, localMembership } = - createMembershipWithConnection(connection); - await flushPromises(); - - expect(localMembership.toggleScreenSharing).not.toBeNull(); - expect(localMembership.screenShareError$.value).toBeNull(); - - localMembership.toggleScreenSharing!(); - await flushPromises(); - - expect(setScreenShareEnabled).toHaveBeenCalledWith( - true, - expect.any(Object), - undefined, - ); - expect(localMembership.screenShareError$.value).toBe(error); - - localMembership.dismissScreenShareError(); - expect(localMembership.screenShareError$.value).toBeNull(); - - scope.end(); - }); - - it("does nothing when there is no local participant", async () => { - // No connection means participant$ never resolves to a participant. - const { scope, localMembership } = createMembershipWithConnection(null); - await flushPromises(); - - expect(localMembership.toggleScreenSharing).not.toBeNull(); - localMembership.toggleScreenSharing!(); - await flushPromises(); - - expect(localMembership.screenShareError$.value).toBeNull(); - - scope.end(); - }); - }); -}); diff --git a/src/state/CallViewModel/localMember/LocalMember.ts b/src/state/CallViewModel/localMember/LocalMember.ts deleted file mode 100644 index baa6b473d..000000000 --- a/src/state/CallViewModel/localMember/LocalMember.ts +++ /dev/null @@ -1,1031 +0,0 @@ -/* -Copyright 2025 Element Creations Ltd. - -SPDX-License-IdFentifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - ConnectionState as LivekitConnectionState, - type Participant, - ParticipantEvent, - type LocalParticipant, - type ScreenShareCaptureOptions, - type TrackPublishOptions, - RoomEvent, - MediaDeviceFailure, -} from "livekit-client"; -import { observeParticipantEvents } from "@livekit/components-core"; -import { - Status as RTCSessionStatus, - type UnstableLivekitTransport, - type MatrixRTCSession, - type RTCCallIntent, - type RTCNotificationType, -} from "matrix-js-sdk/lib/matrixrtc"; -import { - BehaviorSubject, - catchError, - combineLatest, - distinctUntilChanged, - from, - fromEvent, - map, - type Observable, - of, - pairwise, - startWith, - switchMap, - tap, - NEVER, - concat, - race, - Subject, -} from "rxjs"; -import { type Logger } from "matrix-js-sdk/lib/logger"; -import { deepCompare } from "matrix-js-sdk/lib/utils"; -import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; - -import { type Behavior } from "../../Behavior.ts"; -import { type ConnectionMap } from "../remoteMembers/ConnectionMap.ts"; -import { type Epoch, type ObservableScope } from "../../ObservableScope.ts"; -import { type Publisher } from "./Publisher.ts"; -import { type MuteStates } from "../../MuteStates.ts"; -import { - ElementCallError, - FailToStartLivekitConnection, - MembershipManagerError, - UnknownCallError, -} from "../../../utils/errors.ts"; -import { type HostBridge } from "../../../HostBridge.ts"; - -import { PosthogAnalytics } from "../../../analytics/PosthogAnalytics.ts"; -import { - advancedScreenShare, - screenShareResolution, - screenShareFramerate, - screenShareBitrate, - screenShareCodec, - parseResolution, -} from "../../../settings/settings.ts"; -import { - MatrixRTCMode, - type ResolvedDelayedLeaveTimings, -} from "../../../config/ConfigOptions.ts"; -import { Config } from "../../../config/Config.ts"; -import { - type ConnectionState, - type Connection, -} from "../remoteMembers/Connection.ts"; -import { type HomeserverConnected } from "./HomeserverConnected.ts"; -import { - type ClientDelegationParts, - delegateDelayedLeave, - type ClientOpenIDParts, - chooseAuthFlow, - type TransportLocator, -} from "../../../livekit/auth"; - -export enum TransportState { - /** Not even a transport is available to the LocalMembership */ - Waiting = "transport_waiting", -} - -export enum PublishState { - WaitingForUser = "publish_waiting_for_user", - // XXX: This state is removed for now since we do not have full control over - // track publication anymore with the publisher abstraction, might come back in the future? - // /** Implies lk connection is connected */ - // Starting = "publish_start_publishing", - /** Implies lk connection is connected */ - Publishing = "publish_publishing", -} - -// TODO not sure how to map that correctly with the -// new publisher that does not manage tracks itself anymore -export enum TrackState { - /** The track is waiting for user input to create tracks (waiting to call `startTracks()`) */ - WaitingForUser = "tracks_waiting_for_user", - // XXX: This state is removed for now since we do not have full control over - // track creation anymore with the publisher abstraction, might come back in the future? - // /** Implies lk connection is connected */ - // Creating = "tracks_creating", - /** Implies lk connection is connected */ - Ready = "tracks_ready", -} - -export type LocalMemberMediaState = - | { - tracks: TrackState; - connection: ConnectionState; - } - | PublishState - | ElementCallError; -export type LocalMemberState = - | ElementCallError - | TransportState.Waiting - | { - media: LocalMemberMediaState; - matrix: ElementCallError | RTCSessionStatus; - }; - -/* - * - get transport to use - * - get openId + jwt token - * - wait for createTrack() call - * - create tracks - * - wait for join() call - * - Publisher.publishTracks() - * - send join state/sticky event - */ - -interface Props { - // TODO add a comment into some code style readme or file header callviewmodel - // that the inputs for those createSomething$() functions should NOT contain any js-sdk objectes - scope: ObservableScope; - muteStates: MuteStates; - connectionMap$: Behavior>; - createPublisherFactory: (connection: Connection) => Publisher; - joinMatrixRTC: ( - transport: UnstableLivekitTransport, - delayedLeaveTimings: ResolvedDelayedLeaveTimings, - ) => void; - homeserverConnected: HomeserverConnected; - roomId: string; - ownMembershipIdentity: CallMembershipIdentityParts; - localTransport$: Observable; - client: ClientDelegationParts & ClientOpenIDParts; - matrixRTCSession: Pick< - MatrixRTCSession, - "slotId" | "updateCallIntent" | "leave" - >; - /** Whether to hide the screen-sharing button. */ - hideScreensharing: boolean; - /** The application hosting Element Call, to be kept informed of join/leave. */ - hostBridge: HostBridge; - baseUrl: string; - delayId$: Behavior; - logger: Logger; -} - -/** - * This class is responsible for managing the own membership in a room. - * We want - * - a publisher - * - - * @param props The properties required to create the local membership. - * @param props.scope The observable scope to use. - * @param props.connectionMap$ The connection map to get connections from. - * @param props.createPublisherFactory Factory to create a publisher once we have a connection. - * @param props.joinMatrixRTC Callback to join the matrix RTC session once we have a transport. - * @param props.homeserverConnected The homeserver connected state. - * @param props.localTransport The transport to advertise in our membership. - * @param props.logger The logger to use. - * @param props.muteStates The mute states for video and audio. - * @param props.matrixRTCSession The matrix RTC session to join. - * @param props.baseUrl Base URL of the homeserver. - * @param props.delayId$ ID of the delayed leave event to delegate to the SFU. - * @param props.roomId The room ID used as the call identifier in analytics events. - * @param props.hideScreensharing Whether to hide the screen-sharing button. - * @param props.hostBridge The application hosting Element Call. - * @returns - * - publisher: The handle to create tracks and publish them to the room. - * - connected$: the current connection state. Including matrix server and livekit server connection. (only considering the livekit server we are using for our own media publication) - * - transport$: the transport object the ownMembership$ ended up using. - * - connectionState: the current connection state. Including matrix server and livekit server connection. - * - sharingScreen$: Whether we are sharing our screen. `undefined` if we cannot share the screen. - */ -export const createLocalMembership$ = ({ - scope, - connectionMap$, - localTransport$: localTransportWithErrors$, - homeserverConnected, - createPublisherFactory, - joinMatrixRTC, - logger: parentLogger, - muteStates, - client, - matrixRTCSession, - baseUrl, - roomId, - hideScreensharing, - hostBridge, - ownMembershipIdentity, - delayId$, -}: Props): { - /** - * This request to start audio and video tracks. - * Can be called early to pre-emptively get media permissions and start devices. - */ - startTracks: () => void; - /** - * This sets a inner state (shouldPublish) to true and instructs the js-sdk and livekit to keep the user - * connected to matrix and livekit. - */ - requestJoinAndPublish: () => void; - requestDisconnect: () => void; - localMemberState$: Behavior; - sharingScreen$: Behavior; - /** - * Callback to toggle screen sharing. If null, screen sharing is not possible. - */ - toggleScreenSharing: (() => void) | null; - /** - * The last error from toggling screen sharing, until dismissed. - */ - screenShareError$: Behavior; - dismissScreenShareError: () => void; - // tracks$: Behavior; - participant$: Behavior; - connection$: Behavior; - /** - * Tracks the homserver and livekit connected state and based on that computes reconnecting. - */ - reconnecting$: Behavior; - /** Shorthand for homeserverConnected.rtcSession === Status.Disconnected - * Direct translation to the js-sdk membership manager connection `Status`. - */ - disconnected$: Behavior; - /** - * Fully connected - */ - connected$: Behavior; - internalLoggerRef: Logger; -} => { - const logger = parentLogger.getChild("[LocalMembership]"); - logger.debug(`Creating local membership..`); - - // Unwrap the local transport and set the state of the LocalMembership to error in case the transport is an error. - const fatalTransportError$ = new Subject(); - const localTransport$ = localTransportWithErrors$.pipe( - catchError((e: unknown) => { - let error: ElementCallError; - if (e instanceof ElementCallError) { - error = e; - } else { - error = new UnknownCallError( - e instanceof Error - ? e - : new Error("Unknown error from localTransport"), - ); - } - fatalTransportError$.next(error); - return NEVER; // Make this Observable swallow the error - }), - ); - - async function checkDelegationSupport( - endpointUrl: string, - serviceName: string, - ): Promise { - logger.info(`Checking whether ${serviceName} supports delegation…`); - try { - // Bluntly hit the endpoint without auth to check for a 404. Unfortunately - // we can't wrap this in a retry loop, as many servers don't just disable - // delegation support, but in fact are from a time before the endpoint - // existed at all, therefore we can hit CORS errors which would just gum - // up the retry loop. (May be revisited after Matrix 2.0.) - const res = await fetch(endpointUrl, { method: "POST" }); - if (res.status === 404) { - logger.warn(`${serviceName} does not support delegation`); - return false; - } else { - logger.info(`${serviceName} supports delegation`); - return true; - } - } catch (e) { - logger.warn( - `Failed to determine whether ${serviceName} supports delegation, assuming no support`, - e, - ); - return false; - } - } - - const homeserverSupportsDelegation = checkDelegationSupport( - baseUrl + - "/_matrix/client/unstable/io.element.msc4195/rtc/livekit/delegate_delayed_leave", - "homeserver", - ); - - // The transport that we will advertise in our membership, paired with info as - // to whether delayed event delegation is supported - const joinParams$ = scope.behavior( - localTransport$.pipe( - switchMap(async ({ transport }) => { - // Delegation goes through whichever service authenticates us as a - // publisher, so only that service's support counts - const flow = chooseAuthFlow(transport, "publisher"); - const delegationSupported = - flow.kind === "openID" - ? await checkDelegationSupport( - flow.serviceUrl + "/delegate_delayed_leave", - `transport ${flow.serviceUrl}`, - ) - : await homeserverSupportsDelegation; - return { transport, delegationSupported }; - }), - ), - null, - ); - - // Drop Epoch data here since we will not combine this anymore - const localConnection$ = scope.behavior( - combineLatest( - [connectionMap$, localTransport$], - ({ value: connectionMap }, transport) => - connectionMap.getConnectionForTransport(transport), - ).pipe( - tap((connection) => { - logger.info( - `Local connection updated: ${JSON.stringify(connection?.transport)}`, - ); - }), - ), - null, - ); - - // Tracks error that happen when creating the local tracks. - const mediaErrors$ = localConnection$.pipe( - switchMap((connection) => { - if (!connection) { - return of(null); - } else { - return fromEvent( - connection.livekitRoom, - RoomEvent.MediaDevicesError, - (error: Error) => { - return MediaDeviceFailure.getFailure(error) ?? null; - }, - ); - } - }), - ); - - mediaErrors$.pipe(scope.bind()).subscribe((error) => { - if (error) { - // This is a MediaDevice error, can be PermissionDenied, NotFound, DeviceInUse, Other. - // Will also occurs if you cancel screen sharing browser prompt. - // This is not necessarily fatal, since the user might be able to join without media. - // XXX We might want to give some user feedback here to let them know their media is not working. - logger.error(`Failed to create local tracks:`, error); - } - }); - // MATRIX RELATED - - // This should be used in a combineLatest with publisher$ to connect. - // to make it possible to call startTracks before the preferredTransport$ has resolved. - const trackStartRequested = Promise.withResolvers(); - - // This should be used in a combineLatest with publisher$ to connect. - // to make it possible to call startTracks before the preferredTransport$ has resolved. - const joinAndPublishRequested$ = new BehaviorSubject(false); - - /** - * The publisher is stored in here an abstracts creating and publishing tracks. - */ - const publisher$ = new BehaviorSubject(null); - - const startTracks = (): void => { - trackStartRequested.resolve(); - // This used to return the tracks, but now they are only accessible via the publisher. - }; - - const requestJoinAndPublish = (): void => { - trackStartRequested.resolve(); - joinAndPublishRequested$.next(true); - }; - - const requestDisconnect = (): void => { - joinAndPublishRequested$.next(false); - }; - - // Take care of the publisher$ - // create a new one as soon as a local Connection is available - // - // Recreate a new one once the local connection changes - // - stop publishing - // - destruct all current streams - // - overwrite current publisher - scope.reconcile(localConnection$, async (connection) => { - logger.info( - "reconcile based on new localConnection:", - JSON.stringify(connection?.transport), - ); - if (connection !== null) { - const publisher = createPublisherFactory(connection); - publisher$.next(publisher); - - // Clean-up callback - return Promise.resolve(async (): Promise => { - await publisher.destroy(); - }); - } - }); - - // Use reconcile here to not run concurrent createAndSetupTracks calls - // `tracks$` will update once they are ready. - scope.reconcile( - scope.behavior( - combineLatest([ - publisher$ /*, tracks$*/, - from(trackStartRequested.promise), - ]), - null, - ), - async (valueIfReady) => { - if (!valueIfReady) return; - const [publisher] = valueIfReady; - if (publisher) { - await publisher.createAndSetupTracks().catch((e) => logger.error(e)); - } - }, - ); - - // Based on `connectRequested$` we start publishing tracks. (once they are there!) - scope.reconcile( - scope.behavior(combineLatest([publisher$, joinAndPublishRequested$])), - async ([publisher, shouldJoinAndPublish]) => { - // Get the current publishing state to avoid redundant calls. - const isPublishing = publisher?.shouldPublish === true; - if (shouldJoinAndPublish && !isPublishing) { - try { - await publisher?.startPublishing(); - } catch (error) { - const message = - error instanceof Error ? error.message : String(error); - setPublishError(new FailToStartLivekitConnection(message)); - } - } else if (isPublishing) { - try { - await publisher?.stopPublishing(); - } catch (error) { - setPublishError(new UnknownCallError(error as Error)); - } - } - }, - ); - - // STATE COMPUTATION - - // These are non fatal since we can join a room and concume media even though publishing failed. - const publishError$ = new BehaviorSubject(null); - const setPublishError = (e: ElementCallError): void => { - if (publishError$.value !== null) { - logger.error("Multiple Media Errors:", e); - } else { - publishError$.next(e); - } - }; - - const localConnectionState$ = localConnection$.pipe( - switchMap((connection) => - connection ? connection.state$ : of({ state: "initializing" as const }), - ), - ); - - const authenticated$ = scope.behavior( - localConnectionState$.pipe(map(({ state }) => state === "authenticated")), - ); - - const mediaState$: Behavior = scope.behavior( - combineLatest([ - localConnectionState$, - joinAndPublishRequested$, - from(trackStartRequested.promise).pipe( - map(() => true), - startWith(false), - ), - ]).pipe( - map(([localConnectionState, shouldPublish, shouldStartTracks]) => { - const trackState: TrackState = shouldStartTracks - ? TrackState.Ready - : TrackState.WaitingForUser; - - if ( - localConnectionState.state !== "authenticated" || - localConnectionState.livekitState !== - LivekitConnectionState.Connected || - trackState !== TrackState.Ready - ) - return { - connection: localConnectionState, - tracks: trackState, - }; - if (!shouldPublish) return PublishState.WaitingForUser; - // if (!publishing) return PublishState.Starting; - return PublishState.Publishing; - }), - distinctUntilChanged(deepCompare), - ), - ); - const fatalMatrixError$ = new BehaviorSubject(null); - const setMatrixError = (e: ElementCallError): void => { - if (fatalMatrixError$.value !== null) { - logger.error("Multiple Matrix Errors:", e); - } else { - fatalMatrixError$.next(e); - } - }; - - const localMemberState$ = scope.behavior( - concat( - // Waiting until… - of(TransportState.Waiting), - race( - // either there is a fatal transport error - fatalTransportError$, - // or the transport is available. - localTransport$.pipe( - switchMap(() => - // Once available, track session/media state. - combineLatest([ - mediaState$, - homeserverConnected.rtsSession$, - fatalMatrixError$, - publishError$, - ]).pipe( - map( - ([ - mediaState, - rtcSessionStatus, - fatalMatrixError, - publishError, - ]) => ({ - matrix: fatalMatrixError ?? rtcSessionStatus, - media: publishError ?? mediaState, - }), - ), - ), - ), - ), - ), - ), - ); - - /** - * The disconnect reason for the combined Matrix + LiveKit connection, or null - * when fully connected. Homeserver reasons take priority over livekit. - * Both connectivity state and reason come from the same combineLatest emission, - * avoiding any race between the two. - */ - const connectionDisconnectReason$ = scope.behavior( - combineLatest([ - homeserverConnected.combined$, - localConnectionState$.pipe( - map( - (state) => - state.state === "authenticated" && - state.livekitState === LivekitConnectionState.Connected, - ), - ), - ]).pipe( - map(([[hsConnected, hsReason], livekitConnected]) => { - if (!hsConnected) return hsReason!; - if (!livekitConnected) return "livekit" as const; - return null; - }), - tap((v) => logger.debug("livekit+matrix: Connected state changed", v)), - ), - ); - - /** - * Whether we are "fully" connected to the call. Accounts for both the - * connection to the MatrixRTC session and the LiveKit publish connection. - */ - const matrixAndLivekitConnected$ = scope.behavior( - connectionDisconnectReason$.pipe(map((reason) => reason === null)), - ); - - /** - * Whether we should tell the user that we're reconnecting to the call. - */ - const reconnecting$ = scope.behavior( - matrixAndLivekitConnected$.pipe( - pairwise(), - map(([prev, current]) => prev === true && current === false), - ), - false, - ); - - let reconnectStart: { - time: number; - reason: NonNullable<(typeof connectionDisconnectReason$)["value"]>; - } | null = null; - connectionDisconnectReason$ - .pipe(distinctUntilChanged(), pairwise(), scope.bind()) - .subscribe(([prev, reason]) => { - if (reason !== null) { - // Only begin tracking when transitioning FROM connected (null → non-null). - // This prevents the initial startup phase — where we may be non-null before - // the first real connection — from being counted as a reconnect. - if (prev === null) { - reconnectStart ??= { time: Date.now(), reason }; - } - } else if (reconnectStart !== null) { - PosthogAnalytics.instance.eventCallReconnecting.track( - roomId, - reconnectStart.reason, - (Date.now() - reconnectStart.time) / 1000, - ); - PosthogAnalytics.instance.eventCallEnded.cacheReconnecting( - reconnectStart.reason, - ); - reconnectStart = null; - } - }); - - // inform the host about the connect and disconnect intent from the user. - scope - .behavior(joinAndPublishRequested$.pipe(pairwise(), scope.bind()), [ - undefined, - joinAndPublishRequested$.value, - ]) - .subscribe(([prev, current]) => { - // JOIN prev=false (was left) => current-true (now joiend) - if (!prev && current) { - hostBridge.notifyJoined().catch((e) => { - logger.error("Failed to notify the host that we joined", e); - }); - } - // LEAVE prev=false (was joined) => current-true (now left) - if (prev && !current) { - hostBridge.notifyHungUp().catch((e) => { - logger.error("Failed to notify the host that we hung up", e); - }); - } - }); - - muteStates.video.enabled$.pipe(scope.bind()).subscribe((videoEnabled) => { - void matrixRTCSession - .updateCallIntent(videoEnabled ? "video" : "audio") - .catch((e) => { - if (e instanceof Error && e.message === "Not connected yet") { - logger.debug( - "'not connected yet' while updating the call intent (this is expected on startup)", - ); - } else { - throw e; - } - }); - }); - - // Join and leave the session as needed - scope.reconcile( - scope.behavior( - combineLatest([joinParams$, joinAndPublishRequested$, authenticated$]), - ), - async ([joinParams, shouldConnect, authenticated]) => { - const sessionConfig = Config.get().matrix_rtc_session; - - // Only join once authenticated, to ensure the LiveKit room exists first. - // Otherwise, subscribers authenticating with the OpenID flow may fail to - // join (they lack permission to create the room) and never retry. - if (joinParams && shouldConnect && authenticated) { - try { - joinMatrixRTC( - joinParams.transport, - joinParams.delegationSupported - ? sessionConfig.delegated_delayed_leave - : sessionConfig.delayed_leave, - ); - } catch (error) { - logger.error("Error entering RTC session", error); - if (error instanceof Error) - setMatrixError(new MembershipManagerError(error)); - } - - // Clean-up callback to leave the session - return Promise.resolve(async (): Promise => { - try { - await matrixRTCSession.leave(1000); - } catch (e) { - logger.error("Error leaving RTC session", e); - } - }); - } - }, - ); - - const participant$ = scope.behavior( - localConnection$.pipe( - map((c) => c?.livekitRoom?.localParticipant ?? null), - tap((p) => { - logger.debug("participant$ updated:", p?.identity); - }), - ), - ); - - // Delegate delayed leaves to the SFU - scope.reconcile( - scope.behavior(combineLatest([joinParams$, delayId$])), - async ([joinParams, delayId]) => { - if (joinParams?.delegationSupported && delayId !== null) { - try { - logger.info( - `Delegating delayed leave to ${JSON.stringify(joinParams.transport)}…`, - ); - await delegateDelayedLeave({ - client, - membership: ownMembershipIdentity, - transport: joinParams.transport, - slotId: matrixRTCSession.slotId, - roomId, - delayId, - logger, - }); - logger.info("Delayed leave successfully delegated"); - } catch (e) { - // TODO: Surface this to the user as a service interruption? - logger.error(`Failed to delegate leave`, e); - } - } - }, - ); - - // Pause upstream of all local media tracks when we're disconnected from - // MatrixRTC, because it can be an unpleasant surprise for the app to say - // 'reconnecting' and yet still be transmitting your media to others. - // We use matrixConnected$ rather than reconnecting$ because we want to - // pause tracks during the initial joining sequence too until we're sure - // that our own media is displayed on screen. - // TODO refactor this based no livekitState$ - combineLatest([participant$, homeserverConnected.combined$]) - .pipe(scope.bind()) - .subscribe(([participant, [connected]]) => { - if (!participant) return; - const publications = participant.trackPublications.values(); - if (connected) { - for (const p of publications) { - if (p.track?.isUpstreamPaused === true) { - const kind = p.track.kind; - logger.info( - `Resuming ${kind} track (MatrixRTC connection present)`, - ); - p.track - .resumeUpstream() - .catch((e) => - logger.error( - `Failed to resume ${kind} track after MatrixRTC reconnection`, - e, - ), - ); - } - } - } else { - for (const p of publications) { - if (p.track?.isUpstreamPaused === false) { - const kind = p.track.kind; - logger.info( - `Pausing ${kind} track (uncertain MatrixRTC connection)`, - ); - p.track - .pauseUpstream() - .catch((e) => - logger.error( - `Failed to pause ${kind} track after entering uncertain MatrixRTC connection`, - e, - ), - ); - } - } - } - }); - - /** - * Whether the user is currently sharing their screen. - */ - const sharingScreen$ = scope.behavior( - participant$.pipe( - switchMap((p) => (p !== null ? observeSharingScreen$(p) : of(false))), - ), - ); - - const screenShareError$ = new BehaviorSubject(null); - let toggleScreenSharing: (() => void) | null = null; - if ( - "getDisplayMedia" in (navigator.mediaDevices ?? {}) && - !hideScreensharing - ) { - toggleScreenSharing = (): void => { - const screenshareSettings: ScreenShareCaptureOptions = { - // Screen share audio shouldn't have any filtering. - // "echoCancellation" is purposely excluded, as setting it to - // false causes the screen share audio track to include - // an echo of the incoming participant's voice - audio: { - autoGainControl: false, - noiseSuppression: false, - voiceIsolation: false, - }, - selfBrowserSurface: "include", - surfaceSwitching: "include", - systemAudio: "include", - }; - - let publishOptions: TrackPublishOptions | undefined; - - if (advancedScreenShare.getValue()) { - // User has advanced screen share settings enabled - const { width, height } = parseResolution( - screenShareResolution.getValue(), - ); - const fps = screenShareFramerate.getValue(); - const bps = screenShareBitrate.getValue(); - const codec = screenShareCodec.getValue(); - - screenshareSettings.resolution = { - width, - height, - frameRate: fps, - }; - - publishOptions = { - screenShareEncoding: { - maxBitrate: bps, - maxFramerate: fps, - }, - videoCodec: codec, - }; - } else { - // Fall back to config.json settings if available - const screenConf = Config.get().media_quality?.screen_share; - if (screenConf?.max_resolution) { - screenshareSettings.resolution = { - width: Math.round((screenConf.max_resolution * 16) / 9), - height: screenConf.max_resolution, - frameRate: screenConf.max_framerate ?? 30, - }; - } - } - - const targetScreenshareState = !sharingScreen$.value; - logger.info( - `toggleScreenSharing called. Switching ${ - targetScreenshareState ? "On" : "Off" - }`, - ); - // If a connection is ready, toggle screen sharing. - // We deliberately do nothing in the case of a null connection because - // it looks nice for the call control buttons to all become available - // at once upon joining the call, rather than introducing a disabled - // state. The user can just click again. - // We also allow screen sharing to be toggled even if the connection - // is still initializing or publishing tracks, because there's no - // technical reason to disallow this. LiveKit will publish if it can. - const participant = participant$.value; - if (!participant) return; - watchScreenShareToggle( - participant.setScreenShareEnabled( - targetScreenshareState, - screenshareSettings, - publishOptions, - ), - targetScreenshareState, - logger, - (e) => screenShareError$.next(e), - ); - }; - } - - return { - startTracks, - requestJoinAndPublish, - requestDisconnect, - localMemberState$, - participant$, - reconnecting$, - connected$: matrixAndLivekitConnected$, - disconnected$: scope.behavior( - homeserverConnected.rtsSession$.pipe( - map((state) => state === RTCSessionStatus.Disconnected), - ), - ), - sharingScreen$, - toggleScreenSharing, - screenShareError$, - dismissScreenShareError: () => screenShareError$.next(null), - connection$: localConnection$, - internalLoggerRef: logger, - }; -}; - -/** - * Logs the outcome of a screen share toggle and reports failures. - * - * getDisplayMedia may legitimately take a long time (the user is choosing - * what to share) or never settle at all, so nothing is inferred from silence: - * the request and its completion are logged with the elapsed time so that a - * hang is visible in the logs, and only an explicit rejection is reported. - * - * The user cancelling the picker rejects with a NotAllowedError; that is - * logged but not reported. - */ -export function watchScreenShareToggle( - toggle: Promise, - enable: boolean, - logger: Logger, - onError: (e: Error) => void, -): void { - const what = `Screen share ${enable ? "start" : "stop"}`; - const started = Date.now(); - const elapsed = (): string => `${Date.now() - started} ms`; - logger.info(`${what} requested`); - toggle.then( - () => logger.info(`${what} completed in ${elapsed()}`), - (e: unknown) => { - logger.error(`${what} failed after ${elapsed()}:`, e); - if (e instanceof DOMException && e.name === "NotAllowedError") return; - onError(e instanceof Error ? e : new Error(String(e))); - }, - ); -} - -export function observeSharingScreen$(p: Participant): Observable { - return observeParticipantEvents( - p, - ParticipantEvent.TrackPublished, - ParticipantEvent.TrackUnpublished, - ParticipantEvent.LocalTrackPublished, - ParticipantEvent.LocalTrackUnpublished, - ).pipe(map((p) => p.isScreenShareEnabled)); -} - -interface EnterRTCSessionOptions { - encryptMedia: boolean; - matrixRTCMode: MatrixRTCMode; - delayedLeaveTimings: ResolvedDelayedLeaveTimings; - /** Whether and what kind of notification to send when joining. */ - sendNotificationType?: RTCNotificationType; - /** The kind of call being placed. */ - callIntent?: RTCCallIntent; -} - -/** - * Does the necessary steps to enter the RTC session on the matrix side: - * - Preparing the membership info (FOCUS to use, options) - * - Sends the matrix event to join the call, and starts the membership manager: - * - Delay events management - * - Handles retries (fails only after several attempts) - * - * @param rtcSession - The MatrixRTCSession to join. - * @param ownMembershipIdentity - Options for entering the RTC session. - * @param transport - The UnstableLivekitTransport to use for this session. - * @param options - `encryptMedia`: Whether to encrypt media. `matrixRTCMode`: The - * Matrix RTC mode to use. `delayedLeaveTimings`: The preferred timings for - * delayed leave events. `sendNotificationType`: Whether and what kind of - * notification to send on join. `callIntent`: The kind of call being placed. - * @throws If the host could not be told that we are joining. - */ -// Exported for unit testing -export function enterRTCSession( - rtcSession: MatrixRTCSession, - ownMembershipIdentity: CallMembershipIdentityParts, - transport: UnstableLivekitTransport, - { - encryptMedia, - matrixRTCMode, - delayedLeaveTimings, - sendNotificationType: notificationType, - callIntent, - }: EnterRTCSessionOptions, -): void { - PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date()); - PosthogAnalytics.instance.eventCallStarted.track(rtcSession.room.roomId); - - // This must be called before we start trying to join the call, as we need to - // have started tracking by the time calls start getting created. - // groupCallOTelMembership?.onJoinCall(); - - const { - sync_disconnect_grace_period_ms: gracePeriod, - matrix_rtc_session: sessionConfig, - } = Config.get(); - const retryInterval = sessionConfig.network_error_retry_ms; - - // Set maximumNetworkErrorRetryCount such that we will consider the client - // disconnected as soon as either it fails to sync for longer than the grace - // period, or it is likely that a delayed leave event has been sent. - // Math.min is used to account for the respective worst case: /sync not available or leave event emitted. - const maxWaitTime = Math.min(gracePeriod, delayedLeaveTimings.delay_ms); - const maximumNetworkErrorRetryCount = - Math.ceil(maxWaitTime / retryInterval) + 1; - - // Multi-sfu does not need a preferred foci list. just the focus that is actually used. - // TODO where/how do we track errors originating from the ongoing rtcSession? - - rtcSession.join(ownMembershipIdentity, [transport], { - notificationType, - callIntent, - manageMediaKeys: encryptMedia, - delayedLeaveEventRestartMs: delayedLeaveTimings.restart_ms, - delayedLeaveEventDelayMs: delayedLeaveTimings.delay_ms, - delayedLeaveEventRestartLocalTimeoutMs: - delayedLeaveTimings.restart_timeout_ms, - networkErrorRetryMs: sessionConfig.network_error_retry_ms, - membershipEventExpiryMs: sessionConfig.membership_event_expiry_ms, - keyRotationParticipantLimit: sessionConfig.key_rotation_participant_limit, - unstableSendStickyEvents: matrixRTCMode === MatrixRTCMode.Matrix_2_0, - maximumNetworkErrorRetryCount: maximumNetworkErrorRetryCount, - }); -} diff --git a/src/state/CallViewModel/localMember/LocalTransport.test.ts b/src/state/CallViewModel/localMember/LocalTransport.test.ts deleted file mode 100644 index ed6654e2f..000000000 --- a/src/state/CallViewModel/localMember/LocalTransport.test.ts +++ /dev/null @@ -1,126 +0,0 @@ -/* -Copyright 2025 Element Creations Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - afterEach, - beforeEach, - describe, - expect, - it, - type MockedObject, - vi, -} from "vitest"; -import fetchMock from "fetch-mock"; - -import { mockConfig } from "../../../utils/test"; -import { getLocalTransport } from "./LocalTransport"; -import { MatrixRTCTransportMissingError } from "../../../utils/errors"; -import { customTransport } from "../../../settings/settings"; -import { type MatrixClient } from "matrix-js-sdk"; - -describe("LocalTransport", () => { - beforeEach(() => vi.clearAllMocks()); - - it("throws if config is missing", async () => { - await expect( - getLocalTransport({ - _unstable_getRTCTransports: async () => Promise.resolve([]), - getDomain: () => "example.org", - }), - ).rejects.toThrow(new MatrixRTCTransportMissingError("example.org")); - }); - - it("returns preferred transport", async () => { - // Use config so transport discovery succeeds - mockConfig({ - livekit: { livekit_service_url: "https://lk.example.org" }, - }); - - expect( - await getLocalTransport({ - _unstable_getRTCTransports: async () => Promise.resolve([]), - getDomain: () => "example.org", - }), - ).toStrictEqual({ - transport: { - livekit_service_url: "https://lk.example.org", - type: "livekit", - }, - serverName: "example.org", - }); - }); - - describe("transport configuration mechanisms", () => { - let client: MockedObject< - Pick - >; - beforeEach(() => { - mockConfig({}); - customTransport.setValue(customTransport.defaultValue); - client = { - getDomain: vi.fn().mockReturnValue("example.org"), - _unstable_getRTCTransports: vi.fn().mockResolvedValue([]), - }; - }); - - afterEach(() => { - fetchMock.reset(); - }); - - it("supports getting transport via application config", async () => { - mockConfig({ - livekit: { livekit_service_url: "https://lk.example.org" }, - }); - - expect(await getLocalTransport(client)).toStrictEqual({ - transport: { - livekit_service_url: "https://lk.example.org", - type: "livekit", - }, - serverName: "example.org", - }); - }); - - it("supports getting transport via user settings", async () => { - customTransport.setValue({ - type: "livekit", - livekit_service_url: "https://lk.example.org", - }); - - expect(await getLocalTransport(client)).toStrictEqual({ - transport: { - livekit_service_url: "https://lk.example.org", - type: "livekit", - }, - serverName: "example.org", - }); - }); - - it("supports getting transport via backend", async () => { - client._unstable_getRTCTransports.mockResolvedValue([ - { type: "livekit", livekit_service_url: "https://lk.example.org" }, - ]); - - expect(await getLocalTransport(client)).toStrictEqual({ - transport: { - livekit_service_url: "https://lk.example.org", - type: "livekit", - }, - serverName: "example.org", - }); - }); - - it("throws if no options are available", async () => { - await expect( - getLocalTransport({ - getDomain: () => "example.org", - _unstable_getRTCTransports: async () => Promise.resolve([]), - }), - ).rejects.toThrow(new MatrixRTCTransportMissingError("example.org")); - }); - }); -}); diff --git a/src/state/CallViewModel/localMember/LocalTransport.ts b/src/state/CallViewModel/localMember/LocalTransport.ts deleted file mode 100644 index 3b43f8112..000000000 --- a/src/state/CallViewModel/localMember/LocalTransport.ts +++ /dev/null @@ -1,41 +0,0 @@ -/* -Copyright 2025 Element Creations Ltd. - -SPDX-License-IdFentifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { type MatrixClient } from "matrix-js-sdk"; -import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; - -import { Config } from "../../../config/Config.ts"; -import { MatrixRTCTransportMissingError } from "../../../utils/errors.ts"; -import { customTransport as customTransportSetting } from "../../../settings/settings.ts"; -import { RtcTransportAutoDiscovery } from "./RtcTransportAutoDiscovery.ts"; -import { type TransportLocator } from "../../../livekit/auth/types.ts"; - -/** - * Determines the transport to advertise in our MatrixRTC membership and publish - * media on. - */ -export async function getLocalTransport( - client: Pick, -): Promise { - const serverName = client.getDomain(); - if (serverName === null) throw new Error("No server name"); - - const discovery = new RtcTransportAutoDiscovery({ - client, - resolvedConfig: Config.get(), - logger: rootLogger.getChild("[LocalTransport]"), - }); - - // Respect the user's custom transport, if set - const transport = - customTransportSetting.value$.value ?? - (await discovery.discoverPreferredTransport()); - - if (transport === null) throw new MatrixRTCTransportMissingError(serverName); - - return { transport, serverName }; -} diff --git a/src/state/CallViewModel/localMember/Publisher.test.ts b/src/state/CallViewModel/localMember/Publisher.test.ts deleted file mode 100644 index 5d9f03784..000000000 --- a/src/state/CallViewModel/localMember/Publisher.test.ts +++ /dev/null @@ -1,402 +0,0 @@ -/* -Copyright 2025 Element Creations Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { afterEach, beforeEach, describe, expect, it, test, vi } from "vitest"; -import { - ConnectionState as LivekitConnectionState, - LocalParticipant, - type LocalTrack, - type LocalTrackPublication, - ParticipantEvent, - Track, -} from "livekit-client"; -import { BehaviorSubject } from "rxjs"; -import { logger } from "matrix-js-sdk/lib/logger"; - -import { ObservableScope } from "../../ObservableScope"; -import { constant } from "../../Behavior"; -import { - flushPromises, - mockLivekitRoom, - mockMediaDevices, -} from "../../../utils/test"; -import { Publisher } from "./Publisher"; -import { type Connection } from "../remoteMembers/Connection"; -import { type MuteStates } from "../../MuteStates"; - -let scope: ObservableScope; - -beforeEach(() => { - scope = new ObservableScope(); -}); - -afterEach(() => scope.end()); - -function createMockLocalTrack(source: Track.Source): LocalTrack { - const track = { - source, - isMuted: false, - isUpstreamPaused: false, - } as Partial as LocalTrack; - - vi.mocked(track).mute = vi.fn().mockImplementation(() => { - track.isMuted = true; - }); - vi.mocked(track).unmute = vi.fn().mockImplementation(() => { - track.isMuted = false; - }); - vi.mocked(track).pauseUpstream = vi.fn().mockImplementation(() => { - // @ts-expect-error - for that test we want to set isUpstreamPaused directly - track.isUpstreamPaused = true; - }); - vi.mocked(track).resumeUpstream = vi.fn().mockImplementation(() => { - // @ts-expect-error - for that test we want to set isUpstreamPaused directly - track.isUpstreamPaused = false; - }); - - return track; -} - -function createMockMuteState(enabled$: BehaviorSubject): { - enabled$: BehaviorSubject; - syncing$: BehaviorSubject; - setHandler: (h: (enabled: boolean) => void) => void; - unsetHandler: () => void; -} { - let currentHandler = (enabled: boolean): void => {}; - - const ms = { - enabled$, - syncing$: new BehaviorSubject(false), - setHandler: vi.fn().mockImplementation((h: (enabled: boolean) => void) => { - currentHandler = h; - }), - unsetHandler: vi.fn().mockImplementation(() => { - currentHandler = (enabled: boolean): void => {}; - }), - }; - // forward enabled$ emissions to the current handler - enabled$.subscribe((enabled) => { - logger.info(`MockMuteState: enabled changed to ${enabled}`); - currentHandler(enabled); - }); - - return ms; -} - -let connection: Connection; -let muteStates: MuteStates; -let localParticipant: LocalParticipant; -let audioEnabled$: BehaviorSubject; -let videoEnabled$: BehaviorSubject; -let trackPublications: LocalTrackPublication[]; -// use it to control when track creation resolves, default to resolved -let createTrackLock: Promise; - -beforeEach(() => { - trackPublications = []; - audioEnabled$ = new BehaviorSubject(false); - videoEnabled$ = new BehaviorSubject(false); - createTrackLock = Promise.resolve(); - - muteStates = { - audio: createMockMuteState(audioEnabled$), - video: createMockMuteState(videoEnabled$), - } as unknown as MuteStates; - - const mockSendDataPacket = vi.fn(); - const mockEngine = { - client: { - sendUpdateLocalMetadata: vi.fn(), - }, - on: vi.fn().mockReturnThis(), - sendDataPacket: mockSendDataPacket, - }; - - localParticipant = new LocalParticipant( - "local-sid", - "local-identity", - // @ts-expect-error - for that test we want a real LocalParticipant to have the pending publications logic - mockEngine, - { - adaptiveStream: true, - dynacase: false, - audioCaptureDefaults: {}, - videoCaptureDefaults: {}, - stopLocalTrackOnUnpublish: true, - reconnectPolicy: "always", - disconnectOnPageLeave: true, - }, - new Map(), - {}, - {}, - {}, - ); - - vi.mocked(localParticipant).createTracks = vi - .fn() - .mockImplementation(async (opts) => { - const tracks: LocalTrack[] = []; - if (opts.audio) { - tracks.push(createMockLocalTrack(Track.Source.Microphone)); - } - if (opts.video) { - tracks.push(createMockLocalTrack(Track.Source.Camera)); - } - await createTrackLock; - return tracks; - }); - - vi.mocked(localParticipant).publishTrack = vi - .fn() - .mockImplementation(async (track: LocalTrack) => { - const pub = { - track, - source: track.source, - mute: track.mute, - unmute: track.unmute, - } as Partial as LocalTrackPublication; - trackPublications.push(pub); - localParticipant.emit(ParticipantEvent.LocalTrackPublished, pub); - return Promise.resolve(pub); - }); - - vi.mocked(localParticipant).getTrackPublication = vi - .fn() - .mockImplementation((source: Track.Source) => { - return trackPublications.find((pub) => pub.track?.source === source); - }); - - connection = { - state$: constant({ - state: "ConnectedToLkRoom", - livekitConnectionState$: constant(LivekitConnectionState.Connected), - }), - livekitRoom: mockLivekitRoom({ - localParticipant: localParticipant, - }), - } as unknown as Connection; -}); - -describe("Publisher", () => { - let publisher: Publisher; - - beforeEach(() => { - publisher = new Publisher( - connection, - mockMediaDevices({}), - muteStates, - constant({ supported: false, processor: undefined }), - logger, - false, - ); - }); - - afterEach(async () => { - await publisher.destroy(); - }); - - it("Should not create tracks if started muted to avoid unneeded permission requests", async () => { - const createTracksSpy = vi.spyOn( - connection.livekitRoom.localParticipant, - "createTracks", - ); - - audioEnabled$.next(false); - videoEnabled$.next(false); - await publisher.createAndSetupTracks(); - - expect(createTracksSpy).not.toHaveBeenCalled(); - }); - - it("should unsetHandler and stop tracks on destroy", async () => { - // setup all spies - const unsetVideoSpy = vi.spyOn( - ( - publisher as unknown as { - muteStates: { video: { unsetHandler: () => void } }; - } - ).muteStates.video, - "unsetHandler", - ); - const unsetAudioSpy = vi.spyOn( - ( - publisher as unknown as { - muteStates: { audio: { unsetHandler: () => void } }; - } - ).muteStates.audio, - "unsetHandler", - ); - const scopeEndSpy = vi.spyOn( - (publisher as unknown as { scope: { end: () => void } }).scope, - "end", - ); - const stopTracksSpy = vi.spyOn(publisher, "stopTracks"); - // destroy publisher - await publisher.destroy(); - - expect(stopTracksSpy).toHaveBeenCalledOnce(); - expect(unsetVideoSpy).toHaveBeenCalledOnce(); - expect(unsetAudioSpy).toHaveBeenCalledOnce(); - expect(scopeEndSpy).toHaveBeenCalled(); - }); - - it("Should minimize permission request by querying create at once", async () => { - const enableCameraAndMicrophoneSpy = vi.spyOn( - localParticipant, - "enableCameraAndMicrophone", - ); - const createTracksSpy = vi.spyOn(localParticipant, "createTracks"); - - audioEnabled$.next(true); - videoEnabled$.next(true); - await publisher.createAndSetupTracks(); - await flushPromises(); - - expect(enableCameraAndMicrophoneSpy).toHaveBeenCalled(); - - // It should create both at once - expect(createTracksSpy).toHaveBeenCalledWith({ - audio: true, - video: true, - }); - }); - - it("Ensure no data is streamed until publish has been called", async () => { - audioEnabled$.next(true); - await publisher.createAndSetupTracks(); - - // The track should be created and paused - expect(localParticipant.createTracks).toHaveBeenCalledWith({ - audio: true, - video: undefined, - }); - await flushPromises(); - expect(localParticipant.publishTrack).toHaveBeenCalled(); - - await flushPromises(); - const track = localParticipant.getTrackPublication( - Track.Source.Microphone, - )?.track; - expect(track).toBeDefined(); - expect(track!.pauseUpstream).toHaveBeenCalled(); - expect(track!.isUpstreamPaused).toBe(true); - }); - - it("Ensure resume upstream when published is called", async () => { - videoEnabled$.next(true); - await publisher.createAndSetupTracks(); - // await flushPromises(); - await publisher.startPublishing(); - - const track = localParticipant.getTrackPublication( - Track.Source.Camera, - )?.track; - expect(track).toBeDefined(); - // expect(track.pauseUpstream).toHaveBeenCalled(); - expect(track!.isUpstreamPaused).toBe(false); - }); - - describe("Mute states", () => { - let publisher: Publisher; - beforeEach(() => { - publisher = new Publisher( - connection, - mockMediaDevices({}), - muteStates, - constant({ supported: false, processor: undefined }), - logger, - false, - ); - }); - afterEach(async () => { - await publisher.destroy(); - }); - - test.each([ - { mutes: { audioEnabled: true, videoEnabled: false } }, - { mutes: { audioEnabled: true, videoEnabled: false } }, - ])("only create the tracks that are unmuted $mutes", async ({ mutes }) => { - // Ensure all muted - audioEnabled$.next(mutes.audioEnabled); - videoEnabled$.next(mutes.videoEnabled); - - vi.mocked(connection.livekitRoom.localParticipant).createTracks = vi - .fn() - .mockResolvedValue([]); - - await publisher.createAndSetupTracks(); - - expect( - connection.livekitRoom.localParticipant.createTracks, - ).toHaveBeenCalledOnce(); - - expect( - connection.livekitRoom.localParticipant.createTracks, - ).toHaveBeenCalledWith({ - audio: mutes.audioEnabled ? true : undefined, - video: mutes.videoEnabled ? true : undefined, - }); - }); - }); - - it("does mute unmute audio", async () => {}); -}); - -describe("Bug fix", () => { - // There is a race condition when creating and publishing tracks while the mute state changes. - // This race condition could cause tracks to be published even though they are muted at the - // beginning of a call coming from lobby. - // This is caused by our stack using manually the low level API to create and publish tracks, - // but also using the higher level setMicrophoneEnabled and setCameraEnabled functions that also create - // and publish tracks, and managing pending publications. - // Race is as follow, on creation of the Publisher we create the tracks then publish them. - // If in the middle of that process the mute state changes: - // - the `setMicrophoneEnabled` will be no-op because it is not aware of our created track and can't see any pending publication - // - If start publication is requested it will publish the track even though there was a mute request. - it("wrongly publish tracks while muted", async () => { - // setLogLevel(`debug`); - const publisher = new Publisher( - connection, - mockMediaDevices({}), - muteStates, - constant({ supported: false, processor: undefined }), - logger, - false, - ); - audioEnabled$.next(true); - - const resolvers = Promise.withResolvers(); - createTrackLock = resolvers.promise; - - // Initially the audio is unmuted, so creating tracks should publish the audio track - const createTracks = publisher.createAndSetupTracks(); - void publisher.startPublishing(); - void createTracks.then(() => { - void publisher.startPublishing(); - }); - // now mute the audio before allowing track creation to complete - audioEnabled$.next(false); - resolvers.resolve(undefined); - await createTracks; - - await flushPromises(); - - const track = localParticipant.getTrackPublication( - Track.Source.Microphone, - )?.track; - expect(track).toBeDefined(); - - try { - expect(localParticipant.publishTrack).not.toHaveBeenCalled(); - } catch { - expect(track!.mute).toHaveBeenCalled(); - expect(track!.isMuted).toBe(true); - } - await publisher.destroy(); - }); -}); diff --git a/src/state/CallViewModel/localMember/Publisher.ts b/src/state/CallViewModel/localMember/Publisher.ts deleted file mode 100644 index 5353a98d4..000000000 --- a/src/state/CallViewModel/localMember/Publisher.ts +++ /dev/null @@ -1,445 +0,0 @@ -/* -Copyright 2025 Element Creations Ltd. -Copyright 2025 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ -import { - ConnectionState as LivekitConnectionState, - type LocalTrackPublication, - LocalVideoTrack, - ParticipantEvent, - type Room as LivekitRoom, - Track, -} from "livekit-client"; -import { - map, - NEVER, - type Observable, - type Subscription, - switchMap, -} from "rxjs"; -import { type Logger } from "matrix-js-sdk/lib/logger"; - -import type { Behavior } from "../../Behavior.ts"; -import type { MediaDevices, SelectedDevice } from "../../MediaDevices.ts"; -import type { MuteStates } from "../../MuteStates.ts"; -import { - type ProcessorState, - trackProcessorSync, -} from "../../../livekit/TrackProcessorContext.tsx"; - -import { observeTrackReference$ } from "../../observeTrackReference"; -import { type Connection } from "../remoteMembers/Connection.ts"; -import { ObservableScope } from "../../ObservableScope.ts"; - -/** - * A wrapper for a Connection object. - * This wrapper will manage the connection used to publish to the LiveKit room. - * The Publisher is also responsible for creating the media tracks. - */ -export class Publisher { - /** - * By default, livekit will start publishing tracks as soon as they are created. - * In the matrix RTC world, we want to control when tracks are published based - * on whether the user is part of the RTC session or not. - */ - public shouldPublish = false; - - private readonly scope = new ObservableScope(); - - /** - * Creates a new Publisher. - * @param connection - The connection to use for publishing. - * @param devices - The media devices to use for audio and video input. - * @param muteStates - The mute states for audio and video. - * @param trackerProcessorState$ - The processor state for the video track processor (e.g. background blur). - * @param logger - The logger to use for logging :D. - * @param controlledAudioDevices - Whether the app hosting Element Call - * controls the audio output devices, rather than the browser. - */ - public constructor( - private connection: Pick, //setE2EEEnabled, - devices: MediaDevices, - private readonly muteStates: MuteStates, - trackerProcessorState$: Behavior, - private logger: Logger, - controlledAudioDevices: boolean, - ) { - const room = connection.livekitRoom; - - room.setE2EEEnabled(room.options.e2ee !== undefined)?.catch((e: Error) => { - this.logger.error("Failed to set E2EE enabled on room", e); - }); - - // Setup track processor syncing (blur) - this.observeTrackProcessors(this.scope, room, trackerProcessorState$); - // Observe media device changes and update LiveKit active devices accordingly - this.observeMediaDevices(this.scope, devices, controlledAudioDevices); - - this.workaroundRestartAudioInputTrackChrome(devices, this.scope); - - this.connection.livekitRoom.localParticipant.on( - ParticipantEvent.LocalTrackPublished, - this.onLocalTrackPublished.bind(this), - ); - } - - public async destroy(): Promise { - this.scope.end(); - this.logger.info("Scope ended -> unset handler"); - this.muteStates.audio.unsetHandler(); - this.muteStates.video.unsetHandler(); - - this.logger.info(`Start to stop tracks`); - try { - await this.stopTracks(); - this.logger.info(`Done to stop tracks`); - } catch (e) { - this.logger.error(`Failed to stop tracks: ${e}`); - } - } - - // LiveKit will publish the tracks as soon as they are created - // but we want to control when tracks are published. - // We cannot just mute the tracks, even if this will effectively stop the publishing, - // it would also prevent the user from seeing their own video/audio preview. - // So for that we use pauseUpStream(): Stops sending media to the server by replacing - // the sender track with null, but keeps the local MediaStreamTrack active. - // The user can still see/hear themselves locally, but remote participants see nothing. - private onLocalTrackPublished( - localTrackPublication: LocalTrackPublication, - ): void { - this.logger.info("Local track published", localTrackPublication); - const lkRoom = this.connection.livekitRoom; - if (!this.shouldPublish) { - this.logger.debug("Not publishing, pausing upstream"); - this.pauseUpstreams(lkRoom, [localTrackPublication.source]).catch((e) => { - this.logger.error(`Failed to pause upstreams`, e); - }); - } - if (localTrackPublication.source === Track.Source.Microphone) { - const muteState = this.muteStates.audio; - // skip this if a sync is in progress: enabled$ still reflects the old - // state while the handler is mid-flight, so the handler itself will apply - // the correct mute state once it completes. - if (!muteState.syncing$.value) { - const enabled = muteState.enabled$.value; - if (!enabled) { - this.logger.info( - "Local audio track just published but muted meanwhile, setting enabled to false", - ); - lkRoom.localParticipant.setMicrophoneEnabled(false).catch((e) => { - this.logger.error( - `Failed to enable microphone track, enabled:${enabled}`, - e, - ); - }); - } - } - } else if (localTrackPublication.source === Track.Source.Camera) { - const muteState = this.muteStates.video; - // skip this if a sync is in progress: enabled$ still reflects the old - // state while the handler is mid-flight, so the handler itself will apply - // the correct mute state once it completes. - if (!muteState.syncing$.value) { - const enabled = muteState.enabled$.value; - if (!enabled) { - this.logger.info( - "Local video track just published but muted meanwhile, setting enabled to false", - ); - lkRoom.localParticipant.setCameraEnabled(false).catch((e) => { - this.logger.error( - `Failed to enable camera track, enabled:${enabled}`, - e, - ); - }); - } - } - } - } - /** - * Create and setup local audio and video tracks based on the current mute states. - * It creates the tracks only if audio and/or video is enabled, to avoid unnecessary - * permission prompts. - * - * It also observes mute state changes to update LiveKit microphone/camera states accordingly. - * If a track is not created initially because disabled, it will be created when unmuting. - * - * This call is not blocking anymore, instead callers can listen to the - * `RoomEvent.MediaDevicesError` event in the LiveKit room to be notified of any errors. - * - */ - public async createAndSetupTracks(): Promise { - this.logger.debug("createAndSetupTracks called"); - const lkRoom = this.connection.livekitRoom; - // Observe mute state changes and update LiveKit microphone/camera states accordingly - this.observeMuteStates(); - - // Check if audio and/or video is enabled. We only create tracks if enabled, - // because it could prompt for permission, and we don't want to do that unnecessarily. - const audio = this.muteStates.audio.enabled$.value; - const video = this.muteStates.video.enabled$.value; - - // We don't await the creation, because livekit could block until the tracks - // are fully published, and not only that they are created. - // We don't have control on that, localParticipant creates and publishes the tracks - // asap. - // We are using the `ParticipantEvent.LocalTrackPublished` to be notified - // when tracks are actually published, and at that point - // we can pause upstream if needed (depending on if startPublishing has been called). - if (audio && video) { - // Enable both at once in order to have a single permission prompt! - void lkRoom.localParticipant.enableCameraAndMicrophone(); - } else if (audio) { - void lkRoom.localParticipant.setMicrophoneEnabled(true); - } else if (video) { - void lkRoom.localParticipant.setCameraEnabled(true); - } - - return Promise.resolve(); - } - - private async pauseUpstreams( - lkRoom: LivekitRoom, - sources: Track.Source[], - ): Promise { - for (const source of sources) { - const track = lkRoom.localParticipant.getTrackPublication(source)?.track; - if (track) { - await track.pauseUpstream(); - } else { - this.logger.warn( - `No track found for source ${source} to pause upstream`, - ); - } - } - } - - private async resumeUpstreams( - lkRoom: LivekitRoom, - sources: Track.Source[], - ): Promise { - for (const source of sources) { - const track = lkRoom.localParticipant.getTrackPublication(source)?.track; - if (track) { - await track.resumeUpstream(); - } else { - this.logger.warn( - `No track found for source ${source} to resume upstream`, - ); - } - } - } - - /** - * - * Request to publish local tracks to the LiveKit room. - * This will wait for the connection to be ready before publishing. - * Livekit also have some local retry logic for publishing tracks. - * Can be called multiple times, localparticipant manages the state of published tracks (or pending publications). - * - * @returns - */ - public async startPublishing(): Promise { - if (this.shouldPublish) { - this.logger.debug(`Already publishing, ignoring startPublishing call`); - return; - } - this.shouldPublish = true; - this.logger.debug("startPublishing called"); - - const lkRoom = this.connection.livekitRoom; - - // Resume upstream for both audio and video tracks - // We need to call it explicitly because call setTrackEnabled does not always - // resume upstream. It will only if you switch the track from disabled to enabled, - // but if the track is already enabled but upstream is paused, it won't resume it. - // TODO what about screen share? - try { - await this.resumeUpstreams(lkRoom, [ - Track.Source.Microphone, - Track.Source.Camera, - ]); - } catch (e) { - this.logger.error(`Failed to resume upstreams`, e); - } - } - - public async stopPublishing(): Promise { - this.logger.debug("stopPublishing called"); - this.shouldPublish = false; - // Pause upstream will stop sending media to the server, while keeping - // the local MediaStreamTrack active, so the user can still see themselves. - await this.pauseUpstreams(this.connection.livekitRoom, [ - Track.Source.Microphone, - Track.Source.Camera, - Track.Source.ScreenShare, - ]); - } - - public async stopTracks(): Promise { - const lkRoom = this.connection.livekitRoom; - for (const source of [ - Track.Source.Microphone, - Track.Source.Camera, - Track.Source.ScreenShare, - ]) { - const localPub = lkRoom.localParticipant.getTrackPublication(source); - if (localPub?.track) { - // stops and unpublishes the track - await lkRoom.localParticipant.unpublishTrack(localPub!.track, true); - } - } - } - - /// Private methods - - // Restart the audio input track whenever we detect that the active media - // device has changed to refer to a different hardware device. We do this - // for the sake of Chrome, which provides a "default" device that is meant - // to match the system's default audio input, whatever that may be. - // This is special-cased for only audio inputs because we need to dig around - // in the LocalParticipant object for the track object and there's not a nice - // way to do that generically. There is usually no OS-level default video capture - // device anyway, and audio outputs work differently. - private workaroundRestartAudioInputTrackChrome( - devices: MediaDevices, - scope: ObservableScope, - ): void { - const lkRoom = this.connection.livekitRoom; - devices.audioInput.selected$ - .pipe( - switchMap((device) => device?.hardwareDeviceChange$ ?? NEVER), - scope.bind(), - ) - .subscribe(() => { - if (lkRoom.state != LivekitConnectionState.Connected) return; - const activeMicTrack = Array.from( - lkRoom.localParticipant.audioTrackPublications.values(), - ).find((d) => d.source === Track.Source.Microphone)?.track; - - if ( - activeMicTrack && - // only restart if the stream is still running: LiveKit will detect - // when a track stops & restart appropriately, so this is not our job. - // Plus, we need to avoid restarting again if the track is already in - // the process of being restarted. - activeMicTrack.mediaStreamTrack.readyState !== "ended" - ) { - this.logger?.info( - "Restarting audio device track due to active media device changed (workaroundRestartAudioInputTrackChrome)", - ); - // Restart the track, which will cause Livekit to do another - // getUserMedia() call with deviceId: default to get the *new* default device. - // Note that room.switchActiveDevice() won't work: Livekit will ignore it because - // the deviceId hasn't changed (was & still is default). - lkRoom.localParticipant - .getTrackPublication(Track.Source.Microphone) - ?.audioTrack?.restartTrack() - .catch((e) => { - this.logger.error(`Failed to restart audio device track`, e); - }); - } - }); - } - - // Observe changes in the selected media devices and update the LiveKit room accordingly. - private observeMediaDevices( - scope: ObservableScope, - devices: MediaDevices, - controlledAudioDevices: boolean, - ): void { - const lkRoom = this.connection.livekitRoom; - const syncDevice = ( - kind: MediaDeviceKind, - selected$: Observable, - ): Subscription => - selected$.pipe(scope.bind()).subscribe((device) => { - if (lkRoom.state != LivekitConnectionState.Connected) return; - // if (this.connectionState$.value !== ConnectionState.Connected) return; - this.logger.info( - "[LivekitRoom] syncDevice room.getActiveDevice(kind) !== d.id :", - lkRoom.getActiveDevice(kind), - " !== ", - device?.id, - ); - if ( - device !== undefined && - lkRoom.getActiveDevice(kind) !== device.id - ) { - lkRoom - .switchActiveDevice(kind, device.id) - .catch((e: Error) => - this.logger.error( - `Failed to sync ${kind} device with LiveKit`, - e, - ), - ); - } - }); - - syncDevice("audioinput", devices.audioInput.selected$); - if (!controlledAudioDevices) - syncDevice("audiooutput", devices.audioOutput.selected$); - syncDevice("videoinput", devices.videoInput.selected$); - } - - /** - * Observe changes in the mute states and update the LiveKit room accordingly. - * @private - */ - private observeMuteStates(): void { - const lkRoom = this.connection.livekitRoom; - this.muteStates.audio.setHandler(async (enable) => { - try { - this.logger.debug( - `handler: Setting LiveKit microphone enabled: ${enable}`, - ); - await lkRoom.localParticipant.setMicrophoneEnabled(enable); - // Unmute will restart the track if it was paused upstream, - // but until explicitly requested, we want to keep it paused. - if (!this.shouldPublish && enable) { - await this.pauseUpstreams(lkRoom, [Track.Source.Microphone]); - } - return enable; - } catch (e) { - this.logger.error("Failed to update LiveKit audio input mute state", e); - return lkRoom.localParticipant.isMicrophoneEnabled; - } - }); - this.muteStates.video.setHandler(async (enable) => { - try { - this.logger.debug(`handler: Setting LiveKit camera enabled: ${enable}`); - await lkRoom.localParticipant.setCameraEnabled(enable); - // Unmute will restart the track if it was paused upstream, - // but until explicitly requested, we want to keep it paused. - if (!this.shouldPublish && enable) { - await this.pauseUpstreams(lkRoom, [Track.Source.Camera]); - } - return enable; - } catch (e) { - this.logger.error("Failed to update LiveKit video input mute state", e); - return lkRoom.localParticipant.isCameraEnabled; - } - }); - } - - private observeTrackProcessors( - scope: ObservableScope, - room: LivekitRoom, - trackerProcessorState$: Behavior, - ): void { - const track$ = scope.behavior( - observeTrackReference$(room.localParticipant, Track.Source.Camera).pipe( - map((trackRef) => { - const track = trackRef?.publication.track; - return track instanceof LocalVideoTrack ? track : null; - }), - ), - null, - ); - trackProcessorSync(scope, track$, trackerProcessorState$); - } -} diff --git a/src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.test.ts b/src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.test.ts deleted file mode 100644 index d3f246c97..000000000 --- a/src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.test.ts +++ /dev/null @@ -1,153 +0,0 @@ -/* -Copyright 2025 Element Creations Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - beforeEach, - describe, - expect, - it, - type MockedObject, - vi, -} from "vitest"; -import { MatrixError } from "matrix-js-sdk"; -import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; -import { - type UnstableLivekitTransport, - type Transport, -} from "matrix-js-sdk/lib/matrixrtc"; - -import type { ResolvedConfigOptions } from "../../../config/ConfigOptions.ts"; -import { - RtcTransportAutoDiscovery, - type RtcTransportAutoDiscoveryProps, -} from "./RtcTransportAutoDiscovery.ts"; - -type DiscoveryClient = RtcTransportAutoDiscoveryProps["client"]; - -const backendTransport: UnstableLivekitTransport = { - type: "livekit", - livekit_service_url: "https://backend.example.org", -}; - -const configTransport: UnstableLivekitTransport = { - type: "livekit", - livekit_service_url: "https://config.example.org", -}; - -function makeClient(): MockedObject { - return { - getDomain: vi.fn().mockReturnValue("example.org"), - baseUrl: "https://matrix.example.org", - _unstable_getRTCTransports: vi.fn().mockResolvedValue([]), - getOpenIdToken: vi.fn(), - getDeviceId: vi.fn(), - } as unknown as MockedObject; -} - -function makeResolvedConfig(livekitServiceUrl?: string): ResolvedConfigOptions { - return { - livekit: livekitServiceUrl - ? { - livekit_service_url: livekitServiceUrl, - } - : undefined, - } as ResolvedConfigOptions; -} - -describe("RtcTransportAutoDiscovery", () => { - beforeEach(() => { - vi.clearAllMocks(); - }); - const VALID_TEST_CASES: Array<{ transports: Transport[] }> = [ - { transports: [backendTransport] }, - // will pick the first livekit transport in the list, even if there are other non-livekit transports - { transports: [{ type: "not_livekit" }, backendTransport] }, - ]; - it.each(VALID_TEST_CASES)( - "prefers backend transport other app config $transports", - async ({ transports }) => { - const client = makeClient(); - client._unstable_getRTCTransports.mockResolvedValue(transports); - - const discovery = new RtcTransportAutoDiscovery({ - client, - resolvedConfig: makeResolvedConfig(configTransport.livekit_service_url), - logger: rootLogger, - }); - - const discoveredTransport = await discovery.discoverPreferredTransport(); - - expect(discoveredTransport).toStrictEqual(backendTransport); - expect(discoveredTransport).not.toStrictEqual(configTransport); - - expect(client._unstable_getRTCTransports).toHaveBeenCalledTimes(1); - }, - ); - - it("Retries limit_exceeded backend transport", async () => { - const client = makeClient(); - client._unstable_getRTCTransports - .mockRejectedValueOnce( - new MatrixError( - { - errcode: "M_LIMIT_EXCEEDED", - error: "Too many requests", - retry_after_ms: 100, - }, - 429, - ), - ) - .mockResolvedValue([backendTransport]); - - const discovery = new RtcTransportAutoDiscovery({ - client, - resolvedConfig: makeResolvedConfig("https://config.example.org"), - logger: rootLogger, - }); - - await expect(discovery.discoverPreferredTransport()).resolves.toStrictEqual( - backendTransport, - ); - - expect(client._unstable_getRTCTransports).toHaveBeenCalledTimes(2); - }); - - const INVALID_TEST_CASES: Array<{ transports: Transport[] }> = [ - { transports: [] }, - { transports: [{ type: "not_livekit" }] }, - ]; - it.each(INVALID_TEST_CASES)( - "falls back to config when backend has no (valid) livekit transports $transports", - async ({ transports }) => { - const client = makeClient(); - client._unstable_getRTCTransports.mockResolvedValue(transports); - - const discovery = new RtcTransportAutoDiscovery({ - client, - resolvedConfig: makeResolvedConfig(configTransport.livekit_service_url), - logger: rootLogger, - }); - - const discoveredTransport = await discovery.discoverPreferredTransport(); - expect(discoveredTransport).not.toStrictEqual(backendTransport); - expect(discoveredTransport).toStrictEqual(configTransport); - }, - ); - - it("returns null when backend and config are all unavailable", async () => { - const client = makeClient(); - client._unstable_getRTCTransports.mockResolvedValue([]); - - const discovery = new RtcTransportAutoDiscovery({ - client, - resolvedConfig: makeResolvedConfig(undefined), - logger: rootLogger, - }); - - await expect(discovery.discoverPreferredTransport()).resolves.toBeNull(); - }); -}); diff --git a/src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.ts b/src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.ts deleted file mode 100644 index a5bedcd80..000000000 --- a/src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.ts +++ /dev/null @@ -1,103 +0,0 @@ -/* -Copyright 2026 Element Creations Ltd. - -SPDX-License-IdFentifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ -import { - isUnstableLivekitTransport, - type UnstableLivekitTransport, -} from "matrix-js-sdk/lib/matrixrtc"; -import { type MatrixClient } from "matrix-js-sdk"; -import { type Logger } from "matrix-js-sdk/lib/logger"; - -import type { ResolvedConfigOptions } from "../../../config/ConfigOptions.ts"; -import { doNetworkOperationWithRetry } from "../../../utils/matrix.ts"; - -type TransportDiscoveryClient = Pick< - MatrixClient, - "getDomain" | "_unstable_getRTCTransports" ->; - -export interface RtcTransportAutoDiscoveryProps { - client: TransportDiscoveryClient; - resolvedConfig: ResolvedConfigOptions; - logger: Logger; -} - -export class RtcTransportAutoDiscovery { - private readonly client: TransportDiscoveryClient; - private readonly resolvedConfig: ResolvedConfigOptions; - private readonly logger: Logger; - - public constructor({ - client, - resolvedConfig, - logger, - }: RtcTransportAutoDiscoveryProps) { - this.client = client; - this.resolvedConfig = resolvedConfig; - this.logger = logger.getChild("[RtcTransportAutoDiscovery]"); - } - - public async discoverPreferredTransport(): Promise { - // 1) backend transports - const backendTransport = await this.tryBackendTransports(); - if (backendTransport) { - this.logger.info( - `Found backend transport: ${JSON.stringify(backendTransport)}`, - ); - return backendTransport; - } - - // 2) app config URL - const configTransport = this.tryConfigTransport(); - if (configTransport) { - this.logger.info( - `Found app config transport: ${JSON.stringify(configTransport)}`, - ); - return configTransport; - } - - return null; - } - - /** - * Fetches the first rtc_foci from the backend. - * This will not throw errors, but instead just log them and return null if the expected config is not found or malformed. - * @private - */ - private async tryBackendTransports(): Promise { - const client = this.client; - // MSC4143: Attempt to fetch transports from backend. - this.logger.info("First try to use getRTCTransports end point ..."); - try { - const transportList = await doNetworkOperationWithRetry(async () => - client._unstable_getRTCTransports(), - ); - const first = transportList.find(isUnstableLivekitTransport); - if (first) { - return first; - } else { - this.logger.info( - `No livekit transport found in getRTCTransports end point`, - transportList, - ); - } - } catch (ex) { - this.logger.info(`Failed to use getRTCTransports end point: ${ex}`); - } - return null; - } - - private tryConfigTransport(): UnstableLivekitTransport | null { - const url = this.resolvedConfig.livekit?.livekit_service_url; - if (url) { - return { - type: "livekit", - livekit_service_url: url, - }; - } - return null; - } -} diff --git a/src/state/CallViewModel/remoteMembers/Connection.test.ts b/src/state/CallViewModel/remoteMembers/Connection.test.ts deleted file mode 100644 index bdab8e7af..000000000 --- a/src/state/CallViewModel/remoteMembers/Connection.test.ts +++ /dev/null @@ -1,749 +0,0 @@ -/* -Copyright 2025 Element Creations Ltd. -Copyright 2025 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - afterEach, - beforeEach, - describe, - expect, - it, - type MockedObject, - onTestFinished, - vi, -} from "vitest"; -import { - type LocalParticipant, - type RemoteParticipant, - type RemoteTrack, - type RemoteTrackPublication, - type Room as LivekitRoom, - RoomEvent, - Track, - ConnectionState as LivekitConnectionState, -} from "livekit-client"; -import fetchMock from "fetch-mock"; -import EventEmitter from "events"; -import { MatrixError } from "matrix-js-sdk"; -import { logger, type Logger } from "matrix-js-sdk/lib/logger"; -import { - type LivekitGetTokenResponse, - type UnstableLivekitTransport, -} from "matrix-js-sdk/lib/matrixrtc"; - -import { - Connection, - type ConnectionState, - type ConnectionOpts, -} from "./Connection.ts"; -import { ObservableScope } from "../../ObservableScope.ts"; -import { FailToGetOpenIdToken } from "../../../utils/errors.ts"; -import { testJWTToken } from "../../../utils/test-fixtures.ts"; -import { mockRemoteParticipant, ownMemberMock } from "../../../utils/test.ts"; -import { - type ClientGetTokenParts, - type ClientOpenIDParts, -} from "../../../livekit/auth"; - -let testScope: ObservableScope; - -let client: MockedObject; - -let fakeLivekitRoom: MockedObject; - -let localParticipantEventEmiter: EventEmitter; -let fakeLocalParticipant: MockedObject; - -const ROOM_ID = "!roomID:example.org"; -const SLOT_ID = "m.call#ROOM"; - -const transport: UnstableLivekitTransport = { - type: "livekit", - url: "wss://matrix-rtc.m.localhost/livekit/sfu", -}; - -function setupTest(): void { - testScope = new ObservableScope(); - client = vi.mocked({ - getOpenIdToken: vi.fn().mockResolvedValue({ - access_token: "rYsmGUEwNjKgJYyeNUkZseJN", - token_type: "Bearer", - matrix_server_name: "example.org", - expires_in: 3600, - }), - _unstable_getLivekitToken: vi.fn().mockResolvedValue({ jwt: testJWTToken }), - }); - - localParticipantEventEmiter = new EventEmitter(); - - fakeLocalParticipant = vi.mocked({ - identity: "@me:example.org", - isMicrophoneEnabled: vi.fn().mockReturnValue(true), - getTrackPublication: vi.fn().mockReturnValue(undefined), - on: localParticipantEventEmiter.on.bind(localParticipantEventEmiter), - off: localParticipantEventEmiter.off.bind(localParticipantEventEmiter), - addListener: localParticipantEventEmiter.addListener.bind( - localParticipantEventEmiter, - ), - removeListener: localParticipantEventEmiter.removeListener.bind( - localParticipantEventEmiter, - ), - removeAllListeners: localParticipantEventEmiter.removeAllListeners.bind( - localParticipantEventEmiter, - ), - } as unknown as LocalParticipant); - - const fakeRoomEventEmitter = new EventEmitter(); - fakeLivekitRoom = vi.mocked({ - connect: vi.fn(), - disconnect: vi.fn(), - remoteParticipants: new Map(), - localParticipant: fakeLocalParticipant, - state: LivekitConnectionState.Disconnected, - on: fakeRoomEventEmitter.on.bind(fakeRoomEventEmitter), - off: fakeRoomEventEmitter.off.bind(fakeRoomEventEmitter), - addListener: fakeRoomEventEmitter.addListener.bind(fakeRoomEventEmitter), - removeListener: - fakeRoomEventEmitter.removeListener.bind(fakeRoomEventEmitter), - removeAllListeners: - fakeRoomEventEmitter.removeAllListeners.bind(fakeRoomEventEmitter), - setE2EEEnabled: vi.fn().mockResolvedValue(undefined), - emit: (eventName: string | symbol, ...args: unknown[]) => { - fakeRoomEventEmitter.emit(eventName, ...args); - }, - } as unknown as LivekitRoom); -} - -function setupRemoteConnection(): Connection { - const opts: ConnectionOpts = { - client, - roomId: ROOM_ID, - slotId: SLOT_ID, - role: "subscriber", - transport, - serverName: "m.localhost", - scope: testScope, - ownMembershipIdentity: ownMemberMock, - livekitRoomFactory: () => fakeLivekitRoom, - }; - - fakeLivekitRoom.connect.mockImplementation(async (): Promise => { - const changeEv = RoomEvent.ConnectionStateChanged; - - fakeLivekitRoom.state = LivekitConnectionState.Connecting; - fakeLivekitRoom.emit(changeEv, fakeLivekitRoom.state); - fakeLivekitRoom.state = LivekitConnectionState.Connected; - fakeLivekitRoom.emit(changeEv, fakeLivekitRoom.state); - - return Promise.resolve(); - }); - - return new Connection(opts, logger); -} - -afterEach(() => { - vi.useRealTimers(); - vi.clearAllMocks(); - fetchMock.reset(); -}); - -describe("Start connection states", () => { - beforeEach(() => { - fetchMock.post( - `https://matrix-rtc.example.org/livekit/jwt/get_token`, - () => { - return { - // Return a non-retryable error, if not, the retry logic will - // wait and fail the test with a timeout. - status: 404, - }; - }, - ); - }); - - it("start in initialized state", () => { - setupTest(); - - const opts: ConnectionOpts = { - client, - roomId: ROOM_ID, - slotId: SLOT_ID, - role: "subscriber", - transport, - serverName: "m.localhost", - scope: testScope, - ownMembershipIdentity: ownMemberMock, - livekitRoomFactory: () => fakeLivekitRoom, - }; - const connection = new Connection(opts, logger); - - expect(connection.state$.getValue()).toEqual({ state: "initialized" }); - }); - - it("fail to get SFU token then error state", async () => { - setupTest(); - vi.useFakeTimers(); - - const opts: ConnectionOpts = { - client, - roomId: ROOM_ID, - slotId: SLOT_ID, - role: "subscriber", - transport, - serverName: "m.localhost", - scope: testScope, - ownMembershipIdentity: ownMemberMock, - livekitRoomFactory: () => fakeLivekitRoom, - }; - - const connection = new Connection(opts, logger); - - const capturedStates: ConnectionState[] = []; - const s = connection.state$.subscribe((value) => { - capturedStates.push(value); - }); - onTestFinished(() => s.unsubscribe()); - - const deferred = Promise.withResolvers(); - - client._unstable_getLivekitToken.mockImplementation( - async (): Promise => { - return await deferred.promise; - }, - ); - - connection.start().catch(() => { - // expected to throw - }); - - let capturedState = capturedStates.pop(); - expect(capturedState).toBeDefined(); - expect(capturedState).toEqual({ state: "authenticating" }); - - deferred.reject(new FailToGetOpenIdToken(new Error("Failed to get token"))); - - await vi.runAllTimersAsync(); - - capturedState = capturedStates.pop(); - if (capturedState?.state === "error") { - expect(capturedState.error.message).toEqual("Something went wrong"); - expect(connection.transport).toEqual(transport); - } else { - expect.fail( - "Expected FailedToStart state but got " + JSON.stringify(capturedState), - ); - } - }); - - it("fail to get JWT token and error state", async () => { - setupTest(); - vi.useFakeTimers(); - - const opts: ConnectionOpts = { - client, - roomId: ROOM_ID, - slotId: SLOT_ID, - role: "subscriber", - transport, - serverName: "m.localhost", - scope: testScope, - ownMembershipIdentity: ownMemberMock, - livekitRoomFactory: () => fakeLivekitRoom, - }; - - const connection = new Connection(opts, logger); - - const capturedStates: ConnectionState[] = []; - const s = connection.state$.subscribe((value) => { - capturedStates.push(value); - }); - onTestFinished(() => s.unsubscribe()); - - const deferredSFU = Promise.withResolvers(); - client._unstable_getLivekitToken.mockImplementation(async () => { - await deferredSFU.promise; - throw new MatrixError( - { - errcode: "M_LOOKUP_FAILED", - error: "Failed to look up user info from homeserver", - }, - 500, - ); - }); - - connection.start().catch(() => { - // expected to throw - }); - - let capturedState = capturedStates.pop(); - expect(capturedState).toBeDefined(); - expect(capturedState).toEqual({ state: "authenticating" }); - - deferredSFU.resolve(); - await vi.runAllTimersAsync(); - - capturedState = capturedStates.pop(); - - if ( - capturedState?.state === "error" && - capturedState.error.cause instanceof Error - ) { - expect(capturedState.error.cause.message).toContain( - "Failed to look up user info from homeserver", - ); - expect(connection.transport).toEqual(transport); - } else { - expect.fail( - "Expected FailedToStart state but got " + JSON.stringify(capturedState), - ); - } - }); - - it("fail to connect to livekit error state", async () => { - setupTest(); - vi.useFakeTimers(); - - const opts: ConnectionOpts = { - client, - roomId: ROOM_ID, - slotId: SLOT_ID, - role: "subscriber", - transport, - serverName: "m.localhost", - scope: testScope, - ownMembershipIdentity: ownMemberMock, - livekitRoomFactory: () => fakeLivekitRoom, - }; - - const connection = new Connection(opts, logger); - - const capturedStates: ConnectionState[] = []; - const s = connection.state$.subscribe((value) => { - capturedStates.push(value); - }); - onTestFinished(() => s.unsubscribe()); - - const deferredSFU = Promise.withResolvers(); - fakeLivekitRoom.connect.mockImplementation(async () => { - await deferredSFU.promise; - throw new Error("Failed to connect to livekit"); - }); - - connection.start().catch(() => { - // expected to throw - }); - - let capturedState = capturedStates.pop(); - expect(capturedState).toBeDefined(); - - expect(capturedState).toEqual({ state: "authenticating" }); - - deferredSFU.resolve(); - await vi.runAllTimersAsync(); - - capturedState = capturedStates.pop(); - - if ( - capturedState?.state === "error" && - capturedState.error.cause instanceof Error - ) { - expect(capturedState.error.cause.message).toContain( - "Failed to connect to livekit", - ); - expect(connection.transport).toEqual(transport); - } else { - expect.fail( - "Expected FailedToStart state but got " + JSON.stringify(capturedState), - ); - } - }); - - it("connection states happy path", async () => { - vi.useFakeTimers(); - setupTest(); - - const connection = setupRemoteConnection(); - - const capturedStates: ConnectionState[] = []; - const s = connection.state$.subscribe((value) => { - capturedStates.push(value); - }); - onTestFinished(() => s.unsubscribe()); - - await connection.start(); - await vi.runAllTimersAsync(); - - const initialState = capturedStates.shift(); - expect(initialState).toEqual({ state: "initialized" }); - const fetchingState = capturedStates.shift(); - expect(fetchingState).toEqual({ state: "authenticating" }); - const disconnectedState = capturedStates.shift(); - expect(disconnectedState).toEqual({ - state: "authenticated", - livekitState: LivekitConnectionState.Disconnected, - }); - const connectingState = capturedStates.shift(); - expect(connectingState).toEqual({ - state: "authenticated", - livekitState: LivekitConnectionState.Connecting, - }); - const connectedState = capturedStates.shift(); - expect(connectedState).toEqual({ - state: "authenticated", - livekitState: LivekitConnectionState.Connected, - }); - }); - - it("stopping while connecting does not report an error", async () => { - setupTest(); - const connection = setupRemoteConnection(); - - const capturedStates: ConnectionState[] = []; - const s = connection.state$.subscribe((value) => { - capturedStates.push(value); - }); - onTestFinished(() => s.unsubscribe()); - - // livekit-client rejects a pending connect() when disconnect() is called. - const pendingConnect = Promise.withResolvers(); - fakeLivekitRoom.connect.mockReturnValue(pendingConnect.promise); - fakeLivekitRoom.disconnect.mockResolvedValue(undefined); - - const started = connection.start(); - await vi.waitFor(() => expect(fakeLivekitRoom.connect).toHaveBeenCalled()); - const stopping = connection.stop(); - pendingConnect.reject(new Error("Client initiated disconnect")); - await stopping; - - // start() resolves rather than rejecting (it is not awaited by the caller, - // so a rejection would be unhandled). - await expect(started).resolves.toBeUndefined(); - expect(capturedStates.at(-1)).toEqual({ state: "stopped" }); - expect(capturedStates.some((st) => st instanceof Error)).toBe(false); - }); - - it("shutting down the scope should stop the connection", async () => { - setupTest(); - vi.useFakeTimers(); - - const connection = setupRemoteConnection(); - await connection.start(); - - const stopSpy = vi.spyOn(connection, "stop"); - testScope.end(); - - expect(stopSpy).toHaveBeenCalled(); - expect(fakeLivekitRoom.disconnect).toHaveBeenCalled(); - }); -}); - -describe("remote participants", () => { - it("emits the list of remote participants", () => { - setupTest(); - - const connection = setupRemoteConnection(); - - const observedParticipants: RemoteParticipant[][] = []; - const s = connection.remoteParticipants$.subscribe((participants) => { - observedParticipants.push(participants); - }); - onTestFinished(() => s.unsubscribe()); - // The remoteParticipants$ observable is derived from the current members of the - // livekitRoom and the rtc membership in order to publish the members that are publishing - // on this connection. - - let participants: RemoteParticipant[] = [ - mockRemoteParticipant({ identity: "@alice:example.org:DEV000" }), - mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }), - mockRemoteParticipant({ identity: "@carol:example.org:DEV222" }), - // Mock Dan to have no published tracks. We want him to still show show up - // in the participants list. - mockRemoteParticipant({ - identity: "@dan:example.org:DEV333", - getTrackPublication: () => undefined, - getTrackPublications: () => [], - }), - ]; - - // Let's simulate 3 members on the livekitRoom - vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockImplementation( - () => new Map(participants.map((p) => [p.identity, p])), - ); - - participants.forEach((p) => - fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, p), - ); - - // At this point there should be ~~no~~ publishers - // We do have publisher now, since we do not filter for publishers anymore (to also have participants with only data tracks) - // The filtering we do is just based on the matrixRTC member events. - expect(observedParticipants.pop()!.length).toEqual(4); - - participants = [ - mockRemoteParticipant({ identity: "@alice:example.org:DEV000" }), - mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }), - mockRemoteParticipant({ identity: "@carol:example.org:DEV222" }), - mockRemoteParticipant({ identity: "@dan:example.org:DEV333" }), - ]; - participants.forEach((p) => - fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, p), - ); - - // At this point there should be no publishers - expect(observedParticipants.pop()!.length).toEqual(4); - }); - - it("should be scoped to parent scope", (): void => { - setupTest(); - - const connection = setupRemoteConnection(); - - let observedParticipants: RemoteParticipant[][] = []; - const s = connection.remoteParticipants$.subscribe((participants) => { - observedParticipants.push(participants); - }); - onTestFinished(() => s.unsubscribe()); - - let participants: RemoteParticipant[] = [ - mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }), - ]; - - // Let's simulate 3 members on the livekitRoom - vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockImplementation( - () => new Map(participants.map((p) => [p.identity, p])), - ); - - for (const participant of participants) { - fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, participant); - } - - // We should have bob as a participant now - const ps = observedParticipants.pop(); - expect(ps?.length).toEqual(1); - expect(ps?.[0]?.identity).toEqual("@bob:example.org:DEV111"); - - // end the parent scope - testScope.end(); - observedParticipants = []; - - // SHOULD NOT emit any more participants as the scope is ended - participants = participants.filter( - (p) => p.identity !== "@bob:example.org:DEV111", - ); - - fakeLivekitRoom.emit( - RoomEvent.ParticipantDisconnected, - mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }), - ); - - expect(observedParticipants.length).toEqual(0); - }); -}); - -describe("remote track logging", () => { - it("logs remote participant and track events on the connection logger", () => { - setupTest(); - const info = vi.fn(); - const warn = vi.fn(); - const testLogger = { - getChild: (): unknown => testLogger, - info, - debug: vi.fn(), - warn, - error: vi.fn(), - } as unknown as Logger; - new Connection( - { - client, - roomId: ROOM_ID, - slotId: SLOT_ID, - role: "subscriber", - transport, - serverName: "m.localhost", - scope: testScope, - ownMembershipIdentity: ownMemberMock, - livekitRoomFactory: () => fakeLivekitRoom, - }, - testLogger, - ); - info.mockClear(); // drop the constructor log line - - const bob = mockRemoteParticipant({ - identity: "@bob:example.org:DEV111", - sid: "PA_bob", - }); - const pub = { - kind: "audio", - source: "microphone", - trackSid: "TR_mic", - isMuted: false, - isEncrypted: true, - } as unknown as RemoteTrackPublication; - const messages = (): string[] => info.mock.calls.map((c) => c[0] as string); - - fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, bob); - fakeLivekitRoom.emit( - RoomEvent.TrackSubscribed, - {} as RemoteTrack, - pub, - bob, - ); - fakeLivekitRoom.emit(RoomEvent.TrackMuted, pub, bob); - fakeLivekitRoom.emit( - RoomEvent.TrackStreamStateChanged, - pub, - Track.StreamState.Paused, - bob, - ); - expect(messages()).toEqual([ - "Participant connected: @bob:example.org:DEV111 (PA_bob)", - "Subscribed: audio microphone TR_mic of @bob:example.org:DEV111 encrypted=true muted=false", - "Muted: audio microphone TR_mic of @bob:example.org:DEV111 encrypted=true", - "Stream paused: audio microphone TR_mic of @bob:example.org:DEV111 encrypted=true", - ]); - - // Encryption status changes are logged; cryptor errors are warnings - fakeLivekitRoom.emit( - RoomEvent.ParticipantEncryptionStatusChanged, - false, - bob, - ); - expect(messages().at(-1)).toBe( - "Encryption status of @bob:example.org:DEV111: encrypted=false", - ); - const cryptorError = new Error("missing key at index 3"); - fakeLivekitRoom.emit(RoomEvent.EncryptionError, cryptorError, bob); - expect(warn).toHaveBeenCalledWith( - "Encryption error for @bob:example.org:DEV111:", - cryptorError, - ); - - // Local mute events are already logged by the Publisher - fakeLivekitRoom.emit(RoomEvent.TrackMuted, pub, { - ...fakeLocalParticipant, - isLocal: true, - } as unknown as LocalParticipant); - expect(messages().filter((m) => m.startsWith("Muted"))).toHaveLength(1); - - // Listeners are removed when the scope ends - testScope.end(); - fakeLivekitRoom.emit(RoomEvent.ParticipantDisconnected, bob); - expect( - messages().filter((m) => m.startsWith("Participant disconnected")), - ).toHaveLength(0); - }); -}); - -// -// NOT USED ANYMORE ? -// -// This setup look like sth for the Publisher. Not a connection. - -// describe("PublishConnection", () => { -// // let fakeBlurProcessor: ProcessorWrapper; -// let roomFactoryMock: Mock<() => LivekitRoom>; -// let muteStates: MockedObject; - -// function setUpPublishConnection(): void { -// setupTest(); - -// roomFactoryMock = vi.fn().mockReturnValue(fakeLivekitRoom); - -// muteStates = mockMuteStates(); - -// // fakeBlurProcessor = vi.mocked>({ -// // name: "BackgroundBlur", -// // restart: vi.fn().mockResolvedValue(undefined), -// // setOptions: vi.fn().mockResolvedValue(undefined), -// // getOptions: vi.fn().mockReturnValue({ strength: 0.5 }), -// // isRunning: vi.fn().mockReturnValue(false) -// // }); -// } - -// describe("Livekit room creation", () => { -// function createSetup(): void { -// setUpPublishConnection(); - -// const fakeTrackProcessorSubject$ = new BehaviorSubject({ -// supported: true, -// processor: undefined, -// }); - -// const opts: ConnectionOpts = { -// client: client, -// transport: livekitFocus, -// scope: testScope, -// livekitRoomFactory: roomFactoryMock, -// }; - -// const audioInput = { -// available$: of(new Map([["mic1", { id: "mic1" }]])), -// selected$: new BehaviorSubject({ id: "mic1" }), -// select(): void {}, -// }; - -// const videoInput = { -// available$: of(new Map([["cam1", { id: "cam1" }]])), -// selected$: new BehaviorSubject({ id: "cam1" }), -// select(): void {}, -// }; - -// const audioOutput = { -// available$: of(new Map([["speaker", { id: "speaker" }]])), -// selected$: new BehaviorSubject({ id: "speaker" }), -// select(): void {}, -// }; - -// // TODO understand what is wrong with our mocking that requires ts-expect-error -// const fakeDevices = mockMediaDevices({ -// // @ts-expect-error Mocking only -// audioInput, -// // @ts-expect-error Mocking only -// videoInput, -// // @ts-expect-error Mocking only -// audioOutput, -// }); - -// new Connection( -// opts, -// fakeDevices, -// muteStates, -// undefined, -// fakeTrackProcessorSubject$, -// ); -// } - -// it("should create room with proper initial audio and video settings", () => { -// createSetup(); - -// expect(roomFactoryMock).toHaveBeenCalled(); - -// const lastCallArgs = -// roomFactoryMock.mock.calls[roomFactoryMock.mock.calls.length - 1]; - -// const roomOptions = lastCallArgs.pop() as unknown as RoomOptions; -// expect(roomOptions).toBeDefined(); - -// expect(roomOptions!.videoCaptureDefaults?.deviceId).toEqual("cam1"); -// expect(roomOptions!.audioCaptureDefaults?.deviceId).toEqual("mic1"); -// expect(roomOptions!.audioOutput?.deviceId).toEqual("speaker"); -// }); - -// it("respect controlledAudioDevices", () => { -// // TODO: Refactor the code to make it testable. -// // The UrlParams module is a singleton has a cache and is very hard to test. -// // This breaks other tests as well if not handled properly. -// // vi.mock(import("./../UrlParams"), () => { -// // return { -// // getUrlParams: vi.fn().mockReturnValue({ -// // controlledAudioDevices: true -// // }) -// // }; -// // }); -// }); -// }); -// }); diff --git a/src/state/CallViewModel/remoteMembers/Connection.ts b/src/state/CallViewModel/remoteMembers/Connection.ts deleted file mode 100644 index 7e94b39f1..000000000 --- a/src/state/CallViewModel/remoteMembers/Connection.ts +++ /dev/null @@ -1,417 +0,0 @@ -/* -Copyright 2025 Element Creations Ltd. -Copyright 2025 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - connectedParticipantsObserver, - connectionStateObserver, -} from "@livekit/components-core"; -import { - ConnectionError, - ConnectionErrorReason, - type ConnectionState as LivekitConnectionState, - type Participant, - type RemoteParticipant, - type RemoteTrackPublication, - type Room as LivekitRoom, - RoomEvent, - type SubscriptionError, - type Track, - type TrackPublication, -} from "livekit-client"; -import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; -import { BehaviorSubject } from "rxjs"; -import { type Logger } from "matrix-js-sdk/lib/logger"; -import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; - -import { type Behavior } from "../../Behavior.ts"; -import { type ObservableScope } from "../../ObservableScope.ts"; -import { - ElementCallError, - InsufficientCapacityError, - LivekitConnectionError, - PeerConnectionTimeoutError, - SFURoomCreationRestrictedError, - UnknownCallError, -} from "../../../utils/errors.ts"; -import { - type ClientGetTokenParts, - getSFUConfig, - type ClientOpenIDParts, - type SFUConfig, -} from "../../../livekit/auth"; - -export interface ConnectionOpts { - /** The identity parts to use on this connection */ - ownMembershipIdentity: CallMembershipIdentityParts; - /** Whether we want to publish or only subscribe on this connection. */ - role: "publisher" | "subscriber"; - /** The media transport to connect to. */ - transport: UnstableLivekitTransport; - /** The name of the homeserver to which the {@link transport} belongs. */ - serverName: string; - /** The Matrix client to use for OpenID and SFU config requests. */ - client: ClientGetTokenParts & ClientOpenIDParts; - /** The ID of the Matrix room in which the session takes place. */ - roomId: string; - /** The ID of the MatrixRTC slot in which the session takes place. */ - slotId: string; - /** The observable scope to use for this connection. */ - scope: ObservableScope; - - /** Optional factory to create the LiveKit room, mainly for testing purposes. */ - livekitRoomFactory: () => LivekitRoom; -} - -export type ConnectionState = - // The start state of a connection. It has been created but nothing has loaded yet. - | { state: "initialized" } - // `start` has been called on the connection. It acquires the JWT token to connect to the LK room. - | { state: "authenticating" } - | { state: "authenticated"; livekitState: LivekitConnectionState } - | { state: "error"; error: ElementCallError } - | { state: "stopped" }; - -/** - * A connection to a Matrix RTC LiveKit backend. - * - * Expose observables for participants and connection state. - */ -export class Connection { - // Private Behavior - private readonly _state$ = new BehaviorSubject({ - state: "initialized", - }); - - /** - * The current state of the connection to the media transport. - */ - public readonly state$: Behavior = this._state$; - - /** - * Whether we want to publish or only subscribe on this connection. - */ - public readonly role: "publisher" | "subscriber"; - - /** - * The media transport to connect to. - */ - public readonly transport: UnstableLivekitTransport; - - /** - * The name of the homeserver to which the {@link transport} belongs. - */ - public readonly serverName: string; - - public readonly livekitRoom: LivekitRoom; - - private scope: ObservableScope; - - /** - * The remote LiveKit participants that are visible on this connection. - * - * Note that this may include participants that are connected only to - * subscribe, or publishers that are otherwise unattested in MatrixRTC state. - * It is therefore more low-level than what should be presented to the user. - */ - public readonly remoteParticipants$: Behavior; - - /** - * The alias of the LiveKit room. - */ - public get livekitAlias(): string | undefined { - return this._livekitAlias; - } - private _livekitAlias?: string; - - /** - * Whether the connection has been stopped. - * @see Connection.stop - * */ - protected stopped = false; - - // TODO: can we just keep the ConnectionOpts object instead of spreading? - private readonly client: ClientGetTokenParts & ClientOpenIDParts; - private readonly roomId: string; - private readonly slotId: string; - private readonly logger: Logger; - private readonly ownMembershipIdentity: CallMembershipIdentityParts; - /** - * Creates a new connection to a matrix RTC LiveKit backend. - * - * @param opts - Connection options {@link ConnectionOpts}. - * - * @param logger - The logger to use. - */ - public constructor(opts: ConnectionOpts, logger: Logger) { - this.ownMembershipIdentity = opts.ownMembershipIdentity; - this.roomId = opts.roomId; - this.slotId = opts.slotId; - this.logger = logger.getChild( - `[Connection ${JSON.stringify(opts.transport)}]`, - ); - this.logger.debug(`constructor called`); - const { role, transport, serverName, client, scope } = opts; - - this.scope = scope; - this.livekitRoom = opts.livekitRoomFactory(); - this.role = role; - this.transport = transport; - this.serverName = serverName; - this.client = client; - - this.remoteParticipants$ = scope.behavior( - // Only tracks remote participants - connectedParticipantsObserver(this.livekitRoom), - ); - this.logRemoteTrackEvents(); - - scope.onEnd(() => { - this.logger.info(`Connection scope ended, stopping connection`); - void this.stop(); - }); - } - - /** - * Logs the lifecycle of remote participants and their tracks as seen by - * livekit-client, so that rageshakes show what a tile should have been - * rendering (published, subscribed, muted, paused) for each remote member. - */ - private logRemoteTrackEvents(): void { - const room = this.livekitRoom; - const log = this.logger.getChild("[RemoteTracks]"); - // The encryption flag matters: if the publisher encrypts but this client - // believes the track is unencrypted, the cryptor is bypassed and raw - // ciphertext reaches the decoder (audible as loud noise bursts). - const track = (pub: TrackPublication, p: Participant): string => - `${pub.kind} ${pub.source} ${pub.trackSid} of ${p.identity} encrypted=${pub.isEncrypted}`; - - const onParticipantConnected = (p: RemoteParticipant): void => - log.info(`Participant connected: ${p.identity} (${p.sid})`); - const onParticipantDisconnected = (p: RemoteParticipant): void => - log.info(`Participant disconnected: ${p.identity} (${p.sid})`); - const onTrackPublished = ( - pub: RemoteTrackPublication, - p: RemoteParticipant, - ): void => log.info(`Published: ${track(pub, p)} muted=${pub.isMuted}`); - const onTrackUnpublished = ( - pub: RemoteTrackPublication, - p: RemoteParticipant, - ): void => log.info(`Unpublished: ${track(pub, p)}`); - const onTrackSubscribed = ( - _t: Track, - pub: RemoteTrackPublication, - p: RemoteParticipant, - ): void => log.info(`Subscribed: ${track(pub, p)} muted=${pub.isMuted}`); - const onTrackUnsubscribed = ( - _t: Track, - pub: RemoteTrackPublication, - p: RemoteParticipant, - ): void => log.info(`Unsubscribed: ${track(pub, p)}`); - const onTrackSubscriptionFailed = ( - trackSid: string, - p: RemoteParticipant, - reason?: SubscriptionError, - ): void => - log.warn( - `Subscription failed: ${trackSid} of ${p.identity} reason=${reason}`, - ); - // Mute events also fire for the local participant, which the Publisher - // already logs. - const onTrackMuted = (pub: TrackPublication, p: Participant): void => { - if (!p.isLocal) log.info(`Muted: ${track(pub, p)}`); - }; - const onTrackUnmuted = (pub: TrackPublication, p: Participant): void => { - if (!p.isLocal) log.info(`Unmuted: ${track(pub, p)}`); - }; - const onTrackStreamStateChanged = ( - pub: RemoteTrackPublication, - state: Track.StreamState, - p: RemoteParticipant, - ): void => log.info(`Stream ${state}: ${track(pub, p)}`); - const onEncryptionStatusChanged = ( - encrypted: boolean, - p?: Participant, - ): void => - log.info( - `Encryption status of ${p?.identity ?? "unknown participant"}: encrypted=${encrypted}`, - ); - // livekit-client throttles these per cryptor; they indicate frames being - // dropped (missing/invalid key). - const onEncryptionError = (error: Error, p?: Participant): void => - log.warn( - `Encryption error for ${p?.identity ?? "unknown participant"}:`, - error, - ); - - room - .on(RoomEvent.ParticipantConnected, onParticipantConnected) - .on(RoomEvent.ParticipantDisconnected, onParticipantDisconnected) - .on(RoomEvent.TrackPublished, onTrackPublished) - .on(RoomEvent.TrackUnpublished, onTrackUnpublished) - .on(RoomEvent.TrackSubscribed, onTrackSubscribed) - .on(RoomEvent.TrackUnsubscribed, onTrackUnsubscribed) - .on(RoomEvent.TrackSubscriptionFailed, onTrackSubscriptionFailed) - .on(RoomEvent.TrackMuted, onTrackMuted) - .on(RoomEvent.TrackUnmuted, onTrackUnmuted) - .on(RoomEvent.TrackStreamStateChanged, onTrackStreamStateChanged) - .on( - RoomEvent.ParticipantEncryptionStatusChanged, - onEncryptionStatusChanged, - ) - .on(RoomEvent.EncryptionError, onEncryptionError); - this.scope.onEnd(() => { - room - .off(RoomEvent.ParticipantConnected, onParticipantConnected) - .off(RoomEvent.ParticipantDisconnected, onParticipantDisconnected) - .off(RoomEvent.TrackPublished, onTrackPublished) - .off(RoomEvent.TrackUnpublished, onTrackUnpublished) - .off(RoomEvent.TrackSubscribed, onTrackSubscribed) - .off(RoomEvent.TrackUnsubscribed, onTrackUnsubscribed) - .off(RoomEvent.TrackSubscriptionFailed, onTrackSubscriptionFailed) - .off(RoomEvent.TrackMuted, onTrackMuted) - .off(RoomEvent.TrackUnmuted, onTrackUnmuted) - .off(RoomEvent.TrackStreamStateChanged, onTrackStreamStateChanged) - .off( - RoomEvent.ParticipantEncryptionStatusChanged, - onEncryptionStatusChanged, - ) - .off(RoomEvent.EncryptionError, onEncryptionError); - }); - } - - /** - * Starts the connection. - * - * This will: - * 1. Request an OpenId token `request_token` (allows matrix users to verify their identity with a third-party service.) - * 2. Use this token to request the SFU config to the MatrixRtc authentication service. - * 3. Connect to the configured LiveKit room. - * - * The errors are also represented as a state in the `state$` observable. - * It is safe to ignore those errors and handle them accordingly via the `state$` observable. - * @throws {InsufficientCapacityError} if the LiveKit server indicates that it has insufficient capacity to accept the connection. - * @throws {SFURoomCreationRestrictedError} if the LiveKit server indicates that the room does not exist and cannot be created. - */ - // TODO consider an autostart pattern... - public async start(): Promise { - this.logger.debug("Starting Connection"); - this.stopped = false; - try { - this._state$.next({ state: "authenticating" }); - const { url, jwt, livekitAlias } = await this.getSFUConfig(); - this.logger.debug( - "Starting Connection - jwt: ", - jwt, - "wss: ", - url, - "livekitAlias: ", - livekitAlias, - ); - this._livekitAlias = livekitAlias; - // If we were stopped while fetching the config, don't proceed to connect - if (this.stopped) return; - - // Setup observer once we are done with getSFUConfig - connectionStateObserver(this.livekitRoom) - .pipe(this.scope.bind()) - .subscribe((livekitState) => - this._state$.next({ state: "authenticated", livekitState }), - ); - - try { - this.logger.info(`livekitRoom.connect ${url}`); - await this.livekitRoom.connect(url, jwt); - this.logger.info(`livekitRoom.connect SUCCESS ${url}`); - } catch (e) { - this.logger.info(`livekitRoom.connect FAILED ${url}`, e); - // LiveKit uses 503 to indicate that the server has hit its track limits. - // https://github.com/livekit/livekit/blob/fcb05e97c5a31812ecf0ca6f7efa57c485cea9fb/pkg/service/rtcservice.go#L171 - // It also errors with a status code of 200 (yes, really) for room - // participant limits. - // LiveKit Cloud uses 429 for connection limits. - // Either way, all these errors can be explained as "insufficient capacity". - if (e instanceof ConnectionError) { - if (e.status === 503 || e.status === 200 || e.status === 429) { - throw new InsufficientCapacityError(); - } - if (e.status === 404) { - // error msg is "Failed to create call" - // error description is "Call creation might be restricted to authorized users only. Try again later, or contact your server admin if the problem persists." - // The room does not exist. There are two different modes of operation for the SFU: - // - the room is created on the fly when connecting (livekit `auto_create` option) - // - Only authorized users can create rooms, so the room must exist before connecting (done by the auth jwt service) - // In the first case there will not be a 404, so we are in the second case. - throw new SFURoomCreationRestrictedError(); - } - - if (e.reason === ConnectionErrorReason.Timeout) { - // Unabled to establish peer connection within the timeout - throw new PeerConnectionTimeoutError(); - } - - throw new LivekitConnectionError(e); - } - throw e; - } - // If we were stopped while connecting, don't proceed to update state. - if (this.stopped) return; - } catch (error) { - if (this.stopped) { - // stop() was called while we were connecting, which makes the pending - // connect reject. That is the abort we asked for, not a failure, so - // don't record an error state on a stopped connection or rethrow it - // (start() is not necessarily awaited by the caller, so a throw here - // becomes an unhandled promise rejection). - this.logger.debug(`Connect aborted because the connection was stopped`); - return; - } - this.logger.debug(`Failed to connect to LiveKit room: ${error}`); - this._state$.next({ - state: "error", - error: - error instanceof ElementCallError - ? error - : error instanceof Error - ? new UnknownCallError(error) - : new UnknownCallError(new Error(`${error}`)), - }); - // Its okay to ignore the throw. The error is part of the state. - throw error; - } - } - - protected async getSFUConfig(): Promise { - return await getSFUConfig({ - client: this.client, - membership: this.ownMembershipIdentity, - role: this.role, - transport: this.transport, - serverName: this.serverName, - roomId: this.roomId, - slotId: this.slotId, - logger: this.logger, - }); - } - - /** - * Stops the connection. - * - * This will disconnect from the LiveKit room. - * If the connection is already stopped, this is a no-op. - */ - public async stop(): Promise { - this.logger.debug("stop: disconnecing from lk room"); - if (this.stopped) return; - // Mark as stopped before disconnecting so that a connect() aborted by the - // disconnect sees the flag and does not report the abort as an error. - this.stopped = true; - await this.livekitRoom.disconnect(); - this._state$.next({ state: "stopped" }); - this.logger.debug("stop: DONE disconnecing from lk room"); - } -} diff --git a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts deleted file mode 100644 index e5240f096..000000000 --- a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts +++ /dev/null @@ -1,201 +0,0 @@ -/* -Copyright 2025 Element Creations Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - Room as LivekitRoom, - type RoomOptions, - type BaseKeyProvider, - type E2EEManagerOptions, - type BaseE2EEManager, -} from "livekit-client"; -import { logger, type Logger } from "matrix-js-sdk/lib/logger"; -// imported as inline to support worker when loaded from a cdn (cross domain) -import E2EEWorker from "livekit-client/e2ee-worker?worker&inline"; -import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; -import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; - -import { type ObservableScope } from "../../ObservableScope.ts"; -import { Connection } from "./Connection.ts"; -import type { MediaDevices } from "../../MediaDevices.ts"; -import type { Behavior } from "../../Behavior.ts"; -import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx"; -import { getLiveKitOptions } from "../../../livekit/options.ts"; -import { - advancedCamera, - cameraResolution, - cameraFramerate, - cameraBitrate, - cameraCodec, - parseResolution, - echoCancellationSetting, - noiseSuppressionSetting, - autoGainControlSetting, -} from "../../../settings/settings.ts"; -import { - type ClientGetTokenParts, - type ClientOpenIDParts, -} from "../../../livekit/auth"; - -// TODO evaluate if this should be done like the Publisher Factory -export interface ConnectionFactory { - createConnection( - scope: ObservableScope, - role: "publisher" | "subscriber", - transport: UnstableLivekitTransport, - serverName: string, - ownMembershipIdentity: CallMembershipIdentityParts, - logger: Logger, - ): Connection; -} - -export class ECConnectionFactory implements ConnectionFactory { - private readonly livekitRoomFactory: () => LivekitRoom; - - /** - * Creates a ConnectionFactory for LiveKit connections. - * - * @param client - The OpenID client parts for authentication, needed to get openID and JWT tokens. - * @param roomId - The ID of the Matrix room in which the session takes place. - * @param slotId - The ID of the MatrixRTC slot in which the session takes place. - * @param devices - Used for video/audio out/in capture options. - * @param processorState$ - Effects like background blur (only for publishing connection?) - * @param livekitKeyProvider - Optional key provider for end-to-end encryption. - * @param controlledAudioDevices - Option to indicate whether audio output device is controlled externally (native mobile app). - * @param livekitRoomFactory - Optional factory function (for testing) to create LivekitRoom instances. If not provided, a default factory is used. - */ - public constructor( - private client: ClientGetTokenParts & ClientOpenIDParts, - private readonly roomId: string, - private readonly slotId: string, - private devices: MediaDevices, - private processorState$: Behavior, - livekitKeyProvider: BaseKeyProvider | undefined, - private controlledAudioDevices: boolean, - livekitRoomFactory?: () => LivekitRoom, - ) { - const defaultFactory = (): LivekitRoom => { - const roomOptions = generateRoomOption({ - devices: this.devices, - processorState: this.processorState$.value, - e2eeLivekitOptions: livekitKeyProvider && { - keyProvider: livekitKeyProvider, - // It's important that every room use a separate E2EE worker. - // They get confused if given streams from multiple rooms. - worker: new E2EEWorker(), - }, - controlledAudioDevices: this.controlledAudioDevices, - }); - logger.info("[ECConnectionFactory] livekit room options: ", roomOptions); - return new LivekitRoom(roomOptions); - }; - this.livekitRoomFactory = livekitRoomFactory ?? defaultFactory; - } - - /** - * - * @param scope The observable scope (used for clean-up) - * @param transport The transport to use for this connection. - * @param serverName The name of the homeserver to which the {@link transport} belongs. - * @param ownMembershipIdentity required to connect (using the jwt service) with the SFU. - * @param logger The logger instance to use for this connection. - * @param sfuConfig optional config in case we already have a token for this connection. - * @returns - */ - public createConnection( - scope: ObservableScope, - role: "publisher" | "subscriber", - transport: UnstableLivekitTransport, - serverName: string, - ownMembershipIdentity: CallMembershipIdentityParts, - logger: Logger, - ): Connection { - return new Connection( - { - roomId: this.roomId, - slotId: this.slotId, - role, - transport, - serverName, - client: this.client, - scope: scope, - livekitRoomFactory: this.livekitRoomFactory, - ownMembershipIdentity, - }, - logger, - ); - } -} - -/** - * Generate the initial LiveKit RoomOptions based on the current media devices and processor state. - * Reads audio processing and camera quality settings directly from Settings. - */ -function generateRoomOption({ - devices, - processorState, - e2eeLivekitOptions, - controlledAudioDevices, -}: { - devices: MediaDevices; - processorState: ProcessorState; - e2eeLivekitOptions: - | E2EEManagerOptions - | { e2eeManager: BaseE2EEManager } - | undefined; - controlledAudioDevices: boolean; -}): RoomOptions { - const liveKitOptions = getLiveKitOptions(); - - // Apply advanced camera settings if enabled - let videoCaptureDefaults = { - ...liveKitOptions.videoCaptureDefaults, - deviceId: devices.videoInput.selected$.value?.id, - processor: processorState.processor, - }; - let publishDefaults = liveKitOptions.publishDefaults; - - if (advancedCamera.getValue()) { - const { width, height } = parseResolution(cameraResolution.getValue()); - const fps = cameraFramerate.getValue(); - const bps = cameraBitrate.getValue(); - const codec = cameraCodec.getValue(); - - videoCaptureDefaults = { - ...videoCaptureDefaults, - resolution: { width, height, frameRate: fps }, - }; - publishDefaults = { - ...publishDefaults, - videoEncoding: { maxBitrate: bps, maxFramerate: fps }, - videoCodec: codec, - }; - } - - return { - ...liveKitOptions, - videoCaptureDefaults, - publishDefaults, - audioCaptureDefaults: { - ...liveKitOptions.audioCaptureDefaults, - deviceId: devices.audioInput.selected$.value?.id, - echoCancellation: echoCancellationSetting.getValue(), - noiseSuppression: noiseSuppressionSetting.getValue(), - autoGainControl: autoGainControlSetting.getValue(), - }, - audioOutput: { - // When using controlled audio devices, we don't want to set the - // deviceId here, because it will be set by the native app. - // (also the id does not need to match a browser device id) - deviceId: controlledAudioDevices - ? undefined - : devices.audioOutput.selected$.value?.id, - }, - e2ee: e2eeLivekitOptions, - // TODO test and consider this: - // webAudioMix: true, - }; -} diff --git a/src/state/CallViewModel/remoteMembers/ConnectionMap.test.ts b/src/state/CallViewModel/remoteMembers/ConnectionMap.test.ts deleted file mode 100644 index 2344c1e55..000000000 --- a/src/state/CallViewModel/remoteMembers/ConnectionMap.test.ts +++ /dev/null @@ -1,453 +0,0 @@ -/* -Copyright 2025 Element Creations Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; -import { BehaviorSubject } from "rxjs"; -import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; -import { type RemoteParticipant } from "livekit-client"; -import { logger } from "matrix-js-sdk/lib/logger"; - -import { Epoch, mapEpoch, ObservableScope } from "../../ObservableScope.ts"; -import { - createConnectionMap$, - ConnectionMap, - keyFor, -} from "./ConnectionMap.ts"; -import { type ConnectionFactory } from "./ConnectionFactory.ts"; -import { type Connection } from "./Connection.ts"; -import { ownMemberMock, withTestScheduler } from "../../../utils/test.ts"; -import { type Behavior } from "../../Behavior.ts"; -import { type TransportLocator } from "../../../livekit/auth"; - -// Some test constants - -const TRANSPORT_1: TransportLocator = { - transport: { - type: "livekit", - livekit_service_url: "https://lk.example.org", - }, - serverName: "example.org", -}; - -const TRANSPORT_2: TransportLocator = { - transport: { - type: "livekit", - livekit_service_url: "https://lk.sample.com", - }, - serverName: "sample.com", -}; - -function findConnection( - transport: TransportLocator, - connections: Connection[], -): Connection | undefined { - const key = keyFor(transport); - return connections.find((c) => keyFor(c) === key); -} - -let fakeConnectionFactory: ConnectionFactory; -let testScope: ObservableScope; - -// Can be useful to track all created connections in tests, even the disposed ones -let allCreatedConnections: Connection[]; - -beforeEach(() => { - testScope = new ObservableScope(); - allCreatedConnections = []; - fakeConnectionFactory = {} as unknown as ConnectionFactory; - vi.mocked(fakeConnectionFactory).createConnection = vi - .fn() - .mockImplementation( - ( - scope: ObservableScope, - _role: unknown, - transport: UnstableLivekitTransport, - serverName: string, - ) => { - const mockConnection = { - transport, - serverName, - remoteParticipants$: new BehaviorSubject([]), - } as unknown as Connection; - vi.mocked(mockConnection).start = vi.fn(); - vi.mocked(mockConnection).stop = vi.fn(); - // Tie the connection's lifecycle to the scope to test scope lifecycle management - scope.onEnd(() => { - void mockConnection.stop(); - }); - allCreatedConnections.push(mockConnection); - return mockConnection; - }, - ); -}); - -afterEach(() => { - testScope.end(); -}); - -describe("connections$ stream", () => { - test("Should create and start new connections for each transports", () => { - withTestScheduler(({ behavior, expectObservable }) => { - const connectionMap$ = createConnectionMap$({ - scope: testScope, - connectionFactory: fakeConnectionFactory, - localTransport: TRANSPORT_1, - remoteTransports$: behavior("a", { - a: new Epoch([TRANSPORT_1, TRANSPORT_2], 0), - }), - logger: logger, - ownMembershipIdentity: ownMemberMock, - }); - - expectObservable( - connectionMap$.pipe(mapEpoch((d) => d.getConnections())), - ).toBe("a", { - a: expect.toSatisfy((e: Epoch) => { - const connections = e.value; - expect(connections.length).toBe(2); - - expect( - vi.mocked(fakeConnectionFactory).createConnection, - ).toHaveBeenCalledTimes(2); - - const conn1 = findConnection(TRANSPORT_1, connections); - expect(conn1).toBeDefined(); - expect(conn1!.start).toHaveBeenCalled(); - - const conn2 = findConnection(TRANSPORT_2, connections); - expect(conn2).toBeDefined(); - expect(conn2!.start).toHaveBeenCalled(); - return true; - }), - }); - }); - }); - - test("Shares one connection between homeservers advertising the same JWT service", () => { - withTestScheduler(({ behavior, expectObservable }) => { - const sameServiceElsewhere: TransportLocator = { - transport: TRANSPORT_1.transport, - serverName: "elsewhere.org", - }; - const connectionMap$ = createConnectionMap$({ - scope: testScope, - connectionFactory: fakeConnectionFactory, - localTransport: TRANSPORT_1, - remoteTransports$: behavior("a", { - a: new Epoch([sameServiceElsewhere], 0), - }), - logger: logger, - ownMembershipIdentity: ownMemberMock, - }); - - expectObservable( - connectionMap$.pipe(mapEpoch((d) => d.getConnections())), - ).toBe("a", { - a: expect.toSatisfy((e: Epoch) => { - expect(e.value.length).toBe(1); - // The JWT service is reached directly, so the server name plays no - // part and the local locator, which came first, is the one used - expect( - vi.mocked(fakeConnectionFactory).createConnection, - ).toHaveBeenCalledExactlyOnceWith( - expect.anything(), - "publisher", - TRANSPORT_1.transport, - "example.org", - expect.anything(), - expect.anything(), - ); - return true; - }), - }); - }); - }); - - test("Keeps one connection per homeserver for transports with a url", () => { - withTestScheduler(({ behavior, expectObservable }) => { - const urlTransport: TransportLocator = { - transport: { type: "livekit", url: "wss://sfu.example.org" }, - serverName: "example.org", - }; - const sameUrlElsewhere: TransportLocator = { - transport: urlTransport.transport, - serverName: "elsewhere.org", - }; - const connectionMap$ = createConnectionMap$({ - scope: testScope, - connectionFactory: fakeConnectionFactory, - localTransport: urlTransport, - remoteTransports$: behavior("a", { - a: new Epoch([sameUrlElsewhere], 0), - }), - logger: logger, - ownMembershipIdentity: ownMemberMock, - }); - - expectObservable( - connectionMap$.pipe(mapEpoch((d) => d.getConnections())), - ).toBe("a", { - a: expect.toSatisfy((e: Epoch) => { - // Our homeserver asks the named server for the token, so each - // server name needs its own connection - expect(e.value.length).toBe(2); - expect(findConnection(urlTransport, e.value)).toBeDefined(); - expect(findConnection(sameUrlElsewhere, e.value)).toBeDefined(); - return true; - }), - }); - }); - }); - - test("Should start connection only once", () => { - withTestScheduler(({ behavior, expectObservable }) => { - const connectionMap$ = createConnectionMap$({ - scope: testScope, - connectionFactory: fakeConnectionFactory, - localTransport: TRANSPORT_1, - remoteTransports$: behavior("abcdef", { - a: new Epoch([TRANSPORT_1], 0), - b: new Epoch([TRANSPORT_1], 1), - c: new Epoch([TRANSPORT_1], 2), - d: new Epoch([TRANSPORT_1], 3), - e: new Epoch([TRANSPORT_1], 4), - f: new Epoch([TRANSPORT_1, TRANSPORT_2], 5), - }), - logger: logger, - ownMembershipIdentity: ownMemberMock, - }); - - expectObservable( - connectionMap$.pipe(mapEpoch((d) => d.getConnections())), - ).toBe("xxxxxa", { - x: expect.anything(), - a: expect.toSatisfy((e: Epoch) => { - const connections = e.value; - - expect(connections.length).toBe(2); - expect( - vi.mocked(fakeConnectionFactory).createConnection, - ).toHaveBeenCalledTimes(2); - - const conn2 = findConnection(TRANSPORT_2, connections); - expect(conn2).toBeDefined(); - - const conn1 = findConnection(TRANSPORT_1, connections); - expect(conn1).toBeDefined(); - expect(conn1!.start).toHaveBeenCalledOnce(); - - return true; - }), - }); - }); - }); - - test("Should cleanup connections when not needed anymore", () => { - withTestScheduler(({ behavior, expectObservable }) => { - const connectionMap$ = createConnectionMap$({ - scope: testScope, - connectionFactory: fakeConnectionFactory, - localTransport: TRANSPORT_1, - remoteTransports$: behavior("abc", { - a: new Epoch([TRANSPORT_1], 0), - b: new Epoch([TRANSPORT_1, TRANSPORT_2], 1), - c: new Epoch([TRANSPORT_1], 2), - }), - logger: logger, - ownMembershipIdentity: ownMemberMock, - }); - - expectObservable( - connectionMap$.pipe(mapEpoch((d) => d.getConnections())), - ).toBe("xab", { - x: expect.anything(), - a: expect.toSatisfy((e: Epoch) => { - const connections = e.value; - expect(connections.length).toBe(2); - return true; - }), - b: expect.toSatisfy((e: Epoch) => { - const connections = e.value; - - expect(connections.length).toBe(1); - // The second connection should have been stopped has it is no longer needed. - const connection2 = findConnection( - TRANSPORT_2, - allCreatedConnections, - ); - expect(connection2).toBeDefined(); - expect(connection2!.stop).toHaveBeenCalled(); - - // The first connection should still be active - const conn1 = connections[0]; - expect(conn1.stop).not.toHaveBeenCalledOnce(); - - return true; - }), - }); - }); - }); -}); - -describe("ConnectionMap", () => { - test("warns when a second connection to the same URL is merged", () => { - const warn = vi.spyOn(logger, "warn").mockImplementation(() => {}); - const data = new ConnectionMap(logger); - const connection = TRANSPORT_1 as unknown as Connection; - const p = (identity: string): RemoteParticipant => - ({ identity }) as unknown as RemoteParticipant; - data.add(connection, [p("a")]); - data.add({ ...connection } as Connection, [p("b")]); - expect(warn).toHaveBeenCalledWith( - expect.stringContaining( - 'second connection to [null,"https://lk.example.org",null]: existing [a], adding [b]', - ), - ); - expect(data.getParticipantsForTransport(TRANSPORT_1)).toHaveLength(2); - }); -}); - -describe("connectionMap$", () => { - // Used in test to control fake connections' remoteParticipants$ streams - let fakeRemoteParticipantsStreams: Map>; - - beforeEach(() => { - fakeRemoteParticipantsStreams = new Map(); - - function getRemoteParticipantsFor( - transport: TransportLocator, - ): Behavior { - return ( - fakeRemoteParticipantsStreams.get(keyFor(transport)) ?? - new BehaviorSubject([]) - ); - } - - // need a more advanced fake connection factory - vi.mocked(fakeConnectionFactory).createConnection = vi - .fn() - .mockImplementation( - ( - scope: ObservableScope, - _role: unknown, - transport: UnstableLivekitTransport, - serverName: string, - ) => { - const fakeRemoteParticipants$ = new BehaviorSubject< - RemoteParticipant[] - >([]); - const mockConnection = { - transport, - serverName, - remoteParticipants$: getRemoteParticipantsFor({ - transport, - serverName, - }), - } as unknown as Connection; - vi.mocked(mockConnection).start = vi.fn(); - vi.mocked(mockConnection).stop = vi.fn(); - // Tie the connection's lifecycle to the scope to test scope lifecycle management - scope.onEnd(() => { - void mockConnection.stop(); - }); - - fakeRemoteParticipantsStreams.set( - keyFor({ transport, serverName }), - fakeRemoteParticipants$, - ); - return mockConnection; - }, - ); - }); - - test("Should report connections with the remote participants", () => { - withTestScheduler(({ expectObservable, schedule, behavior }) => { - // Setup the fake participants streams behavior - // ============================== - fakeRemoteParticipantsStreams.set( - keyFor(TRANSPORT_1), - behavior("oa-b", { - o: [], - a: [{ identity: "user1A" } as RemoteParticipant], - b: [ - { identity: "user1A" } as RemoteParticipant, - { identity: "user1B" } as RemoteParticipant, - ], - }), - ); - - fakeRemoteParticipantsStreams.set( - keyFor(TRANSPORT_2), - behavior("o-a", { - o: [], - a: [{ identity: "user2A" } as RemoteParticipant], - }), - ); - // ============================== - - const connectionMap$ = createConnectionMap$({ - scope: testScope, - connectionFactory: fakeConnectionFactory, - localTransport: TRANSPORT_1, - remoteTransports$: behavior("a", { - a: new Epoch([TRANSPORT_1, TRANSPORT_2], 0), - }), - logger, - ownMembershipIdentity: ownMemberMock, - }); - - expectObservable(connectionMap$).toBe("abcd", { - a: expect.toSatisfy((e) => { - const data: ConnectionMap = e.value; - expect(data.getConnections().length).toBe(2); - expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(0); - expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(0); - return true; - }), - b: expect.toSatisfy((e) => { - const data: ConnectionMap = e.value; - expect(data.getConnections().length).toBe(2); - expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(1); - expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(0); - expect( - data.getParticipantsForTransport(TRANSPORT_1)[0].identity, - ).toBe("user1A"); - return true; - }), - c: expect.toSatisfy((e) => { - const data: ConnectionMap = e.value; - expect(data.getConnections().length).toBe(2); - expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(1); - expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(1); - expect( - data.getParticipantsForTransport(TRANSPORT_1)[0].identity, - ).toBe("user1A"); - expect( - data.getParticipantsForTransport(TRANSPORT_2)[0].identity, - ).toBe("user2A"); - return true; - }), - d: expect.toSatisfy((e) => { - const data: ConnectionMap = e.value; - expect(data.getConnections().length).toBe(2); - expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(2); - expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(1); - expect( - data.getParticipantsForTransport(TRANSPORT_1)[0].identity, - ).toBe("user1A"); - expect( - data.getParticipantsForTransport(TRANSPORT_1)[1].identity, - ).toBe("user1B"); - expect( - data.getParticipantsForTransport(TRANSPORT_2)[0].identity, - ).toBe("user2A"); - return true; - }), - }); - }); - }); -}); diff --git a/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts b/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts deleted file mode 100644 index be6c149f4..000000000 --- a/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts +++ /dev/null @@ -1,287 +0,0 @@ -/* -Copyright 2025 Element Creations Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; -import { Room as LivekitRoom } from "livekit-client"; -import { BehaviorSubject } from "rxjs"; -import fetchMock from "fetch-mock"; -import { logger } from "matrix-js-sdk/lib/logger"; -import EventEmitter from "events"; - -import { ObservableScope } from "../../ObservableScope.ts"; -import { ECConnectionFactory } from "./ConnectionFactory.ts"; -import { - type ClientGetTokenParts, - type ClientOpenIDParts, -} from "../../../livekit/auth"; -import { - exampleTransport, - mockMediaDevices, - ownMemberMock, -} from "../../../utils/test.ts"; -import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx"; -import { constant } from "../../Behavior"; -import { - echoCancellationSetting, - noiseSuppressionSetting, - autoGainControlSetting, - advancedCamera, - cameraResolution, - cameraFramerate, - cameraBitrate, - cameraCodec, -} from "../../../settings/settings.ts"; -import { testJWTToken } from "../../../utils/test-fixtures.ts"; - -// At the top of your test file, after imports -vi.mock("livekit-client", async (importOriginal) => { - return { - ...(await importOriginal()), - Room: vi.fn().mockImplementation(function (this: LivekitRoom, options) { - const emitter = new EventEmitter(); - return { - on: emitter.on.bind(emitter), - off: emitter.off.bind(emitter), - emit: emitter.emit.bind(emitter), - disconnect: vi.fn(), - remoteParticipants: new Map(), - } as unknown as LivekitRoom; - }), - }; -}); - -let testScope: ObservableScope; -let mockClient: ClientGetTokenParts & ClientOpenIDParts; - -beforeEach(() => { - testScope = new ObservableScope(); - mockClient = { - getOpenIdToken: vi.fn().mockReturnValue(""), - _unstable_getLivekitToken: vi.fn().mockResolvedValue({ jwt: testJWTToken }), - }; -}); - -describe("ECConnectionFactory - Audio inputs options", () => { - test.each([ - { echo: true, noise: true }, - { echo: true, noise: false }, - { echo: false, noise: true }, - { echo: false, noise: false }, - ])( - "it sets echoCancellation=$echo and noiseSuppression=$noise based on settings", - ({ echo, noise }) => { - const RoomConstructor = vi.mocked(LivekitRoom); - - // Set audio processing settings - echoCancellationSetting.setValue(echo); - noiseSuppressionSetting.setValue(noise); - - const ecConnectionFactory = new ECConnectionFactory( - mockClient, - "!roomid:example.org", - "m.call#ROOM", - mockMediaDevices({}), - new BehaviorSubject({ - supported: true, - processor: undefined, - }), - undefined, - false, - ); - ecConnectionFactory.createConnection( - testScope, - "subscriber", - exampleTransport, - "example.org", - ownMemberMock, - logger, - ); - - // Check if Room was constructed with expected options - expect(RoomConstructor).toHaveBeenCalledWith( - expect.objectContaining({ - audioCaptureDefaults: expect.objectContaining({ - echoCancellation: echo, - noiseSuppression: noise, - }), - }), - ); - }, - ); -}); - -describe("ECConnectionFactory - ControlledAudioDevice", () => { - test.each([{ controlled: true }, { controlled: false }])( - "it sets controlledAudioDevice=$controlled then uses deviceId accordingly", - ({ controlled }) => { - const RoomConstructor = vi.mocked(LivekitRoom); - - // Explicitly set audio settings so the test doesn't depend on defaults - echoCancellationSetting.setValue(true); - noiseSuppressionSetting.setValue(true); - autoGainControlSetting.setValue(true); - - const ecConnectionFactory = new ECConnectionFactory( - mockClient, - "!roomid:example.org", - "m.call#ROOM", - mockMediaDevices({ - audioOutput: { - available$: constant(new Map()), - selected$: constant({ id: "DEV00", virtualEarpiece: false }), - select: () => {}, - }, - }), - new BehaviorSubject({ - supported: true, - processor: undefined, - }), - undefined, - controlled, - ); - ecConnectionFactory.createConnection( - testScope, - "subscriber", - exampleTransport, - "example.org", - ownMemberMock, - logger, - ); - - // Check if Room was constructed with expected options - expect(RoomConstructor).toHaveBeenCalledWith( - expect.objectContaining({ - audioOutput: expect.objectContaining({ - deviceId: controlled ? undefined : "DEV00", - }), - }), - ); - }, - ); -}); - -describe("ECConnectionFactory - Camera quality settings", () => { - test("it uses default video options when advancedCamera is disabled", () => { - const RoomConstructor = vi.mocked(LivekitRoom); - advancedCamera.setValue(false); - - const ecConnectionFactory = new ECConnectionFactory( - mockClient, - "!roomid:example.org", - "m.call#ROOM", - mockMediaDevices({}), - new BehaviorSubject({ - supported: true, - processor: undefined, - }), - undefined, - false, - ); - ecConnectionFactory.createConnection( - testScope, - "subscriber", - exampleTransport, - "example.org", - ownMemberMock, - logger, - ); - - // publishDefaults should use config defaults (vp8), not custom settings - expect(RoomConstructor).toHaveBeenCalledWith( - expect.objectContaining({ - publishDefaults: expect.objectContaining({ - videoCodec: "vp8", - }), - }), - ); - }); - - test("it applies custom camera resolution, encoding, and codec when advancedCamera is enabled", () => { - const RoomConstructor = vi.mocked(LivekitRoom); - - advancedCamera.setValue(true); - cameraResolution.setValue("1920x1080"); - cameraFramerate.setValue(60); - cameraBitrate.setValue(4_000_000); - cameraCodec.setValue("vp9"); - - const ecConnectionFactory = new ECConnectionFactory( - mockClient, - "!roomid:example.org", - "m.call#ROOM", - mockMediaDevices({}), - new BehaviorSubject({ - supported: true, - processor: undefined, - }), - undefined, - false, - ); - ecConnectionFactory.createConnection( - testScope, - "subscriber", - exampleTransport, - "example.org", - ownMemberMock, - logger, - ); - - expect(RoomConstructor).toHaveBeenCalledWith( - expect.objectContaining({ - videoCaptureDefaults: expect.objectContaining({ - resolution: { width: 1920, height: 1080, frameRate: 60 }, - }), - publishDefaults: expect.objectContaining({ - videoEncoding: { maxBitrate: 4_000_000, maxFramerate: 60 }, - videoCodec: "vp9", - }), - }), - ); - }); - - test("it applies autoGainControl from settings", () => { - const RoomConstructor = vi.mocked(LivekitRoom); - - autoGainControlSetting.setValue(false); - echoCancellationSetting.setValue(true); - noiseSuppressionSetting.setValue(true); - - const ecConnectionFactory = new ECConnectionFactory( - mockClient, - "!roomid:example.org", - "m.call#ROOM", - mockMediaDevices({}), - new BehaviorSubject({ - supported: true, - processor: undefined, - }), - undefined, - false, - ); - ecConnectionFactory.createConnection( - testScope, - "subscriber", - exampleTransport, - "example.org", - ownMemberMock, - logger, - ); - - expect(RoomConstructor).toHaveBeenCalledWith( - expect.objectContaining({ - audioCaptureDefaults: expect.objectContaining({ - autoGainControl: false, - }), - }), - ); - }); -}); - -afterEach(() => { - testScope.end(); - fetchMock.reset(); -}); diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts deleted file mode 100644 index 1c6a32790..000000000 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts +++ /dev/null @@ -1,400 +0,0 @@ -/* -Copyright 2025 Element Creations Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { describe, test, expect, beforeEach, afterEach, vi } from "vitest"; -import { logger } from "matrix-js-sdk/lib/logger"; -import { - type CallMembership, - type UnstableLivekitTransport, -} from "matrix-js-sdk/lib/matrixrtc"; -import { BehaviorSubject, combineLatest, map, type Observable } from "rxjs"; - -import { - type RemoteMatrixLivekitMember, - createRemoteMatrixLivekitMembers$, -} from "./MatrixLivekitMembers.ts"; -import { Epoch, ObservableScope, trackEpoch } from "../../ObservableScope.ts"; -import { ConnectionMap } from "./ConnectionMap.ts"; -import { - flushPromises, - mockRtcMembership, - mockRemoteParticipant, -} from "../../../utils/test.ts"; -import { type Connection } from "./Connection.ts"; -import { constant } from "../../Behavior.ts"; -import { localRtcMember } from "../../../utils/test-fixtures.ts"; -import { type TransportLocator } from "../../../livekit/auth"; -import { membershipsAndTransports$ } from "../../SessionBehaviors.ts"; - -let testScope: ObservableScope; - -const fallbackMemberId = (userId: string, deviceId: string): string => - `${userId}:${deviceId}`; - -const transportA: UnstableLivekitTransport = { - type: "livekit", - livekit_service_url: "https://lk.example.org", -}; - -const transportB: UnstableLivekitTransport = { - type: "livekit", - livekit_service_url: "https://lk.sample.com", -}; - -const bobMembership = mockRtcMembership("@bob:example.org", "DEV000", { - fociPreferred: [transportA], -}); -const carlMembership = mockRtcMembership("@carl:sample.com", "DEV111", { - fociPreferred: [transportB], -}); - -beforeEach(() => { - testScope = new ObservableScope(); -}); - -afterEach(() => { - testScope.end(); -}); - -function epochMeWith$( - source$: Observable>, - me$: Observable, -): Observable> { - return combineLatest([source$, me$]).pipe( - map(([ep, cd]) => { - return new Epoch(cd, ep.epoch); - }), - ); -} - -test("should signal participant not yet connected to livekit", async () => { - const info = vi.spyOn(logger, "info"); - const mockedMemberships$ = new BehaviorSubject([bobMembership]); - const mockConnectionMap$ = new BehaviorSubject(new ConnectionMap()); - const { memberships$, membershipsWithTransport$ } = - createEpochedMemberships$(mockedMemberships$); - - const connectionMap$ = testScope.behavior( - epochMeWith$(memberships$, mockConnectionMap$), - ); - - const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({ - scope: testScope, - membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), - connectionMap$, - localUser: localRtcMember, - }); - - await flushPromises(); - expect(remoteMatrixLivekitMembers$.value.value).toSatisfy( - (data: RemoteMatrixLivekitMember[]) => { - expect(data.length).toEqual(1); - expect(data[0].membership$.value).toBe(bobMembership); - expect(data[0].participant.value$.value).toBe(null); - expect(data[0].connection$.value).toBe(null); - return true; - }, - ); - expect(info).toHaveBeenCalledWith( - "[RemoteMatrixLivekitMembers] @bob:example.org:DEV000: LiveKit participant missing on no connection", - ); -}); - -// Helper to create epoch'ed memberships$ and membershipsWithTransport$ from memberships observable. -function createEpochedMemberships$(m$: Observable): { - memberships$: Observable>; - membershipsWithTransport$: Observable< - Epoch< - { membership: CallMembership; transport: TransportLocator | undefined }[] - > - >; -} { - const memberships$ = testScope.behavior(m$.pipe(trackEpoch())); - const { membershipsWithTransport$ } = membershipsAndTransports$( - testScope, - memberships$, - ); - return { - memberships$, - membershipsWithTransport$, - }; -} - -test("should signal participant on a connection that is publishing", async () => { - const bobParticipantId = fallbackMemberId( - bobMembership.userId, - bobMembership.deviceId, - ); - - const { memberships$, membershipsWithTransport$ } = createEpochedMemberships$( - constant([bobMembership]), - ); - - const connection = { - transport: bobMembership.getTransport(), - serverName: "example.org", - } as unknown as Connection; - const dataWithPublisher = new ConnectionMap(); - dataWithPublisher.add(connection, [ - mockRemoteParticipant({ identity: bobParticipantId }), - ]); - - const connectionMap$ = testScope.behavior( - epochMeWith$(memberships$, constant(dataWithPublisher)), - ); - - const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({ - scope: testScope, - membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), - connectionMap$, - localUser: localRtcMember, - }); - - await flushPromises(); - expect(remoteMatrixLivekitMembers$.value.value).toSatisfy( - (data: RemoteMatrixLivekitMember[]) => { - expect(data.length).toEqual(1); - expect(data[0].membership$.value).toBe(bobMembership); - expect(data[0].participant.value$.value).toSatisfy((participant) => { - expect(participant).toBeDefined(); - expect(participant!.identity).toEqual(bobParticipantId); - return true; - }); - expect(data[0].connection$.value).toBe(connection); - return true; - }, - ); -}); - -// A member advertising both a `url` and a `livekit_service_url` may have -// authenticated through either flow, so it has two candidate identities. -const bothTransport: UnstableLivekitTransport = { - type: "livekit", - url: "wss://sfu.example.org", - livekit_service_url: "https://lk.example.org", -}; -const bobLegacyId = fallbackMemberId("@bob:example.org", "DEV000"); -const bobHashedId = "2qkXfWz1S7jQmU0bJ6yN8aVtR3pLcDgHe9oZxKwMiA4"; -const bobWithBothIdentities = mockRtcMembership("@bob:example.org", "DEV000", { - fociPreferred: [bothTransport], - backendIdentities: [bobHashedId, bobLegacyId], -}); - -test.each([ - ["legacy", bobLegacyId], - ["hashed", bobHashedId], -])( - "matches a member with two candidate identities publishing under the %s one", - async (_name, identity) => { - const { memberships$, membershipsWithTransport$ } = - createEpochedMemberships$(constant([bobWithBothIdentities])); - - const connection = { - transport: bothTransport, - serverName: "example.org", - } as unknown as Connection; - const dataWithPublisher = new ConnectionMap(); - dataWithPublisher.add(connection, [mockRemoteParticipant({ identity })]); - - const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({ - scope: testScope, - membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), - connectionMap$: testScope.behavior( - epochMeWith$(memberships$, constant(dataWithPublisher)), - ), - localUser: localRtcMember, - }); - - await flushPromises(); - expect(remoteMatrixLivekitMembers$.value.value).toSatisfy( - (data: RemoteMatrixLivekitMember[]) => { - expect(data.length).toEqual(1); - expect(data[0].participant.value$.value?.identity).toEqual(identity); - expect(data[0].connection$.value).toBe(connection); - return true; - }, - ); - }, -); - -test("does not match a member with two candidate identities to anyone else", async () => { - const { memberships$, membershipsWithTransport$ } = createEpochedMemberships$( - constant([bobWithBothIdentities]), - ); - - const connection = { - transport: bothTransport, - serverName: "example.org", - } as unknown as Connection; - const dataWithStranger = new ConnectionMap(); - dataWithStranger.add(connection, [ - mockRemoteParticipant({ identity: "@stranger:example.org:DEV999" }), - ]); - - const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({ - scope: testScope, - membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), - connectionMap$: testScope.behavior( - epochMeWith$(memberships$, constant(dataWithStranger)), - ), - localUser: localRtcMember, - }); - - await flushPromises(); - expect(remoteMatrixLivekitMembers$.value.value).toSatisfy( - (data: RemoteMatrixLivekitMember[]) => { - expect(data.length).toEqual(1); - expect(data[0].participant.value$.value).toBe(null); - return true; - }, - ); -}); - -test("should signal participant on a connection that is not publishing", async () => { - const { memberships$, membershipsWithTransport$ } = createEpochedMemberships$( - constant([bobMembership]), - ); - - const connection = { - transport: bobMembership.getTransport(), - serverName: "example.org", - } as unknown as Connection; - const dataWithPublisher = new ConnectionMap(); - dataWithPublisher.add(connection, []); - - const connectionMap$ = testScope.behavior( - epochMeWith$(memberships$, constant(dataWithPublisher)), - ); - - const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({ - scope: testScope, - membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), - connectionMap$, - localUser: localRtcMember, - }); - await flushPromises(); - expect(remoteMatrixLivekitMembers$.value.value).toSatisfy( - (data: RemoteMatrixLivekitMember[]) => { - expect(data.length).toEqual(1); - expect(data[0].membership$.value).toBe(bobMembership); - expect(data[0].participant.value$.value).toBe(null); - expect(data[0].connection$.value).toBe(connection); - return true; - }, - ); -}); - -describe("Publication edge case", () => { - test("bob is publishing in several connections", async () => { - const { memberships$, membershipsWithTransport$ } = - createEpochedMemberships$(constant([bobMembership, carlMembership])); - - const connectionWithPublisher = new ConnectionMap(); - const bobParticipantId = fallbackMemberId( - bobMembership.userId, - bobMembership.deviceId, - ); - const connectionA = { - transport: transportA, - serverName: "example.org", - } as unknown as Connection; - const connectionB = { - transport: transportB, - serverName: "sample.com", - } as unknown as Connection; - - connectionWithPublisher.add(connectionA, [ - mockRemoteParticipant({ identity: bobParticipantId }), - ]); - connectionWithPublisher.add(connectionB, [ - mockRemoteParticipant({ identity: bobParticipantId }), - ]); - - const connectionMap$ = testScope.behavior( - epochMeWith$(memberships$, constant(connectionWithPublisher)), - ); - - const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({ - scope: testScope, - membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), - connectionMap$, - localUser: localRtcMember, - }); - await flushPromises(); - expect(remoteMatrixLivekitMembers$.value.value).toSatisfy( - (data: RemoteMatrixLivekitMember[]) => { - expect(data.length).toEqual(2); - expect(data[0].membership$.value).toBe(bobMembership); - expect(data[0].connection$.value).toBe(connectionA); - expect(data[0].participant.value$.value).toSatisfy((participant) => { - expect(participant).toBeDefined(); - expect(participant!.identity).toEqual(bobParticipantId); - return true; - }); - - return true; - }, - ); - }); -}); - -test("bob is publishing in the wrong connection", async () => { - const mockedMemberships$ = new BehaviorSubject([ - bobMembership, - carlMembership, - ]); - - const { memberships$, membershipsWithTransport$ } = - createEpochedMemberships$(mockedMemberships$); - - const connectionWithPublisher = new ConnectionMap(); - - const bobParticipantId = fallbackMemberId( - bobMembership.userId, - bobMembership.deviceId, - ); - const connectionA = { - transport: transportA, - serverName: "example.org", - } as unknown as Connection; - const connectionB = { - transport: transportB, - serverName: "sample.com", - } as unknown as Connection; - - // Bob is not publishing on A - connectionWithPublisher.add(connectionA, []); - // Bob is publishing on B but his membership says A - connectionWithPublisher.add(connectionB, [ - mockRemoteParticipant({ identity: bobParticipantId }), - ]); - - const connectionsWithPublisher$ = new BehaviorSubject( - connectionWithPublisher, - ); - const connectionMap$ = testScope.behavior( - epochMeWith$(memberships$, connectionsWithPublisher$), - ); - - const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({ - scope: testScope, - membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), - connectionMap$, - localUser: localRtcMember, - }); - - await flushPromises(); - expect(remoteMatrixLivekitMembers$.value.value).toSatisfy( - (data: RemoteMatrixLivekitMember[]) => { - expect(data.length).toEqual(2); - expect(data[0].membership$.value).toBe(bobMembership); - expect(data[0].connection$.value).toBe(connectionA); - expect(data[0].participant.value$.value).toBe(null); - return true; - }, - ); -}); diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts deleted file mode 100644 index 2f2aa8e0c..000000000 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts +++ /dev/null @@ -1,162 +0,0 @@ -/* -Copyright 2025 Element Creations Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { type LocalParticipant, type RemoteParticipant } from "livekit-client"; -import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; -import { combineLatest, filter, map } from "rxjs"; -import { logger } from "matrix-js-sdk/lib/logger"; - -import { type Behavior } from "../../Behavior"; -import { type ConnectionMap } from "./ConnectionMap"; -import { Epoch, type ObservableScope } from "../../ObservableScope"; -import { type Connection } from "./Connection"; -import { generateItemsWithEpoch } from "../../../utils/observable"; -import { type TransportLocator } from "../../../livekit/auth"; - -interface LocalTaggedParticipant { - type: "local"; - value$: Behavior; -} -interface RemoteTaggedParticipant { - type: "remote"; - value$: Behavior; -} -export type TaggedParticipant = - | LocalTaggedParticipant - | RemoteTaggedParticipant; - -export interface MatrixLivekitMember { - membership$: Behavior; - connection$: Behavior; - // participantId: string; We do not want a participantId here since it will be generated by the jwt - // TODO decide if we can also drop the userId. Its in the matrix membership anyways. - userId: string; -} - -/** - * Represents the local Matrix call member and their associated LiveKit participation. - * `livekitParticipant` can be null if the member is not yet connected to the livekit room - * or if it has no livekit transport at all. - */ -export interface LocalMatrixLivekitMember extends MatrixLivekitMember { - participant: LocalTaggedParticipant; -} - -/** - * Represents a remote Matrix call member and their associated LiveKit participation. - * `livekitParticipant` can be null if the member is not yet connected to the livekit room - * or if it has no livekit transport at all. - */ -export interface RemoteMatrixLivekitMember extends MatrixLivekitMember { - participant: RemoteTaggedParticipant; -} - -interface Props { - scope: ObservableScope; - membershipsWithTransport$: Behavior< - Epoch< - { membership: CallMembership; transport: TransportLocator | undefined }[] - > - >; - connectionMap$: Behavior>; - localUser: { deviceId: string; userId: string }; -} - -/** - * Combines MatrixRTC and Livekit worlds. - * - * It has a small public interface: - * - in (via constructor): - * - an observable of CallMembership[] to track the call members (The matrix side) - * - a `ConnectionMap` for the lk rooms (The livekit side) - * - out (via public Observable): - * - `remoteMatrixLivekitMember` an observable of MatrixLivekitMember[] to track the remote members and associated livekit data. - */ -export function createRemoteMatrixLivekitMembers$({ - scope, - membershipsWithTransport$, - connectionMap$, - localUser, -}: Props): Behavior> { - /** - * Behavior of all the remote call members and their associated livekit data (if available). - */ - return scope.behavior( - combineLatest([membershipsWithTransport$, connectionMap$]).pipe( - filter((values) => - values.every((value) => value.epoch === values[0].epoch), - ), - map(([ms, data]) => new Epoch([ms.value, data.value] as const, ms.epoch)), - generateItemsWithEpoch( - "RemoteMatrixLivekitMembers", - // Generator function. - // creates an array of `{key, data}[]` - // Each change in the keys (new key) will result in a call to the factory function. - function* ([membershipsWithTransport, managerData]) { - for (const { membership, transport } of membershipsWithTransport) { - // Exclude the local membership - if ( - membership.userId === localUser.userId && - membership.deviceId === localUser.deviceId - ) - continue; - - const participants = transport - ? managerData.getParticipantsForTransport(transport) - : []; - const matches = participants.filter((p) => - membership.backendIdentities.includes(p.identity), - ); - const participant = matches[0] ?? null; - const connection = transport - ? managerData.getConnectionForTransport(transport) - : null; - if (matches.length > 1) - logger.warn( - `[RemoteMatrixLivekitMembers] ${membership.rtcBackendIdentity}: ${matches.length} LiveKit participants match (sids ${matches.map((p) => p.sid).join(", ")}), using ${participant?.sid}`, - ); - - yield { - // This could just be the backend identity without the other keys. - // The user ID, device ID, and member ID are included however so - // they show up in debug logs. - keys: [ - membership.userId, - membership.deviceId, - membership.memberId, - membership.rtcBackendIdentity, - ], - data: { membership, participant, connection }, - }; - } - }, - // Each update where the key of the generator array do not change will result in updates to the `data$` behavior. - (scope, data$, userId, _deviceId, _memberId, rtcBackendIdentity) => { - const { participant$, ...rest } = scope.splitBehavior(data$); - // Log whether the member could be matched to a LiveKit participant, - // since a tile shows "waiting for media" for as long as it cannot. - participant$.pipe(scope.bind()).subscribe((p) => { - const transport = data$.value.connection?.transport; - logger.info( - `[RemoteMatrixLivekitMembers] ${rtcBackendIdentity}: LiveKit participant ${p ? `matched (${p.sid})` : "missing"} on ${transport ? JSON.stringify(transport) : "no connection"}`, - ); - }); - // will only get called once per backend identity. - // updates to data$ and as a result to displayName$ and mxcAvatarUrl$ are more frequent. - return { - userId, - participant: { type: "remote" as const, value$: participant$ }, - ...rest, - }; - }, - ), - ), - new Epoch([], -1), - ); -} - -// TODO add back in the callviewmodel pauseWhen(this.pretendToBeDisconnected$) diff --git a/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.test.ts b/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.test.ts deleted file mode 100644 index 79026e211..000000000 --- a/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.test.ts +++ /dev/null @@ -1,632 +0,0 @@ -/* -Copyright 2025 Element Creations Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { afterEach, beforeEach, describe, vi } from "vitest"; -import { - type MatrixEvent, - type RoomMember, - type RoomState, - RoomStateEvent, -} from "matrix-js-sdk"; -import EventEmitter from "events"; -import { it } from "vitest"; - -import { ObservableScope } from "../../ObservableScope.ts"; -import type { Room as MatrixRoom } from "matrix-js-sdk/lib/models/room"; -import { - mockRtcMembership, - mockMatrixRoomMember, - withTestScheduler, -} from "../../../utils/test.ts"; -import { - createMatrixMemberMetadata$, - createRoomMembers$, -} from "./MatrixMemberMetadata.ts"; -let testScope: ObservableScope; -let mockMatrixRoom: MatrixRoom; - -describe("MatrixMemberMetadata", () => { - /* - * To be populated in the test setup. - * Maps userId to a partial/mock RoomMember object. - */ - let fakeMembersMap: Map>; - - beforeEach(() => { - testScope = new ObservableScope(); - fakeMembersMap = new Map>(); - - const roomEmitter = new EventEmitter(); - mockMatrixRoom = { - on: roomEmitter.on.bind(roomEmitter), - off: roomEmitter.off.bind(roomEmitter), - emit: roomEmitter.emit.bind(roomEmitter), - // addListener: roomEmitter.addListener.bind(roomEmitter), - // removeListener: roomEmitter.removeListener.bind(roomEmitter), - getMember: vi.fn().mockImplementation((userId: string) => { - const member = fakeMembersMap.get(userId); - if (member) { - return member as RoomMember; - } - return null; - }), - getMembers: vi.fn().mockImplementation(() => { - const members = Array.from(fakeMembersMap.values()); - return members; - }), - getMembersWithMembership: vi.fn().mockImplementation(() => { - const members = Array.from(fakeMembersMap.values()); - return members; - }), - } as unknown as MatrixRoom; - }); - - function fakeMemberWith(data: Partial): void { - const userId = data.userId || "@alice:example.com"; - const member: Partial = { - userId: userId, - rawDisplayName: data.rawDisplayName ?? userId, - getMxcAvatarUrl: - data.getMxcAvatarUrl || - vi.fn().mockImplementation(() => { - return `mxc://example.com/${userId}`; - }), - ...data, - } as unknown as RoomMember; - fakeMembersMap.set(userId, member); - } - - afterEach(() => { - fakeMembersMap.clear(); - }); - - describe("displayname", () => { - function updateDisplayName( - userId: `@${string}:${string}`, - newDisplayName: string, - ): void { - const member = fakeMembersMap.get(userId); - if (member) { - member.rawDisplayName = newDisplayName; - // Emit the event to notify listeners - mockMatrixRoom.emit( - RoomStateEvent.Members, - {} as unknown as MatrixEvent, - {} as unknown as RoomState, - member as RoomMember, - ); - } else { - throw new Error(`No member found with userId: ${userId}`); - } - } - - it("should show our own user if present in rtc session and room", () => { - withTestScheduler(({ scope, behavior, expectObservable }) => { - fakeMemberWith({ - userId: "@local:example.com", - rawDisplayName: "it's a me", - }); - const memberships$ = behavior("a", { - a: [mockRtcMembership("@local:example.com", "DEVICE1")], - }); - const metadataStore = createMatrixMemberMetadata$( - testScope, - memberships$, - createRoomMembers$(testScope, mockMatrixRoom), - ); - const dn$ = metadataStore.createDisplayNameBehavior$( - scope, - "@local:example.com", - ); - - expectObservable(dn$).toBe("a", { - a: "it's a me", - }); - expectObservable(metadataStore.displaynameMap$).toBe("a", { - a: new Map([["@local:example.com", "it's a me"]]), - }); - }); - }); - - function setUpBasicRoom(): void { - fakeMemberWith({ - userId: "@local:example.com", - rawDisplayName: "it's a me", - }); - fakeMemberWith({ userId: "@alice:example.com", rawDisplayName: "Alice" }); - fakeMemberWith({ userId: "@bob:example.com", rawDisplayName: "Bob" }); - fakeMemberWith({ userId: "@carl:example.com", rawDisplayName: "Carl" }); - fakeMemberWith({ userId: "@evil:example.com", rawDisplayName: "Carl" }); - fakeMemberWith({ userId: "@bob:foo.bar", rawDisplayName: "Bob" }); - fakeMemberWith({ userId: "@no-name:foo.bar" }); - } - - it("should get displayName for users", () => { - setUpBasicRoom(); - - withTestScheduler(({ scope, behavior, expectObservable }) => { - const memberships$ = behavior("a", { - a: [ - mockRtcMembership("@alice:example.com", "DEVICE1"), - mockRtcMembership("@bob:example.com", "DEVICE1"), - ], - }); - const metadataStore = createMatrixMemberMetadata$( - testScope, - memberships$, - createRoomMembers$(testScope, mockMatrixRoom), - ); - const aliceDispName$ = metadataStore.createDisplayNameBehavior$( - scope, - "@alice:example.com", - ); - - expectObservable(aliceDispName$).toBe("a", { - a: "Alice", - }); - - expectObservable(metadataStore.displaynameMap$).toBe("a", { - a: new Map([ - ["@alice:example.com", "Alice"], - ["@bob:example.com", "Bob"], - ]), - }); - }); - }); - - it("should use userId if no display name", () => { - withTestScheduler(({ behavior, expectObservable }) => { - setUpBasicRoom(); - - const memberships$ = behavior("a", { - a: [mockRtcMembership("@no-name:foo.bar", "D000")], - }); - const metadataStore = createMatrixMemberMetadata$( - testScope, - memberships$, - createRoomMembers$(testScope, mockMatrixRoom), - ); - - expectObservable(metadataStore.displaynameMap$).toBe("a", { - a: new Map([ - ["@no-name:foo.bar", "@no-name:foo.bar"], - ]), - }); - }); - }); - - it("should disambiguate users with same display name", () => { - withTestScheduler(({ behavior, expectObservable }) => { - setUpBasicRoom(); - - const memberships$ = behavior("a", { - a: [ - mockRtcMembership("@bob:example.com", "DEVICE1"), - mockRtcMembership("@bob:example.com", "DEVICE2"), - mockRtcMembership("@bob:foo.bar", "BOB000"), - mockRtcMembership("@carl:example.com", "C000"), - mockRtcMembership("@evil:example.com", "E000"), - ], - }); - - const metadataStore = createMatrixMemberMetadata$( - testScope, - memberships$, - createRoomMembers$(testScope, mockMatrixRoom), - ); - - expectObservable(metadataStore.displaynameMap$).toBe("a", { - a: new Map([ - // ["@local:example.com", "it's a me"], - ["@bob:example.com", "Bob (@bob:example.com)"], - ["@bob:example.com", "Bob (@bob:example.com)"], - ["@bob:foo.bar", "Bob (@bob:foo.bar)"], - ["@carl:example.com", "Carl (@carl:example.com)"], - ["@evil:example.com", "Carl (@evil:example.com)"], - ]), - }); - }); - }); - - it("should start to disambiguate reactivly when needed", () => { - withTestScheduler(({ behavior, expectObservable }) => { - setUpBasicRoom(); - - const memberships$ = behavior("ab", { - a: [mockRtcMembership("@bob:example.com", "DEVICE1")], - b: [ - mockRtcMembership("@bob:example.com", "DEVICE1"), - mockRtcMembership("@bob:foo.bar", "BOB000"), - ], - }); - - const metadataStore = createMatrixMemberMetadata$( - testScope, - memberships$, - createRoomMembers$(testScope, mockMatrixRoom), - ); - - expectObservable(metadataStore.displaynameMap$).toBe("ab", { - a: new Map([["@bob:example.com", "Bob"]]), - b: new Map([ - ["@bob:example.com", "Bob (@bob:example.com)"], - ["@bob:foo.bar", "Bob (@bob:foo.bar)"], - ]), - }); - }); - }); - - it("should keep disambiguated name when other leave", () => { - withTestScheduler(({ behavior, expectObservable }) => { - setUpBasicRoom(); - - const memberships$ = behavior("ab", { - a: [ - mockRtcMembership("@bob:example.com", "DEVICE1"), - mockRtcMembership("@bob:foo.bar", "BOB000"), - ], - b: [mockRtcMembership("@bob:example.com", "DEVICE1")], - }); - - const metadataStore = createMatrixMemberMetadata$( - testScope, - memberships$, - createRoomMembers$(testScope, mockMatrixRoom), - ); - - expectObservable(metadataStore.displaynameMap$).toBe("ab", { - a: new Map([ - ["@bob:example.com", "Bob (@bob:example.com)"], - ["@bob:foo.bar", "Bob (@bob:foo.bar)"], - ]), - b: new Map([ - ["@bob:example.com", "Bob (@bob:example.com)"], - ]), - }); - }); - }); - - it("should disambiguate on name change", () => { - withTestScheduler(({ behavior, schedule, expectObservable }) => { - setUpBasicRoom(); - - const memberships$ = behavior("a", { - a: [ - mockRtcMembership("@bob:example.com", "B000"), - mockRtcMembership("@carl:example.com", "C000"), - ], - }); - const metadataStore = createMatrixMemberMetadata$( - testScope, - memberships$, - createRoomMembers$(testScope, mockMatrixRoom), - ); - - schedule("-a", { - a: () => { - updateDisplayName("@carl:example.com", "Bob"); - }, - }); - - expectObservable(metadataStore.displaynameMap$).toBe("ab", { - a: new Map([ - ["@bob:example.com", "Bob"], - ["@carl:example.com", "Carl"], - ]), - b: new Map([ - ["@bob:example.com", "Bob (@bob:example.com)"], - ["@carl:example.com", "Bob (@carl:example.com)"], - ]), - }); - }); - }); - - it("should track individual member id with createDisplayNameBehavior", () => { - withTestScheduler(({ scope, behavior, schedule, expectObservable }) => { - setUpBasicRoom(); - const BOB = "@bob:example.com"; - const CARL = "@carl:example.com"; - // for this test we build a mock environment that does all possible changes: - // - memberships join/leave - // - room join/leave - // - disambiguate - const memberships$ = behavior("ab-d", { - a: [mockRtcMembership(CARL, "C000")], - b: [ - mockRtcMembership(CARL, "C000"), - // bob joins - mockRtcMembership(BOB, "B000"), - ], - // c carl gets renamed to BOB - d: [ - // carl leaves - mockRtcMembership(BOB, "B000"), - ], - }); - schedule("--a-", { - a: () => { - // carl renames - updateDisplayName(CARL, "Bob"); - }, - }); - - const metadataStore = createMatrixMemberMetadata$( - testScope, - memberships$, - createRoomMembers$(testScope, mockMatrixRoom), - ); - - const bob$ = metadataStore.createDisplayNameBehavior$(scope, BOB); - const carl$ = metadataStore.createDisplayNameBehavior$(scope, CARL); - - expectObservable(bob$).toBe("abc-", { - a: undefined, - b: "Bob", - c: "Bob (@bob:example.com)", - // bob stays disambiguate even though carl left - // d: "Bob (@bob:example.com)", - }); - - expectObservable(carl$).toBe("a-cd", { - a: "Carl", - // b: "Carl", - // carl gets renamed and disambiguate - c: "Bob (@carl:example.com)", - d: undefined, - }); - }); - }); - - it("should disambiguate users with invisible characters", () => { - withTestScheduler(({ scope, behavior, expectObservable }) => { - const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB"); - const bobZeroWidthSpaceRtcMember = mockRtcMembership( - "@bob2:example.org", - "BBBB", - ); - const bob = mockMatrixRoomMember(bobRtcMember, { - rawDisplayName: "Bob", - }); - const bobZeroWidthSpace = mockMatrixRoomMember( - bobZeroWidthSpaceRtcMember, - { - rawDisplayName: "Bo\u200bb", - }, - ); - fakeMemberWith(bob); - fakeMemberWith(bobZeroWidthSpace); - fakeMemberWith({ userId: "@carol:example.org" }); - const memberships$ = behavior("ab", { - a: [mockRtcMembership("@carol:example.org", "1111"), bobRtcMember], - b: [ - mockRtcMembership("@carol:example.org", "1111"), - bobRtcMember, - bobZeroWidthSpaceRtcMember, - ], - }); - - const metadataStore = createMatrixMemberMetadata$( - testScope, - memberships$, - createRoomMembers$(testScope, mockMatrixRoom), - ); - - const bob$ = metadataStore.createDisplayNameBehavior$( - scope, - "@bob:example.org", - ); - const bob2$ = metadataStore.createDisplayNameBehavior$( - scope, - "@bob2:example.org", - ); - const carol$ = metadataStore.createDisplayNameBehavior$( - scope, - "@carol:example.org", - ); - expectObservable(bob$).toBe("ab", { - a: "Bob", - b: "Bob (@bob:example.org)", - }); - expectObservable(bob2$).toBe("ab", { - a: undefined, - b: "Bo\u200bb (@bob2:example.org)", - }); - expectObservable(carol$).toBe("a-", { - a: "@carol:example.org", - }); - - expectObservable(metadataStore.displaynameMap$).toBe("ab", { - // Carol has no displayname - So userId is used. - a: new Map([ - ["@carol:example.org", "@carol:example.org"], - ["@bob:example.org", "Bob"], - ]), - // Other Bob joins, and should handle zero width hacks. - b: new Map([ - ["@carol:example.org", "@carol:example.org"], - [bobRtcMember.userId, `Bob (@bob:example.org)`], - [ - bobZeroWidthSpace.userId, - `${bobZeroWidthSpace.rawDisplayName} (${bobZeroWidthSpace.userId})`, - ], - ]), - }); - }); - }); - - it("should strip RTL characters from displayname", () => { - withTestScheduler(({ behavior, expectObservable }) => { - const daveRtcMember = mockRtcMembership("@dave:example.org", "DDDD"); - const daveRTLRtcMember = mockRtcMembership( - "@dave2:example.org", - "DDDD", - ); - const dave = mockMatrixRoomMember(daveRtcMember, { - rawDisplayName: "Dave", - }); - const daveRTL = mockMatrixRoomMember(daveRTLRtcMember, { - rawDisplayName: "\u202eevaD", - }); - - fakeMemberWith({ userId: "@carol:example.org" }); - fakeMemberWith(daveRTL); - fakeMemberWith(dave); - const memberships$ = behavior("ab", { - a: [mockRtcMembership("@carol:example.org", "DDDD")], - b: [ - mockRtcMembership("@carol:example.org", "DDDD"), - daveRtcMember, - daveRTLRtcMember, - ], - }); - - const metadataStore = createMatrixMemberMetadata$( - testScope, - memberships$, - createRoomMembers$(testScope, mockMatrixRoom), - ); - - expectObservable(metadataStore.displaynameMap$).toBe("ab", { - // Carol has no displayname - So userId is used. - a: new Map([["@carol:example.org", "@carol:example.org"]]), - // Both Dave's join. Since after stripping - b: new Map([ - ["@carol:example.org", "@carol:example.org"], - // Not disambiguated - ["@dave:example.org", "Dave"], - // This one is, since it's using RTL. - ["@dave2:example.org", "evaD (@dave2:example.org)"], - ]), - }); - }); - }); - }); - - describe("avatarUrl", () => { - function updateAvatarUrl( - userId: `@${string}:${string}`, - avatarUrl: string, - ): void { - const member = fakeMembersMap.get(userId); - if (member) { - member.getMxcAvatarUrl = vi.fn().mockReturnValue(avatarUrl); - // Emit the event to notify listeners - mockMatrixRoom.emit( - RoomStateEvent.Members, - {} as unknown as MatrixEvent, - {} as unknown as RoomState, - member as RoomMember, - ); - } else { - throw new Error(`No member found with userId: ${userId}`); - } - } - - it("should use avatar url from room members", () => { - withTestScheduler(({ scope, behavior, expectObservable }) => { - fakeMemberWith({ - userId: "@local:example.com", - }); - fakeMemberWith({ - userId: "@alice:example.com", - getMxcAvatarUrl: vi.fn().mockReturnValue("mxc://custom.url/avatar"), - }); - const memberships$ = behavior("a", { - a: [ - mockRtcMembership("@local:example.com", "DEVICE1"), - mockRtcMembership("@alice:example.com", "DEVICE1"), - ], - }); - const metadataStore = createMatrixMemberMetadata$( - testScope, - memberships$, - createRoomMembers$(testScope, mockMatrixRoom), - ); - const local$ = metadataStore.createAvatarUrlBehavior$( - scope, - "@local:example.com", - ); - - const alice$ = metadataStore.createAvatarUrlBehavior$( - scope, - "@alice:example.com", - ); - - expectObservable(local$).toBe("a", { - a: "mxc://example.com/@local:example.com", - }); - expectObservable(alice$).toBe("a", { - a: "mxc://custom.url/avatar", - }); - expectObservable(metadataStore.avatarMap$).toBe("a", { - a: new Map([ - ["@local:example.com", "mxc://example.com/@local:example.com"], - ["@alice:example.com", "mxc://custom.url/avatar"], - ]), - }); - }); - }); - - it("should update on avatar change and user join/leave", () => { - withTestScheduler(({ scope, behavior, schedule, expectObservable }) => { - fakeMemberWith({ userId: "@carl:example.com" }); - fakeMemberWith({ userId: "@bob:example.com" }); - const memberships$ = behavior("ab-d", { - a: [mockRtcMembership("@bob:example.com", "B000")], - b: [ - mockRtcMembership("@bob:example.com", "B000"), - mockRtcMembership("@carl:example.com", "C000"), - ], - d: [mockRtcMembership("@carl:example.com", "C000")], - }); - - const metadataStore = createMatrixMemberMetadata$( - testScope, - memberships$, - createRoomMembers$(testScope, mockMatrixRoom), - ); - - schedule("--c-", { - c: () => { - updateAvatarUrl( - "@carl:example.com", - "mxc://updated.me/updatedAvatar", - ); - }, - }); - - const bob$ = metadataStore.createAvatarUrlBehavior$( - scope, - "@bob:example.com", - ); - const carl$ = metadataStore.createAvatarUrlBehavior$( - scope, - "@carl:example.com", - ); - expectObservable(bob$).toBe("a---", { - a: "mxc://example.com/@bob:example.com", - }); - expectObservable(carl$).toBe("a-c-", { - a: "mxc://example.com/@carl:example.com", - - c: "mxc://updated.me/updatedAvatar", - }); - expectObservable(metadataStore.avatarMap$).toBe("a-c-", { - a: new Map([ - ["@bob:example.com", "mxc://example.com/@bob:example.com"], - ["@carl:example.com", "mxc://example.com/@carl:example.com"], - ]), - // expect an update once we update the avatar URL - c: new Map([ - ["@bob:example.com", "mxc://example.com/@bob:example.com"], - ["@carl:example.com", "mxc://updated.me/updatedAvatar"], - ]), - }); - }); - }); - }); -}); diff --git a/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.ts b/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.ts deleted file mode 100644 index 6a448128c..000000000 --- a/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.ts +++ /dev/null @@ -1,160 +0,0 @@ -/* -Copyright 2025 Element Creations Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { type RoomMember, RoomStateEvent } from "matrix-js-sdk"; -import { combineLatest, fromEvent, map } from "rxjs"; -import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; -import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; -import { - KnownMembership, - type Room as MatrixRoom, -} from "matrix-js-sdk/lib/matrix"; -// eslint-disable-next-line rxjs/no-internal - -import { type ObservableScope } from "../../ObservableScope"; -import { - calculateDisplayName, - shouldDisambiguate, -} from "../../../utils/displayname"; -import { type Behavior } from "../../Behavior"; - -export type RoomMemberMap = Map< - string, - Pick ->; -export function roomToMembersMap(matrixRoom: MatrixRoom): RoomMemberMap { - const members = matrixRoom - .getMembersWithMembership(KnownMembership.Join) - .concat(matrixRoom.getMembersWithMembership(KnownMembership.Invite)); - return members.reduce((acc, member) => { - acc.set(member.userId, { - userId: member.userId, - getMxcAvatarUrl: member.getMxcAvatarUrl.bind(member), - rawDisplayName: member.rawDisplayName, - }); - return acc; - }, new Map()); -} - -export function createRoomMembers$( - scope: ObservableScope, - matrixRoom: MatrixRoom, -): Behavior { - return scope.behavior( - fromEvent(matrixRoom, RoomStateEvent.Members).pipe( - map(() => roomToMembersMap(matrixRoom)), - ), - roomToMembersMap(matrixRoom), - ); -} - -/** - * Displayname for each member of the call. This will disambiguate - * any displayname that clashes with another member. Only members - * joined to the call are considered here. - * - * @returns Map uses the Matrix user ID as the key. - */ -// don't do this work more times than we need to. This is achieved by converting to a behavior: -export const memberDisplaynames$ = ( - scope: ObservableScope, - memberships$: Behavior[]>, - roomMembers$: Behavior, -): Behavior> => { - const logger = rootLogger.getChild("[MatrixMemberMetadata]"); - // This map tracks userIds that at some point needed disambiguation. - // This is a memory leak bound to the number of participants. - // A call application will always increase the memory if there have been more members in a call. - // Its capped by room member participants. - const shouldDisambiguateTrackerMap = new Set(); - return scope.behavior( - combineLatest([ - // Handle call membership changes - memberships$, - // Additionally handle display name changes (implicitly reacting to them) - roomMembers$, - // TODO: do we need: pauseWhen(this.pretendToBeDisconnected$), - ]).pipe( - map(([memberships, roomMembers]) => { - const displaynameMap = new Map(); - // We only consider RTC members for disambiguation as they are the only visible members. - for (const rtcMember of memberships) { - const member = roomMembers.get(rtcMember.userId); - if (!member) { - logger.error(`Could not find member for user ${rtcMember.userId}`); - continue; - } - const disambiguateComputed = shouldDisambiguate( - member, - memberships, - roomMembers, - ); - - const disambiguate = - shouldDisambiguateTrackerMap.has(rtcMember.userId) || - disambiguateComputed; - if (disambiguate) shouldDisambiguateTrackerMap.add(rtcMember.userId); - displaynameMap.set( - rtcMember.userId, - calculateDisplayName(member, disambiguate), - ); - } - return displaynameMap; - }), - ), - ); -}; - -export const createMatrixMemberMetadata$ = ( - scope: ObservableScope, - memberships$: Behavior[]>, - roomMembers$: Behavior, -): { - createDisplayNameBehavior$: ( - scope: ObservableScope, - userId: string, - ) => Behavior; - createAvatarUrlBehavior$: ( - scope: ObservableScope, - userId: string, - ) => Behavior; - displaynameMap$: Behavior>; - avatarMap$: Behavior>; -} => { - const displaynameMap$ = memberDisplaynames$( - scope, - memberships$, - roomMembers$, - ); - const avatarMap$ = scope.behavior( - roomMembers$.pipe( - map((roomMembers) => - Array.from(roomMembers.keys()).reduce((acc, key) => { - acc.set(key, roomMembers.get(key)?.getMxcAvatarUrl()); - return acc; - }, new Map()), - ), - ), - ); - return { - createDisplayNameBehavior$: (scope: ObservableScope, userId: string) => - scope.behavior( - displaynameMap$.pipe( - map((displaynameMap) => displaynameMap.get(userId)), - ), - ), - createAvatarUrlBehavior$: (scope: ObservableScope, userId: string) => - scope.behavior( - roomMembers$.pipe( - map((roomMembers) => roomMembers.get(userId)?.getMxcAvatarUrl()), - ), - ), - // mostly for testing purposes - displaynameMap$, - avatarMap$, - }; -}; diff --git a/src/state/CallViewModel/remoteMembers/integration.test.ts b/src/state/CallViewModel/remoteMembers/integration.test.ts deleted file mode 100644 index bf90e9eb4..000000000 --- a/src/state/CallViewModel/remoteMembers/integration.test.ts +++ /dev/null @@ -1,223 +0,0 @@ -/* -Copyright 2025 Element Creations Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { test, vi, expect, beforeEach, afterEach } from "vitest"; -import { BehaviorSubject } from "rxjs"; -import { type Room as LivekitRoom } from "livekit-client"; -import EventEmitter from "events"; -import fetchMock from "fetch-mock"; -import { logger } from "matrix-js-sdk/lib/logger"; - -import { - type Epoch, - ObservableScope, - trackEpoch, -} from "../../ObservableScope.ts"; -import { ECConnectionFactory } from "./ConnectionFactory.ts"; -import { - type ClientGetTokenParts, - type ClientOpenIDParts, -} from "../../../livekit/auth"; -import { - exampleTransportLocator, - mockMediaDevices, - mockRtcMembership, - ownMemberMock, - withTestScheduler, -} from "../../../utils/test.ts"; -import { type ProcessorState } from "../../../livekit/TrackProcessorContext.tsx"; -import { - createRemoteMatrixLivekitMembers$, - type RemoteMatrixLivekitMember, -} from "./MatrixLivekitMembers.ts"; -import { createConnectionMap$ } from "./ConnectionMap.ts"; -import { membershipsAndTransports$ } from "../../SessionBehaviors.ts"; -import { localRtcMember, testJWTToken } from "../../../utils/test-fixtures.ts"; -import { deepCompare } from "matrix-js-sdk/lib/utils"; - -// Test the integration of ConnectionMap and MatrixLivekitMembers - -let testScope: ObservableScope; -let ecConnectionFactory: ECConnectionFactory; -let mockClient: ClientGetTokenParts & ClientOpenIDParts; -let lkRoomFactory: () => LivekitRoom; - -const createdMockLivekitRooms: Map = new Map(); - -beforeEach(() => { - testScope = new ObservableScope(); - mockClient = { - getOpenIdToken: vi.fn().mockReturnValue(""), - _unstable_getLivekitToken: vi.fn().mockResolvedValue({ jwt: testJWTToken }), - }; - - lkRoomFactory = vi.fn().mockImplementation(() => { - const emitter = new EventEmitter(); - const base = { - on: emitter.on.bind(emitter), - off: emitter.off.bind(emitter), - emit: emitter.emit.bind(emitter), - disconnect: vi.fn(), - remoteParticipants: new Map(), - } as unknown as LivekitRoom; - - vi.mocked(base).connect = vi.fn().mockImplementation(({ url }) => { - createdMockLivekitRooms.set(url, base); - }); - return base; - }); - - ecConnectionFactory = new ECConnectionFactory( - mockClient, - "!roomid:example.org", - "m.call#ROOM", - mockMediaDevices({}), - new BehaviorSubject({ - supported: true, - processor: undefined, - }), - undefined, - false, - lkRoomFactory, - ); - - //TODO a bit annoying to have to do a http mock? - fetchMock.post(`path:/sfu/get`, (url) => { - const domain = new URL(url).hostname; // Extract the domain from the URL - return { - status: 200, - body: { - url: `wss://${domain}/livekit/sfu`, - jwt: testJWTToken, - }, - }; - }); -}); - -afterEach(() => { - testScope.end(); - fetchMock.reset(); -}); - -test("bob, carl, then bob joining no tracks yet", () => { - withTestScheduler(({ expectObservable, behavior, scope }) => { - const bobMembership = mockRtcMembership("@bob:example.com", "BDEV000"); - const carlMembership = mockRtcMembership("@carl:example.com", "CDEV000"); - const daveMembership = mockRtcMembership("@dave:foo.bar", "DDEV000"); - - const eMarble = "abc"; - const vMarble = "abc"; - const memberships$ = scope.behavior( - behavior(eMarble, { - a: [bobMembership], - b: [bobMembership, carlMembership], - c: [bobMembership, carlMembership, daveMembership], - }).pipe(trackEpoch()), - ); - - const membershipsAndTransports = membershipsAndTransports$( - testScope, - memberships$, - ); - - const connectionMap$ = createConnectionMap$({ - scope: testScope, - connectionFactory: ecConnectionFactory, - localTransport: exampleTransportLocator, - remoteTransports$: membershipsAndTransports.transports$, - logger: logger, - ownMembershipIdentity: ownMemberMock, - }); - - const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({ - scope: testScope, - membershipsWithTransport$: - membershipsAndTransports.membershipsWithTransport$, - connectionMap$, - localUser: localRtcMember, - }); - - expectObservable(remoteMatrixLivekitMembers$).toBe(vMarble, { - a: expect.toSatisfy((e: Epoch) => { - const items = e.value; - expect(items.length).toBe(1); - const item = items[0]!; - expectObservable(item.membership$).toBe("a", { - a: bobMembership, - }); - expectObservable(item.connection$).toBe("a", { - a: expect.toSatisfy((co) => - deepCompare(co.transport, bobMembership.transports[0]), - ), - }); - expectObservable(item.participant.value$).toBe("a", { - a: null, - }); - return true; - }), - b: expect.toSatisfy((e: Epoch) => { - const items = e.value; - expect(items.length).toBe(2); - - { - const item = items[0]!; - expectObservable(item.membership$).toBe("a", { - a: bobMembership, - }); - expectObservable(item.participant.value$).toBe("a", { - a: null, - }); - } - - { - const item = items[1]!; - - expectObservable(item.membership$).toBe("a", { - a: carlMembership, - }); - expectObservable(item.participant.value$).toBe("a", { - a: null, - }); - expectObservable(item.connection$).toBe("a", { - a: expect.toSatisfy((connection) => - deepCompare(connection.transport, carlMembership.transports[0]), - ), - }); - } - return true; - }), - c: expect.toSatisfy((e: Epoch) => { - const items = e.value; - expect(items.length).toBe(3); - - expectObservable(items[0].membership$).toBe("a", { - a: bobMembership, - }); - expectObservable(items[1].membership$).toBe("b", { - a: carlMembership, - }); - - { - const item = items[2]!; - expectObservable(item.membership$).toBe("a", { - a: daveMembership, - }); - expectObservable(item.connection$).toBe("a", { - a: expect.toSatisfy((connection) => - deepCompare(connection.transport, daveMembership.transports[0]), - ), - }); - expectObservable(item.participant.value$).toBe("a", { - a: null, - }); - } - return true; - }), - x: expect.anything(), - }); - }); -}); diff --git a/src/state/CallViewModelWidget.test.ts b/src/state/CallViewModelWidget.test.ts index dd2e75ab8..5a7a80d6f 100644 --- a/src/state/CallViewModelWidget.test.ts +++ b/src/state/CallViewModelWidget.test.ts @@ -8,7 +8,6 @@ Please see LICENSE in the repository root for full details. import { it, vi, expect } from "vitest"; import { Subject } from "rxjs"; -// import * as ComponentsCore from "@livekit/components-core"; import { withCallViewModel } from "./CallViewModel/CallViewModelTestUtils.ts"; import { type CallViewModel } from "./CallViewModel/CallViewModel.ts"; import { constant } from "./Behavior.ts"; @@ -21,8 +20,6 @@ import { import { E2eeType } from "../e2ee/e2eeType.ts"; import { MatrixRTCMode } from "../config/ConfigOptions.ts"; -vi.mock("@livekit/components-core", { spy: true }); - it.each([[MatrixRTCMode.Compatibility], [MatrixRTCMode.Matrix_2_0]])( "expect leave when the host asks us to hang up (%s mode)", async (mode) => { diff --git a/src/state/ObservableScope.test.ts b/src/state/ObservableScope.test.ts deleted file mode 100644 index 25bb14324..000000000 --- a/src/state/ObservableScope.test.ts +++ /dev/null @@ -1,293 +0,0 @@ -/* -Copyright 2025 Element Creations Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { BehaviorSubject, combineLatest, Subject } from "rxjs"; -import { logger } from "matrix-js-sdk/lib/logger"; -import { sleep } from "matrix-js-sdk/lib/utils"; - -import { - Epoch, - mapEpoch, - ObservableScope, - trackEpoch, -} from "./ObservableScope"; -import { testScope, withTestScheduler } from "../utils/test"; - -describe("Epoch", () => { - it("should map the value correctly", () => { - const epoch = new Epoch(1); - const mappedEpoch = epoch.mapInner((v) => v + 1); - expect(mappedEpoch.value).toBe(2); - expect(mappedEpoch.epoch).toBe(0); - }); - - it("should be tracked from an observable", () => { - withTestScheduler(({ expectObservable, behavior }) => { - const observable$ = behavior("abc", { - a: 1, - b: 2, - c: 3, - }); - const epochObservable$ = observable$.pipe(trackEpoch()); - expectObservable(epochObservable$).toBe("abc", { - a: expect.toSatisfy((e) => e.epoch === 0 && e.value === 1), - b: expect.toSatisfy((e) => e.epoch === 1 && e.value === 2), - c: expect.toSatisfy((e) => e.epoch === 2 && e.value === 3), - }); - }); - }); - - it("can be mapped without loosing epoch information", () => { - withTestScheduler(({ expectObservable, behavior }) => { - const observable$ = behavior("abc", { - a: "A", - b: "B", - c: "C", - }); - const epochObservable$ = observable$.pipe(trackEpoch()); - const derivedEpoch$ = epochObservable$.pipe( - mapEpoch((e) => e + "-mapped"), - ); - - expectObservable(derivedEpoch$).toBe("abc", { - a: new Epoch("A-mapped", 0), - b: new Epoch("B-mapped", 1), - c: new Epoch("C-mapped", 2), - }); - }); - }); - - it("diamonds emits in a predictable order", () => { - const sb$ = new BehaviorSubject("initial"); - const root$ = sb$.pipe(trackEpoch()); - const derivedA$ = root$.pipe(mapEpoch((e) => e + "-A")); - const derivedB$ = root$.pipe(mapEpoch((e) => e + "-B")); - combineLatest([root$, derivedB$, derivedA$]).subscribe( - ([root, derivedA, derivedB]) => { - logger.log( - "combined" + - root.epoch + - root.value + - "\n" + - derivedA.epoch + - derivedA.value + - "\n" + - derivedB.epoch + - derivedB.value, - ); - }, - ); - sb$.next("updated"); - sb$.next("ANOTERUPDATE"); - }); - - it("behavior test", () => { - const scope = testScope(); - const s$ = new Subject(); - const behavior$ = scope.behavior(s$, 0); - behavior$.subscribe((value) => { - logger.log(`Received value: ${value}`); - }); - s$.next(1); - s$.next(2); - s$.next(3); - s$.next(3); - s$.next(3); - s$.next(3); - s$.next(3); - s$.complete(); - }); -}); - -describe("Reconcile", () => { - beforeEach(() => { - vi.useFakeTimers(); - }); - - afterEach(() => { - vi.useRealTimers(); - }); - - it("should wait clean up before processing next", async () => { - vi.useFakeTimers(); - const scope = testScope(); - const behavior$ = new BehaviorSubject(0); - - const setup = vi.fn().mockImplementation(async () => await sleep(100)); - const cleanup = vi - .fn() - .mockImplementation(async (n: number) => await sleep(100)); - scope.reconcile(behavior$, async (value) => { - await setup(); - return async (): Promise => { - await cleanup(value); - }; - }); - // Let the initial setup process - await vi.advanceTimersByTimeAsync(120); - expect(setup).toHaveBeenCalledTimes(1); - expect(cleanup).toHaveBeenCalledTimes(0); - - // Send next value - behavior$.next(1); - await vi.advanceTimersByTimeAsync(50); - // Should not have started setup for 1 yet - expect(setup).toHaveBeenCalledTimes(1); - expect(cleanup).toHaveBeenCalledTimes(1); - expect(cleanup).toHaveBeenCalledWith(0); - - // Let cleanup finish - await vi.advanceTimersByTimeAsync(50); - // Now setup for 1 should have started - expect(setup).toHaveBeenCalledTimes(2); - }); - - it("should skip intermediates values that are not setup", async () => { - vi.useFakeTimers(); - const scope = testScope(); - const behavior$ = new BehaviorSubject(0); - - const setup = vi - .fn() - .mockImplementation(async (n: number) => await sleep(100)); - - const cleanupLock = Promise.withResolvers(); - const cleanup = vi - .fn() - .mockImplementation(async (n: number) => await cleanupLock.promise); - - scope.reconcile(behavior$, async (value) => { - await setup(value); - return async (): Promise => { - await cleanup(value); - }; - }); - // Let the initial setup process (0) - await vi.advanceTimersByTimeAsync(120); - - // Send 4 next values quickly - behavior$.next(1); - behavior$.next(2); - behavior$.next(3); - behavior$.next(4); - - await vi.advanceTimersByTimeAsync(3000); - // should have only called cleanup for 0 - expect(cleanup).toHaveBeenCalledTimes(1); - expect(cleanup).toHaveBeenCalledWith(0); - // Let cleanup finish - cleanupLock.resolve(undefined); - await vi.advanceTimersByTimeAsync(120); - - // Now setup for 4 should have started, skipping 1,2,3 - expect(setup).toHaveBeenCalledTimes(2); - expect(setup).toHaveBeenCalledWith(4); - expect(setup).not.toHaveBeenCalledWith(1); - expect(setup).not.toHaveBeenCalledWith(2); - expect(setup).not.toHaveBeenCalledWith(3); - }); - - it("should wait for setup to complete before starting cleanup", async () => { - vi.useFakeTimers(); - const scope = testScope(); - const behavior$ = new BehaviorSubject(0); - - const setup = vi - .fn() - .mockImplementation(async (n: number) => await sleep(3000)); - - const cleanupLock = Promise.withResolvers(); - const cleanup = vi - .fn() - .mockImplementation(async (n: number) => await cleanupLock.promise); - - scope.reconcile(behavior$, async (value) => { - await setup(value); - return async (): Promise => { - await cleanup(value); - }; - }); - - await vi.advanceTimersByTimeAsync(500); - // Setup for 0 should be in progress - expect(setup).toHaveBeenCalledTimes(1); - - behavior$.next(1); - await vi.advanceTimersByTimeAsync(500); - - // Should not have started setup for 1 yet - expect(setup).not.toHaveBeenCalledWith(1); - // Should not have called cleanup yet, because the setup for 0 is not done - expect(cleanup).toHaveBeenCalledTimes(0); - - // Let setup for 0 finish - await vi.advanceTimersByTimeAsync(2500 + 100); - // Now cleanup for 0 should have started - expect(cleanup).toHaveBeenCalledTimes(1); - expect(cleanup).toHaveBeenCalledWith(0); - - cleanupLock.resolve(undefined); - await vi.advanceTimersByTimeAsync(100); - // Now setup for 1 should have started - expect(setup).toHaveBeenCalledWith(1); - }); -}); - -describe("behavior", () => { - it("warns with the tag and nesting depth, logging each new depth once", () => { - const warn = vi.spyOn(logger, "warn").mockImplementation(() => {}); - const scope = testScope(); - const source$ = new Subject(); - const behavior$ = scope.behavior(source$, 0, "tagged"); - // Re-enter once on value 1 (depth 2), then a nested re-entry on value 2 - // (depth 3), which is the shape needed to strand a middle subscriber. - behavior$.subscribe((v) => { - if (v === 1) source$.next(2); - }); - behavior$.subscribe((v) => { - if (v === 2) source$.next(3); - }); - const seen: number[] = []; - behavior$.subscribe((v) => seen.push(v)); - - source$.next(1); - - const messages = warn.mock.calls.map((c) => c[0] as string); - expect(messages).toEqual([ - expect.stringContaining("Behavior (tagged) re-entered at depth 2"), - expect.stringContaining("Behavior (tagged) re-entered at depth 3"), - ]); - warn.mock.calls.forEach((c) => expect(c[1]).toEqual(expect.any(String))); - // The behavior settles on the newest value while the last subscriber is - // left stranded on the oldest. - expect(behavior$.value).toBe(3); - expect(seen.at(-1)).toBe(1); - }); - - it("warns when the scope is ended while a behavior is mid-delivery", () => { - const warn = vi.spyOn(logger, "warn").mockImplementation(() => {}); - const scope = new ObservableScope(); - const source$ = new Subject(); - const behavior$ = scope.behavior(source$, 0); - behavior$.subscribe((v) => { - if (v === 1) scope.end(); - }); - // A derived behavior is bound to the scope, so ending the scope - // mid-delivery unsubscribes it before the value reaches it. - const derived$ = scope.behavior(behavior$); - - source$.next(1); - - expect(warn).toHaveBeenCalledWith( - expect.stringContaining("Scope ended while 1 of its behaviors"), - expect.any(String), - ); - expect(behavior$.value).toBe(1); - expect(derived$.value).toBe(0); - }); -}); diff --git a/src/state/ObservableScope.ts b/src/state/ObservableScope.ts index 8f09cb6ef..0cabefbc9 100644 --- a/src/state/ObservableScope.ts +++ b/src/state/ObservableScope.ts @@ -1,334 +1,20 @@ /* -Copyright 2024 New Vector Ltd. +Copyright 2026 Element Creations Ltd. SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { - BehaviorSubject, - catchError, - distinctUntilChanged, - EMPTY, - endWith, - filter, - map, - type Observable, - type OperatorFunction, - share, - take, - takeUntil, -} from "rxjs"; +import { ObservableScope } from "@element-hq/matrixrtc-sdk"; -import { logger } from "matrix-js-sdk/lib/logger"; - -import { type Behavior } from "./Behavior"; - -type MonoTypeOperator = (o: Observable) => Observable; - -type SplitBehavior = keyof T extends string | number - ? { [K in keyof T as `${K}$`]: Behavior } - : never; - -const nothing = Symbol("nothing"); - -/** - * A scope which limits the execution lifetime of its bound Observables. - */ -export class ObservableScope { - /** - * Number of this scope's behaviors currently mid-delivery. Used to detect a - * scope being torn down synchronously from within one of its own emissions. - */ - private delivering = 0; - - private readonly ended$ = new BehaviorSubject(false); - - private readonly bindImpl: MonoTypeOperator = takeUntil( - this.ended$.pipe(filter((ended) => ended)), - ); - - /** - * Binds an Observable to this scope, so that it completes when the scope - * ends. - */ - public bind(): MonoTypeOperator { - return this.bindImpl; - } - - private readonly shareImpl: MonoTypeOperator = share({ - resetOnError: false, - resetOnComplete: false, - resetOnRefCountZero: false, - }); - /** - * Shares (multicasts) the Observable as a hot Observable. - */ - public readonly share: MonoTypeOperator = (input$) => - input$.pipe(this.bindImpl, this.shareImpl); - - /** - * Converts an Observable to a Behavior. If no initial value is specified, the - * Observable must synchronously emit an initial value. - */ - public behavior( - setValue$: Observable, - initialValue: T | typeof nothing = nothing, - // An optional label used only in the re-entrancy warning below, so that a - // torn behavior can be identified from a rageshake. - tag?: string, - ): Behavior { - const subject$ = new BehaviorSubject(initialValue); - // Push values from the Observable into the BehaviorSubject. - // BehaviorSubjects have an undesirable feature where if you call 'complete', - // they will no longer re-emit their current value upon subscription. We want - // to support Observables that complete (for example `of({})`), so we have to - // take care to not propagate the completion event. - // If a subscriber synchronously causes this same behavior to emit again, - // rxjs delivers the nested value to every subscriber first and then - // resumes delivering the outer (older) value to the remaining subscribers, - // leaving them permanently out of sync with the others. A single such - // re-entry can only strand a contiguous run of subscribers; stranding a - // subscriber in the middle of the list needs a nested (deeper) re-entry, so - // we report the depth and log each new depth (with a stack trace) rather - // than only the first occurrence, to make a nested re-entry visible. - let depth = 0; - let maxReportedDepth = 1; - setValue$.pipe(this.bind(), distinctUntilChanged()).subscribe({ - next: (value) => { - if (depth > 0 && depth + 1 > maxReportedDepth) { - maxReportedDepth = depth + 1; - logger.warn( - `Behavior${tag ? ` (${tag})` : ""} re-entered at depth ${ - depth + 1 - } while delivering a value; later subscribers will be left with a stale value`, - new Error().stack, - ); - } - depth++; - this.delivering++; - try { - subject$.next(value); - } finally { - depth--; - this.delivering--; - } - }, - error: (err: unknown) => { - subject$.error(err); - }, - }); - if (subject$.value === nothing) - throw new Error("Behavior failed to synchronously emit an initial value"); - return subject$ as Behavior; - } - - /** - * Ends the scope, causing any bound Observables to complete. - */ - public end(): void { - if (this.delivering > 0) - logger.warn( - `Scope ended while ${this.delivering} of its behaviors were still delivering a value; later subscribers will be left with a stale value`, - new Error().stack, - ); - this.ended$.next(true); - } - - /** - * Register a callback to be executed when the scope is ended. - */ - public onEnd(callback: () => void): void { - this.ended$ - .pipe( - filter((ended) => ended), - take(1), - ) - .subscribe(callback); - } - - /** - * For the duration of the scope, sync some external state with the value of - * the provided Behavior by way of an async function which attempts to update - * (reconcile) the external state. The reconciliation function may return a - * clean-up callback which will be called and awaited before the next change - * in value (or the end of the scope). - * - * All calls to the function and its clean-up callbacks are serialized. If the - * value changes faster than the handlers can keep up with, intermediate - * values may be skipped. - * - * Basically, this is like React's useEffect but async and for Behaviors. - */ - public reconcile( - value$: Behavior, - callback: (value: T) => Promise<(() => Promise) | void>, - ): void { - let latestValue: T | typeof nothing = nothing; - let reconcilePromise: Promise | undefined = undefined; - let cleanUp: (() => Promise) | void = undefined; - let prevVal: T | typeof nothing = nothing; - - // While this loop runs it will process the latest from `value$` until it caught up with the updates. - // It might skip updates from `value$` and only process the newest value after callback has resolved. - const reconcileLoop = async (): Promise => { - while (latestValue !== prevVal) { - await cleanUp?.(); // Call the previous value's clean-up handler - prevVal = latestValue; - - if (latestValue !== nothing) cleanUp = await callback(latestValue); // Sync current value... - // `latestValue` might have gotten updated during the `await callback`. That is why we loop here - } - }; - - value$ - .pipe( - catchError(() => EMPTY), // Ignore errors - this.bind(), // Limit to the duration of the scope - endWith(nothing), // Clean up when the scope ends - ) - .subscribe((value) => { - // Always track the latest value! The `reconcileLoop` will run until it "processed" the "last" `latestValue`. - latestValue = value; - // There's already an instance of the below 'reconcileLoop' loop running - // concurrently. So lets let the loop handle it. NEVER instanciate two `reconcileLoop`s. - if (reconcilePromise) return; - - reconcilePromise = reconcileLoop().finally(() => { - reconcilePromise = undefined; - }); - }); - } - - /** - * Splits a Behavior of objects with static properties into an object with - * Behavior properties. - * - * For example, splitting a `Behavior<{ name: string; age: number }>` results - * in an object of type `{ name$: Behavior; age$: Behavior }`. - */ - public splitBehavior( - input$: Behavior, - ): SplitBehavior { - return Object.fromEntries( - Object.keys(input$.value).map((key) => [ - `${key}$`, - this.behavior( - input$.pipe(map((input) => input[key as keyof T])), - nothing, - key, - ), - ]), - ) as SplitBehavior; - } -} +export { + Epoch, + mapEpoch, + ObservableScope, + trackEpoch, +} from "@element-hq/matrixrtc-sdk"; /** * The global scope, a scope which never ends. */ export const globalScope = new ObservableScope(); - -/** - * `Epoch`'s can be used to create `Behavior`s and `Observable`s which derivitives can be merged - * with `combinedLatest` without duplicated emissions. - * - * This is useful in the following example: - * ``` - * const rootObs$ = of("red","green","blue"); - * const derivedObs$ = rootObs$.pipe( - * map((v)=> {red:"fire", green:"grass", blue:"water"}[v]) - * ); - * const otherDerivedObs$ = rootObs$.pipe( - * map((v)=> {red:"tomatoes", green:"leaves", blue:"sky"}[v]) - * ); - * const mergedObs$ = combineLatest([rootObs$, derivedObs$, otherDerivedObs$]).pipe( - * map(([color, a,b]) => color + " like " + a + " and " + b) - * ); - * - * ``` - * will result in 6 emissions with mismatching items like "red like fire and leaves" - * - * # Use Epoch - * ``` - * const ancestorObs$ = of(1,2,3).pipe(trackEpoch()); - * const derivedObs$ = ancestorObs$.pipe( - * mapEpoch((v)=> "this number: " + v) - * ); - * const otherDerivedObs$ = ancestorObs$.pipe( - * mapEpoch((v)=> "multiplied by: " + v) - * ); - * const mergedObs$ = combineLatest([derivedObs$, otherDerivedObs$]).pipe( - * filter((values) => values.every((v) => v.epoch === values[0].v)), - * map(([color, a, b]) => color + " like " + a + " and " + b) - * ); - * - * ``` - * will result in 3 emissions all matching (e.g. "blue like water and sky") - */ -export class Epoch { - public readonly epoch: number; - public readonly value: T; - - public constructor(value: T, epoch?: number) { - this.value = value; - this.epoch = epoch ?? 0; - } - /** - * Maps the value inside the epoch to a new value while keeping the epoch number. - * # usage - * ``` - * const myEpoch$ = myObservable$.pipe( - * map(trackEpoch()), - * // this is the preferred way using mapEpoch - * mapEpoch((v)=> v+1) - * // This is how inner map can be used: - * map((epoch) => epoch.innerMap((v)=> v+1)) - * // It is equivalent to: - * map((epoch) => new Epoch(epoch.value + 1, epoch.epoch)) - * ) - * ``` - * See also `Epoch` - */ - public mapInner(map: (value: T) => U): Epoch { - return new Epoch(map(this.value), this.epoch); - } -} - -/** - * A `pipe` compatible map oparator that keeps the epoch in tact but allows mapping the value. - * # usage - * ``` - * const myEpoch$ = myObservable$.pipe( - * map(trackEpoch()), - * // this is the preferred way using mapEpoch - * mapEpoch((v)=> v+1) - * // This is how inner map can be used: - * map((epoch) => epoch.innerMap((v)=> v+1)) - * // It is equivalent to: - * map((epoch) => new Epoch(epoch.value + 1, epoch.epoch)) - * ) - * ``` - * See also `Epoch` - */ -export function mapEpoch( - mapFn: (value: T) => U, -): OperatorFunction, Epoch> { - return map((e) => e.mapInner(mapFn)); -} - -/** - * # usage - * ``` - * const myEpoch$ = myObservable$.pipe( - * map(trackEpoch()), - * map((epoch) => epoch.innerMap((v)=> v+1)) - * ) - * const derived = myEpoch$.pipe( - * mapEpoch((v)=>v^2) - * ) - * ``` - * See also `Epoch` - */ -export function trackEpoch(): OperatorFunction> { - return map>((value, number) => new Epoch(value, number)); -} diff --git a/src/state/SessionBehaviors.test.ts b/src/state/SessionBehaviors.test.ts deleted file mode 100644 index d9da6f830..000000000 --- a/src/state/SessionBehaviors.test.ts +++ /dev/null @@ -1,100 +0,0 @@ -/* -Copyright 2025, 2026 Element Creations Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { describe, expect, it, vi } from "vitest"; -import { - MatrixRTCSessionEvent, - type Transport, - type UnstableLivekitTransport, -} from "matrix-js-sdk/lib/matrixrtc"; -import { EventEmitter } from "events"; - -import { - createKeyRotationSuppressed$, - membershipsAndTransports$, -} from "./SessionBehaviors"; -import { Epoch } from "./ObservableScope"; -import { constant } from "./Behavior"; -import { mockRtcMembership, testScope } from "../utils/test"; - -describe("SessionBehaviors", () => { - describe("createKeyRotationSuppressed$", () => { - it("emits initial value from isKeyRotationSuppressed and updates when KeyRotationSuppressedChanged event is emitted", () => { - const scope = testScope(); - const emitter = new EventEmitter(); - const mockSession = Object.assign(emitter, { - on: vi.fn(), - off: vi.fn(), - isKeyRotationSuppressed: false, - }); - - const keyRotationSuppressed$ = createKeyRotationSuppressed$( - scope, - mockSession as any, - ); - - expect(keyRotationSuppressed$.value).toBe(false); - emitter.emit(MatrixRTCSessionEvent.KeyRotationSuppressedChanged, true); - - expect(keyRotationSuppressed$.value).toBe(true); - - emitter.emit(MatrixRTCSessionEvent.KeyRotationSuppressedChanged, false); - - expect(keyRotationSuppressed$.value).toBe(false); - }); - }); - - describe("membershipsAndTransports$", () => { - const serviceTransport: UnstableLivekitTransport = { - type: "livekit", - livekit_service_url: "https://lk.example.org", - }; - const urlTransport: UnstableLivekitTransport = { - type: "livekit", - url: "wss://sfu.sample.com", - }; - - it("locates each member's transport on the member's homeserver", () => { - const bob = mockRtcMembership("@bob:example.org", "DEV000", { - fociPreferred: [serviceTransport], - }); - const carl = mockRtcMembership("@carl:sample.com", "DEV111", { - fociPreferred: [urlTransport], - }); - - const { membershipsWithTransport$, transports$ } = - membershipsAndTransports$( - testScope(), - constant(new Epoch([bob, carl])), - ); - - expect(transports$.value.value).toEqual([ - { transport: serviceTransport, serverName: "example.org" }, - { transport: urlTransport, serverName: "sample.com" }, - ]); - expect( - membershipsWithTransport$.value.value.map( - (m) => m.transport?.serverName, - ), - ).toEqual(["example.org", "sample.com"]); - }); - - it("leaves out members without a LiveKit transport", () => { - const dave = mockRtcMembership("@dave:example.org", "DEV222", { - fociPreferred: [{ type: "something-else" } as Transport], - }); - - const { membershipsWithTransport$, transports$ } = - membershipsAndTransports$(testScope(), constant(new Epoch([dave]))); - - expect(transports$.value.value).toEqual([]); - expect(membershipsWithTransport$.value.value).toEqual([ - { membership: dave, transport: undefined }, - ]); - }); - }); -}); diff --git a/src/state/SessionBehaviors.ts b/src/state/SessionBehaviors.ts deleted file mode 100644 index 6620350cf..000000000 --- a/src/state/SessionBehaviors.ts +++ /dev/null @@ -1,106 +0,0 @@ -/* -Copyright 2025 Element Creations Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - type CallMembership, - type MatrixRTCSession, - MatrixRTCSessionEvent, - isUnstableLivekitTransport, -} from "matrix-js-sdk/lib/matrixrtc"; -import { fromEvent } from "rxjs"; - -import { - Epoch, - mapEpoch, - trackEpoch, - type ObservableScope, -} from "./ObservableScope"; -import { type Behavior } from "./Behavior"; -import { type TransportLocator } from "../livekit/auth"; - -/** - * Tracks the transports used by ourselves, plus all other MatrixRTC session - * members. - */ -export const membershipsAndTransports$ = ( - scope: ObservableScope, - memberships$: Behavior>, -): { - membershipsWithTransport$: Behavior< - Epoch< - { membership: CallMembership; transport: TransportLocator | undefined }[] - > - >; - transports$: Behavior>; -} => { - const membershipsWithTransport$: Behavior< - Epoch< - { membership: CallMembership; transport: TransportLocator | undefined }[] - > - > = scope.behavior( - memberships$.pipe( - mapEpoch((memberships) => { - return memberships.map((membership) => { - const transport = membership.getTransport(); - return { - membership, - transport: isUnstableLivekitTransport(transport) - ? { - transport, - serverName: membership.userId.replace(/^.*?:/, ""), - } - : undefined, - }; - }); - }), - ), - ); - - const transports$: Behavior> = scope.behavior( - membershipsWithTransport$.pipe( - mapEpoch((mts) => mts.flatMap(({ transport: t }) => (t ? [t] : []))), - ), - ); - - return { - membershipsWithTransport$, - transports$, - }; -}; - -export const createMemberships$ = ( - scope: ObservableScope, - matrixRTCSession: MatrixRTCSession, -): Behavior> => { - return scope.behavior( - fromEvent( - matrixRTCSession, - MatrixRTCSessionEvent.MembershipsChanged, - (_, memberships: CallMembership[]) => memberships, - ).pipe(trackEpoch()), - new Epoch(matrixRTCSession.memberships), - ); -}; - -/** - * Whether the session has grown large enough that MatrixRTC has stopped rotating the media - * encryption key. While this is true the key in use is still shared with new joiners, but no new - * key is generated when someone joins or leaves. - */ -export const createKeyRotationSuppressed$ = ( - scope: ObservableScope, - matrixRTCSession: MatrixRTCSession, -): Behavior => { - return scope.behavior( - fromEvent( - matrixRTCSession, - MatrixRTCSessionEvent.KeyRotationSuppressedChanged, - (suppressed: boolean) => suppressed, - ), - matrixRTCSession.isKeyRotationSuppressed, - ); -}; diff --git a/src/state/media/LocalScreenShareViewModel.ts b/src/state/media/LocalScreenShareViewModel.ts index b31739d92..1a675019d 100644 --- a/src/state/media/LocalScreenShareViewModel.ts +++ b/src/state/media/LocalScreenShareViewModel.ts @@ -6,23 +6,19 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type LocalParticipant } from "livekit-client"; +import { type ObservableScope } from "@element-hq/matrixrtc-sdk"; -import { type Behavior } from "../Behavior"; import { type BaseScreenShareInputs, type BaseScreenShareViewModel, createBaseScreenShare, } from "./ScreenShareViewModel"; -import { type ObservableScope } from "../ObservableScope"; export interface LocalScreenShareViewModel extends BaseScreenShareViewModel { local: true; } -export interface LocalScreenShareInputs extends BaseScreenShareInputs { - participant$: Behavior; -} +export type LocalScreenShareInputs = BaseScreenShareInputs; export function createLocalScreenShare( scope: ObservableScope, diff --git a/src/state/media/LocalUserMediaViewModel.ts b/src/state/media/LocalUserMediaViewModel.ts index fd21428bc..289e091d3 100644 --- a/src/state/media/LocalUserMediaViewModel.ts +++ b/src/state/media/LocalUserMediaViewModel.ts @@ -7,29 +7,18 @@ Please see LICENSE in the repository root for full details. */ import { - facingModeFromLocalTrack, - type LocalParticipant, - LocalVideoTrack, - TrackEvent, -} from "livekit-client"; -import { - fromEvent, - map, - merge, - type Observable, - of, - startWith, - switchMap, -} from "rxjs"; + type Behavior, + type LocalMemberMedia, + type ObservableScope, +} from "@element-hq/matrixrtc-sdk"; +import { map, of, switchMap } from "rxjs"; import { logger } from "matrix-js-sdk/lib/logger"; -import { type Behavior } from "../Behavior"; import { type BaseUserMediaInputs, type BaseUserMediaViewModel, createBaseUserMedia, } from "./UserMediaViewModel"; -import { type ObservableScope } from "../ObservableScope"; import { alwaysShowSelf } from "../../settings/settings"; import { platform } from "../../Platform"; import { type MediaDevices } from "../MediaDevices"; @@ -51,9 +40,9 @@ export interface LocalUserMediaViewModel extends BaseUserMediaViewModel { export interface LocalUserMediaInputs extends Omit< BaseUserMediaInputs, - "statsType" + "media$" > { - participant$: Behavior; + media$: Behavior; mediaDevices: MediaDevices; } @@ -61,76 +50,43 @@ export function createLocalUserMedia( scope: ObservableScope, { mediaDevices, ...inputs }: LocalUserMediaInputs, ): LocalUserMediaViewModel { - const baseUserMedia = createBaseUserMedia(scope, { - ...inputs, - statsType: "outbound-rtp", - }); - - /** - * The local video track as an observable that emits whenever the track - * changes, the camera is switched, or the track is muted. - */ - const videoTrack$: Observable = + const baseUserMedia = createBaseUserMedia(scope, inputs); + const facingMode$ = scope.behavior( baseUserMedia.video$.pipe( - switchMap((v) => { - const track = v?.publication.track; - if (!(track instanceof LocalVideoTrack)) return of(null); - return merge( - // Watch for track restarts because they indicate a camera switch. - // This event is also emitted when unmuting the track object. - fromEvent(track, TrackEvent.Restarted).pipe( - startWith(null), - map(() => track), - ), - // When the track object is muted, reset it to null. - fromEvent(track, TrackEvent.Muted).pipe(map(() => null)), - ); - }), - ); + switchMap((video) => video?.facingMode$ ?? of(undefined)), + ), + ); return { ...baseUserMedia, local: true, + // Mirror only front-facing cameras (those that face the user) mirror$: scope.behavior( - videoTrack$.pipe( - // Mirror only front-facing cameras (those that face the user) - map( - (track) => - track !== null && - facingModeFromLocalTrack(track).facingMode === "user", - ), - ), + facingMode$.pipe(map((facingMode) => facingMode === "user")), ), alwaysShow$: alwaysShowSelf.value$, setAlwaysShow: alwaysShowSelf.setValue, switchCamera$: scope.behavior( platform === "desktop" ? of(null) - : videoTrack$.pipe( - map((track) => { - if (track === null) return null; - const facingMode = facingModeFromLocalTrack(track).facingMode; - // If the camera isn't front or back-facing, don't provide a switch - // camera shortcut at all - if (facingMode !== "user" && facingMode !== "environment") - return null; - // Restart the track with a camera facing the opposite direction - return (): void => - void track - .restartTrack({ - facingMode: facingMode === "user" ? "environment" : "user", - }) - .then(() => { - // Inform the MediaDevices which camera was chosen - const deviceId = - track.mediaStreamTrack.getSettings().deviceId; - if (deviceId !== undefined) - mediaDevices.videoInput.select(deviceId); - }) - .catch((e) => - logger.error("Failed to switch camera", facingMode, e), - ); - }), + : facingMode$.pipe( + // If the camera isn't front or back-facing, don't provide a switch + // camera shortcut at all + map((facingMode) => + facingMode === undefined + ? null + : (): void => + void inputs.media$.value + ?.switchCamera() + .then((deviceId) => { + // Inform the MediaDevices which camera was chosen + if (deviceId !== undefined) + mediaDevices.videoInput.select(deviceId); + }) + .catch((e) => + logger.error("Failed to switch camera", facingMode, e), + ), + ), ), ), }; diff --git a/src/state/media/MediaViewModel.test.ts b/src/state/media/MediaViewModel.test.ts index 872fdd3ed..eae61f378 100644 --- a/src/state/media/MediaViewModel.test.ts +++ b/src/state/media/MediaViewModel.test.ts @@ -7,35 +7,26 @@ Please see LICENSE in the repository root for full details. import { expect, onTestFinished, test, vi } from "vitest"; import { - type LocalTrackPublication, - LocalVideoTrack, - Track, - TrackEvent, -} from "livekit-client"; + type Behavior, + constant, + type MediaStreamStats, +} from "@element-hq/matrixrtc-sdk"; import { waitFor } from "@testing-library/dom"; +import { BehaviorSubject } from "rxjs"; import { - mockLocalParticipant, - mockMediaDevices, - mockRtcMembership, + mockAudioTrack, mockLocalMedia, + mockMediaDevices, + mockMemberMedia, mockRemoteMedia, - withTestScheduler, - mockRemoteParticipant, mockRemoteScreenShare, + mockRtcMembership, + mockVideoTrack, + withTestScheduler, } from "../../utils/test"; -import { constant } from "../Behavior"; import { showConnectionStats } from "../../settings/settings"; -global.MediaStreamTrack = class {} as unknown as { - new (): MediaStreamTrack; - prototype: MediaStreamTrack; -}; -global.MediaStream = class {} as unknown as { - new (): MediaStream; - prototype: MediaStream; -}; - const platformMock = vi.hoisted(() => vi.fn(() => "desktop")); vi.mock("../../Platform", () => ({ get platform(): string { @@ -43,49 +34,45 @@ vi.mock("../../Platform", () => ({ }, })); -const observeRtpStreamStatsMock = vi.hoisted(() => vi.fn()); -vi.mock("./observeRtpStreamStats", async (importOriginal) => ({ - ...(await importOriginal>()), - observeRtpStreamStats$: observeRtpStreamStatsMock, -})); - const rtcMembership = mockRtcMembership("@alice:example.org", "AAAA"); test("control a participant's volume", () => { - const setVolumeSpy = vi.fn(); + const setVolume = vi.fn(); const vm = mockRemoteMedia( rtcMembership, {}, - mockRemoteParticipant({ setVolume: setVolumeSpy }), + mockMemberMedia({ + microphone$: constant(mockAudioTrack({ setVolume })), + }), ); withTestScheduler(({ expectObservable, schedule }) => { schedule("-ab---c---d|", { a() { // Try muting by toggling vm.togglePlaybackMuted(); - expect(setVolumeSpy).toHaveBeenLastCalledWith(0); + expect(setVolume).toHaveBeenLastCalledWith(0); }, b() { // Try unmuting by dragging the slider back up vm.adjustPlaybackVolume(0.6); vm.adjustPlaybackVolume(0.8); vm.commitPlaybackVolume(); - expect(setVolumeSpy).toHaveBeenCalledWith(0.6); - expect(setVolumeSpy).toHaveBeenLastCalledWith(0.8); + expect(setVolume).toHaveBeenCalledWith(0.6); + expect(setVolume).toHaveBeenLastCalledWith(0.8); }, c() { // Try muting by dragging the slider back down vm.adjustPlaybackVolume(0.2); vm.adjustPlaybackVolume(0); vm.commitPlaybackVolume(); - expect(setVolumeSpy).toHaveBeenCalledWith(0.2); - expect(setVolumeSpy).toHaveBeenLastCalledWith(0); + expect(setVolume).toHaveBeenCalledWith(0.2); + expect(setVolume).toHaveBeenLastCalledWith(0); }, d() { // Try unmuting by toggling vm.togglePlaybackMuted(); // The volume should return to the last non-zero committed volume - expect(setVolumeSpy).toHaveBeenLastCalledWith(0.8); + expect(setVolume).toHaveBeenLastCalledWith(0.8); }, }); expectObservable(vm.playbackVolume$).toBe("ab(cd)(ef)g", { @@ -101,58 +88,39 @@ test("control a participant's volume", () => { }); test("control a participant's screen share volume", () => { - const setVolumeSpy = vi.fn(); + const setVolume = vi.fn(); const vm = mockRemoteScreenShare( rtcMembership, {}, - mockRemoteParticipant({ setVolume: setVolumeSpy }), + mockMemberMedia({ + screenShareAudio$: constant( + mockAudioTrack({ source: "screenShareAudio", setVolume }), + ), + }), ); withTestScheduler(({ expectObservable, schedule }) => { schedule("-ab---c---d|", { a() { - // Try muting by toggling vm.togglePlaybackMuted(); - expect(setVolumeSpy).toHaveBeenLastCalledWith( - 0, - Track.Source.ScreenShareAudio, - ); + expect(setVolume).toHaveBeenLastCalledWith(0); }, b() { - // Try unmuting by dragging the slider back up vm.adjustPlaybackVolume(0.6); vm.adjustPlaybackVolume(0.8); vm.commitPlaybackVolume(); - expect(setVolumeSpy).toHaveBeenCalledWith( - 0.6, - Track.Source.ScreenShareAudio, - ); - expect(setVolumeSpy).toHaveBeenLastCalledWith( - 0.8, - Track.Source.ScreenShareAudio, - ); + expect(setVolume).toHaveBeenCalledWith(0.6); + expect(setVolume).toHaveBeenLastCalledWith(0.8); }, c() { - // Try muting by dragging the slider back down vm.adjustPlaybackVolume(0.2); vm.adjustPlaybackVolume(0); vm.commitPlaybackVolume(); - expect(setVolumeSpy).toHaveBeenCalledWith( - 0.2, - Track.Source.ScreenShareAudio, - ); - expect(setVolumeSpy).toHaveBeenLastCalledWith( - 0, - Track.Source.ScreenShareAudio, - ); + expect(setVolume).toHaveBeenCalledWith(0.2); + expect(setVolume).toHaveBeenLastCalledWith(0); }, d() { - // Try unmuting by toggling vm.togglePlaybackMuted(); - // The volume should return to the last non-zero committed volume - expect(setVolumeSpy).toHaveBeenLastCalledWith( - 0.8, - Track.Source.ScreenShareAudio, - ); + expect(setVolume).toHaveBeenLastCalledWith(0.8); }, }); expectObservable(vm.playbackVolume$).toBe("ab(cd)(ef)g", { @@ -165,25 +133,25 @@ test("control a participant's screen share volume", () => { g: 0.8, }); }); + expect(vm.audioEnabled$.value).toBe(true); }); test("local media remembers whether it should always be shown", () => { const vm1 = mockLocalMedia( rtcMembership, {}, - mockLocalParticipant({}), + mockMemberMedia({ local: true }), mockMediaDevices({}), ); withTestScheduler(({ expectObservable, schedule }) => { schedule("-a|", { a: () => vm1.setAlwaysShow(false) }); expectObservable(vm1.alwaysShow$).toBe("ab", { a: true, b: false }); }); - // Next local media should start out *not* always shown const vm2 = mockLocalMedia( rtcMembership, {}, - mockLocalParticipant({}), + mockMemberMedia({ local: true }), mockMediaDevices({}), ); withTestScheduler(({ expectObservable, schedule }) => { @@ -196,43 +164,24 @@ test("switch cameras", async () => { // Camera switching is only available on mobile platformMock.mockReturnValue("android"); onTestFinished(() => void platformMock.mockReset()); - - // Construct a mock video track which knows how to be restarted - const track = new LocalVideoTrack({ - getConstraints() {}, - addEventListener() {}, - removeEventListener() {}, - } as unknown as MediaStreamTrack); - - let deviceId = "front camera"; - const restartTrack = vi.fn(async ({ facingMode }) => { - deviceId = facingMode === "user" ? "front camera" : "back camera"; - track.emit(TrackEvent.Restarted); - return Promise.resolve(); + const facingMode$ = new BehaviorSubject<"user" | "environment" | undefined>( + "user", + ); + // The SDK restarts the camera the other way round and says which device it + // ended up on + const switchCamera = vi.fn(async (): Promise => { + const back = facingMode$.value === "user"; + facingMode$.next(back ? "environment" : "user"); + return Promise.resolve(back ? "back camera" : "front camera"); }); - track.restartTrack = restartTrack; - - Object.defineProperty(track, "mediaStreamTrack", { - get() { - return { - label: "Video", - getSettings: (): object => ({ - deviceId, - facingMode: deviceId === "front camera" ? "user" : "environment", - }), - }; - }, - }); - const selectVideoInput = vi.fn(); - const vm = mockLocalMedia( rtcMembership, {}, - mockLocalParticipant({ - getTrackPublication() { - return { track } as unknown as LocalTrackPublication; - }, + mockMemberMedia({ + local: true, + camera$: constant(mockVideoTrack({ facingMode$ })), + switchCamera, }), mockMediaDevices({ videoInput: { @@ -242,69 +191,82 @@ test("switch cameras", async () => { }, }), ); + expect(vm.mirror$.value).toBe(true); // Switch to back camera vm.switchCamera$.value!(); - expect(restartTrack).toHaveBeenCalledExactlyOnceWith({ - facingMode: "environment", - }); + expect(switchCamera).toHaveBeenCalledTimes(1); await waitFor(() => { - expect(selectVideoInput).toHaveBeenCalledTimes(1); expect(selectVideoInput).toHaveBeenCalledWith("back camera"); }); - expect(deviceId).toBe("back camera"); + expect(vm.mirror$.value).toBe(false); // Switch to front camera vm.switchCamera$.value!(); - expect(restartTrack).toHaveBeenCalledTimes(2); - expect(restartTrack).toHaveBeenLastCalledWith({ facingMode: "user" }); + expect(switchCamera).toHaveBeenCalledTimes(2); await waitFor(() => { - expect(selectVideoInput).toHaveBeenCalledTimes(2); expect(selectVideoInput).toHaveBeenLastCalledWith("front camera"); }); - expect(deviceId).toBe("front camera"); + expect(vm.mirror$.value).toBe(true); }); -test("remote media is in waiting state when participant has not yet connected", () => { - const vm = mockRemoteMedia(rtcMembership, {}, null); // null participant +test("no camera switch where the facing mode is unknown", () => { + platformMock.mockReturnValue("android"); + onTestFinished(() => void platformMock.mockReset()); + const vm = mockLocalMedia( + rtcMembership, + {}, + mockMemberMedia({ local: true, camera$: constant(mockVideoTrack()) }), + mockMediaDevices({}), + ); + expect(vm.switchCamera$.value).toBeNull(); +}); + +test("remote media is in waiting state while its media has not arrived", () => { + const vm = mockRemoteMedia(rtcMembership, {}, null); expect(vm.waitingForMedia$.value).toBe(true); }); -test("remote media is not in waiting state when participant is connected", () => { - const vm = mockRemoteMedia(rtcMembership, {}, mockRemoteParticipant({})); - expect(vm.waitingForMedia$.value).toBe(false); -}); - -test("remote media is not in waiting state when participant is connected with no publications", () => { - const vm = mockRemoteMedia( - rtcMembership, - {}, - mockRemoteParticipant({ - getTrackPublication: () => undefined, - getTrackPublications: () => [], - }), - ); +test("remote media is not in waiting state once its media is there", () => { + const vm = mockRemoteMedia(rtcMembership, {}, mockMemberMedia()); expect(vm.waitingForMedia$.value).toBe(false); }); test("remote media is not in waiting state when user does not intend to publish anywhere", () => { - const vm = mockRemoteMedia( - rtcMembership, - {}, - mockRemoteParticipant({}), - undefined, // No room (no advertised transport) - ); + const vm = mockRemoteMedia(rtcMembership, {}, null, {}); expect(vm.waitingForMedia$.value).toBe(false); }); +test("audio and video follow the tracks' mute state", () => { + const vm = mockRemoteMedia( + rtcMembership, + {}, + mockMemberMedia({ + microphone$: constant(mockAudioTrack({ muted$: constant(false) })), + camera$: constant(mockVideoTrack({ muted$: constant(true) })), + }), + ); + expect(vm.audioEnabled$.value).toBe(true); + expect(vm.videoEnabled$.value).toBe(false); +}); + test("user media polls stream stats only while the setting is on", () => { - const participant = mockRemoteParticipant({}); - const vm = mockRemoteMedia(rtcMembership, {}, participant); onTestFinished(() => showConnectionStats.setValue(false)); withTestScheduler(({ cold, expectObservable, schedule }) => { const stats = { type: "inbound-rtp" } as RTCInboundRtpStreamStats; - observeRtpStreamStatsMock.mockImplementation(() => - cold("-s", { s: stats }), + // A behavior that only runs its source while subscribed, as the SDK's does + const vm = mockRemoteMedia( + rtcMembership, + {}, + mockMemberMedia({ + microphone$: constant( + mockAudioTrack({ + stats$: cold("-s", { + s: stats, + }) as unknown as Behavior, + }), + ), + }), ); schedule("-a-b", { a() { @@ -319,14 +281,4 @@ test("user media polls stream stats only while the setting is on", () => { s: stats, }); }); - expect(observeRtpStreamStatsMock).toHaveBeenCalledWith( - participant, - Track.Source.Microphone, - "inbound-rtp", - ); - expect(observeRtpStreamStatsMock).toHaveBeenCalledWith( - participant, - Track.Source.Camera, - "inbound-rtp", - ); }); diff --git a/src/state/media/MemberMediaViewModel.ts b/src/state/media/MemberMediaViewModel.ts index e57dcc636..cbf9a05cf 100644 --- a/src/state/media/MemberMediaViewModel.ts +++ b/src/state/media/MemberMediaViewModel.ts @@ -7,18 +7,16 @@ Please see LICENSE in the repository root for full details. */ import { - type Room as LivekitRoom, - RoomEvent as LivekitRoomEvent, - type Participant, - type Track, -} from "livekit-client"; -import { - type AudioSource, - roomEventSelector, - type TrackReference, - type VideoSource, -} from "@livekit/components-core"; -import { type LocalParticipant, type RemoteParticipant } from "livekit-client"; + type AudioMediaTrack, + type Behavior, + E2eeType, + type EncryptionError, + type EncryptionSystem, + type MediaTrack, + type MemberMedia, + type ObservableScope, + type VideoMediaTrack, +} from "@element-hq/matrixrtc-sdk"; import { combineLatest, distinctUntilChanged, @@ -31,13 +29,7 @@ import { throttleTime, } from "rxjs"; -import { type Behavior } from "../Behavior"; import { type BaseMediaViewModel, createBaseMedia } from "./MediaViewModel"; -import { type EncryptionSystem } from "../../e2ee/sharedKeyManagement"; -import { type ObservableScope } from "../ObservableScope"; -import { observeTrackReference$ } from "../observeTrackReference"; -import { E2eeType } from "../../e2ee/e2eeType"; -import { observeInboundRtpStreamStats$ } from "./observeRtpStreamStats"; import { type UserMediaViewModel } from "./UserMediaViewModel"; import { type ScreenShareViewModel } from "./ScreenShareViewModel"; import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; @@ -56,12 +48,12 @@ export enum EncryptionStatus { */ export interface BaseMemberMediaViewModel extends BaseMediaViewModel { /** - * The LiveKit video track for this media. + * The video track for this media. */ - video$: Behavior; + video$: Behavior; /** - * The LiveKit transport on which this member should be publishing. Exposed - * for debugging. + * The URL of the transport on which this member should be publishing. + * Exposed for debugging. */ transport$: Behavior; /** @@ -72,19 +64,31 @@ export interface BaseMemberMediaViewModel extends BaseMediaViewModel { } export interface MemberMediaInputs extends BaseMediaViewModel { - participant$: Behavior; - livekitRoom$: Behavior; - audioSource: AudioSource; - videoSource: VideoSource; - transport$: Behavior; + media$: Behavior; + audioSource: "microphone" | "screenShareAudio"; + videoSource: "camera" | "screenShare"; + focusUrl$: Behavior; encryptionSystem: EncryptionSystem; } +/** The member's track for a source, undefined while there is none. */ +export function memberTrack$( + media$: Behavior, + source: "microphone" | "camera" | "screenShare" | "screenShareAudio", +): Observable { + return media$.pipe( + switchMap((media) => + media === null + ? of(undefined) + : (media[`${source}$`] as Behavior), + ), + ); +} + export function createMemberMedia( scope: ObservableScope, { - participant$, - livekitRoom$, + media$, audioSource, videoSource, transport$, @@ -92,183 +96,104 @@ export function createMemberMedia( ...inputs }: MemberMediaInputs, ): BaseMemberMediaViewModel { - const trackBehavior$ = ( - scope: ObservableScope, - source: Track.Source, - ): Behavior => - scope.behavior( - participant$.pipe( - switchMap((p) => - !p ? of(undefined) : observeTrackReference$(p, source), - ), + const audio$ = scope.behavior( + memberTrack$(media$, audioSource), + ); + const video$ = scope.behavior( + memberTrack$(media$, videoSource), + ); + const unencrypted$ = ( + track$: Behavior, + ): Observable => + track$.pipe( + switchMap((track) => + track === undefined + ? of(false) + : track.encrypted$.pipe(map((encrypted) => !encrypted)), ), ); - const audio$ = trackBehavior$(scope, audioSource); - const video$ = trackBehavior$(scope, videoSource); - return { ...createBaseMedia(inputs), video$, transport$, unencryptedWarning$: scope.behavior( combineLatest( - [audio$, video$], - (a, v) => - encryptionSystem.kind !== E2eeType.NONE && - (a?.publication.isEncrypted === false || - v?.publication.isEncrypted === false), + [unencrypted$(audio$), unencrypted$(video$)], + (a, v) => encryptionSystem.kind !== E2eeType.NONE && (a || v), ), ), encryptionStatus$: scope.behavior( - participant$.pipe( - switchMap((participant): Observable => { - if (!participant) { - return of(EncryptionStatus.Connecting); - } else if ( - participant.isLocal || - encryptionSystem.kind === E2eeType.NONE - ) { + media$.pipe( + switchMap((media): Observable => { + if (media === null) return of(EncryptionStatus.Connecting); + if (media.local || encryptionSystem.kind === E2eeType.NONE) return of(EncryptionStatus.Okay); - } else if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) { - return combineLatest([ - encryptionErrorObservable$( - livekitRoom$, - participant, - encryptionSystem, - "MissingKey", - ), - encryptionErrorObservable$( - livekitRoom$, - participant, - encryptionSystem, - "InvalidKey", - ), - observeRemoteTrackReceivingOkay$(participant, audioSource), - observeRemoteTrackReceivingOkay$(participant, videoSource), - ]).pipe( - map(([keyMissing, keyInvalid, audioOkay, videoOkay]) => { - if (keyMissing) return EncryptionStatus.KeyMissing; - if (keyInvalid) return EncryptionStatus.KeyInvalid; - if (audioOkay || videoOkay) return EncryptionStatus.Okay; - return undefined; // no change - }), - filter((x) => !!x), - startWith(EncryptionStatus.Connecting), - ); - } else { - return combineLatest([ - encryptionErrorObservable$( - livekitRoom$, - participant, - encryptionSystem, - "InvalidKey", - ), - observeRemoteTrackReceivingOkay$(participant, audioSource), - observeRemoteTrackReceivingOkay$(participant, videoSource), - ]).pipe( - map( - ([keyInvalid, audioOkay, videoOkay]): - | EncryptionStatus - | undefined => { - if (keyInvalid) return EncryptionStatus.PasswordInvalid; - if (audioOkay || videoOkay) return EncryptionStatus.Okay; - return undefined; // no change - }, - ), - filter((x) => !!x), - startWith(EncryptionStatus.Connecting), - ); - } + const perParticipant = + encryptionSystem.kind === E2eeType.PER_PARTICIPANT; + return combineLatest([ + encryptionError$(media, "MissingKey"), + encryptionError$(media, "InvalidKey"), + receivingOkay$(audio$), + receivingOkay$(video$), + ]).pipe( + map(([keyMissing, keyInvalid, audioOkay, videoOkay]) => { + if (perParticipant && keyMissing) + return EncryptionStatus.KeyMissing; + if (keyInvalid) + return perParticipant + ? EncryptionStatus.KeyInvalid + : EncryptionStatus.PasswordInvalid; + if (audioOkay || videoOkay) return EncryptionStatus.Okay; + return undefined; // no change + }), + filter((x) => x !== undefined), + startWith(EncryptionStatus.Connecting), + ); }), ), ), }; } -function encryptionErrorObservable$( - room$: Behavior, - participant: Participant, - encryptionSystem: EncryptionSystem, - criteria: string, +function encryptionError$( + media: MemberMedia, + criteria: EncryptionError, ): Observable { - return room$.pipe( - switchMap((room) => { - if (room === undefined) return of(false); - return roomEventSelector(room, LivekitRoomEvent.EncryptionError).pipe( - map((e) => { - const [err] = e; - if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) { - return ( - // Ideally we would pull the participant identity from the field on the error. - // However, it gets lost in the serialization process between workers. - // So, instead we do a string match - (err?.message.includes(participant.identity) && - err?.message.includes(criteria)) ?? - false - ); - } else if (encryptionSystem.kind === E2eeType.SHARED_KEY) { - return !!err?.message.includes(criteria); - } - - return false; - }), - ); - }), + return media.encryptionError$.pipe( + map((error) => error === criteria), distinctUntilChanged(), throttleTime(1000), // Throttle to avoid spamming the UI startWith(false), ); } -function observeRemoteTrackReceivingOkay$( - participant: Participant, - source: Track.Source, +/** + * Whether frames arrive and decode, which is the only sign that the key in + * use is the right one. + */ +function receivingOkay$( + track$: Behavior, ): Observable { - let lastStats: { - framesDecoded: number | undefined; - framesDropped: number | undefined; - framesReceived: number | undefined; - } = { - framesDecoded: undefined, - framesDropped: undefined, - framesReceived: undefined, - }; - - return observeInboundRtpStreamStats$(participant, source).pipe( - map((stats) => { - if (!stats) return undefined; - const { framesDecoded, framesDropped, framesReceived } = stats; - return { - framesDecoded, - framesDropped, - framesReceived, - }; - }), - filter((newStats) => !!newStats), - map((newStats): boolean | undefined => { - const oldStats = lastStats; - lastStats = newStats; + let last: { framesDecoded?: number; framesReceived?: number } = {}; + return track$.pipe( + switchMap((track) => track?.stats$ ?? of(undefined)), + map((stats): boolean | undefined => { + if (stats === undefined || stats.type !== "inbound-rtp") return undefined; + const { framesDecoded, framesReceived } = + stats as RTCInboundRtpStreamStats; + const previous = last; + last = { framesDecoded, framesReceived }; if ( - typeof newStats.framesReceived === "number" && - typeof oldStats.framesReceived === "number" && - typeof newStats.framesDecoded === "number" && - typeof oldStats.framesDecoded === "number" - ) { - const framesReceivedDelta = - newStats.framesReceived - oldStats.framesReceived; - const framesDecodedDelta = - newStats.framesDecoded - oldStats.framesDecoded; - - // if we received >0 frames and managed to decode >0 frames then we treat that as success - - if (framesReceivedDelta > 0) { - return framesDecodedDelta > 0; - } - } - - // no change - return undefined; + framesReceived === undefined || + previous.framesReceived === undefined || + framesDecoded === undefined || + previous.framesDecoded === undefined + ) + return undefined; + const received = framesReceived - previous.framesReceived; + if (received > 0) return framesDecoded - previous.framesDecoded > 0; + return undefined; // no change }), filter((x) => typeof x === "boolean"), startWith(undefined), diff --git a/src/state/media/RemoteScreenShareViewModel.ts b/src/state/media/RemoteScreenShareViewModel.ts index 8c46aeb33..b7b48e6a5 100644 --- a/src/state/media/RemoteScreenShareViewModel.ts +++ b/src/state/media/RemoteScreenShareViewModel.ts @@ -6,18 +6,20 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { Track, type RemoteParticipant } from "livekit-client"; -import { map, of, switchMap } from "rxjs"; +import { + type AudioMediaTrack, + type Behavior, + type ObservableScope, +} from "@element-hq/matrixrtc-sdk"; +import { map } from "rxjs"; -import { type Behavior } from "../Behavior"; import { type BaseScreenShareInputs, type BaseScreenShareViewModel, createBaseScreenShare, } from "./ScreenShareViewModel"; -import { type ObservableScope } from "../ObservableScope"; import { createVolumeControls, type VolumeControls } from "../VolumeControls"; -import { observeTrackReference$ } from "../observeTrackReference"; +import { memberTrack$ } from "./MemberMediaViewModel"; export interface RemoteScreenShareViewModel extends BaseScreenShareViewModel, VolumeControls { @@ -33,7 +35,6 @@ export interface RemoteScreenShareViewModel } export interface RemoteScreenShareInputs extends BaseScreenShareInputs { - participant$: Behavior; pretendToBeDisconnected$: Behavior; } @@ -41,16 +42,16 @@ export function createRemoteScreenShare( scope: ObservableScope, { pretendToBeDisconnected$, ...inputs }: RemoteScreenShareInputs, ): RemoteScreenShareViewModel { + const audio$ = scope.behavior( + memberTrack$(inputs.media$, "screenShareAudio"), + ); return { ...createBaseScreenShare(scope, inputs), ...createVolumeControls(scope, { pretendToBeDisconnected$, sink$: scope.behavior( - inputs.participant$.pipe( - map( - (p) => (volume) => - p?.setVolume(volume, Track.Source.ScreenShareAudio), - ), + audio$.pipe( + map((track) => (volume: number) => track?.setVolume(volume)), ), ), }), @@ -59,14 +60,7 @@ export function createRemoteScreenShare( pretendToBeDisconnected$.pipe(map((disconnected) => !disconnected)), ), audioEnabled$: scope.behavior( - inputs.participant$.pipe( - switchMap((p) => - p - ? observeTrackReference$(p, Track.Source.ScreenShareAudio) - : of(null), - ), - map(Boolean), - ), + audio$.pipe(map((track) => track !== undefined)), ), }; } diff --git a/src/state/media/RemoteUserMediaViewModel.ts b/src/state/media/RemoteUserMediaViewModel.ts index 6daa381a7..71b215c83 100644 --- a/src/state/media/RemoteUserMediaViewModel.ts +++ b/src/state/media/RemoteUserMediaViewModel.ts @@ -6,34 +6,33 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type RemoteParticipant } from "livekit-client"; +import { + type AudioMediaTrack, + type Behavior, + type ObservableScope, +} from "@element-hq/matrixrtc-sdk"; import { combineLatest, map, of, switchMap } from "rxjs"; import { logger } from "matrix-js-sdk/lib/logger"; -import { type Behavior } from "../Behavior"; import { createVolumeControls, type VolumeControls } from "../VolumeControls"; import { type BaseUserMediaInputs, type BaseUserMediaViewModel, createBaseUserMedia, } from "./UserMediaViewModel"; -import { type ObservableScope } from "../ObservableScope"; +import { memberTrack$ } from "./MemberMediaViewModel"; export interface RemoteUserMediaViewModel extends BaseUserMediaViewModel, VolumeControls { local: false; /** - * Whether we are waiting for this user's LiveKit participant to exist. This - * could be because either we or the remote party are still connecting. + * Whether we are waiting for this user's media to arrive. This could be + * because either we or the remote party are still connecting. */ waitingForMedia$: Behavior; } -export interface RemoteUserMediaInputs extends Omit< - BaseUserMediaInputs, - "statsType" -> { - participant$: Behavior; +export interface RemoteUserMediaInputs extends BaseUserMediaInputs { pretendToBeDisconnected$: Behavior; } @@ -41,19 +40,15 @@ export function createRemoteUserMedia( scope: ObservableScope, { pretendToBeDisconnected$, ...inputs }: RemoteUserMediaInputs, ): RemoteUserMediaViewModel { - const baseUserMedia = createBaseUserMedia(scope, { - ...inputs, - statsType: "inbound-rtp", - }); + const baseUserMedia = createBaseUserMedia(scope, inputs); const waitingForMedia$ = scope.behavior( combineLatest( - [inputs.livekitRoom$, inputs.participant$], - (livekitRoom, participant) => - // If livekitRoom is undefined, the user is not attempting to publish on - // any transport and so we shouldn't expect a participant. (They might - // be a subscribe-only bot for example.) - livekitRoom !== undefined && participant === null, + [inputs.focusUrl$, inputs.media$], + // Without a transport the user is not attempting to publish anywhere + // and so we shouldn't expect media. (They might be a subscribe-only bot + // for example.) + (focusUrl, media) => focusUrl !== undefined && media === null, ), ); waitingForMedia$.pipe(scope.bind()).subscribe((waiting) => { @@ -65,7 +60,9 @@ export function createRemoteUserMedia( ...createVolumeControls(scope, { pretendToBeDisconnected$, sink$: scope.behavior( - inputs.participant$.pipe(map((p) => (volume) => p?.setVolume(volume))), + memberTrack$(inputs.media$, "microphone").pipe( + map((track) => (volume: number) => track?.setVolume(volume)), + ), ), }), local: false, diff --git a/src/state/media/ScreenShareViewModel.ts b/src/state/media/ScreenShareViewModel.ts index 8336f0a6b..3db18e510 100644 --- a/src/state/media/ScreenShareViewModel.ts +++ b/src/state/media/ScreenShareViewModel.ts @@ -6,9 +6,8 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { Track } from "livekit-client"; +import { type ObservableScope } from "@element-hq/matrixrtc-sdk"; -import { type ObservableScope } from "../ObservableScope"; import { type LocalScreenShareViewModel } from "./LocalScreenShareViewModel"; import { createMemberMedia, @@ -43,8 +42,8 @@ export function createBaseScreenShare( return { ...createMemberMedia(scope, { ...inputs, - audioSource: Track.Source.ScreenShareAudio, - videoSource: Track.Source.ScreenShare, + audioSource: "screenShareAudio", + videoSource: "screenShare", }), type: "screen share", }; diff --git a/src/state/media/UserMediaViewModel.ts b/src/state/media/UserMediaViewModel.ts index fb664f0bb..08c43a42a 100644 --- a/src/state/media/UserMediaViewModel.ts +++ b/src/state/media/UserMediaViewModel.ts @@ -6,6 +6,12 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ +import { + type Behavior, + type MediaStreamStats, + type MediaTrack, + type ObservableScope, +} from "@element-hq/matrixrtc-sdk"; import { BehaviorSubject, combineLatest, @@ -14,24 +20,17 @@ import { Subject, switchMap, } from "rxjs"; -import { - observeParticipantEvents, - observeParticipantMedia, -} from "@livekit/components-core"; -import { ParticipantEvent, Track } from "livekit-client"; import { type ReactionOption } from "../../reactions"; -import { type Behavior } from "../Behavior"; import { type LocalUserMediaViewModel } from "./LocalUserMediaViewModel"; import { createMemberMedia, type MemberMediaInputs, + memberTrack$, type BaseMemberMediaViewModel, } from "./MemberMediaViewModel"; import { type RemoteUserMediaViewModel } from "./RemoteUserMediaViewModel"; -import { type ObservableScope } from "../ObservableScope"; import { showConnectionStats } from "../../settings/settings"; -import { observeRtpStreamStats$ } from "./observeRtpStreamStats"; /** * A participant's user media (i.e. their microphone and camera feed). @@ -48,17 +47,13 @@ export interface BaseUserMediaViewModel extends BaseMemberMediaViewModel { videoOrientation$: Behavior<"landscape" | "portrait">; toggleCropVideo: () => void; /** - * The expected identity of the LiveKit participant. Exposed for debugging. + * The expected identity of the member on the media backend. Exposed for debugging. */ rtcBackendIdentity: string; handRaised$: Behavior; reaction$: Behavior; - audioStreamStats$: Behavior< - RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined - >; - videoStreamStats$: Behavior< - RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined - >; + audioStreamStats$: Behavior; + videoStreamStats$: Behavior; /** * Set the aspect ratio of the video track to determine the orientation. */ @@ -72,7 +67,6 @@ export interface BaseUserMediaInputs extends Omit< rtcBackendIdentity: string; handRaised$: Behavior; reaction$: Behavior; - statsType: "inbound-rtp" | "outbound-rtp"; } export function createBaseUserMedia( @@ -81,30 +75,37 @@ export function createBaseUserMedia( rtcBackendIdentity, handRaised$, reaction$, - statsType, ...inputs }: BaseUserMediaInputs, ): BaseUserMediaViewModel { - const { participant$ } = inputs; - const media$ = scope.behavior( - participant$.pipe( - switchMap((p) => (p && observeParticipantMedia(p)) ?? of(undefined)), - ), - ); + const { media$ } = inputs; const toggleCropVideo$ = new Subject(); const videoAspectRatio$ = new BehaviorSubject(NaN); + const enabled$ = ( + scope: ObservableScope, + source: "microphone" | "camera", + ): Behavior => + scope.behavior( + memberTrack$(media$, source).pipe( + switchMap((track) => + track === undefined + ? of(false) + : track.muted$.pipe(map((muted) => !muted)), + ), + ), + ); + // The statistics are only polled while the setting asks for them const streamStats$ = ( scope: ObservableScope, - source: Track.Source, - ): Behavior< - RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined - > => + source: "microphone" | "camera", + ): Behavior => scope.behavior( - combineLatest([participant$, showConnectionStats.value$]).pipe( - switchMap(([p, showConnectionStats]) => - p && showConnectionStats - ? observeRtpStreamStats$(p, source, statsType) - : of(undefined), + combineLatest([ + memberTrack$(media$, source), + showConnectionStats.value$, + ]).pipe( + switchMap(([track, show]) => + track !== undefined && show ? track.stats$ : of(undefined), ), ), ); @@ -112,28 +113,17 @@ export function createBaseUserMedia( return { ...createMemberMedia(scope, { ...inputs, - audioSource: Track.Source.Microphone, - videoSource: Track.Source.Camera, + audioSource: "microphone", + videoSource: "camera", }), type: "user", speaking$: scope.behavior( - participant$.pipe( - switchMap((p) => - p - ? observeParticipantEvents( - p, - ParticipantEvent.IsSpeakingChanged, - ).pipe(map((p) => p.isSpeaking)) - : of(false), - ), + media$.pipe( + switchMap((media) => (media === null ? of(false) : media.speaking$)), ), ), - audioEnabled$: scope.behavior( - media$.pipe(map((m) => m?.microphoneTrack?.isMuted === false)), - ), - videoEnabled$: scope.behavior( - media$.pipe(map((m) => m?.cameraTrack?.isMuted === false)), - ), + audioEnabled$: enabled$(scope, "microphone"), + videoEnabled$: enabled$(scope, "camera"), videoOrientation$: scope.behavior( videoAspectRatio$.pipe( map((aspect) => (aspect > 1 ? "landscape" : "portrait")), @@ -144,8 +134,8 @@ export function createBaseUserMedia( rtcBackendIdentity, handRaised$, reaction$, - audioStreamStats$: streamStats$(scope, Track.Source.Microphone), - videoStreamStats$: streamStats$(scope, Track.Source.Camera), + audioStreamStats$: streamStats$(scope, "microphone"), + videoStreamStats$: streamStats$(scope, "camera"), setVideoAspectRatio: (ratio) => videoAspectRatio$.next(ratio), }; } diff --git a/src/state/media/WrappedUserMediaViewModel.ts b/src/state/media/WrappedUserMediaViewModel.ts index e9575d0c0..ed781ef3a 100644 --- a/src/state/media/WrappedUserMediaViewModel.ts +++ b/src/state/media/WrappedUserMediaViewModel.ts @@ -5,20 +5,16 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { combineLatest, map, of, switchMap } from "rxjs"; import { - type LocalParticipant, - ParticipantEvent, - type RemoteParticipant, -} from "livekit-client"; -import { observeParticipantEvents } from "@livekit/components-core"; + type Behavior, + generateItems, + type LocalMemberMedia, + type MemberMedia, + type ObservableScope, +} from "@element-hq/matrixrtc-sdk"; +import { combineLatest, map, of, switchMap } from "rxjs"; -import { type ObservableScope } from "../ObservableScope.ts"; -import type { Behavior } from "../Behavior.ts"; -import type { MediaDevices } from "../MediaDevices.ts"; import { observeSpeaker$ } from "./observeSpeaker.ts"; -import { generateItems } from "../../utils/observable.ts"; -import { type TaggedParticipant } from "../CallViewModel/remoteMembers/MatrixLivekitMembers.ts"; import { type UserMediaViewModel } from "./UserMediaViewModel.ts"; import { type ScreenShareViewModel } from "./ScreenShareViewModel.ts"; import { @@ -82,82 +78,58 @@ export type WrappedUserMediaViewModel = UserMediaViewModel & { bin$: Behavior; }; -interface WrappedUserMediaInputs extends Omit< +type WrappedUserMediaInputs = Omit< LocalUserMediaInputs & RemoteUserMediaInputs, - "participant$" -> { - participant: TaggedParticipant; - mediaDevices: MediaDevices; - pretendToBeDisconnected$: Behavior; -} + "media$" +> & + ( + | { local: true; media$: Behavior } + | { local: false; media$: Behavior } + ); export function createWrappedUserMedia( scope: ObservableScope, - { - participant, - mediaDevices, - pretendToBeDisconnected$, - ...inputs - }: WrappedUserMediaInputs, + { mediaDevices, pretendToBeDisconnected$, ...rest }: WrappedUserMediaInputs, ): WrappedUserMediaViewModel { - const userMedia = - participant.type === "local" - ? createLocalUserMedia(scope, { - participant$: participant.value$, - mediaDevices, - ...inputs, - }) - : createRemoteUserMedia(scope, { - participant$: participant.value$, - pretendToBeDisconnected$, - ...inputs, - }); - + const { local, media$: _media$, ...inputs } = rest; + const userMedia = rest.local + ? createLocalUserMedia(scope, { + media$: rest.media$, + mediaDevices, + ...inputs, + }) + : createRemoteUserMedia(scope, { + media$: rest.media$, + pretendToBeDisconnected$, + ...inputs, + }); // TypeScript needs this widening of the type to happen in a separate statement - const participant$: Behavior = - participant.value$; + const media$: Behavior = _media$; const screenShares$ = scope.behavior( - participant$.pipe( - switchMap((p) => - p === null - ? of([]) - : observeParticipantEvents( - p, - ParticipantEvent.TrackPublished, - ParticipantEvent.TrackUnpublished, - ParticipantEvent.LocalTrackPublished, - ParticipantEvent.LocalTrackUnpublished, - ).pipe( - // Technically more than one screen share might be possible... our - // MediaViewModels don't support it though since they look for a unique - // track for the given source. So generateItems here is a bit overkill. - generateItems( - `${inputs.id} screenShares$`, - function* (p) { - if (p.isScreenShareEnabled) - yield { - keys: ["screen-share"], - data: undefined, - }; - }, - (scope, _data$, key) => { - const id = `${inputs.id}:${key}`; - return participant.type === "local" - ? createLocalScreenShare(scope, { - ...inputs, - id, - participant$: participant.value$, - }) - : createRemoteScreenShare(scope, { - ...inputs, - id, - participant$: participant.value$, - pretendToBeDisconnected$, - }); - }, - ), - ), + media$.pipe( + switchMap((media) => + media === null ? of(false) : media.screenShareEnabled$, + ), + // Technically more than one screen share might be possible... our + // MediaViewModels don't support it though since they look for a unique + // track for the given source. So generateItems here is a bit overkill. + generateItems( + `${inputs.id} screenShares$`, + function* (enabled) { + if (enabled) yield { keys: ["screen-share"], data: undefined }; + }, + (scope, _data$, key) => { + const id = `${inputs.id}:${key}`; + return local + ? createLocalScreenShare(scope, { ...inputs, id, media$ }) + : createRemoteScreenShare(scope, { + ...inputs, + id, + media$, + pretendToBeDisconnected$, + }); + }, ), ), ); diff --git a/src/state/media/observeRtpStreamStats.ts b/src/state/media/observeRtpStreamStats.ts deleted file mode 100644 index afb0da62d..000000000 --- a/src/state/media/observeRtpStreamStats.ts +++ /dev/null @@ -1,74 +0,0 @@ -/* -Copyright 2023, 2024 New Vector Ltd. -Copyright 2026 Element Creations Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - LocalTrack, - type Participant, - RemoteTrack, - type Track, -} from "livekit-client"; -import { - combineLatest, - interval, - type Observable, - startWith, - switchMap, - map, - share, -} from "rxjs"; - -import { observeTrackReference$ } from "../observeTrackReference"; - -// Use a shared timer for all the stats observers so that we don't clog up the -// event loop with hundreds of timers in large calls -const refreshStats$ = interval(1000).pipe(share()); - -export function observeRtpStreamStats$( - participant: Participant, - source: Track.Source, - type: "inbound-rtp" | "outbound-rtp", -): Observable< - RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined -> { - return combineLatest([ - observeTrackReference$(participant, source), - refreshStats$.pipe(startWith(0)), - ]).pipe( - switchMap(async ([trackReference]) => { - const track = trackReference?.publication?.track; - if ( - !track || - !(track instanceof RemoteTrack || track instanceof LocalTrack) - ) { - return undefined; - } - const report = await track.getRTCStatsReport(); - if (!report) { - return undefined; - } - - for (const v of report.values()) { - if (v.type === type) { - return v; - } - } - - return undefined; - }), - startWith(undefined), - ); -} - -export function observeInboundRtpStreamStats$( - participant: Participant, - source: Track.Source, -): Observable { - return observeRtpStreamStats$(participant, source, "inbound-rtp").pipe( - map((x) => x as RTCInboundRtpStreamStats | undefined), - ); -} diff --git a/src/state/observeTrackReference.ts b/src/state/observeTrackReference.ts deleted file mode 100644 index 8e295d057..000000000 --- a/src/state/observeTrackReference.ts +++ /dev/null @@ -1,28 +0,0 @@ -/* -Copyright 2023, 2024 New Vector Ltd. -Copyright 2026 Element Creations Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - observeParticipantMedia, - type TrackReference, -} from "@livekit/components-core"; -import { type Participant, type Track } from "livekit-client"; -import { distinctUntilChanged, map, type Observable } from "rxjs"; - -/** - * Reactively reads a participant's track reference for a given media source. - */ -export function observeTrackReference$( - participant: Participant, - source: Track.Source, -): Observable { - return observeParticipantMedia(participant).pipe( - map(() => participant.getTrackPublication(source)), - distinctUntilChanged(), - map((publication) => publication && { participant, publication, source }), - ); -} diff --git a/src/tile/GridTile.test.tsx b/src/tile/GridTile.test.tsx index 60fbc303c..5d6fb81ae 100644 --- a/src/tile/GridTile.test.tsx +++ b/src/tile/GridTile.test.tsx @@ -5,10 +5,6 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { - type LocalTrackPublication, - type RemoteTrackPublication, -} from "livekit-client"; import { test, expect } from "vitest"; import { act, render, screen } from "@testing-library/react"; import { axe } from "vitest-axe"; @@ -19,10 +15,9 @@ import { GridTile } from "./GridTile"; import { mockRtcMembership, mockRemoteMedia, - mockRemoteParticipant, mockLocalMedia, - mockLocalParticipant, mockMediaDevices, + mockMemberMedia, } from "../utils/test"; import { GridTileViewModel } from "../state/TileViewModel"; import { ReactionsSenderProvider } from "../reactions/useReactionsSender"; @@ -67,11 +62,7 @@ test("GridTile displays remote media", async () => { rawDisplayName: "Alice", getMxcAvatarUrl: () => "mxc://adfsg", }, - mockRemoteParticipant({ - setVolume() {}, - getTrackPublication: () => - ({}) as Partial as RemoteTrackPublication, - }), + mockMemberMedia(), ); const { container } = render( @@ -101,10 +92,7 @@ test("GridTile displays local media", async () => { rawDisplayName: "Alice", getMxcAvatarUrl: () => "mxc://adfsg", }, - mockLocalParticipant({ - getTrackPublication: () => - ({}) as Partial as LocalTrackPublication, - }), + mockMemberMedia({ local: true }), mockMediaDevices({}), ); diff --git a/src/tile/MediaView.test.tsx b/src/tile/MediaView.test.tsx index 660d37a27..0b5653517 100644 --- a/src/tile/MediaView.test.tsx +++ b/src/tile/MediaView.test.tsx @@ -9,26 +9,13 @@ import { describe, expect, it, test } from "vitest"; import { render, screen } from "@testing-library/react"; import { axe } from "vitest-axe"; import { TooltipProvider } from "@vector-im/compound-web"; -import { - type TrackReference, - type TrackReferencePlaceholder, -} from "@livekit/components-core"; -import { type LocalTrackPublication, Track } from "livekit-client"; import { type ComponentProps } from "react"; import { MediaView } from "./MediaView"; -import { mockLocalParticipant } from "../utils/test"; +import { mockVideoTrack } from "../utils/test"; describe("MediaView", () => { - const participant = mockLocalParticipant({}); - const trackReferencePlaceholder: TrackReferencePlaceholder = { - participant, - source: Track.Source.Camera, - }; - const trackReference: TrackReference = { - ...trackReferencePlaceholder, - publication: {} as Partial as LocalTrackPublication, - }; + const track = mockVideoTrack({ source: "camera" }); const baseProps: ComponentProps = { displayName: "some name", @@ -38,7 +25,7 @@ describe("MediaView", () => { mirror: false, unencryptedWarning: false, showNameTags: true, - video: trackReference, + video: track, userId: "@alice:example.com", mxcAvatarUrl: undefined, focusable: true, @@ -49,16 +36,6 @@ describe("MediaView", () => { expect(await axe(container)).toHaveNoViolations(); }); - describe("placeholder track", () => { - test("neither video nor avatar are shown", () => { - render(); - expect(screen.queryByTestId("video")).toBeNull(); - expect( - screen.queryAllByRole("img", { name: "@alice:example.com" }).length, - ).toBe(0); - }); - }); - describe("with no video", () => { it("shows avatar", () => { render(); diff --git a/src/tile/MediaView.tsx b/src/tile/MediaView.tsx index 6776317df..dc477007c 100644 --- a/src/tile/MediaView.tsx +++ b/src/tile/MediaView.tsx @@ -5,18 +5,19 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type TrackReferenceOrPlaceholder } from "@livekit/components-core"; +import { type VideoMediaTrack } from "@element-hq/matrixrtc-sdk"; import { animated } from "@react-spring/web"; import { - type FC, type ComponentProps, + type FC, type ReactNode, type SyntheticEvent, + useEffect, + useRef, useState, } from "react"; import { useTranslation } from "react-i18next"; import classNames from "classnames"; -import { VideoTrack } from "@livekit/components-react"; import { Text, Tooltip } from "@vector-im/compound-web"; import { ErrorSolidIcon } from "@vector-im/compound-design-tokens/assets/web/icons"; @@ -40,7 +41,7 @@ interface Props extends ComponentProps { style?: ComponentProps["style"]; targetWidth: number; targetHeight: number; - video: TrackReferenceOrPlaceholder | undefined; + video: VideoMediaTrack | undefined; /** * How to fit the video content inside the tile. When undefined, MediaView * chooses a smart default based on the aspect ratios of the tile and video. @@ -128,13 +129,23 @@ export const MediaView: FC = ({ setOurVideoAspectRatio(ratio); setTheirVideoAspectRatio?.(ratio); }; + const videoElement = useRef(null); const videoRef = (el: HTMLVideoElement | null) => { + videoElement.current = el; if (el !== null) setVideoAspectRatio(el.videoWidth / el.videoHeight); }; const onResize = (ev: SyntheticEvent) => setVideoAspectRatio( ev.currentTarget.videoWidth / ev.currentTarget.videoHeight, ); + // The SDK plays the track on the element and watches its size, so the + // element has to be handed back before it leaves the DOM + useEffect(() => { + const el = videoElement.current; + if (video === undefined || el === null) return; + video.attach(el); + return (): void => video.detach(el); + }, [video]); const warnings = unencryptedWarning && ( = ({ className={styles.avatar} style={{ display: video && videoEnabled ? "none" : "initial" }} /> - {video?.publication !== undefined && ( - { rawDisplayName: "Alice", getMxcAvatarUrl: () => "mxc://adfsg", }, - mockRemoteParticipant({}), + mockMemberMedia(), ); const vm2 = mockLocalMedia( @@ -51,7 +50,7 @@ test("SpotlightTile is accessible", async () => { rawDisplayName: "Bob", getMxcAvatarUrl: () => "mxc://dlskf", }, - mockLocalParticipant({}), + mockMemberMedia({ local: true }), mockMediaDevices({}), ); @@ -99,7 +98,7 @@ test("Screen share volume UI is shown when screen share has audio", async () => const vm = mockRemoteScreenShare( mockRtcMembership("@alice:example.org", "AAAA"), {}, - mockRemoteParticipant({}), + mockMemberMedia(), ); vi.spyOn(vm, "audioEnabled$", "get").mockReturnValue(constant(true)); @@ -137,7 +136,7 @@ test("Screen share volume UI is hidden when screen share has no audio", async () const vm = mockRemoteScreenShare( mockRtcMembership("@alice:example.org", "AAAA"), {}, - mockRemoteParticipant({}), + mockMemberMedia(), ); vi.spyOn(vm, "audioEnabled$", "get").mockReturnValue(constant(false)); diff --git a/src/tile/SpotlightTile.tsx b/src/tile/SpotlightTile.tsx index 344eed00e..ce2c4e662 100644 --- a/src/tile/SpotlightTile.tsx +++ b/src/tile/SpotlightTile.tsx @@ -29,7 +29,7 @@ import { animated } from "@react-spring/web"; import { BehaviorSubject, type Observable, map } from "rxjs"; import { useTranslation } from "react-i18next"; import classNames from "classnames"; -import { type TrackReferenceOrPlaceholder } from "@livekit/components-core"; +import { type VideoMediaTrack } from "@element-hq/matrixrtc-sdk"; import { Menu, MenuItem, Text } from "@vector-im/compound-web"; import FullScreenMaximiseIcon from "../icons/FullScreenMaximise.svg?react"; @@ -73,7 +73,7 @@ interface SpotlightItemBaseProps { } interface SpotlightMemberMediaItemBaseProps extends SpotlightItemBaseProps { - video: TrackReferenceOrPlaceholder | undefined; + video: VideoMediaTrack | undefined; unencryptedWarning: boolean; transport: UnstableLivekitTransport | undefined; } diff --git a/src/livekit/BlurBackgroundTransformer.ts b/src/tracks/BlurBackgroundTransformer.ts similarity index 100% rename from src/livekit/BlurBackgroundTransformer.ts rename to src/tracks/BlurBackgroundTransformer.ts diff --git a/src/tracks/MemberAudioRenderer.tsx b/src/tracks/MemberAudioRenderer.tsx new file mode 100644 index 000000000..e67e4781b --- /dev/null +++ b/src/tracks/MemberAudioRenderer.tsx @@ -0,0 +1,177 @@ +/* +Copyright 2025 New Vector Ltd. +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + type AudioMediaTrack, + type MemberMedia, + type RemoteRTCMember, +} from "@element-hq/matrixrtc-sdk"; +import { type FC, useEffect, useMemo, useState } from "react"; + +import { useEarpieceAudioConfig } from "../MediaDevicesContext"; +import { useBehavior } from "../useBehavior"; +import * as controls from "../controls"; + +export interface MemberAudioRendererProps { + /** The remote members whose audio should be heard; never the local one. */ + members: RemoteRTCMember[]; + /** + * If set to `true`, mutes every audio track. + * @remarks + * If set to `true`, the server will stop sending audio track data to the client. + */ + muted?: boolean; +} + +/** + * Plays every remote member's microphone and screen share audio, one audio + * element per track. Only members of the session reach here, so anything + * else that publishes on the transport stays inaudible. + * + * It also takes care of the earpiece audio configuration for iOS devices. + * This is done by using the WebAudio API to create a stereo pan effect that + * mimics the earpiece audio. + */ +export const MemberAudioRenderer: FC = ({ + members, + muted, +}) => { + // This component is also (in addition to the "only play audio for session + // members" logic above) responsible for mimicking earpiece audio on iPhones. + // The Safari audio devices enumeration does not expose an earpiece audio device. + // We alternatively use the audioContext pan node to only use one of the stereo channels. + + // This component does get additionally complicated because of a Safari bug. + // (see: https://bugs.webkit.org/show_bug.cgi?id=251532 + // and the related issues: https://bugs.webkit.org/show_bug.cgi?id=237878 + // and https://bugs.webkit.org/show_bug.cgi?id=231105) + // + // AudioContext gets stopped if the webview gets moved into the background. + // Once the phone is in standby audio playback will stop. + // So we can only use the pan trick only works is the phone is not in standby. + // If earpiece mode is not used we do not use audioContext to allow standby playback. + // shouldUseAudioContext is set to false if stereoPan === 0 to allow standby bluetooth playback. + + const { pan: stereoPan, volume: volumeFactor } = useEarpieceAudioConfig(); + const shouldUseAudioContext = stereoPan !== 0; + + // initialize the potentially used audio context. + const [audioContext, setAudioContext] = useState( + undefined, + ); + useEffect(() => { + const ctx = new AudioContext(); + setAudioContext(ctx); + return (): void => { + void ctx.close(); + }; + }, []); + const audioNodes = useMemo( + () => ({ + gain: audioContext?.createGain(), + pan: audioContext?.createStereoPanner(), + }), + [audioContext], + ); + + // Simple effects to update the gain and pan node based on the props + useEffect(() => { + if (audioNodes.pan) audioNodes.pan.pan.value = stereoPan; + }, [audioNodes.pan, stereoPan]); + useEffect(() => { + if (audioNodes.gain) audioNodes.gain.gain.value = volumeFactor; + }, [audioNodes.gain, volumeFactor]); + + const plugins = useMemo( + () => + shouldUseAudioContext && audioNodes.gain && audioNodes.pan + ? { context: audioContext, nodes: [audioNodes.gain, audioNodes.pan] } + : { context: undefined, nodes: [] }, + [shouldUseAudioContext, audioContext, audioNodes], + ); + + return ( + // We add all audio elements into one
for the browser developer tool experience/tidyness. +
+ {members.map((member) => ( + + ))} +
+ ); +}; + +interface AudioPlugins { + context: AudioContext | undefined; + nodes: AudioNode[]; +} + +const MemberAudio: FC<{ + media$: RemoteRTCMember["media$"]; + muted?: boolean; + plugins: AudioPlugins; +}> = ({ media$, muted, plugins }) => { + const media = useBehavior(media$); + return ( + media !== null && ( + <> + + + + ) + ); +}; + +const AudioTrack: FC<{ + track$: MemberMedia["microphone$"]; + muted?: boolean; + plugins: AudioPlugins; +}> = ({ track$, muted, plugins }) => { + const track = useBehavior(track$); + return ( + track !== undefined && ( + + ) + ); +}; + +/** + * One audio element, handed to the SDK to play on. The Web Audio graph for the + * earpiece is applied before the element so that playback starts through it. + */ +const TrackAudio: FC<{ + track: AudioMediaTrack; + muted?: boolean; + plugins: AudioPlugins; +}> = ({ track, muted, plugins }) => { + const [element, setElement] = useState(null); + useEffect(() => { + track.setAudioContext(plugins.context, plugins.nodes); + }, [track, plugins]); + useEffect(() => { + if (element === null) return; + track.attach(element); + controls.setPlaybackStarted(); + return (): void => track.detach(element); + }, [track, element]); + // A live microphone has no captions to offer + // eslint-disable-next-line jsx-a11y/media-has-caption + return