Matrix/element-call-Github
1
0
Fork 0
mirror of https://github.com/vector-im/element-call.git synced 2026-10-16 23:45:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

use the matrixRtcSdk

Browse Source
This commit is contained in:
Timo K.
2026-10-14 10:23:29 +02:00
committed by Timo
parent 10b34b0218
commit 5f3db07b76
94 changed files with 1664 additions and 12281 deletions
Download Patch File Download Diff File Expand all files Collapse all files
component/index.tsx
+1 -1
View File
@@ -74,7 +74,7 @@ import {
import { MediaDevicesContext } from "../src/MediaDevicesContext";
import { MediaDevices } from "../src/state/MediaDevices";
import { ObservableScope } from "../src/state/ObservableScope";
import { ProcessorProvider } from "../src/livekit/TrackProcessorContext";
import { ProcessorProvider } from "../src/tracks/TrackProcessorContext";
import { Config } from "../src/config/Config";
import { type ConfigOptions } from "../src/config/ConfigOptions";
import { i18n } from "../src/utils/i18n";
sdk-target-based-on-call-view-model/README.md
+4 -3
View File
@@ -1,8 +1,9 @@
# SDK mode (EXPERIMENTAL, superseded)
# SDK mode (LEGACY)
> Superseded by the `sdk/` package (`@element-hq/matrixrtc-sdk`) and its harness in
> `sdk/dev`; see `sdk-plan.md`. This folder still builds, with its own script, until
> that package can do what this does.
> `sdk/dev`; see `sdk/SdkArchitecture.md`. This bundle is in production use, so it
> keeps building as a thin layer over that package, with its interface unchanged,
> until its consumers have moved to the package directly. It will then be removed.
EC can be build in sdk mode. This will result in a compiled js file that can be imported in very simple webapps.
sdk-target-based-on-call-view-model/helper.ts
+2 -24
View File
@@ -6,14 +6,12 @@ Please see LICENSE in the repository root for full details.
*/
/**
* This file contains helper functions and types for the MatrixRTC SDK.
* This file contains helper functions and types for the legacy SDK bundle.
*/
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import { scan } from "rxjs";
import { type WidgetHelpers } from "../src/widget";
import { type LivekitRoomItem } from "../src/state/CallViewModel/CallViewModel";
export const tryMakeSticky = (widget: WidgetHelpers): void => {
const logger = rootLogger.getChild("[MatrixRTCSdk]");
@@ -27,25 +25,5 @@ export const tryMakeSticky = (widget: WidgetHelpers): void => {
logger.error("failed to make sticky MatrixRTCSdk", error);
});
};
/** The data channel topic this bundle has always used, so that old and new builds talk. */
export const TEXT_LK_TOPIC = "matrixRTC";
/**
* simple helper operator to combine the last emitted and the current emitted value of a rxjs observable
*
* I think there should be a builtin for this but i did not find it...
*/
export const currentAndPrev = scan<
LivekitRoomItem[],
{
prev: LivekitRoomItem[];
current: LivekitRoomItem[];
}
>(
({ current: lastCurrentVal }, items) => ({
prev: lastCurrentVal,
current: items,
}),
{
prev: [],
current: [],
},
);
sdk-target-based-on-call-view-model/index.html
+1 -1
View File
@@ -30,7 +30,7 @@
div.appendChild(child);
});
sdk.members$.subscribe((memberObjects) => {
sdk.remoteMembers$.subscribe((memberObjects) => {
const div = document.getElementById("members");
div.innerHTML = "<h3>Members:</h3>";
sdk-target-based-on-call-view-model/main.ts
+98 -197
View File
@@ -6,62 +6,74 @@ Please see LICENSE in the repository root for full details.
*/
/**
* EXPERIMENTAL
* LEGACY, in production use
*
* This file is the entrypoint for the sdk build of element call:
* `pnpm build:sdk-target-based-on-call-view-model`
* use in widgets.
* It exposes the `createMatrixRTCSdk` which creates the `MatrixRTCSdk` interface (see below) that
* can be used to join a rtc session and exchange realtime data.
* It takes care of all the tricky bits:
* This file is the entrypoint of the legacy SDK bundle,
* `pnpm build:sdk-target-based-on-call-view-model`, for use in widgets. It
* exposes `createMatrixRTCSdk`, which creates the `MatrixRTCSdk` interface
* (see below) to join an RTC session and exchange realtime data.
*
* It is a thin layer over the `@element-hq/matrixrtc-sdk` package and Element
* Call's call view model, which take care of all the tricky bits:
* - sending delayed events
* - finding the right sfu
* - handling the media stream
* - sending join/leave state or sticky events
* - setting up encryption and scharing keys
* - setting up encryption and sharing keys
*
* The interface is kept as it was for its existing consumers. New consumers
* use the `@element-hq/matrixrtc-sdk` package directly; this bundle goes away
* once the existing ones have moved.
*/
import {
combineLatest,
filter,
map,
type Observable,
of,
shareReplay,
Subject,
switchMap,
tap,
} from "rxjs";
import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import {
type CallMembership,
MatrixRTCSessionEvent,
MatrixRTCSessionManager,
} from "matrix-js-sdk/lib/matrixrtc";
import {
type Room as LivekitRoom,
type TextStreamReader,
type LocalParticipant,
type RemoteParticipant,
} from "livekit-client";
type Behavior,
constant,
createMatrixRTCClient,
E2eeType,
ObservableScope,
type RTCMember,
} from "@element-hq/matrixrtc-sdk";
// TODO how can this get fixed? to just be part of `livekit-client`
// Can this be done in the tsconfig.json
import { type TextStreamInfo } from "../node_modules/livekit-client/dist/src/room/types";
import { type Behavior, constant } from "../src/state/Behavior";
import {
callViewModelOptionsFromParams,
createCallViewModel$,
} from "../src/state/CallViewModel/CallViewModel";
import { ObservableScope } from "../src/state/ObservableScope";
import { getUrlParams } from "../src/UrlParams";
import { MuteStates } from "../src/state/MuteStates";
import { MediaDevices } from "../src/state/MediaDevices";
import { E2eeType } from "../src/e2ee/e2eeType";
import { currentAndPrev, TEXT_LK_TOPIC, tryMakeSticky } from "./helper";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import { TEXT_LK_TOPIC, tryMakeSticky } from "./helper";
import { initializeWidget } from "../src/widget";
import { type Connection } from "../src/state/CallViewModel/remoteMembers/Connection";
import { createWidgetHostBridge } from "../src/HostBridge";
import { observeElementSize$ } from "../src/utils/elementSize";
import {
matrixRTCClientOptions,
selectedDeviceId$,
} from "../src/room/InCallView";
/**
* A member as this bundle has always presented it. The media transport is
* the SDK's business now, so the transport objects are gone: `connection` and
* `participant` are always null and only kept so that existing code keeps
* destructuring. `membership` is what consumers read.
*/
interface MatrixRTCSdkMember {
/** @deprecated Always null: the SDK owns the transport connection. */
connection: null;
membership: CallMembership;
/** @deprecated Always null: the SDK owns the transport participant. */
participant: null;
}
interface MatrixRTCSdk {
/**
@@ -81,21 +93,11 @@ interface MatrixRTCSdk {
/**
* flattened list of remote members
*/
remoteMembers$: Behavior<
{
connection: Connection | null;
membership: CallMembership;
participant: LocalParticipant | RemoteParticipant | null;
}[]
>;
remoteMembers$: Behavior<MatrixRTCSdkMember[]>;
/**
* flattened local member
*/
localMember$: Behavior<{
connection: Connection | null;
membership: CallMembership;
participant: LocalParticipant | null;
} | null>;
localMember$: Behavior<MatrixRTCSdkMember | null>;
/** Use the LocalMemberConnectionState returned from `join` for a more detailed connection state */
connected$: Behavior<boolean>;
sendData?: (data: unknown) => Promise<void>;
@@ -124,12 +126,6 @@ export async function createMatrixRTCSdk(
const room = client.getRoom(roomId);
if (room === null) throw Error("could not get room from client");
// rtc session
const slot = { application, id };
const rtcSessionManager = new MatrixRTCSessionManager(logger, client, slot);
rtcSessionManager.start();
const rtcSession = rtcSessionManager.getRoomSession(room);
// media devices
const mediaDevices = new MediaDevices(scope, {
controlledAudioDevices,
@@ -142,134 +138,63 @@ export async function createMatrixRTCSdk(
hostBridge,
);
// call view model
// rtc client: the session, the transport and the media, as in the app
const encryptionSystem = { kind: E2eeType.PER_PARTICIPANT } as const;
const rtcClient = createMatrixRTCClient(
scope,
client,
room,
{
microphoneEnabled: muteStates.audio.enabled$.value,
cameraEnabled: muteStates.video.enabled$.value,
audioInputDeviceId$: selectedDeviceId$(scope, mediaDevices.audioInput),
videoInputDeviceId$: selectedDeviceId$(scope, mediaDevices.videoInput),
audioOutputDeviceId$: controlledAudioDevices
? constant(undefined)
: selectedDeviceId$(scope, mediaDevices.audioOutput),
videoProcessor$: constant(undefined),
},
{
...matrixRTCClientOptions(urlParams, encryptionSystem),
application,
slot: id,
},
);
// call view model: the host bridge's hang-up handling and the leave flow
const callViewModel = createCallViewModel$(
scope,
rtcSession,
rtcClient,
room,
mediaDevices,
muteStates,
{
...callViewModelOptionsFromParams(urlParams),
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
encryptionSystem,
hostBridge,
// The SDK owns its page, so the body is the space it has
windowSize$: scope.behavior(observeElementSize$(document.body)),
},
of({}),
of({}),
constant({ supported: false, processor: undefined }),
);
logger.info("CallViewModelCreated");
// create data listener
const data$ = new Subject<{ rtcBackendIdentity: string; data: string }>();
const lkTextStreamHandlerFunction = async (
reader: TextStreamReader,
participantInfo: { identity: string },
livekitRoom: LivekitRoom,
): Promise<void> => {
const info = reader.info;
logger.info(
`Received text stream from ${participantInfo.identity}\n` +
` Topic: ${info.topic}\n` +
` Timestamp: ${info.timestamp}\n` +
` ID: ${info.id}\n` +
` Size: ${info.size}`, // Optional, only available if the stream was sent with `sendText`
);
const participants = callViewModel.livekitRoomItems$.value.find(
(i) => i.livekitRoom === livekitRoom,
)?.participants;
if (participants && participants.includes(participantInfo.identity)) {
const text = await reader.readAll();
logger.info(`Received text: ${text}`);
data$.next({ rtcBackendIdentity: participantInfo.identity, data: text });
} else {
logger.warn(
"Received text from unknown participant",
participantInfo.identity,
);
}
};
const livekitRoomItemsSub = callViewModel.livekitRoomItems$
.pipe(
tap((beforecurrentAndPrev) => {
logger.info(
`LiveKit room items updated: ${beforecurrentAndPrev.length}`,
beforecurrentAndPrev,
);
// Data arrives from attested members only; the SDK has matched the sender
const data$ = rtcClient.data$.pipe(
filter(({ topic }) => topic === TEXT_LK_TOPIC),
map(({ member, text }) => {
logger.info(`Received text from ${member.id}: ${text}`);
return { rtcBackendIdentity: member.id, data: text };
}),
currentAndPrev,
tap((aftercurrentAndPrev) => {
logger.info(
`LiveKit room items updated: ${aftercurrentAndPrev.current.length}, ${aftercurrentAndPrev.prev.length}`,
aftercurrentAndPrev,
);
}),
)
.subscribe({
next: ({ prev, current }) => {
const prevRooms = prev.map((i) => i.livekitRoom);
const currentRooms = current.map((i) => i.livekitRoom);
const addedRooms = currentRooms.filter((r) => !prevRooms.includes(r));
const removedRooms = prevRooms.filter((r) => !currentRooms.includes(r));
addedRooms.forEach((r) => {
logger.info(`Registering text stream handler for room `);
r.registerTextStreamHandler(
TEXT_LK_TOPIC,
(reader, participantInfo) =>
void lkTextStreamHandlerFunction(reader, participantInfo, r),
);
});
removedRooms.forEach((r) => {
logger.info(`Unregistering text stream handler for room `);
r.unregisterTextStreamHandler(TEXT_LK_TOPIC);
});
},
complete: () => {
logger.info("Livekit room items subscription completed");
for (const item of callViewModel.livekitRoomItems$.value) {
logger.info("unregistering room item from room", item.transport);
item.livekitRoom.unregisterTextStreamHandler(TEXT_LK_TOPIC);
}
},
});
// create sendData function
const sendFn: Behavior<(data: string) => Promise<TextStreamInfo>> =
scope.behavior(
callViewModel.localMatrixLivekitMember$.pipe(
switchMap((m) => {
if (!m)
return of((data: string): never => {
throw Error("local membership not yet ready.");
});
return m.participant.value$.pipe(
map((p) => {
if (p === null) {
return (data: string): never => {
throw Error("local participant not yet ready to send data.");
};
} else {
return async (data: string): Promise<TextStreamInfo> =>
p.sendText(data, { topic: TEXT_LK_TOPIC });
}
}),
);
}),
),
);
const sendData = async (data: unknown): Promise<void> => {
const dataString = JSON.stringify(data);
logger.info("try sending: ", dataString);
try {
await Promise.resolve();
const info = await sendFn.value(dataString);
logger.info(`Sent text with stream ID: ${info.id}`);
await rtcClient.sendData(TEXT_LK_TOPIC, dataString);
logger.info("sent text");
} catch (e) {
logger.error("failed sending: ", dataString, e);
}
@@ -288,13 +213,12 @@ export async function createMatrixRTCSdk(
// after hangup gets called
const leaveSubs = callViewModel.leave$.subscribe(() => {
const scheduleWidgetCloseOnLeave = async (): Promise<void> => {
const leaveResolver = Promise.withResolvers<void>();
logger.info("waiting for RTC leave");
rtcSession.on(MatrixRTCSessionEvent.JoinStateChanged, (isJoined) => {
logger.info("received RTC join update: ", isJoined);
if (!isJoined) leaveResolver.resolve();
await new Promise<void>((resolve) => {
rtcClient.status$
.pipe(filter((status) => status === "disconnected"))
.subscribe(() => resolve());
});
await leaveResolver.promise;
logger.info("send Unstick");
await hostBridge
.setAlwaysOnScreen(false)
@@ -315,6 +239,15 @@ export async function createMatrixRTCSdk(
logger.info("createMatrixRTCSdk done");
const flatten = (member: RTCMember): Observable<MatrixRTCSdkMember> =>
member.membership$.pipe(
map((membership) => ({
connection: null,
membership,
participant: null,
})),
);
return {
join: (): void => {
// first lets try making the widget sticky
@@ -326,54 +259,22 @@ export async function createMatrixRTCSdk(
},
stop: (): void => {
leaveSubs.unsubscribe();
livekitRoomItemsSub.unsubscribe();
scope.end();
},
data$,
localMember$: scope.behavior(
callViewModel.localMatrixLivekitMember$.pipe(
tap((member) =>
logger.info("localMatrixLivekitMember$ next: ", member),
),
switchMap((member) => {
if (member === null) return of(null);
return combineLatest([
member.connection$,
member.membership$,
member.participant.value$,
]).pipe(
map(([connection, membership, participant]) => ({
connection,
membership,
participant,
})),
);
}),
tap((member) => logger.info("localMember$ next: ", member)),
rtcClient.localMember$.pipe(
switchMap((member) => (member === null ? of(null) : flatten(member))),
),
),
connected$: callViewModel.connected$,
remoteMembers$: scope.behavior(
callViewModel.remoteMatrixLivekitMembers$.pipe(
switchMap((members) => {
const listOfMemberObservables = members.map((member) =>
combineLatest([
member.connection$,
member.membership$,
member.participant.value$,
]).pipe(
map(([connection, membership, participant]) => ({
connection,
membership,
participant,
})),
// using shareReplay instead of a Behavior here because the behavior would need
// a tricky scope.end() setup.
shareReplay({ bufferSize: 1, refCount: true }),
rtcClient.remoteMembers$.pipe(
switchMap((members) =>
members.length === 0
? of([])
: combineLatest(members.map((member) => flatten(member))),
),
);
return combineLatest(listOfMemberObservables);
}),
),
[],
),
src/App.tsx
+1 -1
View File
@@ -35,7 +35,7 @@ import { ErrorPage, LoadingPage } from "./FullScreenView";
import { Initializer } from "./initializer";
import { type WidgetHelpers } from "./widget";
import { useTheme } from "./useTheme";
import { ProcessorProvider } from "./livekit/TrackProcessorContext";
import { ProcessorProvider } from "./tracks/TrackProcessorContext";
import { type AppViewModel } from "./state/AppViewModel";
import { MediaDevicesContext } from "./MediaDevicesContext";
import {
src/button/ReactionToggleButton.test.tsx
+1 -2
View File
@@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details.
*/
import { act, render } from "@testing-library/react";
import { expect, test, vi } from "vitest";
import { expect, test } from "vitest";
import { TooltipProvider } from "@vector-im/compound-web";
import { userEvent } from "@testing-library/user-event";
import { type ReactNode } from "react";
@@ -20,7 +20,6 @@ import { type MockRTCSession } from "../utils/test";
import { ReactionsSenderProvider } from "../reactions/useReactionsSender";
import { initializeWidget } from "../widget";
initializeWidget();
vi.mock("livekit-client/e2ee-worker?worker");
const localIdent = `${localRtcMember.userId}:${localRtcMember.deviceId}`;
src/config/ConfigOptions.ts
+3 -17
View File
@@ -6,23 +6,9 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
/**
* The MatrixRTC mode determines how Element Call interacts with the
* MatrixRTC backend and other participants. Selectable via the Developer
* Settings, or pinned for a deployment via `matrix_rtc_mode` in config.json.
*/
export enum MatrixRTCMode {
/** Multi-SFU transport, legacy JWT endpoint, state events. */
Compatibility = "compatibility",
/**
* Multi-SFU transport with:
* - sticky events
* - hashed RTC backend identity
* - the new endpoint for the jwt token on the local membership (remote memberships will always try the new jwt endpoint first -> then the legacy one)
* - use the hashed identity for the local membership
*/
Matrix_2_0 = "matrix_2_0",
}
import { MatrixRTCMode } from "@element-hq/matrixrtc-sdk";
export { MatrixRTCMode };
export interface DelayedLeaveTimings {
/**
src/e2ee/e2eeType.ts
+2 -6
View File
@@ -1,12 +1,8 @@
/*
Copyright 2023, 2024 New Vector Ltd.
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
export enum E2eeType {
NONE = 0,
PER_PARTICIPANT = 1,
SHARED_KEY = 2,
}
export { E2eeType } from "@element-hq/matrixrtc-sdk";
src/e2ee/matrixKeyProvider.test.ts
-72
View File
@@ -1,72 +0,0 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { describe, expect, test, vi } from "vitest";
import {
type MatrixRTCSession,
MatrixRTCSessionEvent,
} from "matrix-js-sdk/lib/matrixrtc";
import { KeyProviderEvent } from "livekit-client";
import { MatrixKeyProvider } from "./matrixKeyProvider";
function mockRTCSession(): MatrixRTCSession {
return {
on: vi.fn(),
off: vi.fn(),
reemitEncryptionKeys: vi.fn(),
} as unknown as MatrixRTCSession;
}
describe("matrixKeyProvider", () => {
test("initializes", () => {
const keyProvider = new MatrixKeyProvider();
expect(keyProvider).toBeTruthy();
});
test("listens for key requests and emits existing keys", () => {
const keyProvider = new MatrixKeyProvider();
const session = mockRTCSession();
keyProvider.setRTCSession(session);
expect(session.on).toHaveBeenCalledWith(
MatrixRTCSessionEvent.EncryptionKeyChanged,
expect.any(Function),
);
expect(session.off).not.toHaveBeenCalled();
});
test("stops listening when session changes", () => {
const keyProvider = new MatrixKeyProvider();
const session1 = mockRTCSession();
const session2 = mockRTCSession();
keyProvider.setRTCSession(session1);
expect(session1.off).not.toHaveBeenCalled();
keyProvider.setRTCSession(session2);
expect(session1.off).toHaveBeenCalledWith(
MatrixRTCSessionEvent.EncryptionKeyChanged,
expect.any(Function),
);
});
test("emits existing keys", () => {
const keyProvider = new MatrixKeyProvider();
const setKeyListener = vi.fn();
keyProvider.on(KeyProviderEvent.SetKey, setKeyListener);
const session = mockRTCSession();
keyProvider.setRTCSession(session);
expect(session.reemitEncryptionKeys).toHaveBeenCalled();
});
});
src/e2ee/matrixKeyProvider.ts
-75
View File
@@ -1,75 +0,0 @@
/*
Copyright 2023, 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { BaseKeyProvider } from "livekit-client";
import {
type MatrixRTCSession,
MatrixRTCSessionEvent,
} from "matrix-js-sdk/lib/matrixrtc";
import { logger as rootLogger, type Logger } from "matrix-js-sdk/lib/logger";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
export class MatrixKeyProvider extends BaseKeyProvider {
private rtcSession?: MatrixRTCSession;
private logger: Logger;
public constructor() {
super({ ratchetWindowSize: 10, keyringSize: 256 });
this.logger = rootLogger.getChild("[MatrixKeyProvider]");
}
public setRTCSession(rtcSession: MatrixRTCSession): void {
if (this.rtcSession) {
this.rtcSession.off(
MatrixRTCSessionEvent.EncryptionKeyChanged,
this.onEncryptionKeyChanged,
);
}
this.rtcSession = rtcSession;
this.rtcSession.on(
MatrixRTCSessionEvent.EncryptionKeyChanged,
this.onEncryptionKeyChanged,
);
// The new session could be aware of keys of which the old session wasn't,
// so emit key changed events
this.rtcSession.reemitEncryptionKeys();
}
private onEncryptionKeyChanged = (
encryptionKey: Uint8Array<ArrayBuffer>,
encryptionKeyIndex: number,
membershipParts: CallMembershipIdentityParts,
rtcBackendIdentity: string,
): void => {
crypto.subtle
.importKey("raw", encryptionKey, "HKDF", false, [
"deriveBits",
"deriveKey",
])
.then(
(keyMaterial) => {
this.onSetEncryptionKey(
keyMaterial,
rtcBackendIdentity,
encryptionKeyIndex,
);
this.logger.debug(
`Sent new key to livekit room=${this.rtcSession?.room.roomId} participantId=${rtcBackendIdentity} (before hash: ${membershipParts.userId}:${membershipParts.deviceId}) encryptionKeyIndex=${encryptionKeyIndex}`,
);
},
(e) => {
this.logger.error(
`Failed to create key material from buffer for livekit room=${this.rtcSession?.room.roomId} participantId before hash=${membershipParts.userId}:${membershipParts.deviceId} encryptionKeyIndex=${encryptionKeyIndex}`,
e,
);
},
);
};
}
src/e2ee/sharedKeyManagement.ts
+2 -5
View File
@@ -13,7 +13,7 @@ import {
useLocalStorage,
} from "../useLocalStorage";
import { getUrlParams, useUrlParams } from "../UrlParams";
import { E2eeType } from "./e2eeType";
import { E2eeType, type EncryptionSystem } from "@element-hq/matrixrtc-sdk";
import { useClient } from "../ClientContext";
/**
@@ -82,10 +82,7 @@ export function getKeyForRoom(roomId: string): string | null {
return keyForRoom(roomId, paramsRoomId, password);
}
export type Unencrypted = { kind: E2eeType.NONE };
export type SharedSecret = { kind: E2eeType.SHARED_KEY; secret: string };
export type PerParticipantE2EE = { kind: E2eeType.PER_PARTICIPANT };
export type EncryptionSystem = Unencrypted | SharedSecret | PerParticipantE2EE;
export { type EncryptionSystem } from "@element-hq/matrixrtc-sdk";
export function useRoomEncryptionSystem(roomId: string): EncryptionSystem {
const { client } = useClient();
src/livekit/MatrixAudioRenderer.test.tsx
-342
View File
@@ -1,342 +0,0 @@
/*
Copyright 2023, 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { afterEach, beforeEach, expect, it, vi } from "vitest";
import { render, type RenderResult } from "@testing-library/react";
import {
getTrackReferenceId,
type TrackReference,
} from "@livekit/components-core";
import {
type Participant,
type RemoteAudioTrack,
type Room,
Track,
} from "livekit-client";
import { type ReactNode } from "react";
import { useTracks } from "@livekit/components-react";
import { testAudioContext } from "../useAudioContext.test";
import * as MediaDevicesContext from "../MediaDevicesContext";
import { LivekitRoomAudioRenderer } from "./MatrixAudioRenderer";
import {
exampleTransportLocator,
mockLocalParticipant,
mockMediaDevices,
mockRemoteParticipant,
mockTrack,
} from "../utils/test";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import { initializeWidget } from "../widget";
initializeWidget();
export const TestAudioContextConstructor = vi.fn(
class {
public constructor() {
return testAudioContext;
}
},
);
const MediaDevicesProvider = MediaDevicesContext.MediaDevicesContext.Provider;
beforeEach(() => {
vi.stubGlobal("AudioContext", TestAudioContextConstructor);
});
afterEach(() => {
vi.unstubAllGlobals();
vi.clearAllMocks();
});
vi.mock("@livekit/components-react", async (importOriginal) => {
return {
...(await importOriginal()),
AudioTrack: (props: { trackRef: TrackReference }): ReactNode => {
return (
<audio data-testid={"audio"}>
{getTrackReferenceId(props.trackRef)}
</audio>
);
},
useTracks: vi.fn(),
};
});
let tracks: TrackReference[] = [];
/**
* Render the test component with given rtc members and livekit participant identities.
*
* It is possible to have rtc members that are not in livekit (e.g. not yet joined) and vice versa.
*
* @param rtcMembers - Array of active rtc members with userId and deviceId.
* @param livekitParticipantIdentities - Array of livekit participant (that are publishing).
* @param explicitTracks - Array of tracks available in livekit, if not provided, one audio track per livekitParticipantIdentities will be created.
* */
function renderTestComponent(
rtcMembers: { userId: string; deviceId: string }[],
livekitParticipantIdentities: string[],
explicitTracks?: {
participantId: string;
kind: Track.Kind;
source: Track.Source;
}[],
): RenderResult {
const liveKitParticipants = livekitParticipantIdentities.map((identity) =>
mockRemoteParticipant({ identity }),
);
const participants = rtcMembers.flatMap(({ userId, deviceId }) => {
const p = liveKitParticipants.find(
(p) => p.identity === `${userId}:${deviceId}`,
);
return p === undefined ? [] : [p];
});
const livekitRoom = {
remoteParticipants: new Map<string, Participant>(
liveKitParticipants.map((p) => [p.identity, p]),
),
} as unknown as Room;
if ((explicitTracks?.length ?? 0) > 0) {
tracks = explicitTracks!.map(({ participantId, source, kind }) => {
const participant =
liveKitParticipants.find((p) => p.identity === participantId) ??
mockRemoteParticipant({ identity: participantId });
return mockTrack(participant, kind, source);
});
} else {
tracks = participants.map((p) => mockTrack(p));
}
vi.mocked(useTracks).mockReturnValue(tracks);
return render(
<MediaDevicesProvider value={mockMediaDevices({})}>
<LivekitRoomAudioRenderer
validIdentities={participants.map((p) => p.identity)}
livekitRoom={livekitRoom}
transport={exampleTransportLocator}
/>
</MediaDevicesProvider>,
);
}
it("should render for member", () => {
const { container, queryAllByTestId } = renderTestComponent(
[{ userId: "@alice", deviceId: "DEV0" }],
["@alice:DEV0"],
);
expect(container).toBeTruthy();
expect(queryAllByTestId("audio")).toHaveLength(1);
});
function spyOnWarn(): ReturnType<typeof vi.fn> {
const warn = vi.fn();
vi.spyOn(rootLogger, "getChild").mockReturnValue({
warn,
} as unknown as typeof rootLogger);
return warn;
}
it("should not render or warn for the local participant", () => {
const warn = spyOnWarn();
const local = mockLocalParticipant({ identity: "@alice:DEV0" });
vi.mocked(useTracks).mockReturnValue([mockTrack(local)]);
const { queryAllByTestId } = render(
<MediaDevicesProvider value={mockMediaDevices({})}>
<LivekitRoomAudioRenderer
validIdentities={[]}
livekitRoom={{ remoteParticipants: new Map() } as unknown as Room}
transport={exampleTransportLocator}
/>
</MediaDevicesProvider>,
);
expect(queryAllByTestId("audio")).toHaveLength(0);
expect(warn).not.toHaveBeenCalled();
});
it("should warn only once per unexpected participant", () => {
const warn = spyOnWarn();
const { rerender } = renderTestComponent(
[{ userId: "@bob", deviceId: "DEV0" }],
["@alice:DEV0"],
[
{
participantId: "@alice:DEV0",
kind: Track.Kind.Audio,
source: Track.Source.Microphone,
},
],
);
expect(warn).toHaveBeenCalledTimes(1);
rerender(
<MediaDevicesProvider value={mockMediaDevices({})}>
<LivekitRoomAudioRenderer
validIdentities={[]}
livekitRoom={{ remoteParticipants: new Map() } as unknown as Room}
transport={exampleTransportLocator}
/>
</MediaDevicesProvider>,
);
expect(warn).toHaveBeenCalledTimes(1);
});
it("should not render without member", () => {
const { container, queryAllByTestId } = renderTestComponent(
[{ userId: "@bob", deviceId: "DEV0" }],
["@alice:DEV0"],
);
expect(container).toBeTruthy();
expect(queryAllByTestId("audio")).toHaveLength(0);
});
const TEST_CASES: {
name: string;
rtcUsers: { userId: string; deviceId: string }[];
livekitParticipantIdentities: string[];
explicitTracks?: {
participantId: string;
kind: Track.Kind;
source: Track.Source;
}[];
expectedAudioTracks: number;
}[] = [
{
name: "single user single device",
rtcUsers: [
{ userId: "@alice", deviceId: "DEV0" },
{ userId: "@alice", deviceId: "DEV1" },
{ userId: "@bob", deviceId: "DEV0" },
],
livekitParticipantIdentities: ["@alice:DEV0", "@bob:DEV0", "@alice:DEV1"],
expectedAudioTracks: 3,
},
// Charlie is a rtc member but not in livekit
{
name: "Charlie is rtc member but not in livekit",
rtcUsers: [
{ userId: "@alice", deviceId: "DEV0" },
{ userId: "@bob", deviceId: "DEV0" },
{ userId: "@charlie", deviceId: "DEV0" },
],
livekitParticipantIdentities: ["@alice:DEV0", "@bob:DEV0"],
expectedAudioTracks: 2,
},
// Charlie is in livekit but not rtc member
{
name: "Charlie is in livekit but not rtc member",
rtcUsers: [
{ userId: "@alice", deviceId: "DEV0" },
{ userId: "@bob", deviceId: "DEV0" },
],
livekitParticipantIdentities: ["@alice:DEV0", "@bob:DEV0", "@charlie:DEV0"],
expectedAudioTracks: 2,
},
{
name: "no audio track, only video track",
rtcUsers: [{ userId: "@alice", deviceId: "DEV0" }],
livekitParticipantIdentities: ["@alice:DEV0"],
explicitTracks: [
{
participantId: "@alice:DEV0",
kind: Track.Kind.Video,
source: Track.Source.Camera,
},
],
expectedAudioTracks: 0,
},
{
name: "Audio track from unknown source",
rtcUsers: [{ userId: "@alice", deviceId: "DEV0" }],
livekitParticipantIdentities: ["@alice:DEV0"],
explicitTracks: [
{
participantId: "@alice:DEV0",
kind: Track.Kind.Audio,
source: Track.Source.Unknown,
},
],
expectedAudioTracks: 1,
},
{
name: "Audio track from other device",
rtcUsers: [{ userId: "@alice", deviceId: "DEV0" }],
livekitParticipantIdentities: ["@alice:DEV0"],
explicitTracks: [
{
participantId: "@alice:DEV1",
kind: Track.Kind.Audio,
source: Track.Source.Microphone,
},
],
expectedAudioTracks: 0,
},
{
name: "two audio tracks, microphone and screenshare",
rtcUsers: [{ userId: "@alice", deviceId: "DEV0" }],
livekitParticipantIdentities: ["@alice:DEV0"],
explicitTracks: [
{
participantId: "@alice:DEV0",
kind: Track.Kind.Audio,
source: Track.Source.Microphone,
},
{
participantId: "@alice:DEV0",
kind: Track.Kind.Audio,
source: Track.Source.ScreenShareAudio,
},
],
expectedAudioTracks: 2,
},
];
it.each(TEST_CASES)(
`should render sound test cases $name`,
({
rtcUsers,
livekitParticipantIdentities,
explicitTracks,
expectedAudioTracks,
}) => {
const { queryAllByTestId } = renderTestComponent(
rtcUsers,
livekitParticipantIdentities,
explicitTracks,
);
expect(queryAllByTestId("audio")).toHaveLength(expectedAudioTracks);
},
);
it("should not setup audioContext gain and pan if there is no need to.", () => {
renderTestComponent([{ userId: "@bob", deviceId: "DEV0" }], ["@bob:DEV0"]);
const audioTrack = tracks[0].publication.track! as RemoteAudioTrack;
expect(audioTrack.setAudioContext).toHaveBeenCalledTimes(1);
expect(audioTrack.setAudioContext).toHaveBeenCalledWith(undefined);
expect(audioTrack.setWebAudioPlugins).toHaveBeenCalledTimes(1);
expect(audioTrack.setWebAudioPlugins).toHaveBeenCalledWith([]);
expect(testAudioContext.gain.gain.value).toEqual(1);
expect(testAudioContext.pan.pan.value).toEqual(0);
});
it("should setup audioContext gain and pan", () => {
vi.spyOn(MediaDevicesContext, "useEarpieceAudioConfig").mockReturnValue({
pan: 1,
volume: 0.1,
});
renderTestComponent([{ userId: "@bob", deviceId: "DEV0" }], ["@bob:DEV0"]);
const audioTrack = tracks[0].publication.track! as RemoteAudioTrack;
expect(audioTrack.setAudioContext).toHaveBeenCalled();
expect(audioTrack.setWebAudioPlugins).toHaveBeenCalled();
expect(testAudioContext.gain.gain.value).toEqual(0.1);
expect(testAudioContext.pan.pan.value).toEqual(1);
});
src/livekit/MatrixAudioRenderer.tsx
-216
View File
@@ -1,216 +0,0 @@
/*
Copyright 2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { getTrackReferenceId } from "@livekit/components-core";
import { type Room as LivekitRoom } from "livekit-client";
import { type RemoteAudioTrack, Track } from "livekit-client";
import { useEffect, useMemo, useRef, useState, type ReactNode } from "react";
import {
useTracks,
AudioTrack,
type AudioTrackProps,
} from "@livekit/components-react";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import { useEarpieceAudioConfig } from "../MediaDevicesContext";
import { useReactiveState } from "../useReactiveState";
import * as controls from "../controls";
import { type TransportLocator } from "./auth";
export interface MatrixAudioRendererProps {
transport: TransportLocator;
livekitRoom: LivekitRoom;
/**
* The list of participant identities to render audio for.
* This list needs to be composed based on the matrixRTC members so that we do not play audio from users
* that are not expected to be in the rtc session (local user is excluded).
*/
validIdentities: string[];
/**
* If set to `true`, mutes all audio tracks rendered by the component.
* @remarks
* If set to `true`, the server will stop sending audio track data to the client.
*/
muted?: boolean;
}
/**
* Takes care of handling remote participants’ audio tracks and makes sure that microphones and screen share are audible.
*
* It also takes care of the earpiece audio configuration for iOS devices.
* This is done by using the WebAudio API to create a stereo pan effect that mimics the earpiece audio.
* @example
* ```tsx
* <LiveKitRoom>
* <MatrixAudioRenderer />
* </LiveKitRoom>
* ```
* @public
*/
export function LivekitRoomAudioRenderer({
transport,
livekitRoom,
validIdentities,
muted,
}: MatrixAudioRendererProps): ReactNode {
const logger = rootLogger.getChild("[MatrixAudioRenderer]");
// Identities we have already warned about, so that re-renders (which happen
// on every active speaker update) don't repeat the warning.
const warnedIdentities = useRef(new Set<string>());
const tracks = useTracks(
[
Track.Source.Microphone,
Track.Source.ScreenShareAudio,
Track.Source.Unknown,
],
{
updateOnlyOn: [],
onlySubscribed: true,
room: livekitRoom,
},
)
// Only keep remote audio tracks (we never render our own audio)
.filter(
(ref) =>
ref.publication.kind === Track.Kind.Audio && !ref.participant.isLocal,
)
// Only keep tracks from participants that are in the validIdentities list
.filter((ref) => {
const { identity } = ref.participant;
const isValid = validIdentities.includes(identity);
if (!isValid && !warnedIdentities.current.has(identity)) {
warnedIdentities.current.add(identity);
// Log that there is an invalid identity, that means that someone is publishing audio that is not expected to be in the call.
logger.warn(
`Audio track ${identity} from ${JSON.stringify(transport)} has no matching matrix call member`,
`current members: ${validIdentities.join()}`,
`track will not get rendered`,
);
} else if (isValid) {
warnedIdentities.current.delete(identity);
}
return isValid;
});
// This component is also (in addition to the "only play audio for connected members" logic above)
// responsible for mimicking earpiece audio on iPhones.
// The Safari audio devices enumeration does not expose an earpiece audio device.
// We alternatively use the audioContext pan node to only use one of the stereo channels.
// This component does get additionally complicated because of a Safari bug.
// (see: https://bugs.webkit.org/show_bug.cgi?id=251532
// and the related issues: https://bugs.webkit.org/show_bug.cgi?id=237878
// and https://bugs.webkit.org/show_bug.cgi?id=231105)
//
// AudioContext gets stopped if the webview gets moved into the background.
// Once the phone is in standby audio playback will stop.
// So we can only use the pan trick only works is the phone is not in standby.
// If earpiece mode is not used we do not use audioContext to allow standby playback.
// shouldUseAudioContext is set to false if stereoPan === 0 to allow standby bluetooth playback.
const { pan: stereoPan, volume: volumeFactor } = useEarpieceAudioConfig();
const shouldUseAudioContext = stereoPan !== 0;
// initialize the potentially used audio context.
const [audioContext, setAudioContext] = useState<AudioContext | undefined>(
undefined,
);
useEffect(() => {
const ctx = new AudioContext();
setAudioContext(ctx);
return (): void => {
void ctx.close();
};
}, []);
const audioNodes = useMemo(
() => ({
gain: audioContext?.createGain(),
pan: audioContext?.createStereoPanner(),
}),
[audioContext],
);
// Simple effects to update the gain and pan node based on the props
useEffect(() => {
if (audioNodes.pan) audioNodes.pan.pan.value = stereoPan;
}, [audioNodes.pan, stereoPan]);
useEffect(() => {
if (audioNodes.gain) audioNodes.gain.gain.value = volumeFactor;
}, [audioNodes.gain, volumeFactor]);
return (
// We add all audio elements into one <div> for the browser developer tool experience/tidyness.
<div style={{ display: "none" }}>
{tracks.map((trackRef) => (
<AudioTrackWithAudioNodes
key={getTrackReferenceId(trackRef)}
trackRef={trackRef}
muted={muted}
audioContext={shouldUseAudioContext ? audioContext : undefined}
audioNodes={audioNodes}
/>
))}
</div>
);
}
interface StereoPanAudioTrackProps {
muted?: boolean;
audioContext?: AudioContext;
audioNodes: {
gain?: GainNode;
pan?: StereoPannerNode;
};
}
/**
* This wraps `livekit.AudioTrack` to allow adding audio nodes to a track.
* It main purpose is to remount the AudioTrack component when switching from
* audioContext to normal audio playback.
* As of now the AudioTrack component does not support adding audio nodes while being mounted.
* @param props The component props
* @param props.trackRef The track reference
* @param props.muted If the track should be muted
* @param props.audioContext The audio context to use
* @param props.audioNodes The audio nodes to use
* @returns
*/
function AudioTrackWithAudioNodes({
trackRef,
muted,
audioContext,
audioNodes,
...props
}: StereoPanAudioTrackProps &
AudioTrackProps &
React.RefAttributes<HTMLAudioElement>): ReactNode {
// This is used to unmount/remount the AudioTrack component.
// Mounting needs to happen after the audioContext is set.
// (adding the audio context when already mounted did not work outside strict mode)
const [trackReady, setTrackReady] = useReactiveState(
() => false,
// We only want the track to reset once both (audioNodes and audioContext) are set.
// for unsetting the audioContext its enough if one of the two is undefined.
[audioContext && audioNodes],
);
useEffect(() => {
if (!trackRef || trackReady) return;
const track = trackRef.publication.track as RemoteAudioTrack;
const useContext = audioContext && audioNodes.gain && audioNodes.pan;
track.setAudioContext(useContext ? audioContext : undefined);
track.setWebAudioPlugins(
useContext ? [audioNodes.gain!, audioNodes.pan!] : [],
);
setTrackReady(true);
controls.setPlaybackStarted();
}, [audioContext, audioNodes, setTrackReady, trackReady, trackRef]);
return (
trackReady && <AudioTrack trackRef={trackRef} muted={muted} {...props} />
);
}
src/livekit/options.test.ts
-213
View File
@@ -1,213 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { describe, expect, it, vi } from "vitest";
import { VideoPresets, type VideoPreset } from "livekit-client";
import { buildLiveKitOptions, getLiveKitOptions } from "./options";
import { Config } from "../config/Config";
vi.mock("../config/Config", () => ({
Config: {
get: vi.fn(),
},
}));
describe("buildLiveKitOptions", () => {
it("returns sensible defaults with no config", () => {
const opts = buildLiveKitOptions();
expect(opts.adaptiveStream).toBe(true);
expect(opts.dynacast).toBe(true);
expect(opts.videoCaptureDefaults?.resolution).toEqual(
VideoPresets.h720.resolution,
);
expect(opts.publishDefaults?.videoCodec).toBe("vp8");
expect(opts.publishDefaults?.videoEncoding).toEqual({
maxBitrate: 1_700_000,
maxFramerate: 30,
});
expect(opts.publishDefaults?.screenShareEncoding).toEqual({
maxBitrate: 5_000_000,
maxFramerate: 30,
});
expect(opts.publishDefaults?.videoSimulcastLayers).toEqual([
VideoPresets.h180,
VideoPresets.h360,
]);
});
it("applies video codec from config", () => {
const opts = buildLiveKitOptions({ video_codec: "vp9" });
expect(opts.publishDefaults?.videoCodec).toBe("vp9");
});
it("applies video resolution and encoding from config", () => {
const baseVideoConfig = {
max_resolution: 1080,
max_bitrate: 3_000_000,
max_framerate: 60,
};
const opts1080 = buildLiveKitOptions({
video: baseVideoConfig,
});
const opts1440 = buildLiveKitOptions({
video: { ...baseVideoConfig, max_resolution: 1440 },
});
const opts2160 = buildLiveKitOptions({
video: { ...baseVideoConfig, max_resolution: 2160 },
});
expect(opts1080.videoCaptureDefaults?.resolution).toEqual(
VideoPresets.h1080.resolution,
);
expect(opts1440.videoCaptureDefaults?.resolution).toEqual(
VideoPresets.h1440.resolution,
);
expect(opts2160.videoCaptureDefaults?.resolution).toEqual(
VideoPresets.h2160.resolution,
);
expect(opts1080.publishDefaults?.videoEncoding).toEqual({
maxBitrate: 3_000_000,
maxFramerate: 60,
});
});
it("applies screen share encoding from config", () => {
const opts = buildLiveKitOptions({
screen_share: {
max_bitrate: 8_000_000,
max_framerate: 15,
},
});
expect(opts.publishDefaults?.screenShareEncoding).toEqual({
maxBitrate: 8_000_000,
maxFramerate: 15,
});
});
it("uses DEFAULT_CONFIG defaults when only resolution is set", () => {
const opts = buildLiveKitOptions({
screen_share: {
max_resolution: 720,
},
});
// Bitrate and framerate fall back to DEFAULT_CONFIG, not the preset
expect(opts.publishDefaults?.screenShareEncoding).toEqual({
maxBitrate: 5_000_000,
maxFramerate: 30,
});
});
it("maps low resolutions to the closest preset, rounding up", () => {
const expectations: [number, VideoPreset][] = [
[180, VideoPresets.h180],
[360, VideoPresets.h360],
[480, VideoPresets.h540],
[540, VideoPresets.h540],
[720, VideoPresets.h720],
];
for (const [height, preset] of expectations) {
const opts = buildLiveKitOptions({ video: { max_resolution: height } });
expect(opts.videoCaptureDefaults?.resolution).toEqual(preset.resolution);
}
});
it("screen share layers fall back to max_framerate, then 30", () => {
const fromMax = buildLiveKitOptions({
screen_share: {
max_framerate: 15,
simulcast_layers: [{ height: 540, bitrate: 1_000_000 }],
},
});
expect(
fromMax.publishDefaults?.screenShareSimulcastLayers?.[0],
).toMatchObject({ encoding: { maxFramerate: 15 } });
const fromDefault = buildLiveKitOptions({
screen_share: {
simulcast_layers: [{ height: 540, bitrate: 1_000_000 }],
},
});
expect(
fromDefault.publishDefaults?.screenShareSimulcastLayers?.[0],
).toMatchObject({ encoding: { maxFramerate: 30 } });
});
it("applies custom video simulcast layers", () => {
const opts = buildLiveKitOptions({
video: {
simulcast_layers: [
{ height: 180, bitrate: 100_000 },
{ height: 360, bitrate: 300_000 },
{ height: 540, bitrate: 600_000 },
],
max_framerate: 24,
},
});
const layers = opts.publishDefaults?.videoSimulcastLayers;
expect(layers).toHaveLength(3);
expect(layers?.[0]).toMatchObject({
width: 320,
height: 180,
encoding: { maxBitrate: 100_000, maxFramerate: 24 },
});
expect(layers?.[2]).toMatchObject({
width: 960,
height: 540,
encoding: { maxBitrate: 600_000, maxFramerate: 24 },
});
});
it("applies custom screen share simulcast layers", () => {
const opts = buildLiveKitOptions({
screen_share: {
simulcast_layers: [{ height: 540, bitrate: 1_000_000, framerate: 5 }],
},
});
const layers = opts.publishDefaults?.screenShareSimulcastLayers;
expect(layers).toHaveLength(1);
expect(layers?.[0]).toMatchObject({
width: 960,
height: 540,
encoding: { maxBitrate: 1_000_000, maxFramerate: 5 },
});
});
it("does not include screenShareSimulcastLayers when not configured", () => {
const opts = buildLiveKitOptions();
expect(opts.publishDefaults?.screenShareSimulcastLayers).toBeUndefined();
});
it("backupCodec always uses stock VP8 720p encoding", () => {
const opts = buildLiveKitOptions({
video_codec: "av1",
video: { max_bitrate: 10_000_000, max_framerate: 60 },
});
const backup = opts.publishDefaults?.backupCodec as {
codec: string;
encoding: { maxBitrate: number; maxFramerate: number };
};
expect(backup.codec).toBe("vp8");
expect(backup.encoding).toEqual(VideoPresets.h720.encoding);
});
});
describe("getLiveKitOptions", () => {
it("reads from Config singleton", () => {
vi.mocked(Config.get).mockReturnValue({
media_quality: { video_codec: "h264" },
} as ReturnType<typeof Config.get>);
const opts = getLiveKitOptions();
expect(opts.publishDefaults?.videoCodec).toBe("h264");
});
it("throws when Config is not initialized", () => {
vi.mocked(Config.get).mockImplementation(() => {
throw new Error("Config not initialized");
});
expect(() => getLiveKitOptions()).toThrow("Config not initialized");
});
});
src/livekit/options.ts
-152
View File
@@ -1,152 +0,0 @@
/*
Copyright 2023, 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
AudioPresets,
DefaultReconnectPolicy,
type RoomOptions,
type TrackPublishDefaults,
type VideoPreset,
VideoPresets,
VideoPreset as VideoPresetClass,
} from "livekit-client";
import { Config } from "../config/Config";
import { DEFAULT_CONFIG, type ConfigOptions } from "../config/ConfigOptions";
/**
* Find the closest matching VideoPreset for a given height.
*/
function videoPresetForHeight(height: number): VideoPreset {
if (height <= 180) return VideoPresets.h180;
if (height <= 360) return VideoPresets.h360;
if (height <= 540) return VideoPresets.h540;
if (height <= 720) return VideoPresets.h720;
if (height <= 1080) return VideoPresets.h1080;
if (height <= 1440) return VideoPresets.h1440;
return VideoPresets.h2160;
}
/**
* Build LiveKit publish options from config, falling back to sensible defaults.
*/
function buildPublishOptions(
mediaQuality: ConfigOptions["media_quality"],
): TrackPublishDefaults {
const defaults = DEFAULT_CONFIG.media_quality;
const videoConf = mediaQuality?.video;
const screenConf = mediaQuality?.screen_share;
const codec = mediaQuality?.video_codec ?? defaults.video_codec;
// Camera video encoding
const videoEncoding = {
maxBitrate: videoConf?.max_bitrate ?? defaults.video.max_bitrate,
maxFramerate: videoConf?.max_framerate ?? defaults.video.max_framerate,
};
// Camera simulcast layers
let videoSimulcastLayers: VideoPreset[];
if (videoConf?.simulcast_layers) {
videoSimulcastLayers = videoConf.simulcast_layers.map(
(layer) =>
new VideoPresetClass(
Math.round((layer.height * 16) / 9),
layer.height,
layer.bitrate,
videoConf?.max_framerate ?? defaults.video.max_framerate,
),
);
} else {
videoSimulcastLayers = [VideoPresets.h180, VideoPresets.h360];
}
// Screen share encoding
const screenShareEncoding = {
maxBitrate: screenConf?.max_bitrate ?? defaults.screen_share.max_bitrate,
maxFramerate:
screenConf?.max_framerate ?? defaults.screen_share.max_framerate,
};
// Screen share simulcast layers
let screenShareSimulcastLayers: VideoPreset[] | undefined;
if (screenConf?.simulcast_layers) {
screenShareSimulcastLayers = screenConf.simulcast_layers.map(
(layer) =>
new VideoPresetClass(
Math.round((layer.height * 16) / 9),
layer.height,
layer.bitrate,
layer.framerate ?? screenConf?.max_framerate ?? 30,
),
);
}
return {
audioPreset: AudioPresets.music,
dtx: true,
// disable red because the livekit server strips out red packets for clients
// that don't support it (firefox) but of course that doesn't work with e2ee.
red: false,
forceStereo: false,
simulcast: true,
videoSimulcastLayers: videoSimulcastLayers as VideoPreset[],
screenShareEncoding,
...(screenShareSimulcastLayers && {
screenShareSimulcastLayers: screenShareSimulcastLayers as VideoPreset[],
}),
stopMicTrackOnMute: false,
videoCodec: codec,
videoEncoding,
backupCodec: {
codec: "vp8",
encoding: VideoPresets.h720.encoding,
},
} as TrackPublishDefaults;
}
/**
* Build LiveKit RoomOptions from config.
* Call this after Config.init() has resolved.
*/
export function buildLiveKitOptions(
mediaQuality?: ConfigOptions["media_quality"],
): RoomOptions {
const videoHeight =
mediaQuality?.video?.max_resolution ??
DEFAULT_CONFIG.media_quality.video.max_resolution;
const basePreset = videoPresetForHeight(videoHeight);
return {
// automatically manage subscribed video quality
adaptiveStream: true,
// optimize publishing bandwidth and CPU for published tracks
dynacast: true,
// capture settings
videoCaptureDefaults: {
resolution: basePreset.resolution,
},
// publish settings
publishDefaults: buildPublishOptions(mediaQuality),
// default LiveKit options that seem to be sane
stopLocalTrackOnUnpublish: true,
reconnectPolicy: new DefaultReconnectPolicy(),
disconnectOnPageLeave: true,
webAudioMix: false,
};
}
/**
* Get LiveKit options, reading from the loaded Config singleton.
* Requires Config.init() to have resolved first.
*/
export function getLiveKitOptions(): RoomOptions {
return buildLiveKitOptions(Config.get().media_quality);
}
src/room/CallEventAudioRenderer.test.tsx
-1
View File
@@ -41,7 +41,6 @@ import {
import { MAX_PARTICIPANT_COUNT_FOR_SOUND } from "../state/CallViewModel/CallViewModel";
import { initializeWidget } from "../widget";
initializeWidget();
vitest.mock("livekit-client/e2ee-worker?worker");
vitest.mock("../useAudioContext");
vitest.mock("../soundUtils");
vitest.mock("../rtcSessionHelpers", async (importOriginal) => ({
src/room/CallView.test.tsx
+1 -1
View File
@@ -61,7 +61,7 @@ import {
} from "../HostBridge";
import { type JoinCallData } from "../widget";
import { MatrixRTCTransportMissingError } from "../utils/errors";
import { ProcessorProvider } from "../livekit/TrackProcessorContext";
import { ProcessorProvider } from "../tracks/TrackProcessorContext";
import { MediaDevicesContext } from "../MediaDevicesContext";
import { constant } from "../state/Behavior";
src/room/InCallView.test.tsx
+36 -47
View File
@@ -15,22 +15,20 @@ import {
vi,
} from "vitest";
import { act, render, type RenderResult } from "@testing-library/react";
import { type LocalParticipant } from "livekit-client";
import { BehaviorSubject, of } from "rxjs";
import { BehaviorSubject } from "rxjs";
import { BrowserRouter } from "react-router-dom";
import { TooltipProvider } from "@vector-im/compound-web";
import { RoomContext, useLocalParticipant } from "@livekit/components-react";
import userEvent from "@testing-library/user-event";
import { ActiveCall, InCallView } from "./InCallView";
import {
mockLivekitRoom,
mockLocalParticipant,
mockMediaDevices,
mockMuteStates,
mockRemoteParticipant,
mockRtcMembership,
type MockRTCSession,
mockMatrixRTCClient,
mockMemberMedia,
mockRTCMember,
} from "../utils/test";
import { E2eeType } from "../e2ee/e2eeType";
import {
@@ -41,15 +39,17 @@ import {
type CallViewModel,
type CallViewModelOptions,
} from "../state/CallViewModel/CallViewModel";
import { alice, local } from "../utils/test-fixtures";
import { alice, local, aliceRtcMember } from "../utils/test-fixtures";
import { ReactionsSenderProvider } from "../reactions/useReactionsSender";
import { useRoomEncryptionSystem } from "../e2ee/sharedKeyManagement";
import { LivekitRoomAudioRenderer } from "../livekit/MatrixAudioRenderer";
import { MemberAudioRenderer } from "../tracks/MemberAudioRenderer";
import type * as MatrixRTCSdk from "@element-hq/matrixrtc-sdk";
import { constant, type ObservableScope } from "@element-hq/matrixrtc-sdk";
import { MediaDevicesContext } from "../MediaDevicesContext";
import { type MediaDevices as ECMediaDevices } from "../state/MediaDevices";
import { AppBar } from "../AppBar";
import { type MatrixInfo } from "./VideoPreview";
import { ProcessorProvider } from "../livekit/TrackProcessorContext";
import { ProcessorProvider } from "../tracks/TrackProcessorContext";
import { initializeWidget } from "../widget";
import { RootElementProvider } from "../RootElementContext";
@@ -68,21 +68,19 @@ vi.mock("../soundUtils");
vi.mock("../useAudioContext");
vi.mock("../tile/GridTile");
vi.mock("../tile/SpotlightTile");
vi.mock("@livekit/components-react");
vi.mock("livekit-client/e2ee-worker?worker");
vi.mock("../e2ee/sharedKeyManagement");
vi.mock("../livekit/MatrixAudioRenderer");
vi.mock("../tracks/MemberAudioRenderer");
// ActiveCall builds the client itself; the view is what these tests are about
const createMatrixRTCClientMock = vi.hoisted(() => vi.fn());
vi.mock("@element-hq/matrixrtc-sdk", async (importOriginal) => ({
...(await importOriginal<typeof MatrixRTCSdk>()),
createMatrixRTCClient: createMatrixRTCClientMock,
}));
vi.mock("react-use-measure", () => ({
default: (): [() => void, object] => [(): void => {}, {}],
}));
const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC");
const localParticipant = mockLocalParticipant({
identity: "@local:example.org:AAAAAA",
});
const remoteParticipant = mockRemoteParticipant({
identity: "@alice:example.org:AAAAAA",
});
const matrixInfo = {
userId: "",
@@ -100,21 +98,26 @@ let useRoomEncryptionSystemMock: MockedFunction<typeof useRoomEncryptionSystem>;
beforeEach(() => {
vi.clearAllMocks();
// MatrixAudioRenderer is tested separately.
(
LivekitRoomAudioRenderer as MockedFunction<typeof LivekitRoomAudioRenderer>
).mockImplementation((_props) => {
return <div>mocked: MatrixAudioRenderer</div>;
});
(
useLocalParticipant as MockedFunction<typeof useLocalParticipant>
).mockImplementation(
() =>
({
isScreenShareEnabled: false,
localParticipant: localRtcMember as unknown as LocalParticipant,
}) as unknown as ReturnType<typeof useLocalParticipant>,
createMatrixRTCClientMock.mockImplementation((scope: ObservableScope) =>
mockMatrixRTCClient(scope, {
localMember$: constant(
mockRTCMember(true, {
membership: localRtcMember,
media$: constant(mockMemberMedia({ local: true })),
}),
),
remoteMembers$: constant([
mockRTCMember(false, {
membership: aliceRtcMember,
media$: constant(mockMemberMedia()),
}),
]),
}),
);
// MemberAudioRenderer is tested separately.
(
MemberAudioRenderer as MockedFunction<typeof MemberAudioRenderer>
).mockImplementation(() => <div>mocked: MemberAudioRenderer</div>);
useRoomEncryptionSystemMock =
useRoomEncryptionSystem as typeof useRoomEncryptionSystemMock;
useRoomEncryptionSystemMock.mockReturnValue({ kind: E2eeType.NONE });
@@ -131,14 +134,6 @@ function createInCallView(args: CreateInCallViewArgs = {}): RenderResult & {
} {
const mediaDevices = args.mediaDevices ?? mockMediaDevices({});
const muteState = mockMuteStates();
const livekitRoom = mockLivekitRoom(
{
localParticipant,
},
{
remoteParticipants$: of([remoteParticipant]),
},
);
const { vm, footerVm, developerSettingsVm, rtcSession } =
getBasicCallViewModelEnvironment(
[local, alice],
@@ -174,9 +169,7 @@ function createInCallView(args: CreateInCallViewArgs = {}): RenderResult & {
vm={vm}
rtcSession={rtcSession.asMockedSession()}
>
<TooltipProvider>
<RoomContext value={livekitRoom}>{content}</RoomContext>
</TooltipProvider>
<TooltipProvider>{content}</TooltipProvider>
</ReactionsSenderProvider>
</MediaDevicesContext>
</BrowserRouter>,
@@ -244,7 +237,6 @@ describe("ActiveCall", () => {
<MediaDevicesContext value={mediaDevices}>
<ProcessorProvider>
<TooltipProvider>
<RoomContext value={mockLivekitRoom({ localParticipant })}>
<ActiveCall
client={matrixRoom.client}
rtcSession={rtcSession.asMockedSession()}
@@ -255,7 +247,6 @@ describe("ActiveCall", () => {
e2eeSystem={{ kind: E2eeType.NONE }}
onLeft={(): void => {}}
/>
</RoomContext>
</TooltipProvider>
</ProcessorProvider>
</MediaDevicesContext>
@@ -294,7 +285,6 @@ describe("ActiveCall", () => {
<MediaDevicesContext value={mediaDevices}>
<ProcessorProvider>
<TooltipProvider>
<RoomContext value={mockLivekitRoom({ localParticipant })}>
<ActiveCall
client={matrixRoom.client}
rtcSession={rtcSession.asMockedSession()}
@@ -305,7 +295,6 @@ describe("ActiveCall", () => {
e2eeSystem={{ kind: E2eeType.NONE }}
onLeft={(): void => {}}
/>
</RoomContext>
</TooltipProvider>
</ProcessorProvider>
</MediaDevicesContext>
src/room/InCallView.tsx
+156 -45
View File
@@ -46,7 +46,10 @@ import {
} from "../state/CallViewModel/CallViewModel.ts";
import { Grid, type TileProps } from "../grid/Grid";
import { SpotlightTile } from "../tile/SpotlightTile";
import { type EncryptionSystem } from "../e2ee/sharedKeyManagement";
import {
type EncryptionSystem,
type MatrixRTCMode,
} from "@element-hq/matrixrtc-sdk";
import { E2eeType } from "../e2ee/e2eeType";
import { makeGridLayout } from "../grid/GridLayout";
import { type CallLayoutOutputs } from "../grid/CallLayout";
@@ -63,9 +66,36 @@ import {
import { ReactionsAudioRenderer } from "./ReactionAudioRenderer";
import { ReactionsOverlay } from "./ReactionsOverlay";
import { CallEventAudioRenderer } from "./CallEventAudioRenderer";
import { matrixRTCMode as matrixRTCModeSetting } from "../settings/settings";
import {
createMatrixRTCClient,
type MatrixRTCClientOptions,
constant,
type Behavior,
} from "@element-hq/matrixrtc-sdk";
import { map } from "rxjs";
import {
advancedCamera,
advancedScreenShare,
autoGainControlSetting,
cameraBitrate,
cameraCodec,
cameraFramerate,
cameraResolution,
customLivekitUrl,
echoCancellationSetting,
matrixRTCMode as matrixRTCModeSetting,
noiseSuppressionSetting,
parseResolution,
screenShareBitrate,
screenShareCodec,
screenShareFramerate,
screenShareResolution,
} from "../settings/settings";
import { Config } from "../config/Config";
import { type MediaDevice } from "../state/MediaDevices";
import { createSentCallNotification$ } from "../state/CallViewModel/CallNotificationLifecycle.ts";
import { ReactionsReader } from "../reactions/ReactionsReader";
import { LivekitRoomAudioRenderer } from "../livekit/MatrixAudioRenderer.tsx";
import { MemberAudioRenderer } from "../tracks/MemberAudioRenderer.tsx";
import { muteAllAudio$ } from "../state/MuteAllAudioModel.ts";
import { useMediaDevices } from "../MediaDevicesContext.ts";
import { EarpieceOverlay } from "./EarpieceOverlay.tsx";
@@ -76,10 +106,9 @@ import {
} from "../AppBar.tsx";
import { useBehavior } from "../useBehavior.ts";
import { useValueBehavior } from "../useValueBehavior.ts";
import { constant } from "../state/Behavior.ts";
import { Toast } from "../Toast.tsx";
import overlayStyles from "../Overlay.module.css";
import { useTrackProcessorState$ } from "../livekit/TrackProcessorContext.tsx";
import { useTrackProcessorState$ } from "../tracks/TrackProcessorContext.tsx";
import { type Layout } from "../state/layout-types.ts";
import { ObservableScope } from "../state/ObservableScope.ts";
import { CallFooter, type FooterSnapshot } from "../components/CallFooter.tsx";
@@ -100,7 +129,7 @@ declare module "react" {
export interface ActiveCallProps extends Omit<
InCallViewProps,
"vm" | "livekitRoom" | "connState" | "footerVm" | "developerSettingsVm"
"vm" | "footerVm" | "developerSettingsVm"
> {
e2eeSystem: EncryptionSystem;
// TODO refactor those reasons into an enum
@@ -131,9 +160,28 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
const { autoLeaveWhenOthersLeft, waitForCallPickup, sendNotificationType } =
urlParams;
const rtcClient = createMatrixRTCClient(
scope,
props.client,
props.matrixRoom,
{
microphoneEnabled: props.muteStates.audio.enabled$.value,
cameraEnabled: props.muteStates.video.enabled$.value,
audioInputDeviceId$: selectedDeviceId$(scope, mediaDevices.audioInput),
videoInputDeviceId$: selectedDeviceId$(scope, mediaDevices.videoInput),
// A host that routes audio itself leaves the browser's output alone
audioOutputDeviceId$: urlParams.controlledAudioDevices
? constant(undefined)
: selectedDeviceId$(scope, mediaDevices.audioOutput),
videoProcessor$: scope.behavior(
trackProcessorState$.pipe(map((state) => state.processor)),
),
},
matrixRTCClientOptions(urlParams, props.e2eeSystem),
);
const vm = createCallViewModel$(
scope,
props.rtcSession,
rtcClient,
props.matrixRoom,
mediaDevices,
props.muteStates,
@@ -143,14 +191,14 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
hostBridge,
autoLeaveWhenOthersLeft,
waitForCallPickup: waitForCallPickup && sendNotificationType === "ring",
// We merely sample the current mode here, so the user would need to
// manually rejoin to switch to a different one.
matrixRTCMode: matrixRTCModeSetting.value$.value,
sentCallNotification$: createSentCallNotification$(
scope,
props.rtcSession,
),
windowSize$: scope.behavior(observeElementSize$(rootElement)),
},
reactionsReader.raisedHands$,
reactionsReader.reactions$,
trackProcessorState$,
);
// TODO move this somewhere else once we use the callViewModel in the lobby as well!
vm.join();
@@ -250,16 +298,6 @@ export const InCallView: FC<InCallViewProps> = ({
const { sendReaction, toggleRaisedHand } = useReactionsSender();
useWakeLock();
// TODO-MULTI-SFU This is unused now??
// const connectionState = useBehavior(vm.livekitConnectionState$);
// annoyingly we don't get the disconnection reason this way,
// only by listening for the emitted event
// This needs to be done differential. with the vm connection state we start with Disconnected.
// TODO-MULTI-SFU decide how to handle this properly
// @BillCarsonFr
// if (connectionState === ConnectionState.Disconnected)
// throw new ConnectionLostError();
const containerRef1 = useRef<HTMLDivElement | null>(null);
const [containerRef2, bounds] = useMeasure();
@@ -296,7 +334,7 @@ export const InCallView: FC<InCallViewProps> = ({
);
const ringingVm = useBehavior(vm.ringingVm$);
const audioParticipants = useBehavior(vm.livekitRoomItems$);
const remoteMembers = useBehavior(vm.remoteMembers$);
const participantCount = useBehavior(vm.participantCount$);
const reconnecting = useBehavior(vm.reconnecting$);
const screenShareError = useBehavior(vm.screenShareError$);
@@ -646,7 +684,11 @@ export const InCallView: FC<InCallViewProps> = ({
const footer = footerVm !== null && (
<CallFooter className={styles.footer} ref={footerRef} vm={footerVm} />
);
const allConnections = useBehavior(vm.allConnections$);
const connectedTransports = useBehavior(vm.connectedTransports$);
const localMember = useBehavior(vm.localMember$);
const localTransport = useBehavior(
localMember?.transport$ ?? constant(undefined),
);
return (
// The pointer handler here exists to control the visibility of the footer,
@@ -665,15 +707,7 @@ export const InCallView: FC<InCallViewProps> = ({
onPointerOut={onPointerOut}
>
{header}
{audioParticipants.map(({ livekitRoom, transport, participants }) => (
<LivekitRoomAudioRenderer
key={JSON.stringify(transport)}
transport={transport}
livekitRoom={livekitRoom}
validIdentities={participants}
muted={muteAllAudio}
/>
))}
<MemberAudioRenderer members={remoteMembers} muted={muteAllAudio} />
{renderContent()}
<CallEventAudioRenderer vm={vm} muted={muteAllAudio} />
<ReactionsAudioRenderer vm={vm} muted={muteAllAudio} />
@@ -694,22 +728,99 @@ export const InCallView: FC<InCallViewProps> = ({
tab={settingsTab}
onTabChange={setSettingsTab}
developerSettingsVm={developerSettingsVm}
livekitRooms={allConnections.map(
({ livekitRoom: room, livekitAlias, transport, role }) => ({
room,
livekitAlias,
role,
url:
// It should be relatively clear to the user which kind of URL
// this is, based on the scheme (ws:// vs. http://)
"url" in transport
? transport.url
: transport.livekit_service_url,
}),
)}
transports={connectedTransports.map((transport) => ({
transport,
local: transport === localTransport,
}))}
/>
</>
)}
</div>
);
};
/** The id of the device the user picked, for the client to capture or play on. */
export function selectedDeviceId$(
scope: ObservableScope,
device: MediaDevice<unknown, { id: string }>,
): Behavior<string | undefined> {
return scope.behavior(device.selected$.pipe(map((d) => d?.id)));
}
/**
* What the client needs from Element Call's configuration, settings and URL:
* the SDK reads none of them itself. Shared with the legacy SDK bundle so that
* a widget built from it behaves like the app.
*/
export function matrixRTCClientOptions(
urlParams: ReturnType<typeof useUrlParams>,
encryptionSystem: EncryptionSystem,
): MatrixRTCClientOptions {
const config = Config.get();
const session = config.matrix_rtc_session;
const cameraSettings = advancedCamera.getValue()
? {
resolution: {
...parseResolution(cameraResolution.getValue()),
frameRate: cameraFramerate.getValue(),
},
maxBitrate: cameraBitrate.getValue(),
maxFramerate: cameraFramerate.getValue(),
codec: cameraCodec.getValue(),
}
: undefined;
const screenConfig = config.media_quality?.screen_share;
const screenShareSettings = advancedScreenShare.getValue()
? {
resolution: {
...parseResolution(screenShareResolution.getValue()),
frameRate: screenShareFramerate.getValue(),
},
maxBitrate: screenShareBitrate.getValue(),
maxFramerate: screenShareFramerate.getValue(),
codec: screenShareCodec.getValue(),
}
: screenConfig?.max_resolution
? {
resolution: {
width: Math.round((screenConfig.max_resolution * 16) / 9),
height: screenConfig.max_resolution,
frameRate: screenConfig.max_framerate ?? 30,
},
}
: undefined;
return {
encryptionSystem,
// matrix_rtc_mode in config.json overrides the user's Developer Settings
// choice. We merely sample the current mode here, so the user would need
// to manually rejoin to switch to a different one.
matrixRTCMode:
(config.matrix_rtc_mode as MatrixRTCMode | undefined) ??
matrixRTCModeSetting.value$.value,
sendNotificationType: urlParams.sendNotificationType,
applicationData: urlParams.callIntent
? { "m.call.intent": urlParams.callIntent }
: undefined,
timings: {
syncDisconnectGracePeriodMs: config.sync_disconnect_grace_period_ms,
networkErrorRetryMs: session.network_error_retry_ms,
waitForKeyRotationMs: session.wait_for_key_rotation_ms,
membershipEventExpiryMs: session.membership_event_expiry_ms,
keyRotationParticipantLimit: session.key_rotation_participant_limit,
delayedLeave: session.delayed_leave,
delegatedDelayedLeave: session.delegated_delayed_leave,
},
mediaQuality: config.media_quality,
capture: {
audio: {
echoCancellation: echoCancellationSetting.getValue(),
noiseSuppression: noiseSuppressionSetting.getValue(),
autoGainControl: autoGainControlSetting.getValue(),
},
camera: cameraSettings,
screenShare: screenShareSettings,
},
transportUrl: customLivekitUrl.value$.value ?? undefined,
fallbackTransportUrl: config.livekit?.livekit_service_url,
};
}
src/room/KnockLobbyView.test.tsx
+2 -2
View File
@@ -14,14 +14,14 @@ import { type MatrixClient, type RoomSummary } from "matrix-js-sdk";
import { KnockLobbyView } from "./KnockLobbyView";
import { LeaveToHomeProvider } from "../LeaveToHomeContext";
import { MediaDevicesContext } from "../MediaDevicesContext";
import { type ProcessorState } from "../livekit/TrackProcessorContext";
import { type ProcessorState } from "../tracks/TrackProcessorContext";
import { mockMediaDevices } from "../utils/test";
vi.mock("@livekit/components-react", () => ({
usePreviewTracks: (): unknown[] => [],
}));
vi.mock("../livekit/TrackProcessorContext", () => ({
vi.mock("../tracks/TrackProcessorContext", () => ({
useTrackProcessor: (): ProcessorState => ({
supported: false,
processor: undefined,
src/room/LobbyView.test.tsx
+2 -2
View File
@@ -28,7 +28,7 @@ import {
} from "../utils/test";
import { type MediaDevices } from "../state/MediaDevices";
import { MediaDevicesContext } from "../MediaDevicesContext";
import { type ProcessorState } from "../livekit/TrackProcessorContext";
import { type ProcessorState } from "../tracks/TrackProcessorContext";
import { type EncryptionSystem } from "../e2ee/sharedKeyManagement";
import lobbyStyles from "./LobbyView.module.css";
import headerStyles from "../Header.module.css";
@@ -41,7 +41,7 @@ vi.mock("@livekit/components-react", () => ({
usePreviewTracks: (): unknown[] => [],
}));
vi.mock("../livekit/TrackProcessorContext", () => ({
vi.mock("../tracks/TrackProcessorContext", () => ({
useTrackProcessor: (): ProcessorState => ({
supported: false,
processor: undefined,
src/room/LobbyView.tsx
+1 -1
View File
@@ -42,7 +42,7 @@ import { useInitial } from "../useInitial";
import {
useTrackProcessor,
useTrackProcessorSync,
} from "../livekit/TrackProcessorContext";
} from "../tracks/TrackProcessorContext";
import { getValue } from "../utils/observable";
import { useBehavior } from "../useBehavior";
import { CallFooter, type FooterSnapshot } from "../components/CallFooter";
src/room/ReactionAudioRenderer.test.tsx
-1
View File
@@ -48,7 +48,6 @@ function TestComponent({ vm }: { vm: CallViewModel }): ReactNode {
);
}
vitest.mock("livekit-client/e2ee-worker?worker");
vitest.mock("../useAudioContext");
vitest.mock("../soundUtils");
src/room/ReactionsOverlay.test.tsx
+1 -3
View File
@@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details.
*/
import { render } from "@testing-library/react";
import { expect, test, afterEach, vi } from "vitest";
import { expect, test, afterEach } from "vitest";
import { act } from "react";
import { showReactions } from "../settings/settings";
@@ -22,8 +22,6 @@ import { getBasicCallViewModelEnvironment } from "../utils/test-viewmodel";
import { initializeWidget } from "../widget";
initializeWidget();
vi.mock("livekit-client/e2ee-worker?worker");
afterEach(() => {
showReactions.setValue(showReactions.defaultValue);
});
src/room/__snapshots__/InCallView.test.tsx.snap
+3
View File
@@ -84,6 +84,9 @@ exports[`InCallView > rendering > renders 1`] = `
class="_nav_e4b327 _rightNav_e4b327"
/>
</header>
<div>
mocked: MemberAudioRenderer
</div>
<div
class="_scrollingGrid_4e7ff8 _grid_b0d1cd"
>
src/settings/DeveloperSettingsTab.test.tsx
+44 -58
View File
@@ -11,10 +11,10 @@ import userEvent, { type UserEvent } from "@testing-library/user-event";
import { TooltipProvider } from "@vector-im/compound-web";
import type { MatrixClient } from "matrix-js-sdk";
import type { Room as LivekitRoom } from "livekit-client";
import {
DeveloperSettingsTab,
type DeveloperSettingsSnapshot,
type TransportInfo,
} from "./DeveloperSettingsTab";
import { outOfCallDeveloperSettingsTabViewModel } from "./DeveloperSettingsTabViewModel";
import { createStaticViewModel } from "../state/ViewModel";
@@ -23,9 +23,13 @@ import {
enableExtendedLivekitLogs as enableExtendedLivekitLogsSetting,
matrixRTCMode as matrixRTCModeSetting,
} from "./settings";
import { MatrixRTCMode } from "../config/ConfigOptions";
import { exampleTransport, mockConfig } from "../utils/test";
import { getSFUConfig } from "../livekit/auth";
import {
authenticateWithTransport,
constant,
MatrixRTCMode,
} from "@element-hq/matrixrtc-sdk";
import type * as MatrixRTCSdk from "@element-hq/matrixrtc-sdk";
import { mockConfig } from "../utils/test";
// Mock url params hook to avoid environment-dependent snapshot churn.
vi.mock("../UrlParams", () => ({
@@ -36,32 +40,30 @@ vi.mock("../UrlParams", () => ({
}));
// IMPORTANT: mock the same specifier used by DeveloperSettingsTab
vi.mock("../livekit/auth", () => ({
getSFUConfig: vi.fn().mockResolvedValue({
vi.mock("@element-hq/matrixrtc-sdk", async (importOriginal) => ({
...(await importOriginal<typeof MatrixRTCSdk>()),
authenticateWithTransport: vi.fn().mockResolvedValue({
url: "mock-url",
jwt: "mock-jwt",
}),
}));
// Provide a minimal mock of a Livekit Room structure used by the component.
function createMockLivekitRoom(
wsUrl: string,
serverInfo: object,
metadata: string,
): { isLocal: boolean; url: string; room: LivekitRoom; livekitAlias: string } {
const mockRoom = {
serverInfo,
metadata,
engine: { client: { ws: { url: wsUrl } } },
localParticipant: { identity: "localParticipantIdentity" },
remoteParticipants: new Map(),
} as unknown as LivekitRoom;
/** A transport the panel shows, as the client would hand it out. */
function mockTransport(
url: string,
local: boolean,
resolved?: { url: string; roomAlias: string; identity: string },
): TransportInfo {
return {
isLocal: true,
url: wsUrl,
room: mockRoom,
livekitAlias: "TestAlias",
local,
transport: {
type: "livekit",
id: url,
raw: { type: "livekit", livekit_service_url: url },
resolved$: constant(
resolved && { type: "livekit", token: "secret", ...resolved },
),
},
};
}
@@ -96,36 +98,20 @@ describe("DeveloperSettingsTab", () => {
it("renders and matches snapshot", async () => {
const client = createMockMatrixClient();
const livekitRooms: {
room: LivekitRoom;
url: string;
isLocal?: boolean;
livekitAlias: string;
}[] = [
createMockLivekitRoom(
"wss://local-sfu.example.org",
{ region: "local", version: "1.2.3" },
"local-metadata",
),
{
isLocal: false,
livekitAlias: "TestAlias2",
url: "wss://remote-sfu.example.org",
room: {
localParticipant: { identity: "localParticipantIdentity" },
remoteParticipants: new Map(),
serverInfo: { region: "remote", version: "4.5.6" },
metadata: "remote-metadata",
engine: { client: { ws: { url: "wss://remote-sfu.example.org" } } },
} as unknown as LivekitRoom,
},
const transports = [
mockTransport("https://local-sfu.example.org/jwt", true, {
url: "wss://local-sfu.example.org",
roomAlias: "TestAlias",
identity: "localParticipantIdentity",
}),
mockTransport("https://remote-sfu.example.org/jwt", false),
];
const { container } = render(
<DeveloperSettingsTab
client={client}
roomId={"#room:example.org"}
livekitRooms={livekitRooms}
transports={transports}
env={{ MY_MOCK_ENV: 10, ENV: "test" } as unknown as ImportMetaEnv}
vm={outOfCallDeveloperSettingsTabViewModel}
/>,
@@ -169,7 +155,7 @@ describe("DeveloperSettingsTab", () => {
const saveButton = screen.getByRole("button", { name: "Save" });
await user.click(saveButton);
expect(getSFUConfig).not.toHaveBeenCalled();
expect(authenticateWithTransport).not.toHaveBeenCalled();
expect(customTransportSetting.getValue()).toBe(null);
});
@@ -192,7 +178,7 @@ describe("DeveloperSettingsTab", () => {
const saveButton = screen.getByRole("button", { name: "Save" });
await user.click(saveButton);
expect(getSFUConfig).not.toHaveBeenCalled();
expect(authenticateWithTransport).not.toHaveBeenCalled();
expect(customTransportSetting.getValue()).toBe(null);
});
@@ -218,7 +204,7 @@ describe("DeveloperSettingsTab", () => {
name: "Reset overwrite",
});
await user.click(cancelButton);
expect(getSFUConfig).not.toHaveBeenCalled();
expect(authenticateWithTransport).not.toHaveBeenCalled();
expect(customTransportSetting.getValue()).toBe(null);
});
@@ -245,11 +231,11 @@ describe("DeveloperSettingsTab", () => {
const saveButton = screen.getByRole("button", { name: "Save" });
await user.click(saveButton);
expect(getSFUConfig).toHaveBeenCalledWith(
expect.objectContaining({
transport: { type: "livekit", url: "wss://example.livekit.valid" },
roomId: "#testRoom",
}),
expect(authenticateWithTransport).toHaveBeenCalledWith(
expect.anything(),
expect.anything(),
"wss://example.livekit.valid",
"#testRoom",
);
expect(customTransportSetting.getValue()).toEqual({
@@ -279,8 +265,8 @@ describe("DeveloperSettingsTab", () => {
});
const saveButton = screen.getByRole("button", { name: "Save" });
(getSFUConfig as Mock).mockImplementation(() => {
throw new Error("Failed to get SFU config");
(authenticateWithTransport as Mock).mockImplementation(() => {
throw new Error("Invalid URL");
});
await user.click(saveButton);
expect(
src/settings/DeveloperSettingsTab.tsx
+38 -62
View File
@@ -11,7 +11,6 @@ import {
type ReactNode,
useCallback,
useEffect,
useMemo,
useId,
useState,
} from "react";
@@ -32,7 +31,6 @@ import {
RadioControl,
Separator,
} from "@vector-im/compound-web";
import { type Room as LivekitRoom } from "livekit-client";
import { FieldRow, InputField } from "../input/Input";
import { Config } from "../config/Config";
@@ -67,6 +65,10 @@ import styles from "./DeveloperSettingsTab.module.css";
import settingsStyles from "./SettingsModal.module.css";
import { Slider } from "../Slider";
import { useUrlParams } from "../UrlParams";
import {
authenticateWithTransport,
type TransportMetadata,
} from "@element-hq/matrixrtc-sdk";
import { useBehavior } from "../useBehavior";
import { type ViewModel } from "../state/ViewModel.ts";
import { getSFUConfig } from "../livekit/auth/index.ts";
@@ -75,6 +77,13 @@ import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
/**
* The state of MatrixRTC's media key rotation.
*/
/** One connected transport, as the panel shows it. */
export interface TransportInfo {
transport: TransportMetadata;
/** Whether this is the transport the local member publishes on. */
local: boolean;
}
export interface KeyRotationInfo {
/** Whether the call is large enough that MatrixRTC has stopped rotating the media key. */
suppressed: boolean;
@@ -106,19 +115,15 @@ const KeyRotationStatus: FC<{ info: KeyRotationInfo }> = ({ info }) => (
interface Props {
client: MatrixClient;
roomId?: string;
livekitRooms?: {
room: LivekitRoom;
url: string;
isLocal?: boolean;
livekitAlias?: string;
}[];
/** The transports the client is connected to, while in a call. */
transports?: TransportInfo[];
env: ImportMetaEnv;
vm: ViewModel<DeveloperSettingsSnapshot>;
}
export const DeveloperSettingsTab: FC<Props> = ({
client,
livekitRooms,
transports,
roomId,
env,
vm,
@@ -185,17 +190,6 @@ export const DeveloperSettingsTab: FC<Props> = ({
const urlParams = useUrlParams();
const localSfuUrl = useMemo((): URL | null => {
const localRoom = livekitRooms?.find((r) => r.isLocal)?.room;
if (localRoom?.engine.client.ws?.url) {
// strip the URL params
const url = new URL(localRoom.engine.client.ws.url);
url.search = "";
return url;
}
return null;
}, [livekitRooms]);
const MediaQualitySettings: React.FC<{
id: string;
header: string;
@@ -521,9 +515,7 @@ export const DeveloperSettingsTab: FC<Props> = ({
if (userId === null || deviceId === null) {
throw new Error("Invalid user or device ID");
}
const transport = JSON.parse(customTransportTextBuffer);
// Validate the transport by attempting to get a token for it
await getSFUConfig({
await authenticateWithTransport(
client,
membership: { userId, deviceId, memberId: "" },
transport,
@@ -601,39 +593,8 @@ export const DeveloperSettingsTab: FC<Props> = ({
</HelpMessage>
</InlineField>
</Form>
{livekitRooms?.map((livekitRoom) => (
<div className={styles.livekit_room_box}>
<h4>
{t("developer_mode.livekit_sfu", {
url: livekitRoom.url || "unknown",
})}
</h4>
<p>LivekitAlias: {livekitRoom.livekitAlias}</p>
<p>connectionState (wont hot reload): {livekitRoom.room.state}</p>
{livekitRoom.isLocal && <p>ws-url: {localSfuUrl?.href}</p>}
<p>
{t("developer_mode.livekit_server_info")}(
{livekitRoom.isLocal ? "local" : "remote"})
</p>
<pre className={styles.pre}>
{livekitRoom.room.serverInfo
? JSON.stringify(livekitRoom.room.serverInfo, null, 2)
: "undefined"}
{livekitRoom.room.metadata}
</pre>
<p>Local Participant</p>
<pre className={styles.pre}>
{livekitRoom.room.localParticipant.identity}
</pre>
<p>Remote Participants</p>
<ul>
{Array.from(livekitRoom.room.remoteParticipants.keys()).map(
(id) => (
<li key={id}>{id}</li>
),
)}
</ul>
</div>
{transports?.map(({ transport, local }) => (
<TransportBox key={transport.id} transport={transport} local={local} />
))}
<Separator />
<MediaQualitySettings
@@ -698,9 +659,24 @@ export const DeveloperSettingsTab: FC<Props> = ({
);
};
/**
* The text shown in the custom transport field for a given setting value.
*/
function transportText(transport: UnstableLivekitTransport | null): string {
return transport === null ? "" : JSON.stringify(transport);
}
const TransportBox: FC<TransportInfo> = ({ transport, local }) => {
const { t } = useTranslation();
const resolved = useBehavior(transport.resolved$);
return (
<div className={styles.livekit_room_box}>
<h4>{t("developer_mode.livekit_sfu", { url: transport.id })}</h4>
<p>
{t("developer_mode.livekit_server_info")}({local ? "local" : "remote"})
</p>
<pre className={styles.pre}>
{resolved === undefined
? "not connected"
: JSON.stringify(
{ ...resolved, token: resolved.token && "<redacted>" },
null,
2,
)}
</pre>
</div>
);
};
src/settings/SettingsModal.tsx
+6 -9
View File
@@ -9,7 +9,6 @@ import { type FC, type ReactNode, useEffect, useState } from "react";
import { useTranslation } from "react-i18next";
import { type MatrixClient } from "matrix-js-sdk";
import { Button, Root as Form, Separator } from "@vector-im/compound-web";
import { type Room as LivekitRoom } from "livekit-client";
import { Modal } from "../Modal";
import styles from "./SettingsModal.module.css";
@@ -28,10 +27,11 @@ import {
import { PreferencesSettingsTab } from "./PreferencesSettingsTab";
import { Slider } from "../Slider";
import { DeviceSelection } from "./DeviceSelection";
import { useTrackProcessor } from "../livekit/TrackProcessorContext";
import { useTrackProcessor } from "../tracks/TrackProcessorContext";
import {
DeveloperSettingsTab,
type DeveloperSettingsSnapshot,
type TransportInfo,
} from "./DeveloperSettingsTab";
import { FieldRow, InputField } from "../input/Input";
import { useSubmitRageshake } from "./submit-rageshake";
@@ -56,11 +56,8 @@ interface Props {
onTabChange: (tab: SettingsTab) => void;
client: MatrixClient;
roomId?: string;
livekitRooms?: {
room: LivekitRoom;
url: string;
role: "publisher" | "subscriber";
}[];
/** The transports the client is connected to, while in a call. */
transports?: TransportInfo[];
/** Only available while in a call. Used by the developer tab. */
developerSettingsVm?: ViewModel<DeveloperSettingsSnapshot>;
}
@@ -74,7 +71,7 @@ export const SettingsModal: FC<Props> = ({
onTabChange,
client,
roomId,
livekitRooms,
transports,
developerSettingsVm,
}) => {
const { t } = useTranslation();
@@ -227,7 +224,7 @@ export const SettingsModal: FC<Props> = ({
<DeveloperSettingsTab
env={import.meta.env}
client={client}
livekitRooms={livekitRooms}
transports={transports}
roomId={roomId}
vm={developerSettingsVm ?? outOfCallDeveloperSettingsTabViewModel}
/>
src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap
+8 -52
View File
@@ -361,19 +361,8 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
class="_livekit_room_box_2ddec4"
>
<h4>
LiveKit SFU: wss://local-sfu.example.org
LiveKit SFU: https://local-sfu.example.org/jwt
</h4>
<p>
LivekitAlias:
TestAlias
</p>
<p>
connectionState (wont hot reload):
</p>
<p>
ws-url:
wss://local-sfu.example.org/
</p>
<p>
LiveKit Server Info
(
@@ -384,37 +373,20 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
class="_pre_2ddec4"
>
{
"region": "local",
"version": "1.2.3"
"type": "livekit",
"token": "&lt;redacted&gt;",
"url": "wss://local-sfu.example.org",
"roomAlias": "TestAlias",
"identity": "localParticipantIdentity"
}
local-metadata
</pre>
<p>
Local Participant
</p>
<pre
class="_pre_2ddec4"
>
localParticipantIdentity
</pre>
<p>
Remote Participants
</p>
<ul />
</div>
<div
class="_livekit_room_box_2ddec4"
>
<h4>
LiveKit SFU: wss://remote-sfu.example.org
LiveKit SFU: https://remote-sfu.example.org/jwt
</h4>
<p>
LivekitAlias:
TestAlias2
</p>
<p>
connectionState (wont hot reload):
</p>
<p>
LiveKit Server Info
(
@@ -424,24 +396,8 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
<pre
class="_pre_2ddec4"
>
{
"region": "remote",
"version": "4.5.6"
}
remote-metadata
not connected
</pre>
<p>
Local Participant
</p>
<pre
class="_pre_2ddec4"
>
localParticipantIdentity
</pre>
<p>
Remote Participants
</p>
<ul />
</div>
<div
class="_separator_13qwf_8"
src/state/Behavior.ts
+2 -23
View File
@@ -1,29 +1,8 @@
/*
Copyright 2025 New Vector Ltd.
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { BehaviorSubject } from "rxjs";
/**
* A stateful, read-only reactive value. As an Observable, it is "hot" and
* always replays the current value upon subscription.
*
* A Behavior is to BehaviorSubject what Observable is to Subject; it does not
* provide a way to imperatively set new values. For more info on the
* distinction between Behaviors and Observables, see
* https://monoid.dk/post/behaviors-and-streams-why-both/.
*/
export type Behavior<T> = Omit<
BehaviorSubject<T>,
"next" | "observers" | "error"
>;
/**
* Creates a Behavior which never changes in value.
*/
export function constant<T>(value: T): Behavior<T> {
return new BehaviorSubject(value);
}
export { type Behavior, constant } from "@element-hq/matrixrtc-sdk";
src/state/CallViewModel/CallNotificationLifecycle.test.ts
+6 -7
View File
@@ -27,7 +27,6 @@ import {
createCallNotificationLifecycle$,
type RingAttempt,
} from "./CallNotificationLifecycle";
import { Epoch, trackEpoch } from "../ObservableScope";
import { constant } from "../Behavior";
function mockRingEvent(
@@ -44,8 +43,8 @@ function mockRingEvent(
}
const defaultProps = {
memberships$: constant(new Epoch([])),
matrixRoomMembers$: constant(new Map([[alice.userId, alice]])),
memberUserIds$: constant<string[]>([]),
roomMemberUserIds$: constant([alice.userId]),
receivedDecline$: NEVER,
options: {
waitForCallPickup: true,
@@ -122,7 +121,7 @@ test("ring attempt times out after nobody joins", () => {
scope,
...defaultProps,
// No one ever joins (only local user)
memberships$: constant(new Epoch([])),
memberUserIds$: constant<string[]>([]),
sentCallNotification$: hot("-a", {
a: mockRingEvent("$notif1", 30),
}),
@@ -140,8 +139,8 @@ test("ring attempt is accepted once recipient joins", () => {
const { ringAttempts$ } = createCallNotificationLifecycle$({
scope,
...defaultProps,
memberships$: scope.behavior(
behavior("a-b", { a: [], b: [aliceRtcMember] }).pipe(trackEpoch()),
memberUserIds$: scope.behavior(
behavior("a-b", { a: [], b: [aliceRtcMember.userId] }),
),
sentCallNotification$: hot("-a", {
a: mockRingEvent("$notif1", 30),
@@ -160,7 +159,7 @@ test("ring attempt is immediately accepted if recipient is already joined", () =
const { ringAttempts$ } = createCallNotificationLifecycle$({
scope,
...defaultProps,
memberships$: constant(new Epoch([aliceRtcMember])),
memberUserIds$: constant([aliceRtcMember.userId]),
sentCallNotification$: hot("-a", {
a: mockRingEvent("$notif1", 30),
}),
src/state/CallViewModel/CallNotificationLifecycle.ts
+14 -15
View File
@@ -6,7 +6,6 @@ Please see LICENSE in the repository root for full details.
*/
import {
type CallMembership,
type IRTCNotificationContent,
type MatrixRTCSession,
MatrixRTCSessionEvent,
@@ -35,9 +34,7 @@ import {
} from "matrix-js-sdk";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import { type Behavior } from "../Behavior";
import { type Epoch, type ObservableScope } from "../ObservableScope";
import { type RoomMemberMap } from "./remoteMembers/MatrixMemberMetadata";
import { type Behavior, type ObservableScope } from "@element-hq/matrixrtc-sdk";
export type AutoLeaveReason = "allOthersLeft" | "timeout" | "decline";
@@ -81,8 +78,10 @@ export function createReceivedDecline$(
export interface Props {
scope: ObservableScope;
memberships$: Behavior<Epoch<CallMembership[]>>;
matrixRoomMembers$: Behavior<RoomMemberMap>;
/** The user ids of the session's members, ours included. */
memberUserIds$: Behavior<string[]>;
/** The user ids of the room's members, ours included. */
roomMemberUserIds$: Behavior<string[]>;
sentCallNotification$: Observable<CallNotificationWrapper | null>;
receivedDecline$: Observable<
Parameters<EventTimelineSetHandlerMap[RoomEvent.Timeline]>
@@ -93,8 +92,8 @@ export interface Props {
export function createCallNotificationLifecycle$({
scope,
memberships$,
matrixRoomMembers$,
memberUserIds$,
roomMemberUserIds$,
sentCallNotification$,
receivedDecline$,
options,
@@ -127,7 +126,7 @@ export function createCallNotificationLifecycle$({
switchMap((notificationEvent) => {
// We assume that there is only one other user in the room when ringing
// TODO: Respect io.element.functional_members
const recipient = [...matrixRoomMembers$.value.keys()].find(
const recipient = roomMemberUserIds$.value.find(
(userId) => userId !== localUser.userId,
);
if (recipient === undefined) {
@@ -140,8 +139,8 @@ export function createCallNotificationLifecycle$({
map(() => "timeout" as const),
);
// Call is accepted when the recipient joins
const accept$ = memberships$.pipe(
filter((ms) => ms.value.some((m) => m.userId === recipient)),
const accept$ = memberUserIds$.pipe(
filter((userIds) => userIds.includes(recipient)),
map(() => "accept" as const),
);
// Call is declined when we receive a decline event
@@ -171,12 +170,12 @@ export function createCallNotificationLifecycle$({
scope.share,
);
const allOthersLeft$ = memberships$.pipe(
const allOthersLeft$ = memberUserIds$.pipe(
pairwise(),
filter(
([{ value: prev }, { value: current }]) =>
current.every((m) => m.userId === localUser.userId) &&
prev.some((m) => m.userId !== localUser.userId),
([prev, current]) =>
current.every((userId) => userId === localUser.userId) &&
prev.some((userId) => userId !== localUser.userId),
),
map(() => {}),
);
src/state/CallViewModel/CallViewModel.test.ts
+5 -135
View File
@@ -8,30 +8,18 @@ Please see LICENSE in the repository root for full details.
import { test, vi, onTestFinished, it, describe, expect } from "vitest";
import {
BehaviorSubject,
combineLatest,
debounceTime,
distinctUntilChanged,
map,
NEVER,
type Observable,
of,
switchMap,
} from "rxjs";
import { SyncState } from "matrix-js-sdk";
import {
ConnectionState,
type LocalTrackPublication,
type Participant,
type RemoteParticipant,
} from "livekit-client";
import * as ComponentsCore from "@livekit/components-core";
import {
Status,
type CallMembership,
type IRTCNotificationContent,
MatrixRTCSessionEvent,
type UnstableLivekitTransport,
} from "matrix-js-sdk/lib/matrixrtc";
import { deepCompare } from "matrix-js-sdk/lib/utils";
@@ -42,7 +30,6 @@ import {
withTestScheduler,
mockRtcMembership,
testScope,
exampleTransport,
} from "../../utils/test.ts";
import { E2eeType } from "../../e2ee/e2eeType.ts";
import {
@@ -64,6 +51,7 @@ import { getValue } from "../../utils/observable.ts";
import { type Behavior, constant } from "../Behavior.ts";
import {
localParticipant,
type Participant,
withCallViewModel as withCallViewModelInMode,
} from "./CallViewModelTestUtils.ts";
import { MatrixRTCMode } from "../../config/ConfigOptions.ts";
@@ -80,11 +68,6 @@ vi.mock("rxjs", async (importOriginal) => ({
interval: (): Observable<number> => NEVER,
}));
vi.mock("@livekit/components-core");
vi.mock("livekit-client/e2ee-worker?worker");
vi.mock("../e2ee/matrixKeyProvider");
const getPlatform = vi.hoisted(() => vi.fn(() => "desktop"));
vi.mock("../../Platform", () => ({
get platform(): string {
@@ -93,15 +76,6 @@ vi.mock("../../Platform", () => ({
isFirefox: (): boolean => false,
}));
vi.mock(
"../state/CallViewModel/localMember/localTransport",
async (importOriginal) => ({
...(await importOriginal()),
makeTransport: async (): Promise<UnstableLivekitTransport> =>
Promise.resolve(exampleTransport),
}),
);
const yesNo = {
y: true,
n: false,
@@ -278,10 +252,7 @@ describe.each(modes)("CallViewModel (%s mode)", (mode) => {
b: [],
}),
rtcMembers$: constant([localRtcMember, aliceRtcMember, bobRtcMember]),
livekitConnectionState$: behavior(connectionInputMarbles, {
c: ConnectionState.Connected,
s: ConnectionState.Connecting,
}),
connected$: behavior(connectionInputMarbles, { c: true, s: false }),
},
(vm) => {
expectObservable(summarizeLayout$(vm.layout$)).toBe(
@@ -1388,9 +1359,9 @@ describe.each(modes)("CallViewModel (%s mode)", (mode) => {
function participantJoinLeave$(
behavior: (
marbles: string,
values: Record<string, RemoteParticipant[]>,
) => Behavior<RemoteParticipant[]>,
): Behavior<RemoteParticipant[]> {
values: Record<string, Participant[]>,
) => Behavior<Participant[]>,
): Behavior<Participant[]> {
return behavior("a-b-c-d", {
a: [], // Start empty
b: [aliceParticipant], // Alice joins
@@ -1592,10 +1563,6 @@ describe.each(modes)("CallViewModel (%s mode)", (mode) => {
it.skip("audio output changes when toggling earpiece mode", () => {
withTestScheduler(({ schedule, expectObservable }) => {
vi.mocked(ComponentsCore.createMediaDeviceObserver).mockReturnValue(
of([]),
);
const devices = new MediaDevices(testScope(), {
controlledAudioDevices: true,
});
@@ -1627,78 +1594,6 @@ describe.each(modes)("CallViewModel (%s mode)", (mode) => {
});
});
});
it.skip("media tracks are paused while reconnecting to MatrixRTC", () => {
withTestScheduler(({ schedule, expectObservable }) => {
const trackRunning$ = new BehaviorSubject(true);
const originalPublications = localParticipant.trackPublications;
localParticipant.trackPublications = new Map([
[
"video",
{
track: new (class {
public get isUpstreamPaused(): boolean {
return !trackRunning$.value;
}
public async pauseUpstream(): Promise<void> {
trackRunning$.next(false);
return Promise.resolve();
}
public async resumeUpstream(): Promise<void> {
trackRunning$.next(true);
return Promise.resolve();
}
})(),
} as unknown as LocalTrackPublication,
],
]);
onTestFinished(() => {
localParticipant.trackPublications = originalPublications;
});
// There are three indicators that the client might be disconnected from
// MatrixRTC: whether the sync loop is connected, whether the membership is
// present in local room state, and whether the membership manager thinks
// we've hit the timeout for the delayed leave event. Let's test all
// combinations of these conditions.
const syncingMarbles = " nyny----n--y";
const membershipStatusMarbles = " y---ny-n-yn-y";
const probablyLeftMarbles = " n-----y-ny---n";
const expectedReconnectingMarbles = "n-ynyny------n";
const expectedTrackRunningMarbles = "nynynyn------y";
withCallViewModel(
{ initialSyncState: SyncState.Reconnecting },
(vm, rtcSession, _subjects, setSyncState) => {
schedule(syncingMarbles, {
y: () => setSyncState(SyncState.Syncing),
n: () => setSyncState(SyncState.Reconnecting),
});
schedule(membershipStatusMarbles, {
y: () => {
rtcSession.membershipStatus = Status.Connected;
},
});
schedule(probablyLeftMarbles, {
y: () => {
rtcSession.probablyLeft = true;
},
n: () => {
rtcSession.probablyLeft = false;
},
});
expectObservable(vm.reconnecting$).toBe(
expectedReconnectingMarbles,
yesNo,
);
expectObservable(trackRunning$).toBe(
expectedTrackRunningMarbles,
yesNo,
);
},
);
});
});
});
describe("callViewModelOptionsFromParams", () => {
@@ -1709,33 +1604,8 @@ describe("callViewModelOptionsFromParams", () => {
const widgetUrl = (extra: string): string =>
`#?widgetId=id&parentUrl=${encodeURIComponent("http://parent")}&${extra}`;
it("carries an explicitly requested notification type", () => {
const params = computeUrlParams("", widgetUrl("sendNotificationType=ring"));
expect(callViewModelOptionsFromParams(params).sendNotificationType).toBe(
"ring",
);
});
it("carries the notification type an intent implies", () => {
const params = computeUrlParams("", widgetUrl("intent=start_call_dm"));
expect(callViewModelOptionsFromParams(params).sendNotificationType).toBe(
"ring",
);
});
it("carries hideScreensharing", () => {
const params = computeUrlParams("", widgetUrl("hideScreensharing=true"));
expect(callViewModelOptionsFromParams(params).hideScreensharing).toBe(true);
});
it("carries controlledAudioDevices and the call intent", () => {
const params = computeUrlParams(
"",
widgetUrl("controlledAudioDevices=true&intent=start_call_voice"),
);
expect(callViewModelOptionsFromParams(params)).toMatchObject({
controlledAudioDevices: true,
callIntent: "audio",
});
});
});
src/state/CallViewModel/CallViewModel.ts
+171 -533
View File
@@ -7,14 +7,25 @@ Please see LICENSE in the repository root for full details.
*/
import {
type BaseKeyProvider,
type ConnectionState,
ExternalE2EEKeyProvider,
type Room as LivekitRoom,
type RoomOptions,
type RemoteParticipant,
} from "livekit-client";
import { type Room as MatrixRoom } from "matrix-js-sdk";
type Behavior,
constant,
type DisconnectReason,
type EncryptionSystem,
generateItems,
type LocalRTCMember,
type MatrixRTCClient,
type ObservableScope,
pauseWhen,
type RemoteRTCMember,
type RTCMember,
type TransportMetadata,
} from "@element-hq/matrixrtc-sdk";
import {
KnownMembership,
type Room as MatrixRoom,
type RoomMember,
RoomStateEvent,
} from "matrix-js-sdk";
import {
BehaviorSubject,
combineLatest,
@@ -38,40 +49,15 @@ import {
throttleTime,
timer,
takeUntil,
from,
concatWith,
catchError,
switchAll,
} from "rxjs";
import { type Logger, logger as rootLogger } from "matrix-js-sdk/lib/logger";
import {
MembershipManagerEvent,
type UnstableLivekitTransport,
type MatrixRTCSession,
type RTCCallIntent,
type RTCNotificationType,
} from "matrix-js-sdk/lib/matrixrtc";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import { v4 as uuidv4 } from "uuid";
import { type IMembershipManager } from "matrix-js-sdk/lib/matrixrtc/IMembershipManager";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import {
createToggle$,
filterBehavior,
generateItems,
mapScoped,
pauseWhen,
} from "../../utils/observable";
import { createToggle$ } from "../../utils/observable";
import {
duplicateTiles,
playReactionsSound,
showReactions,
} from "../../settings/settings";
import { Config } from "../../config/Config";
import {
MatrixRTCMode,
type ResolvedDelayedLeaveTimings,
} from "../../config/ConfigOptions";
import { isFirefox, platform } from "../../Platform";
import { setPipEnabled$ } from "../../controls";
import { TileStore } from "../TileStore";
@@ -80,7 +66,6 @@ import { spotlightExpandedLayout } from "../SpotlightExpandedLayout";
import { oneOnOneDesktopLayout } from "../OneOnOneDesktopLayout";
import { oneOnOneMobileLayout } from "../OneOnOneMobileLayout";
import { pipLayout } from "../PipLayout";
import { type EncryptionSystem } from "../../e2ee/sharedKeyManagement";
import {
type RaisedHandInfo,
type ReactionInfo,
@@ -89,12 +74,8 @@ import {
import { shallowEquals as shallowArrayEquals } from "../../utils/array";
import { shallowEquals as shallowObjectEquals } from "../../utils/object";
import { type MediaDevices } from "../MediaDevices";
import { constant, type Behavior } from "../Behavior";
import { E2eeType } from "../../e2ee/e2eeType";
import { MatrixKeyProvider } from "../../e2ee/matrixKeyProvider";
import { type MuteStates } from "../MuteStates";
import { HeaderStyle, type UrlParams } from "../../UrlParams";
import { type ProcessorState } from "../../livekit/TrackProcessorContext";
import { type HostBridge, nullHostBridge } from "../../HostBridge";
import {
type Alignment,
@@ -108,45 +89,18 @@ import {
computeSpotlight,
type OneOnOneMedia,
} from "../layoutMedia.ts";
import { ElementCallError, UnknownCallError } from "../../utils/errors.ts";
import { Epoch, type ObservableScope } from "../ObservableScope.ts";
import { createHomeserverConnected$ } from "./localMember/HomeserverConnected.ts";
import {
createLocalMembership$,
enterRTCSession,
TransportState,
} from "./localMember/LocalMember.ts";
import { getLocalTransport } from "./localMember/LocalTransport.ts";
import {
createKeyRotationSuppressed$,
createMemberships$,
membershipsAndTransports$,
} from "../SessionBehaviors.ts";
import {
type ConnectionFactory,
ECConnectionFactory,
} from "./remoteMembers/ConnectionFactory.ts";
import {
ConnectionMap,
createConnectionMap$,
} from "./remoteMembers/ConnectionMap.ts";
import {
createRemoteMatrixLivekitMembers$,
type LocalMatrixLivekitMember,
type RemoteMatrixLivekitMember,
} from "./remoteMembers/MatrixLivekitMembers.ts";
type ElementCallError,
fromMatrixRTCError,
UnknownCallError,
} from "../../utils/errors.ts";
import { PosthogAnalytics } from "../../analytics/PosthogAnalytics.ts";
import {
type AutoLeaveReason,
type CallNotificationWrapper,
createCallNotificationLifecycle$,
createReceivedDecline$,
createSentCallNotification$,
} from "./CallNotificationLifecycle.ts";
import {
createMatrixMemberMetadata$,
createRoomMembers$,
} from "./remoteMembers/MatrixMemberMetadata.ts";
import { Publisher } from "./localMember/Publisher.ts";
import { type Connection } from "./remoteMembers/Connection.ts";
import {
type LayoutSwitchViewModel,
createLayoutSwitchViewModel,
@@ -167,13 +121,6 @@ import {
import { type GridTileViewModel } from "../TileViewModel.ts";
import { type TransportLocator } from "../../livekit/auth";
//TODO
// Larger rename
// member,membership -> rtcMember
// participant -> livekitParticipant
// matrixLivekitItem -> callMember
// js-sdk
// callMembership -> rtcMembership
export interface CallViewModelOptions {
encryptionSystem: EncryptionSystem;
/**
@@ -181,33 +128,23 @@ export interface CallViewModelOptions {
* to know when the user joins or leaves. Defaults to no host.
*/
hostBridge?: HostBridge;
/**
* Whether the app hosting Element Call controls the audio output devices,
* rather than the browser. Defaults to false.
*/
controlledAudioDevices?: boolean;
/** The style of header to show. Defaults to {@link HeaderStyle.Standard}. */
header?: HeaderStyle;
/** Whether the call controls should be shown. Defaults to true. */
showControls?: boolean;
/** Whether to hide the screen-sharing button. Defaults to false. */
hideScreensharing?: boolean;
/**
* Whether and what kind of notification to send when joining the call.
*/
sendNotificationType?: RTCNotificationType;
/** The kind of call being placed. */
callIntent?: RTCCallIntent;
autoLeaveWhenOthersLeft?: boolean;
/**
* If the call is started in a way where we want it to behave like a telephone usecase
* If we sent a notification event, we want the ui to show a ringing state
*/
waitForCallPickup?: boolean;
/** Optional factory to create LiveKit rooms, mainly for testing purposes. */
livekitRoomFactory?: (options?: RoomOptions) => LivekitRoom;
/** Optional behavior overriding the local connection state, mainly for testing purposes. */
connectionState$?: Behavior<ConnectionState>;
/**
* The call notification the client sent with its join, for the ringing UI.
* Defaults to none sent.
*/
sentCallNotification$?: Observable<CallNotificationWrapper | null>;
/**
* The size of the space the call is drawn in: the page when Element Call
* owns it, or the container a host mounted it in when it is a component.
@@ -217,12 +154,6 @@ export interface CallViewModelOptions {
* changed.
*/
windowSize$: Behavior<{ width: number; height: number }>;
/** Optional value overriding the local transport, for testing purposes. */
localTransport?: TransportLocator;
/** Optional value overriding the connection factory, for testing purposes. */
connectionFactory?: ConnectionFactory;
/** The version & compatibility mode of MatrixRTC that we should use. */
matrixRTCMode?: MatrixRTCMode;
/** Optional behavior overriding for the screensharing, for testing */
toggleScreensharing?: () => void;
}
@@ -242,22 +173,11 @@ export interface CallViewModelOptions {
*/
export function callViewModelOptionsFromParams(
params: UrlParams,
): Pick<
CallViewModelOptions,
| "controlledAudioDevices"
| "header"
| "showControls"
| "hideScreensharing"
| "sendNotificationType"
| "callIntent"
> {
): Pick<CallViewModelOptions, "header" | "showControls" | "hideScreensharing"> {
return {
controlledAudioDevices: params.controlledAudioDevices,
header: params.header,
showControls: params.showControls,
hideScreensharing: params.hideScreensharing,
sendNotificationType: params.sendNotificationType,
callIntent: params.callIntent,
};
}
@@ -276,12 +196,6 @@ interface LayoutScanState {
tiles: TileStore;
}
export type LivekitRoomItem = {
livekitRoom: LivekitRoom;
participants: string[];
transport: TransportLocator;
};
/**
* The return of createCallViewModel$
* this interface represents the root source of data for the call view.
@@ -376,12 +290,11 @@ export interface CallViewModel {
* key is generated when someone joins or leaves.
*/
keyRotationSuppressed$: Behavior<boolean>;
allConnections$: Behavior<Connection[]>;
/** Participants sorted by livekit room so they can be used in the audio rendering */
livekitRoomItems$: Behavior<LivekitRoomItem[]>;
/** use the layout instead, this is just for the sdk export. */
remoteMatrixLivekitMembers$: Behavior<RemoteMatrixLivekitMember[]>;
localMatrixLivekitMember$: Behavior<LocalMatrixLivekitMember | null>;
/** The local member, once our membership has been seen in the room. */
localMember$: Behavior<LocalRTCMember | null>;
remoteMembers$: Behavior<RemoteRTCMember[]>;
/** The transports the client holds a connection to, for the developer panel. */
connectedTransports$: Behavior<TransportMetadata[]>;
/** List of participants raising their hand */
handsRaised$: Behavior<Record<string, RaisedHandInfo>>;
/** List of reactions. Keys are: membership.membershipId (currently predefined as: `${membershipEvent.userId}:${membershipEvent.deviceId}`)*/
@@ -464,14 +377,10 @@ export interface CallViewModel {
switch: () => void;
} | null>;
/**
* Whether the app is currently reconnecting to the LiveKit server and/or setting the matrix rtc room state.
*/
/** Connected once, and currently not: to the homeserver, the session or the media transport. */
reconnecting$: Behavior<boolean>;
/**
* Shortcut for not requireing to parse and combine connectionState.matrix and connectionState.livekit
*/
/** Connected to the homeserver, the session and the media transport, all three. */
connected$: Behavior<boolean>;
}
@@ -479,21 +388,20 @@ export interface CallViewModel {
* A view model providing all the application logic needed to show the in-call
* UI (may eventually be expanded to cover the lobby and feedback screens in the
* future).
*
* The session itself — memberships, transports, media — is the client's; this
* adds everything that makes it a call.
*/
// Throughout this class and related code we must distinguish between MatrixRTC
// state and LiveKit state. We use the common terminology of room "members", RTC
// "memberships", and LiveKit "participants".
export function createCallViewModel$(
scope: ObservableScope,
rtcClient: MatrixRTCClient,
// A call is permanently tied to a single Matrix room
matrixRTCSession: MatrixRTCSession,
matrixRoom: MatrixRoom,
mediaDevices: MediaDevices,
muteStates: MuteStates,
options: CallViewModelOptions,
handsRaisedSubject$: Observable<Record<string, RaisedHandInfo>>,
reactionsSubject$: Observable<Record<string, ReactionInfo>>,
trackProcessorState$: Behavior<ProcessorState>,
): CallViewModel {
const logger = rootLogger.getChild("[CallViewModel]");
const client = matrixRoom.client;
@@ -506,277 +414,53 @@ export function createCallViewModel$(
// so that callers which don't care (chiefly tests) behave as they always have.
const {
hostBridge = nullHostBridge,
controlledAudioDevices = false,
header = HeaderStyle.Standard,
showControls = true,
hideScreensharing = false,
sendNotificationType,
callIntent,
sentCallNotification$ = of(null),
} = options;
const livekitKeyProvider = getE2eeKeyProvider(
options.encryptionSystem,
matrixRTCSession,
logger,
);
// matrix_rtc_mode in config.json overrides the user's Developer Settings choice.
// It is validated at config load (src/config/Config.ts) so the cast is safe.
const configMatrixRTCMode = Config.get().matrix_rtc_mode as
| MatrixRTCMode
| undefined;
const matrixRTCMode =
configMatrixRTCMode ?? options.matrixRTCMode ?? MatrixRTCMode.Compatibility;
// Each hbar seperates a block of input variables required for the CallViewModel to function.
// The outputs of this block is written under the hbar.
//
// For mocking purposes it is recommended to only mock the functions creating those outputs.
// All other fields are just temp computations for the mentioned output.
// The class does not need anything except the values underneath the bar.
// The creations of the values under the bar are all tested independently and testing the callViewModel Should
// not test their creation. Call view model only needs:
// - memberships$ via createMemberships$
// - localMembership via createLocalMembership$
// - callLifecycle via createCallNotificationLifecycle$
// - matrixMemberMetadataStore via createMatrixMemberMetadata$
// ------------------------------------------------------------------------
// memberships$
const memberships$ = createMemberships$(scope, matrixRTCSession);
// ------------------------------------------------------------------------
// matrixLivekitMembers$ AND localMembership
const membershipsAndTransports = membershipsAndTransports$(
scope,
memberships$,
);
const ownMembershipIdentity: CallMembershipIdentityParts = {
userId,
deviceId,
// Consumed by the sticky membership manager as `member.id`, *and* stamped
// into every to-device key event by the key transport. A pre-sticky
// membership advertises `${userId}:${deviceId}` as its `membershipID`
// instead, so a uuid there names a member no peer can resolve.
memberId:
matrixRTCMode === MatrixRTCMode.Matrix_2_0
? uuidv4()
: `${userId}:${deviceId}`,
};
const localTransport$ = options.localTransport
? constant(options.localTransport)
: from(getLocalTransport(client));
const connectionFactory =
options.connectionFactory ??
new ECConnectionFactory(
client,
matrixRoom.roomId,
matrixRTCSession.slotId,
mediaDevices,
trackProcessorState$,
livekitKeyProvider,
controlledAudioDevices,
options.livekitRoomFactory,
);
const connectionMap$ = scope.behavior<Epoch<ConnectionMap>>(
localTransport$.pipe(
catchError(() => NEVER), // Swallow errors
concatWith(NEVER), // So the Observable doesn't complete prematurely
mapScoped("connectionMap$", (localTransport, scope) =>
createConnectionMap$({
scope,
connectionFactory,
localTransport,
remoteTransports$: membershipsAndTransports.transports$,
logger,
ownMembershipIdentity,
}),
),
switchAll(),
),
new Epoch(new ConnectionMap()),
);
const remoteMatrixLivekitMembers$: Behavior<
Epoch<RemoteMatrixLivekitMember[]>
> = createRemoteMatrixLivekitMembers$({
scope: scope,
membershipsWithTransport$:
membershipsAndTransports.membershipsWithTransport$,
connectionMap$,
localUser: { userId, deviceId },
// The mute switches drive the client, which reports back what the device
// allowed, so a denied permission flips the switch back
muteStates.audio.setHandler(rtcClient.setMicrophoneEnabled);
muteStates.video.setHandler(rtcClient.setCameraEnabled);
scope.onEnd(() => {
muteStates.audio.unsetHandler();
muteStates.video.unsetHandler();
});
const localMembership = createLocalMembership$({
scope,
homeserverConnected: createHomeserverConnected$(
scope,
client,
matrixRTCSession,
),
muteStates,
joinMatrixRTC: (
transport: UnstableLivekitTransport,
delayedLeaveTimings: ResolvedDelayedLeaveTimings,
) => {
return enterRTCSession(
matrixRTCSession,
ownMembershipIdentity,
transport,
{
encryptMedia: livekitKeyProvider !== undefined,
matrixRTCMode,
delayedLeaveTimings,
sendNotificationType,
callIntent,
},
);
},
createPublisherFactory: (connection: Connection) => {
return new Publisher(
connection,
mediaDevices,
muteStates,
trackProcessorState$,
logger.getChild(`[Publisher ${JSON.stringify(connection.transport)}]`),
controlledAudioDevices,
);
},
connectionMap$,
client,
matrixRTCSession,
localTransport$,
roomId: matrixRoom.roomId,
hideScreensharing,
hostBridge,
baseUrl: client.baseUrl,
ownMembershipIdentity,
delayId$: scope.behavior(
(
fromEvent(
matrixRTCSession,
MembershipManagerEvent.DelayIdChanged,
// The type of reemitted event includes the original emitted as the second arg.
) as Observable<[string | undefined, IMembershipManager]>
).pipe(map(([delayId]) => delayId ?? null)),
matrixRTCSession.delayId ?? null,
),
logger: logger.getChild(`[${Date.now()}]`),
});
const localRtcMembership$ = scope.behavior(
memberships$.pipe(
map(
(memberships) =>
memberships.value.find(
(membership) =>
membership.userId === userId && membership.deviceId === deviceId,
) ?? null,
),
),
);
const localMatrixLivekitMember$: Behavior<LocalMatrixLivekitMember | null> =
scope.behavior(
localRtcMembership$.pipe(
filterBehavior((membership) => membership !== null),
map((membership$) => {
if (membership$ === null) return null;
return {
membership$,
participant: {
type: "local" as const,
value$: localMembership.participant$,
},
connection$: localMembership.connection$,
userId,
};
}),
),
);
const matrixLivekitMembers$ = scope.behavior(
const members$ = scope.behavior<RTCMember[]>(
combineLatest(
[localMatrixLivekitMember$, remoteMatrixLivekitMembers$],
(local, remote) => [...(local === null ? [] : [local]), ...remote.value],
[rtcClient.localMember$, rtcClient.remoteMembers$],
(local, remote) => (local === null ? remote : [local, ...remote]),
),
);
// ------------------------------------------------------------------------
// matrixMemberMetadataStore
const matrixRoomMembers$ = createRoomMembers$(scope, matrixRoom);
const matrixMemberMetadataStore = createMatrixMemberMetadata$(
scope,
scope.behavior(memberships$.pipe(map((mems) => mems.value))),
matrixRoomMembers$,
const matrixRoomMembers$ = scope.behavior(
fromEvent(matrixRoom, RoomStateEvent.Members).pipe(
map(() => roomMembers(matrixRoom)),
),
roomMembers(matrixRoom),
);
// ------------------------------------------------------------------------
// callLifecycle
// TODO if we are in "unknown" state we need a loading rendering (or empty screen)
// Otherwise it looks like we already connected and only than the ringing starts which is weird.
const { ringAttempts$, autoLeave$ } = createCallNotificationLifecycle$({
scope,
memberships$,
matrixRoomMembers$,
sentCallNotification$: createSentCallNotification$(scope, matrixRTCSession),
memberUserIds$: scope.behavior(
members$.pipe(map((members) => members.map((m) => m.userId))),
),
roomMemberUserIds$: scope.behavior(
matrixRoomMembers$.pipe(map((members) => members.map((m) => m.userId))),
),
sentCallNotification$,
receivedDecline$: createReceivedDecline$(matrixRoom),
options,
localUser: { userId, deviceId },
});
const allConnections$ = scope.behavior(
connectionMap$.pipe(map((d) => d.value.getConnections())),
);
const livekitRoomItems$ = scope.behavior<LivekitRoomItem[]>(
remoteMatrixLivekitMembers$.pipe(
// Flatten the members to a snapshot of their current connections and
// participant objects
switchMap(({ value: members }) => {
if (members.length === 0) return of([]);
return combineLatest(
members.map((m) =>
combineLatest(
[m.connection$, m.participant.value$],
(connection, participant) => ({ connection, participant }),
),
),
);
}),
// Keep only those with a defined connection and participant
map((members) =>
members.filter(
(
m,
): m is { connection: Connection; participant: RemoteParticipant } =>
m.connection !== null && m.participant !== null,
),
),
// Group members by connection
map((members) =>
[...Map.groupBy(members, (m) => m.connection)].map(
([connection, members]) => ({
transport: {
transport: connection.transport,
serverName: connection.serverName,
},
livekitRoom: connection.livekitRoom,
participants: members.map((m) => m.participant.identity),
}),
),
),
),
);
const handsRaised$ = scope.behavior(
handsRaisedSubject$.pipe(pauseWhen(localMembership.reconnecting$)),
handsRaisedSubject$.pipe(pauseWhen(rtcClient.reconnecting$)),
);
const reactions$ = scope.behavior(
@@ -789,7 +473,7 @@ export function createCallViewModel$(
]),
),
),
pauseWhen(localMembership.reconnecting$),
pauseWhen(rtcClient.reconnecting$),
),
);
@@ -797,59 +481,44 @@ export function createCallViewModel$(
* List of user media (camera feeds) that we want tiles for.
*/
const userMedia$ = scope.behavior<WrappedUserMediaViewModel[]>(
combineLatest([matrixLivekitMembers$, duplicateTiles.value$]).pipe(
// Generate a collection of user media from the list of expected (whether
// present or missing) LiveKit participants.
combineLatest([members$, duplicateTiles.value$]).pipe(
// Generate a collection of user media from the list of members, whether
// their media is present or still missing.
generateItems(
"CallViewModel userMedia$",
function* ([members, duplicateTiles]) {
for (const {
userId,
participant,
connection$,
membership$,
} of members) {
const rtcId = membership$.value.rtcBackendIdentity;
const mediaId = `${userId}:${membership$.value.deviceId}`;
for (const member of members) {
const mediaId = `${member.userId}:${member.deviceId}`;
for (let dup = 0; dup < 1 + duplicateTiles; dup++) {
yield {
keys: [dup, mediaId, userId, participant, connection$, rtcId],
data: undefined,
};
yield { keys: [dup, mediaId, member.id], data: member };
}
}
},
(scope, _, dup, mediaId, userId, participant, connection$, rtcId) =>
createWrappedUserMedia(scope, {
(scope, member$, dup, mediaId, _memberId) => {
const member = member$.value;
return createWrappedUserMedia(scope, {
id: `${mediaId}:${dup}`,
userId,
rtcBackendIdentity: rtcId,
participant,
userId: member.userId,
rtcBackendIdentity: member.id,
...(member.local
? { local: true, media$: (member as LocalRTCMember).media$ }
: { local: false, media$: member.media$ }),
encryptionSystem: options.encryptionSystem,
livekitRoom$: scope.behavior(
connection$.pipe(map((c) => c?.livekitRoom)),
),
transport$: scope.behavior(
connection$.pipe(map((c) => c?.transport)),
focusUrl$: scope.behavior(
member.transport$.pipe(map((transport) => transport?.id)),
),
mediaDevices,
pretendToBeDisconnected$: localMembership.reconnecting$,
displayName$: scope.behavior(
matrixMemberMetadataStore
.createDisplayNameBehavior$(scope, userId)
.pipe(map((name) => name ?? userId)),
),
mxcAvatarUrl$: matrixMemberMetadataStore.createAvatarUrlBehavior$(
scope,
userId,
),
pretendToBeDisconnected$: rtcClient.reconnecting$,
displayName$: member.displayName$,
mxcAvatarUrl$: member.avatarUrl$,
handRaised$: scope.behavior(
handsRaised$.pipe(map((v) => v[mediaId]?.time ?? null)),
),
reaction$: scope.behavior(
reactions$.pipe(map((v) => v[mediaId] ?? undefined)),
),
}),
});
},
),
),
);
@@ -865,26 +534,23 @@ export function createCallViewModel$(
if (pickupState !== "accept")
yield { keys: [intent, recipient], data: pickupState };
},
(scope, pickupState$, intent, userId) =>
createRingingMedia({
(scope, pickupState$, intent, userId) => {
const member$ = matrixRoomMembers$.pipe(
map((members) => members.find((m) => m.userId === userId)),
);
return createRingingMedia({
id: `ringing:${userId}`,
userId,
displayName$: scope.behavior(
matrixRoomMembers$.pipe(
map(
(members) =>
members.get(userId)?.rawDisplayName || userId,
member$.pipe(map((m) => m?.rawDisplayName || userId)),
),
),
),
mxcAvatarUrl$:
matrixMemberMetadataStore.createAvatarUrlBehavior$(
scope,
userId,
mxcAvatarUrl$: scope.behavior(
member$.pipe(map((m) => m?.getMxcAvatarUrl())),
),
pickupState$,
intent,
}),
});
},
),
map(([media]) => media ?? null),
),
@@ -893,9 +559,6 @@ export function createCallViewModel$(
),
);
/**
* All screen share media that we want to display.
*/
const screenShares$ = scope.behavior<ScreenShareViewModel[]>(
userMedia$.pipe(
switchMap((userMedia) =>
@@ -927,14 +590,9 @@ export function createCallViewModel$(
* - There can be multiple participants for one Matrix user if they join from
* multiple devices.
*/
const participantCount$ = scope.behavior(
matrixLivekitMembers$.pipe(map((ms) => ms.length)),
);
const participantCount$ = rtcClient.memberCount$;
const keyRotationSuppressed$ = createKeyRotationSuppressed$(
scope,
matrixRTCSession,
);
const keyRotationSuppressed$ = rtcClient.keyRotationSuppressed$;
const leaveSoundEffect$ = userMedia$.pipe(
pairwise(),
@@ -1243,7 +901,7 @@ export function createCallViewModel$(
// without a name tag!)
// TODO: Only hide name tags in DMs, not group chats that just
// happen to have only 2 users
members.size > 2,
members.length > 2,
),
)
: of(true),
@@ -1630,55 +1288,73 @@ export function createCallViewModel$(
/**
* Whether we are sharing our screen.
*/
// reassigned here to make it publicly accessible
const sharingScreen$ = localMembership.sharingScreen$;
const sharingScreen$ = scope.behavior(
rtcClient.localMember$.pipe(
switchMap((member) => member?.sharingScreen$ ?? of(false)),
),
);
/**
* Callback to toggle screen sharing. If null, screen sharing is not possible.
* Callback to toggle screen sharing. If null, screen sharing is not possible:
* the platform cannot capture a screen, or the host hides the button.
*/
// reassigned here to make it publicly accessible
const toggleScreenSharing =
options.toggleScreensharing ?? localMembership.toggleScreenSharing;
options.toggleScreensharing ??
(hideScreensharing || !("getDisplayMedia" in (navigator.mediaDevices ?? {}))
? null
: (): void => rtcClient.localMember$.value?.toggleScreenSharing?.());
const errors$ = scope.behavior<{
transportError?: ElementCallError;
matrixError?: ElementCallError;
connectionError?: ElementCallError;
publishError?: ElementCallError;
} | null>(
localMembership.localMemberState$.pipe(
map((value) => {
const returnObject: {
transportError?: ElementCallError;
matrixError?: ElementCallError;
connectionError?: ElementCallError;
publishError?: ElementCallError;
} = {};
if (value instanceof ElementCallError) return { transportError: value };
if (value === TransportState.Waiting) return null;
if (value.matrix instanceof ElementCallError)
returnObject.matrixError = value.matrix;
if (value.media instanceof ElementCallError)
returnObject.publishError = value.media;
else if (
typeof value.media === "object" &&
value.media.connection.state === "error"
)
returnObject.connectionError = value.media.connection.error;
return returnObject;
}),
const screenShareError$ = scope.behavior(
rtcClient.localMember$.pipe(
switchMap((member) => member?.screenShareError$ ?? of(null)),
),
null,
);
// Tell the host and the analytics about the user's joins and leaves
const join = (): void => {
PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date());
PosthogAnalytics.instance.eventCallStarted.track(matrixRoom.roomId);
rtcClient.join();
hostBridge.notifyJoined().catch((e) => {
logger.error("Failed to notify the host that we joined", e);
});
};
const leave = (): void => {
rtcClient.leave();
hostBridge.notifyHungUp().catch((e) => {
logger.error("Failed to notify the host that we hung up", e);
});
};
let reconnectStart: { time: number; reason: DisconnectReason } | null = null;
rtcClient.disconnectReason$
.pipe(distinctUntilChanged(), pairwise(), scope.bind())
.subscribe(([prev, reason]) => {
if (reason !== null) {
// Only the loss of a connection that existed counts as a reconnect,
// not the startup phase
if (prev === null) reconnectStart ??= { time: Date.now(), reason };
} else if (reconnectStart !== null) {
const reason =
reconnectStart.reason === "media" ? "livekit" : reconnectStart.reason;
PosthogAnalytics.instance.eventCallReconnecting.track(
matrixRoom.roomId,
reason,
(Date.now() - reconnectStart.time) / 1000,
);
PosthogAnalytics.instance.eventCallEnded.cacheReconnecting(reason);
reconnectStart = null;
}
});
return {
autoLeave$: autoLeave$,
ringingVm$: ringingMedia$,
ringingStatusLocation: header === HeaderStyle.AppBar ? "app_bar" : "tile",
leave$: leave$,
hangup: (): void => userHangup$.next(),
join: localMembership.requestJoinAndPublish,
leave: localMembership.requestDisconnect,
join,
leave,
toggleScreenSharing: toggleScreenSharing,
sharingScreen$: sharingScreen$,
@@ -1688,23 +1364,15 @@ export function createCallViewModel$(
unhoverScreen: (): void => screenUnhover$.next(),
fatalError$: scope.behavior(
errors$.pipe(
map((errors) => {
logger.debug("errors$ to compute any fatal errors:", errors);
return (
errors?.transportError ??
errors?.matrixError ??
errors?.connectionError ??
null
);
}),
filter((error) => error !== null),
rtcClient.fatalError$.pipe(
map((error) => error && fromMatrixRTCError(error)),
),
null,
),
allConnections$,
participantCount$: participantCount$,
keyRotationSuppressed$: keyRotationSuppressed$,
localMember$: rtcClient.localMember$,
remoteMembers$: rtcClient.remoteMembers$,
connectedTransports$: rtcClient.connectedTransports$,
handsRaised$: handsRaised$,
reactions$: reactions$,
joinSoundEffect$: joinSoundEffect$,
@@ -1720,23 +1388,6 @@ export function createCallViewModel$(
showLayoutSwitch$.pipe(map((show) => (show ? layoutSwitchVm : null))),
),
layout$: layout$,
localMatrixLivekitMember$,
remoteMatrixLivekitMembers$: scope.behavior(
remoteMatrixLivekitMembers$.pipe(
map((members) => members.value),
tap((v) => {
const listForLogs = v
.map(
(m) =>
m.membership$.value.userId + "|" + m.membership$.value.deviceId,
)
.join(",");
logger.debug(
`matrixLivekitMembers$ updated (exported) [${listForLogs}]`,
);
}),
),
),
tileStoreGeneration$: tileStoreGeneration$,
showSpotlightIndicators$: showSpotlightIndicators$,
showSpeakingIndicators$: showSpeakingIndicators$,
@@ -1750,30 +1401,17 @@ export function createCallViewModel$(
overflowing$,
earpieceMode$: earpieceMode$,
audioOutputSwitcher$: audioOutputSwitcher$,
reconnecting$: localMembership.reconnecting$,
livekitRoomItems$,
connected$: localMembership.connected$,
screenShareError$: localMembership.screenShareError$,
dismissScreenShareError: localMembership.dismissScreenShareError,
reconnecting$: rtcClient.reconnecting$,
connected$: rtcClient.connected$,
screenShareError$,
dismissScreenShareError: (): void =>
rtcClient.localMember$.value?.dismissScreenShareError(),
};
}
function getE2eeKeyProvider(
e2eeSystem: EncryptionSystem,
rtcSession: MatrixRTCSession,
logger: Logger,
): BaseKeyProvider | undefined {
if (e2eeSystem.kind === E2eeType.NONE) return undefined;
if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) {
const keyProvider = new MatrixKeyProvider();
keyProvider.setRTCSession(rtcSession);
return keyProvider;
} else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) {
const keyProvider = new ExternalE2EEKeyProvider();
keyProvider
.setKey(e2eeSystem.secret)
.catch((e) => logger.error("Failed to set shared key for E2EE", e));
return keyProvider;
}
/** The members a call can be with: those in the room and those invited. */
function roomMembers(room: MatrixRoom): RoomMember[] {
return room
.getMembersWithMembership(KnownMembership.Join)
.concat(room.getMembersWithMembership(KnownMembership.Invite));
}
src/state/CallViewModel/CallViewModelTestUtils.ts
+128 -108
View File
@@ -7,39 +7,39 @@ Please see LICENSE in the repository root for full details.
*/
import {
ConnectionState,
type Participant,
ParticipantEvent,
type RemoteParticipant,
type Room as LivekitRoom,
type TrackPublication,
} from "livekit-client";
type Behavior,
constant,
E2eeType,
type LocalMemberMedia,
type LocalRTCMember,
type MatrixRTCMode,
type ObservableScope,
type RemoteRTCMember,
} from "@element-hq/matrixrtc-sdk";
import { SyncState } from "matrix-js-sdk/lib/sync";
import { BehaviorSubject, combineLatest, map, of } from "rxjs";
import { onTestFinished, vi } from "vitest";
import { BehaviorSubject, combineLatest, map, of, switchMap } from "rxjs";
import { ClientEvent, type RoomMember, type MatrixClient } from "matrix-js-sdk";
import EventEmitter from "events";
import * as ComponentsCore from "@livekit/components-core";
import type { CallMembership } from "matrix-js-sdk/lib/matrixrtc";
import { E2eeType } from "../../e2ee/e2eeType";
import { type RaisedHandInfo, type ReactionInfo } from "../../reactions";
import {
type CallViewModel,
createCallViewModel$,
type CallViewModelOptions,
} from "./CallViewModel";
import { createSentCallNotification$ } from "./CallNotificationLifecycle";
import {
exampleTransportLocator,
mockConfig,
MockConnection,
mockLivekitRoom,
mockLocalParticipant,
mockMatrixRoom,
mockMatrixRoomMember,
mockMatrixRTCClient,
mockMediaDevices,
mockVideoTrack,
mockMemberMedia,
mockMuteStates,
MockRTCSession,
mockRTCMember,
testScope,
} from "../../utils/test";
import {
@@ -52,10 +52,7 @@ import {
local,
localRtcMember,
} from "../../utils/test-fixtures";
import { type Behavior, constant } from "../Behavior";
import { type ProcessorState } from "../../livekit/TrackProcessorContext";
import { type MediaDevices } from "../MediaDevices";
import { type MatrixRTCMode } from "../../config/ConfigOptions";
mockConfig({
livekit: { livekit_service_url: "http://my-default-service-url.com" },
@@ -65,11 +62,18 @@ const carol = local;
const dave = mockMatrixRoomMember(daveRTLRtcMember, { rawDisplayName: "Dave" });
/** Whoever publishes media: a stand-in for a LiveKit participant, by identity. */
export interface Participant {
identity: string;
}
export interface CallViewModelInputs {
remoteParticipants$: Behavior<RemoteParticipant[]>;
/** The members whose media has arrived. The local member's always has. */
remoteParticipants$: Behavior<Participant[]>;
rtcMembers$: Behavior<Partial<CallMembership>[]>;
roomMembers: RoomMember[];
livekitConnectionState$: Behavior<ConnectionState>;
/** Whether the local transport is connected. */
connected$: Behavior<boolean>;
speaking: Map<Participant, Behavior<boolean>>;
videoEnabled: Map<Participant, Behavior<boolean>>;
sharingScreen: Map<Participant, Behavior<boolean>>;
@@ -78,7 +82,7 @@ export interface CallViewModelInputs {
windowSize$: Behavior<{ width: number; height: number }>;
}
export const localParticipant = mockLocalParticipant({ identity: "" });
export const localParticipant: Participant = { identity: "" };
export function withCallViewModel(mode: MatrixRTCMode) {
return (
@@ -94,9 +98,7 @@ export function withCallViewModel(mode: MatrixRTCMode) {
dave,
daveRTL,
],
livekitConnectionState$: connectionState$ = constant(
ConnectionState.Connected,
),
connected$ = constant(true),
speaking = new Map(),
videoEnabled = new Map(),
sharingScreen = new Map(),
@@ -146,113 +148,131 @@ export function withCallViewModel(mode: MatrixRTCMode) {
const rtcSession = new MockRTCSession(room, []).withMemberships(
rtcMembers$,
);
const participantsSpy = vi
.spyOn(ComponentsCore, "connectedParticipantsObserver")
.mockReturnValue(remoteParticipants$);
const mediaSpy = vi
.spyOn(ComponentsCore, "observeParticipantMedia")
.mockImplementation((p) => {
return (videoEnabled.get(p) ?? constant(false)).pipe(
map((videoEnabled) => ({
participant: p,
isMicrophoneEnabled: false,
isCameraEnabled: videoEnabled,
isScreenShareEnabled: false,
cameraTrack: {
isMuted: !videoEnabled,
} as unknown as TrackPublication,
})),
);
const scope = testScope();
const mediaOf = (
scope: ObservableScope,
participant: Participant,
local: boolean,
): LocalMemberMedia =>
mockMemberMedia({
local,
speaking$: speaking.get(participant) ?? constant(false),
screenShareEnabled$: sharingScreen.get(participant) ?? constant(false),
camera$: scope.behavior(
(videoEnabled.get(participant) ?? constant(false)).pipe(
map((enabled) =>
mockVideoTrack({ source: "camera", muted$: constant(!enabled) }),
),
),
),
screenShare$: scope.behavior(
(sharingScreen.get(participant) ?? constant(false)).pipe(
map((sharing) =>
sharing ? mockVideoTrack({ source: "screenShare" }) : undefined,
),
),
),
});
const eventsSpy = vi
.spyOn(ComponentsCore, "observeParticipantEvents")
.mockImplementation((p, ...eventTypes) => {
return combineLatest([
(eventTypes.includes(ParticipantEvent.IsSpeakingChanged) &&
speaking.get(p)) ||
constant(false),
(eventTypes.includes(ParticipantEvent.TrackPublished) &&
sharingScreen.get(p)) ||
constant(false),
]).pipe(
map(
([isSpeaking, isScreenShareEnabled]) =>
({ ...p, isSpeaking, isScreenShareEnabled }) as Participant,
const participantOf = (
scope: ObservableScope,
membership: CallMembership,
): Behavior<Participant | undefined> =>
scope.behavior(
remoteParticipants$.pipe(
map((participants) =>
participants.find(
(p) => p.identity === membership.rtcBackendIdentity,
),
),
),
);
const memberOf = (
scope: ObservableScope,
membership: CallMembership,
): RemoteRTCMember =>
mockRTCMember(false, {
membership,
roomMember: roomMembers.find((m) => m.userId === membership.userId),
transportUrl: "http://my-default-service-url.com",
media$: scope.behavior(
participantOf(scope, membership).pipe(
map((participant) =>
participant === undefined
? null
: mediaOf(scope, participant, false),
),
),
),
});
const remoteMembers$ = scope.behavior<RemoteRTCMember[]>(
rtcMembers$.pipe(
map((memberships) =>
(memberships as CallMembership[])
.filter(
(m) =>
m.userId !== localRtcMember.userId ||
m.deviceId !== localRtcMember.deviceId,
)
.map((membership) => memberOf(scope, membership)),
),
),
);
const localMember$ = scope.behavior<LocalRTCMember | null>(
rtcMembers$.pipe(
map((memberships) =>
(memberships as CallMembership[]).some(
(m) =>
m.userId === localRtcMember.userId &&
m.deviceId === localRtcMember.deviceId,
)
? mockRTCMember(true, {
membership: localRtcMember,
roomMember: carol,
transportUrl: "http://my-default-service-url.com",
media$: constant(mediaOf(scope, localParticipant, true)),
})
: null,
),
),
);
const rtcClient = mockMatrixRTCClient(scope, {
localMember$,
remoteMembers$,
connected$,
reconnecting$: scope.behavior(
combineLatest([connected$, of(initialSyncState)]).pipe(
switchMap(() => connected$),
map((connected) => !connected),
),
),
});
const roomEventSelectorSpy = vi
.spyOn(ComponentsCore, "roomEventSelector")
.mockImplementation((_room, _eventType) => of());
const muteStates = mockMuteStates();
const raisedHands$ = new BehaviorSubject<Record<string, RaisedHandInfo>>(
{},
);
const reactions$ = new BehaviorSubject<Record<string, ReactionInfo>>({});
const livekitRoomFactory = (): LivekitRoom =>
mockLivekitRoom({
localParticipant,
disconnect: async () => Promise.resolve(),
setE2EEEnabled: async () => Promise.resolve(),
});
const vm = createCallViewModel$(
testScope(),
rtcSession.asMockedSession(),
scope,
rtcClient,
room,
mediaDevices,
muteStates,
{
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
autoLeaveWhenOthersLeft: false,
livekitRoomFactory,
connectionState$,
sentCallNotification$: createSentCallNotification$(
scope,
rtcSession.asMockedSession(),
),
windowSize$,
localTransport: exampleTransportLocator,
connectionFactory: {
createConnection(
scope,
role,
transport,
serverName,
ownMembershipIdentity,
logger,
) {
return new MockConnection(
{
scope,
role,
transport,
serverName,
ownMembershipIdentity,
client: room.client,
roomId: room.roomId,
slotId: "m.call#ROOM",
livekitRoomFactory,
},
logger,
);
},
},
matrixRTCMode: mode,
...options,
},
raisedHands$,
reactions$,
new BehaviorSubject<ProcessorState>({
processor: undefined,
supported: undefined,
}),
);
onTestFinished(() => {
participantsSpy.mockRestore();
mediaSpy.mockRestore();
eventsSpy.mockRestore();
roomEventSelectorSpy.mockRestore();
});
void mode;
continuation(vm, rtcSession, { raisedHands$: raisedHands$ }, setSyncState);
};
src/state/CallViewModel/localMember/HomeserverConnected.test.ts
-336
View File
@@ -1,336 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { describe, it, expect, beforeEach, afterEach } from "vitest";
import { EventEmitter } from "events";
import { ClientEvent, SyncState } from "matrix-js-sdk";
import { MembershipManagerEvent, Status } from "matrix-js-sdk/lib/matrixrtc";
import { ObservableScope } from "../../ObservableScope";
import { createHomeserverConnected$ } from "./HomeserverConnected";
import { testScope, withTestScheduler } from "../../../utils/test";
/**
* Minimal stub of a Matrix client sufficient for our tests:
```
createHomeserverConnected$(
scope: ObservableScope,
client: NodeStyleEventEmitter & Pick<MatrixClient, "getSyncState">,
matrixRTCSession: NodeStyleEventEmitter &
Pick<MatrixRTCSession, "membershipStatus" | "probablyLeft">,
)
```
*/
class MockMatrixClient extends EventEmitter {
private syncState: SyncState;
public constructor(initial: SyncState) {
super();
this.syncState = initial;
}
public setSyncState(state: SyncState): void {
this.syncState = state;
// Matrix's Sync event in createHomeserverConnected$ expects [SyncState]
this.emit(ClientEvent.Sync, [state]);
}
public getSyncState(): SyncState {
return this.syncState;
}
}
/**
* Minimal stub of MatrixRTCSession (membership manager):
```
createHomeserverConnected$(
scope: ObservableScope,
client: NodeStyleEventEmitter & Pick<MatrixClient, "getSyncState">,
matrixRTCSession: NodeStyleEventEmitter &
Pick<MatrixRTCSession, "membershipStatus" | "probablyLeft">,
)
```
*/
class MockMatrixRTCSession extends EventEmitter {
public membershipStatus: Status;
public probablyLeft: boolean;
public constructor(props: {
membershipStatus: Status;
probablyLeft: boolean;
}) {
super();
this.membershipStatus = props.membershipStatus;
this.probablyLeft = props.probablyLeft;
}
public setMembershipStatus(status: Status): void {
this.membershipStatus = status;
this.emit(MembershipManagerEvent.StatusChanged);
}
public setProbablyLeft(flag: boolean): void {
this.probablyLeft = flag;
this.emit(MembershipManagerEvent.ProbablyLeft);
}
}
describe("createHomeserverConnected$", () => {
let scope: ObservableScope;
let client: MockMatrixClient;
let session: MockMatrixRTCSession;
beforeEach(() => {
scope = new ObservableScope();
client = new MockMatrixClient(SyncState.Error); // start disconnected
session = new MockMatrixRTCSession({
membershipStatus: Status.Disconnected,
probablyLeft: false,
});
});
afterEach(() => {
scope.end();
});
// LLM generated test cases. They are a bit overkill but I improved the mocking so it is
// easy enough to read them so I think they can stay.
// Note: gracePeriodMs is set to 0 to avoid debouncing delays in tests
it("reports syncing reason when sync state is not Syncing", () => {
const hsConnected = createHomeserverConnected$(scope, client, session, 0);
expect(hsConnected.combined$.value).toEqual([false, "sync"]);
});
it("reports membership reason when sync is Syncing but membership is not Connected", () => {
const hsConnected = createHomeserverConnected$(scope, client, session, 0);
client.setSyncState(SyncState.Syncing);
expect(hsConnected.combined$.value).toEqual([false, "membership"]);
});
it("reports probablyLeft reason when membership transitions to Connected but ProbablyLeft is true", () => {
const hsConnected = createHomeserverConnected$(scope, client, session, 0);
// Make sync loop OK
client.setSyncState(SyncState.Syncing);
// Indicate probable leave before connection
session.setProbablyLeft(true);
session.setMembershipStatus(Status.Connected);
expect(hsConnected.combined$.value).toEqual([false, "probablyLeft"]);
});
it("becomes null (connected) only when all three conditions are satisfied", () => {
const hsConnected = createHomeserverConnected$(scope, client, session, 0);
// 1. Sync loop connected
client.setSyncState(SyncState.Syncing);
expect(hsConnected.combined$.value).toEqual([false, "membership"]); // not yet membership connected
// 2. Membership connected
session.setMembershipStatus(Status.Connected);
expect(hsConnected.combined$.value).toEqual([true, null]); // probablyLeft is false
});
it("returns syncing reason when sync loop leaves Syncing", () => {
const hsConnected = createHomeserverConnected$(scope, client, session, 0);
// Reach connected state
client.setSyncState(SyncState.Syncing);
session.setMembershipStatus(Status.Connected);
expect(hsConnected.combined$.value).toEqual([true, null]);
// Sync loop error => should report syncing reason
client.setSyncState(SyncState.Error);
expect(hsConnected.combined$.value).toEqual([false, "sync"]);
});
it("returns membershipConnected reason when membership status becomes disconnected", () => {
const hsConnected = createHomeserverConnected$(scope, client, session, 0);
client.setSyncState(SyncState.Syncing);
session.setMembershipStatus(Status.Connected);
expect(hsConnected.combined$.value).toEqual([true, null]);
session.setMembershipStatus(Status.Disconnected);
expect(hsConnected.combined$.value).toEqual([false, "membership"]);
});
it("returns certainlyConnected reason when ProbablyLeft is emitted", () => {
const hsConnected = createHomeserverConnected$(scope, client, session, 0);
client.setSyncState(SyncState.Syncing);
session.setMembershipStatus(Status.Connected);
expect(hsConnected.combined$.value).toEqual([true, null]);
session.setProbablyLeft(true);
expect(hsConnected.combined$.value).toEqual([false, "probablyLeft"]);
});
it("recovers to null (connected) if ProbablyLeft becomes false again while other conditions remain true", () => {
const hsConnected = createHomeserverConnected$(scope, client, session, 0);
client.setSyncState(SyncState.Syncing);
session.setMembershipStatus(Status.Connected);
expect(hsConnected.combined$.value).toEqual([true, null]);
session.setProbablyLeft(true);
expect(hsConnected.combined$.value).toEqual([false, "probablyLeft"]);
// Simulate clearing the flag (in realistic scenario membership manager would update)
session.setProbablyLeft(false);
expect(hsConnected.combined$.value).toEqual([true, null]);
});
it("composite sequence reflects each individual failure reason", () => {
const hsConnected = createHomeserverConnected$(scope, client, session, 0);
// Initially: sync error + membership disconnected → syncing wins (highest priority)
expect(hsConnected.combined$.value).toEqual([false, "sync"]);
// Fix sync only → membershipConnected is now the blocker
client.setSyncState(SyncState.Syncing);
expect(hsConnected.combined$.value).toEqual([false, "membership"]);
// Fix membership → all conditions satisfied
session.setMembershipStatus(Status.Connected);
expect(hsConnected.combined$.value).toEqual([true, null]);
// Introduce probablyLeft → certainlyConnected
session.setProbablyLeft(true);
expect(hsConnected.combined$.value).toEqual([false, "probablyLeft"]);
// Restore notProbablyLeft → connected again
session.setProbablyLeft(false);
expect(hsConnected.combined$.value).toEqual([true, null]);
// Drop sync → syncing reason
client.setSyncState(SyncState.Error);
expect(hsConnected.combined$.value).toEqual([false, "sync"]);
});
});
describe("createHomeserverConnected$ - combined$ reason values", () => {
let scope: ObservableScope;
let client: MockMatrixClient;
let session: MockMatrixRTCSession;
beforeEach(() => {
scope = new ObservableScope();
// Start with sync failing and membership disconnected
client = new MockMatrixClient(SyncState.Error);
session = new MockMatrixRTCSession({
membershipStatus: Status.Disconnected,
probablyLeft: false,
});
});
afterEach(() => {
scope.end();
});
it("is [true, null] when all three conditions are satisfied", () => {
const { combined$ } = createHomeserverConnected$(scope, client, session, 0);
client.setSyncState(SyncState.Syncing);
session.setMembershipStatus(Status.Connected);
expect(combined$.value).toEqual([true, null]);
});
it("reports syncing when sync loop is not Syncing", () => {
const { combined$ } = createHomeserverConnected$(scope, client, session, 0);
// client starts with SyncState.Error, membership also disconnected
expect(combined$.value).toEqual([false, "sync"]);
});
it("reports membershipConnected when sync is fine but membership is not Connected", () => {
const { combined$ } = createHomeserverConnected$(scope, client, session, 0);
client.setSyncState(SyncState.Syncing);
// session still Status.Disconnected
expect(combined$.value).toEqual([false, "membership"]);
});
it("reports certainlyConnected when probablyLeft is true", () => {
const { combined$ } = createHomeserverConnected$(scope, client, session, 0);
client.setSyncState(SyncState.Syncing);
session.setMembershipStatus(Status.Connected);
session.setProbablyLeft(true);
expect(combined$.value).toEqual([false, "probablyLeft"]);
});
it("prioritises syncing over membershipConnected when both fail", () => {
const { combined$ } = createHomeserverConnected$(scope, client, session, 0);
// Both sync (Error) and membership (Disconnected) are failing
expect(combined$.value).toEqual([false, "sync"]);
});
it("updates reason as conditions change", () => {
const { combined$ } = createHomeserverConnected$(scope, client, session, 0);
// Initially: syncing fails
expect(combined$.value).toEqual([false, "sync"]);
// Fix sync → membershipConnected is now the blocker
client.setSyncState(SyncState.Syncing);
expect(combined$.value).toEqual([false, "membership"]);
// Fix membership → probablyLeft makes certainlyConnected fail
session.setProbablyLeft(true);
session.setMembershipStatus(Status.Connected);
expect(combined$.value).toEqual([false, "probablyLeft"]);
// Clear probablyLeft → all conditions satisfied
session.setProbablyLeft(false);
expect(combined$.value).toEqual([true, null]);
});
});
describe("createHomeserverConnected$ - Grace Period", () => {
const GRACE_PERIOD = 5;
function marbleTest(
syncStateMarbles: string,
expectedConnectedMarbles: string,
): void {
withTestScheduler(({ behavior, schedule, expectObservable }) => {
const syncState$ = behavior(syncStateMarbles, {
s: SyncState.Syncing,
e: SyncState.Error,
});
const client = new MockMatrixClient(syncState$.value);
schedule(syncStateMarbles, {
s: () => client.setSyncState(SyncState.Syncing),
e: () => client.setSyncState(SyncState.Error),
});
const session = new MockMatrixRTCSession({
membershipStatus: Status.Connected,
probablyLeft: false,
});
const hsConnected = createHomeserverConnected$(
testScope(),
client,
session,
GRACE_PERIOD,
);
expectObservable(hsConnected.combined$).toBe(expectedConnectedMarbles, {
y: [true, null],
n: [false, "sync"],
});
});
}
it("respects gracePeriodMs: stays true during grace period and flips false after", () => {
// - Initial state: Everything is connected
// - Sync error occurs -> should remain connected due to grace period
// - After grace period, not connected
marbleTest("se", "y-----n");
// If the sync error takes longer to occur, it should take equally long for
// the connection state to change
marbleTest("s--e", "y-------n");
});
it("recovers immediately if sync returns during grace period", () => {
// - Initial state: Connected
// - Sync error occurs
// - Sync recovers BEFORE the grace period expires
// - Connection state remains constant
marbleTest("se--s", "y");
});
it("flips to true IMMEDIATELY even if a grace period was pending", () => {
// - Initial error: connection eventually flips to false
// - Back to Syncing -> Must be connected immediately (synchronously)
marbleTest("e-----s", "y----ny");
});
});
src/state/CallViewModel/localMember/HomeserverConnected.ts
-136
View File
@@ -1,136 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
MembershipManagerEvent,
Status,
type MatrixRTCSession,
} from "matrix-js-sdk/lib/matrixrtc";
import { ClientEvent, type MatrixClient, SyncState } from "matrix-js-sdk";
import {
fromEvent,
startWith,
map,
tap,
type Observable,
distinctUntilChanged,
switchMap,
of,
delay,
combineLatest,
} from "rxjs";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import { Config } from "../../../config/Config";
import { type ObservableScope } from "../../ObservableScope";
import { type Behavior } from "../../Behavior";
import { type NodeStyleEventEmitter } from "../../../utils/test";
export type HomeserverDisconnectReason = "sync" | "membership" | "probablyLeft";
export interface HomeserverConnected {
/**
* Emits `[true, null]` when the homeserver connection is healthy, or
* `[false, reason]` when one of the three sub-conditions fails.
*/
combined$: Behavior<[boolean, HomeserverDisconnectReason | null]>;
rtsSession$: Behavior<Status>;
}
/**
* Behavior representing whether we consider ourselves connected to the Matrix homeserver
* for the purposes of a MatrixRTC session.
*
* `combined$` emits `null` when all conditions are satisfied, or the first failing
* reason (priority: syncing > membershipConnected > certainlyConnected):
* 1. Sync loop is not in SyncState.Syncing (after grace period) → "sync"
* 2. membershipStatus !== Status.Connected → "membership"
* 3. probablyLeft === true → "probablyLeft"
*
* @param scope - The observable scope for lifecycle management.
* @param client - The Matrix client to monitor sync state.
* @param matrixRTCSession - The RTC session to monitor membership.
* @param gracePeriodMs - Grace period in milliseconds to wait before reporting sync disconnect.
* If not provided, uses the config value (default 10000ms).
*/
export function createHomeserverConnected$(
scope: ObservableScope,
client: NodeStyleEventEmitter & Pick<MatrixClient, "getSyncState">,
matrixRTCSession: NodeStyleEventEmitter &
Pick<MatrixRTCSession, "membershipStatus" | "probablyLeft">,
gracePeriodMs?: number,
): HomeserverConnected {
const logger = rootLogger.getChild("[HomeserverConnected]");
// Get grace period from parameter or config (default 10000ms)
const graceMs = gracePeriodMs ?? Config.get().sync_disconnect_grace_period_ms;
const syncing$ = (
fromEvent(client, ClientEvent.Sync) as Observable<[SyncState]>
).pipe(
startWith([client.getSyncState()]),
map(([state]) => state === SyncState.Syncing),
distinctUntilChanged(),
switchMap((isSyncing) => {
if (isSyncing || graceMs <= 0) {
return of(isSyncing);
}
return of(false).pipe(delay(graceMs), startWith(true));
}),
distinctUntilChanged(),
);
const rtsSession$ = scope.behavior<Status>(
fromEvent(matrixRTCSession, MembershipManagerEvent.StatusChanged).pipe(
map(() => matrixRTCSession.membershipStatus ?? Status.Unknown),
),
matrixRTCSession.membershipStatus ?? Status.Unknown,
);
const membershipConnected$ = rtsSession$.pipe(
map((status) => status === Status.Connected),
);
// This is basically notProbablyLeft$
//
// probablyLeft is computed by a local timer that mimics the server delayed event.
// If we locally predict our server event timed out. We consider ourselves as probablyLeft
// even though we might not yet have received the delayed event leave.
//
// If that is not the case we certainly still have a valid membership on the matrix network
// independet if the sync currently works.
const certainlyConnected$ = fromEvent(
matrixRTCSession,
MembershipManagerEvent.ProbablyLeft,
).pipe(
startWith(null),
map(() => matrixRTCSession.probablyLeft !== true),
);
const combined$ = scope.behavior(
combineLatest([syncing$, membershipConnected$, certainlyConnected$]).pipe(
map(
([syncing, membership, certainly]): [
boolean,
HomeserverDisconnectReason | null,
] => {
if (!syncing) return [false, "sync"];
if (!membership) return [false, "membership"];
if (!certainly) return [false, "probablyLeft"];
return [true, null];
},
),
tap(([connected, reason]) => {
logger.info(
`Homeserver connected update: ${connected ? "connected" : reason}`,
);
}),
),
);
return { combined$, rtsSession$ };
}
src/state/CallViewModel/localMember/LocalMember.test.ts
-920
View File
@@ -1,920 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
Status as RTCMemberStatus,
type UnstableLivekitTransport,
type MatrixRTCSession,
} from "matrix-js-sdk/lib/matrixrtc";
import {
describe,
expect,
it,
vi,
beforeAll,
afterAll,
beforeEach,
afterEach,
} from "vitest";
import { BehaviorSubject, of, Subject } from "rxjs";
import { logger } from "matrix-js-sdk/lib/logger";
import {
ConnectionState as LivekitConnectionState,
type LocalParticipant,
type LocalTrack,
} from "livekit-client";
import fetchMock from "fetch-mock";
import { PosthogAnalytics } from "../../../analytics/PosthogAnalytics";
import {
MatrixRTCMode,
type ResolvedDelayedLeaveTimings,
} from "../../../config/ConfigOptions";
import { type HomeserverDisconnectReason } from "./HomeserverConnected";
import {
flushPromises,
mockConfig,
mockLivekitRoom,
mockLocalParticipant,
mockMuteStates,
withTestScheduler,
ownMemberMock,
testScope,
} from "../../../utils/test";
import {
TransportState,
createLocalMembership$,
enterRTCSession,
PublishState,
TrackState,
watchScreenShareToggle,
} from "./LocalMember";
import { MatrixRTCTransportMissingError } from "../../../utils/errors";
import { Epoch, ObservableScope } from "../../ObservableScope";
import { constant } from "../../Behavior";
import { ConnectionMap } from "../remoteMembers/ConnectionMap";
import {
type ConnectionState,
type Connection,
} from "../remoteMembers/Connection";
import { type Publisher } from "./Publisher";
import { initializeWidget } from "../../../widget";
import { nullHostBridge } from "../../../HostBridge";
import * as livekitAuth from "../../../livekit/auth";
initializeWidget();
const MATRIX_RTC_MODE = MatrixRTCMode.Compatibility;
const getUrlParams = vi.hoisted(() => vi.fn(() => ({})));
vi.mock("../../../UrlParams", () => ({ getUrlParams }));
vi.mock("@livekit/components-core", () => ({
observeParticipantEvents: vi
.fn()
.mockReturnValue(of({ isScreenShareEnabled: false })),
}));
describe("watchScreenShareToggle", () => {
it("reports nothing when the toggle completes", async () => {
const onError = vi.fn();
watchScreenShareToggle(Promise.resolve(), true, logger, onError);
await flushPromises();
expect(onError).not.toHaveBeenCalled();
});
it("reports failures other than the user cancelling", async () => {
const onError = vi.fn();
const e = new Error("NotReadableError");
watchScreenShareToggle(Promise.reject(e), true, logger, onError);
await flushPromises();
expect(onError).toHaveBeenCalledWith(e);
});
it("does not report the user cancelling the picker", async () => {
const onError = vi.fn();
const cancelled = new DOMException("Permission denied", "NotAllowedError");
watchScreenShareToggle(Promise.reject(cancelled), true, logger, onError);
await flushPromises();
expect(onError).not.toHaveBeenCalled();
});
});
const timings: ResolvedDelayedLeaveTimings = {
delay_ms: 10000,
restart_ms: 4000,
restart_timeout_ms: 1000,
};
const delegatedTimings: ResolvedDelayedLeaveTimings = {
delay_ms: timings.delay_ms * 10,
restart_ms: timings.restart_ms! * 10,
restart_timeout_ms: timings.restart_timeout_ms! * 10,
};
const mockedClient = {
baseUrl: "https://matrix.example.org",
getDomain: vi.fn().mockReturnValue("example.org"),
getDeviceId: vi.fn().mockReturnValue("AAAA"),
getOpenIdToken: vi.fn().mockResolvedValue({
access_token: "ACCCESS_TOKEN",
token_type: "Bearer",
matrix_server_name: "localhost",
expires_in: 10000,
}),
_unstable_delegateDelayedLeave: vi.fn().mockResolvedValue({}),
};
describe("enterRTCSession", () => {
const transport: UnstableLivekitTransport = {
livekit_service_url: "http://my-livekit-service-url.com",
type: "livekit",
};
const options = {
encryptMedia: true,
matrixRTCMode: MATRIX_RTC_MODE,
delayedLeaveTimings: timings,
};
const mockedSession = vi.mocked({
room: {
roomId: "roomId",
client: mockedClient,
},
memberships: [],
join: vi.fn(),
}) as unknown as MatrixRTCSession;
beforeEach(() =>
mockConfig({
livekit: { livekit_service_url: "http://my-default-service-url.com" },
}),
);
it("It joins the correct Session", () => {
enterRTCSession(mockedSession, ownMemberMock, transport, options);
expect(mockedSession.join).toHaveBeenLastCalledWith(
{
deviceId: "DEVICE",
memberId: "@alice:example.org:DEVICE",
userId: "@alice:example.org",
},
[transport],
expect.objectContaining({ manageMediaKeys: true }),
);
});
it("passes keyRotationParticipantLimit from config to join", () => {
mockConfig({
livekit: { livekit_service_url: "http://my-default-service-url.com" },
matrix_rtc_session: {
network_error_retry_ms: 0,
key_rotation_participant_limit: 50,
delayed_leave: timings,
delegated_delayed_leave: timings,
},
});
enterRTCSession(mockedSession, ownMemberMock, transport, options);
expect(mockedSession.join).toHaveBeenLastCalledWith(
expect.any(Object),
expect.any(Object),
expect.objectContaining({
keyRotationParticipantLimit: 50,
}),
);
});
it("uses the specified delayed leave timings", () => {
enterRTCSession(mockedSession, ownMemberMock, transport, options);
expect(mockedSession.join).toHaveBeenLastCalledWith(
expect.anything(),
expect.anything(),
expect.objectContaining({
delayedLeaveEventRestartMs: timings.restart_ms,
delayedLeaveEventDelayMs: timings.delay_ms,
delayedLeaveEventRestartLocalTimeoutMs: timings.restart_timeout_ms,
}),
);
});
});
describe("LocalMembership", () => {
const defaultCreateLocalMemberValues = {
options: constant({
encryptMedia: false,
matrixRTCMode: MatrixRTCMode.Matrix_2_0,
}),
matrixRTCSession: {
updateCallIntent: vi.fn().mockReturnValue(Promise.resolve()),
leave: vi.fn(),
} as unknown as MatrixRTCSession,
muteStates: mockMuteStates(),
trackProcessorState$: constant({
supported: false,
processor: undefined,
}),
logger: logger,
createPublisherFactory: vi.fn().mockReturnValue({
destroy: vi.fn(),
createAndSetupTracks: vi.fn().mockResolvedValue(undefined),
}),
joinMatrixRTC: async (): Promise<void> => {},
homeserverConnected: {
combined$: constant<[boolean, HomeserverDisconnectReason | null]>([
true,
null,
]),
rtsSession$: constant(RTCMemberStatus.Connected),
},
roomId: "!test-room-id:example.org",
hideScreensharing: false,
hostBridge: nullHostBridge,
baseUrl: "https://matrix.example.org",
ownMembershipIdentity: ownMemberMock,
client: mockedClient,
delayId$: constant(null),
matrixRTCMode: MATRIX_RTC_MODE,
};
beforeEach(() => {
mockConfig({
livekit: { livekit_service_url: "http://my-default-service-url.com" },
matrix_rtc_session: {
network_error_retry_ms: 1000,
delayed_leave: timings,
delegated_delayed_leave: delegatedTimings,
},
});
fetchMock.catch(404);
});
afterEach(async () => {
void (await fetchMock.flush());
fetchMock.reset();
vi.clearAllMocks();
});
it("throws error on missing RTC config error", () => {
withTestScheduler(({ scope, hot, expectObservable }) => {
const localMembership = createLocalMembership$({
scope,
...defaultCreateLocalMemberValues,
// we do not need any connection data since we want to fail before reaching that.
connectionMap$: constant(new Epoch(new ConnectionMap())),
localTransport$: hot(
"1ms #",
{},
new MatrixRTCTransportMissingError("domain.com"),
),
});
expectObservable(localMembership.localMemberState$).toBe("ne", {
n: TransportState.Waiting,
e: expect.toSatisfy((e) => e instanceof MatrixRTCTransportMissingError),
});
});
});
it("logs if callIntent cannot be updated", async () => {
async function reject(): Promise<void> {
return Promise.reject(new Error("Not connected yet"));
}
const localMembership = createLocalMembership$({
scope: testScope(),
...defaultCreateLocalMemberValues,
matrixRTCSession: {
slotId: "m.call#ROOM",
updateCallIntent: vi.fn().mockImplementation(reject),
leave: vi.fn(),
},
connectionMap$: constant(new Epoch(new ConnectionMap())),
localTransport$: constant(mockTransportLocator),
});
const expextedLog =
"'not connected yet' while updating the call intent (this is expected on startup)";
const internalLogger = vi.spyOn(localMembership.internalLoggerRef, "debug");
await flushPromises();
defaultCreateLocalMemberValues.muteStates.video.setEnabled$.value?.(true);
expect(internalLogger).toHaveBeenCalledWith(expextedLog);
});
const mockTransport: UnstableLivekitTransport = {
type: "livekit",
url: "https://sfu.example.org",
livekit_service_url: "https://jwt.example.org",
};
const mockTransportLocator: livekitAuth.TransportLocator = {
transport: mockTransport,
serverName: "example.org",
};
const connectionTransportAConnected = {
livekitRoom: mockLivekitRoom({
localParticipant: {
isScreenShareEnabled: false,
trackPublications: [],
} as unknown as LocalParticipant,
}),
state$: constant({
state: "authenticated",
livekitState: LivekitConnectionState.Connected,
}),
transport: mockTransport,
serverName: "example.org",
} as Connection;
const connectionTransportAConnecting = {
...connectionTransportAConnected,
state$: constant({
state: "authenticated",
livekitState: LivekitConnectionState.Connecting,
}),
livekitRoom: mockLivekitRoom({}),
} as unknown as Connection;
const delegationSpy = vi
.spyOn(livekitAuth, "delegateDelayedLeave")
.mockResolvedValue();
afterEach(() => delegationSpy.mockClear());
const urlOnlyTransport: UnstableLivekitTransport = {
type: "livekit",
url: "https://sfu.example.org",
};
// With a `url` only we publish through the CS API, so the homeserver
// delegates. With a `livekit_service_url` we publish through the JWT
// service, so that delegates.
it.each([
["no", mockTransport, null, timings],
[
"homeserver",
urlOnlyTransport,
"https://matrix.example.org/_matrix/client/unstable/io.element.msc4195/rtc/livekit/delegate_delayed_leave",
delegatedTimings,
],
[
"transport",
mockTransport,
"https://jwt.example.org/delegate_delayed_leave",
delegatedTimings,
],
])(
"joins session with %s delegation support",
async (_serviceName, transport, delegationUrl, delayedLeaveTimings) => {
const scope = testScope();
const joinMatrixRTC = vi.fn();
const delayId$ = new BehaviorSubject<string | null>(null);
if (delegationUrl !== null)
fetchMock.post(delegationUrl, () => ({ status: 401, body: {} }));
const connectionMap = new ConnectionMap();
connectionMap.add(
{ ...connectionTransportAConnected, transport } as Connection,
[],
);
const localMembership = createLocalMembership$({
scope,
...defaultCreateLocalMemberValues,
connectionMap$: constant(new Epoch(connectionMap)),
joinMatrixRTC,
localTransport$: constant({ transport, serverName: "example.org" }),
delayId$,
});
localMembership.requestJoinAndPublish();
void (await fetchMock.flush());
await flushPromises();
// Joins with timings appropriate for the level of delegation support
expect(joinMatrixRTC).toHaveBeenCalledWith(
transport,
delayedLeaveTimings,
);
expect(delegationSpy).not.toHaveBeenCalled();
delayId$.next("leave1");
await flushPromises();
if (delegationUrl === null) {
expect(delegationSpy).not.toHaveBeenCalled();
} else {
// Delegation is supported in this test case, so go on to check that
// LocalMember actually performs delegation
const expectDelegation = (delayId: string) =>
expect(delegationSpy).toHaveBeenLastCalledWith({
client: mockedClient,
membership: ownMemberMock,
transport,
roomId: "!test-room-id:example.org",
delayId,
logger: expect.anything(),
});
expectDelegation("leave1");
delayId$.next("leave2"); // Can change delegated leaves
await flushPromises();
expectDelegation("leave2");
}
},
);
it("only starts tracks if requested", async () => {
const scope = new ObservableScope();
const publishers: Publisher[] = [];
const tracks$ = new BehaviorSubject<LocalTrack[]>([]);
const publishing$ = constant<boolean>(false);
defaultCreateLocalMemberValues.createPublisherFactory.mockImplementation(
() => {
const p = {
// It is enought to check if destroy is called. Destroy itself is tested in the publisher to make sure it does
// all the cleanup we need.
destroy: vi.fn(),
createAndSetupTracks: vi.fn().mockImplementation(async () => {
tracks$.next([{}, {}] as LocalTrack[]);
return Promise.resolve();
}),
tracks$,
publishing$,
};
publishers.push(p as unknown as Publisher);
return p;
},
);
const publisherFactory =
defaultCreateLocalMemberValues.createPublisherFactory as ReturnType<
typeof vi.fn
>;
const connectionMap = new ConnectionMap();
connectionMap.add(connectionTransportAConnected, []);
// connectionMap.add(connectionTransportB, []);
const localMembership = createLocalMembership$({
scope,
...defaultCreateLocalMemberValues,
connectionMap$: constant(new Epoch(connectionMap)),
localTransport$: constant(mockTransportLocator),
});
await flushPromises();
expect(publisherFactory).toHaveBeenCalledOnce();
// expect(localMembership.tracks$.value.length).toBe(0);
expect(publishers[0].createAndSetupTracks).not.toHaveBeenCalled();
localMembership.startTracks();
await flushPromises();
expect(publishers[0].createAndSetupTracks).toHaveBeenCalled();
scope.end();
await flushPromises();
// stop all tracks after ending scopes
expect(publishers[0].destroy).toHaveBeenCalled();
// expect(publishers[0].stopTracks).toHaveBeenCalled();
publisherFactory.mockClear();
});
// TODO add an integration test combining publisher and localMembership
//
it("tracks livekit state correctly", async () => {
const scope = new ObservableScope();
const connectionMap = new ConnectionMap();
const localTransport$ = new Subject<livekitAuth.TransportLocator>();
const connectionMap$ = new BehaviorSubject(new Epoch(connectionMap));
const publishers: Publisher[] = [];
const publishing$ = new BehaviorSubject<boolean>(false);
const createTrackResolver = Promise.withResolvers<void>();
const publishResolver = Promise.withResolvers<void>();
defaultCreateLocalMemberValues.createPublisherFactory.mockImplementation(
() => {
const p = {
// It is enought to check if destroy is called. Destroy itself is tested in the publisher to make sure it does
// all the cleanup we need.
destroy: vi.fn(),
createAndSetupTracks: vi.fn().mockImplementation(async () => {
await createTrackResolver.promise;
}),
startPublishing: vi.fn().mockImplementation(async () => {
await publishResolver.promise;
publishing$.next(true);
}),
publishing$,
};
publishers.push(p as unknown as Publisher);
return p;
},
);
const publisherFactory =
defaultCreateLocalMemberValues.createPublisherFactory as ReturnType<
typeof vi.fn
>;
const localMembership = createLocalMembership$({
scope,
...defaultCreateLocalMemberValues,
connectionMap$: connectionMap$,
localTransport$,
});
await flushPromises();
expect(localMembership.localMemberState$.value).toStrictEqual(
TransportState.Waiting,
);
localTransport$.next(mockTransportLocator);
await flushPromises();
expect(localMembership.localMemberState$.value).toStrictEqual({
matrix: RTCMemberStatus.Connected,
media: {
connection: { state: "initializing" },
tracks: TrackState.WaitingForUser,
},
});
const connectionMap2 = new ConnectionMap();
connectionMap2.add(
// clone because we will mutate this later.
{ ...connectionTransportAConnecting } as unknown as Connection,
[],
);
connectionMap$.next(new Epoch(connectionMap2));
await flushPromises();
expect(localMembership.localMemberState$.value).toStrictEqual({
matrix: RTCMemberStatus.Connected,
media: {
connection: {
state: "authenticated",
livekitState: LivekitConnectionState.Connecting,
},
tracks: TrackState.WaitingForUser,
},
});
(
connectionMap2.getConnectionForTransport(mockTransportLocator)!
.state$ as BehaviorSubject<ConnectionState>
).next({
state: "authenticated",
livekitState: LivekitConnectionState.Connected,
});
expect(localMembership.localMemberState$.value).toStrictEqual({
matrix: RTCMemberStatus.Connected,
media: {
connection: {
state: "authenticated",
livekitState: LivekitConnectionState.Connected,
},
tracks: TrackState.WaitingForUser,
},
});
expect(publisherFactory).toHaveBeenCalledOnce();
// expect(localMembership.tracks$.value.length).toBe(0);
// -------
localMembership.startTracks();
// -------
await flushPromises();
// expect(localMembership.localMemberState$.value).toStrictEqual({
// matrix: RTCMemberStatus.Connected,
// media: {
// tracks: TrackState.Creating,
// connection: ConnectionState.LivekitConnected,
// },
// });
createTrackResolver.resolve();
await flushPromises();
expect(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(localMembership.localMemberState$.value as any).media,
).toStrictEqual(PublishState.WaitingForUser);
// -------
localMembership.requestJoinAndPublish();
// -------
expect(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(localMembership.localMemberState$.value as any).media,
).toStrictEqual(PublishState.Publishing);
publishResolver.resolve();
await flushPromises();
expect(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(localMembership.localMemberState$.value as any).media,
).toStrictEqual(PublishState.Publishing);
expect(publishers[0].destroy).not.toHaveBeenCalled();
expect(localMembership.localMemberState$.isStopped).toBe(false);
scope.end();
await flushPromises();
// stays in connected state because it is stopped before the update to tracks update the state.
expect(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(localMembership.localMemberState$.value as any).media,
).toStrictEqual(PublishState.Publishing);
// stop all tracks after ending scopes
expect(publishers[0].destroy).toHaveBeenCalled();
// expect(publishers[0].stopTracks).toHaveBeenCalled();
});
// TODO add tests for matrix local matrix participation.
describe("reconnecting analytics", () => {
beforeAll(() => {
mockConfig();
});
beforeEach(() => {
vi.restoreAllMocks();
});
afterAll(() => {
PosthogAnalytics.resetInstance();
});
it("does not fire CallReconnecting for the initial non-connected state at startup", async () => {
const scope = new ObservableScope();
const trackSpy = vi.spyOn(
PosthogAnalytics.instance.eventCallReconnecting,
"track",
);
// Simulate startup where membership isn't established yet
const hsReason$ = new BehaviorSubject<
[boolean, HomeserverDisconnectReason | null]
>([false, "membership"]);
const connectionMap = new ConnectionMap();
connectionMap.add(connectionTransportAConnected, []);
createLocalMembership$({
scope,
...defaultCreateLocalMemberValues,
homeserverConnected: {
combined$: hsReason$,
rtsSession$: constant(RTCMemberStatus.Connected),
},
connectionMap$: constant(new Epoch(connectionMap)),
localTransport$: constant(mockTransportLocator),
});
await flushPromises();
// Membership is established — call is now connected
hsReason$.next([true, null]);
expect(trackSpy).not.toHaveBeenCalled();
scope.end();
});
it("fires CallReconnecting with homeserver reason and duration when reconnected", async () => {
const scope = new ObservableScope();
const trackSpy = vi.spyOn(
PosthogAnalytics.instance.eventCallReconnecting,
"track",
);
const hsReason$ = new BehaviorSubject<
[boolean, HomeserverDisconnectReason | null]
>([true, null]);
const connectionMap = new ConnectionMap();
connectionMap.add(connectionTransportAConnected, []);
createLocalMembership$({
scope,
...defaultCreateLocalMemberValues,
homeserverConnected: {
combined$: hsReason$,
rtsSession$: constant(RTCMemberStatus.Connected),
},
connectionMap$: constant(new Epoch(connectionMap)),
localTransport$: constant(mockTransportLocator),
});
await flushPromises();
hsReason$.next([false, "sync"]);
hsReason$.next([true, null]);
expect(trackSpy).toHaveBeenCalledWith(
defaultCreateLocalMemberValues.roomId,
"sync",
expect.any(Number),
);
scope.end();
});
it("reports livekit reason when livekit disconnects then reconnects", async () => {
const scope = new ObservableScope();
const trackSpy = vi.spyOn(
PosthogAnalytics.instance.eventCallReconnecting,
"track",
);
const connectionState$ = new BehaviorSubject<ConnectionState>({
state: "authenticated",
livekitState: LivekitConnectionState.Connected,
});
const mutableConnection = {
...connectionTransportAConnected,
state$: connectionState$,
} as unknown as Connection;
const connectionMap = new ConnectionMap();
connectionMap.add(mutableConnection, []);
createLocalMembership$({
scope,
...defaultCreateLocalMemberValues,
homeserverConnected: {
combined$: constant<[boolean, HomeserverDisconnectReason | null]>([
true,
null,
]),
rtsSession$: constant(RTCMemberStatus.Connected),
},
connectionMap$: constant(new Epoch(connectionMap)),
localTransport$: constant(mockTransportLocator),
});
await flushPromises();
connectionState$.next({
state: "authenticated",
livekitState: LivekitConnectionState.Disconnected,
});
connectionState$.next({
state: "authenticated",
livekitState: LivekitConnectionState.Connected,
});
expect(trackSpy).toHaveBeenCalledWith(
defaultCreateLocalMemberValues.roomId,
"livekit",
expect.any(Number),
);
scope.end();
});
it("fires one event per completed reconnection cycle", async () => {
const scope = new ObservableScope();
const trackSpy = vi.spyOn(
PosthogAnalytics.instance.eventCallReconnecting,
"track",
);
const hsReason$ = new BehaviorSubject<
[boolean, HomeserverDisconnectReason | null]
>([true, null]);
const connectionMap = new ConnectionMap();
connectionMap.add(connectionTransportAConnected, []);
createLocalMembership$({
scope,
...defaultCreateLocalMemberValues,
homeserverConnected: {
combined$: hsReason$,
rtsSession$: constant(RTCMemberStatus.Connected),
},
connectionMap$: constant(new Epoch(connectionMap)),
localTransport$: constant(mockTransportLocator),
});
await flushPromises();
hsReason$.next([false, "membership"]);
hsReason$.next([true, null]);
hsReason$.next([false, "probablyLeft"]);
hsReason$.next([false, "sync"]);
hsReason$.next([false, "membership"]);
hsReason$.next([true, null]);
expect(trackSpy).toHaveBeenCalledTimes(2);
expect(trackSpy).toHaveBeenNthCalledWith(
1,
defaultCreateLocalMemberValues.roomId,
"membership",
expect.any(Number),
);
expect(trackSpy).toHaveBeenNthCalledWith(
2,
defaultCreateLocalMemberValues.roomId,
"probablyLeft",
expect.any(Number),
);
scope.end();
});
});
describe("toggleScreenSharing", () => {
let originalMediaDevices: MediaDevices | undefined;
beforeAll(() => {
mockConfig();
// Screen sharing is only offered when getDisplayMedia is available.
originalMediaDevices = navigator.mediaDevices;
Object.defineProperty(navigator, "mediaDevices", {
configurable: true,
value: { getDisplayMedia: vi.fn() },
});
});
afterAll(() => {
Object.defineProperty(navigator, "mediaDevices", {
configurable: true,
value: originalMediaDevices,
});
});
const createMembershipWithConnection = (
connection: Connection | null,
): {
scope: ObservableScope;
localMembership: ReturnType<typeof createLocalMembership$>;
} => {
const scope = new ObservableScope();
const connectionMap = new ConnectionMap();
if (connection) connectionMap.add(connection, []);
const localMembership = createLocalMembership$({
scope,
...defaultCreateLocalMemberValues,
connectionMap$: constant(new Epoch(connectionMap)),
localTransport$: constant(mockTransportLocator),
});
return { scope, localMembership };
};
it("surfaces a failure and clears it on dismiss", async () => {
const error = new Error("NotReadableError");
const setScreenShareEnabled = vi.fn().mockRejectedValue(error);
const connection = {
state$: constant({
state: "authenticated",
livekitState: LivekitConnectionState.Connected,
}),
transport: mockTransport,
serverName: "example.org",
livekitRoom: mockLivekitRoom({
localParticipant: mockLocalParticipant({
isScreenShareEnabled: false,
setScreenShareEnabled,
}),
}),
} as unknown as Connection;
const { scope, localMembership } =
createMembershipWithConnection(connection);
await flushPromises();
expect(localMembership.toggleScreenSharing).not.toBeNull();
expect(localMembership.screenShareError$.value).toBeNull();
localMembership.toggleScreenSharing!();
await flushPromises();
expect(setScreenShareEnabled).toHaveBeenCalledWith(
true,
expect.any(Object),
undefined,
);
expect(localMembership.screenShareError$.value).toBe(error);
localMembership.dismissScreenShareError();
expect(localMembership.screenShareError$.value).toBeNull();
scope.end();
});
it("does nothing when there is no local participant", async () => {
// No connection means participant$ never resolves to a participant.
const { scope, localMembership } = createMembershipWithConnection(null);
await flushPromises();
expect(localMembership.toggleScreenSharing).not.toBeNull();
localMembership.toggleScreenSharing!();
await flushPromises();
expect(localMembership.screenShareError$.value).toBeNull();
scope.end();
});
});
});
src/state/CallViewModel/localMember/LocalMember.ts
-1031
View File
File diff suppressed because it is too large Load Diff
src/state/CallViewModel/localMember/LocalTransport.test.ts
-126
View File
@@ -1,126 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
afterEach,
beforeEach,
describe,
expect,
it,
type MockedObject,
vi,
} from "vitest";
import fetchMock from "fetch-mock";
import { mockConfig } from "../../../utils/test";
import { getLocalTransport } from "./LocalTransport";
import { MatrixRTCTransportMissingError } from "../../../utils/errors";
import { customTransport } from "../../../settings/settings";
import { type MatrixClient } from "matrix-js-sdk";
describe("LocalTransport", () => {
beforeEach(() => vi.clearAllMocks());
it("throws if config is missing", async () => {
await expect(
getLocalTransport({
_unstable_getRTCTransports: async () => Promise.resolve([]),
getDomain: () => "example.org",
}),
).rejects.toThrow(new MatrixRTCTransportMissingError("example.org"));
});
it("returns preferred transport", async () => {
// Use config so transport discovery succeeds
mockConfig({
livekit: { livekit_service_url: "https://lk.example.org" },
});
expect(
await getLocalTransport({
_unstable_getRTCTransports: async () => Promise.resolve([]),
getDomain: () => "example.org",
}),
).toStrictEqual({
transport: {
livekit_service_url: "https://lk.example.org",
type: "livekit",
},
serverName: "example.org",
});
});
describe("transport configuration mechanisms", () => {
let client: MockedObject<
Pick<MatrixClient, "getDomain" | "_unstable_getRTCTransports">
>;
beforeEach(() => {
mockConfig({});
customTransport.setValue(customTransport.defaultValue);
client = {
getDomain: vi.fn().mockReturnValue("example.org"),
_unstable_getRTCTransports: vi.fn().mockResolvedValue([]),
};
});
afterEach(() => {
fetchMock.reset();
});
it("supports getting transport via application config", async () => {
mockConfig({
livekit: { livekit_service_url: "https://lk.example.org" },
});
expect(await getLocalTransport(client)).toStrictEqual({
transport: {
livekit_service_url: "https://lk.example.org",
type: "livekit",
},
serverName: "example.org",
});
});
it("supports getting transport via user settings", async () => {
customTransport.setValue({
type: "livekit",
livekit_service_url: "https://lk.example.org",
});
expect(await getLocalTransport(client)).toStrictEqual({
transport: {
livekit_service_url: "https://lk.example.org",
type: "livekit",
},
serverName: "example.org",
});
});
it("supports getting transport via backend", async () => {
client._unstable_getRTCTransports.mockResolvedValue([
{ type: "livekit", livekit_service_url: "https://lk.example.org" },
]);
expect(await getLocalTransport(client)).toStrictEqual({
transport: {
livekit_service_url: "https://lk.example.org",
type: "livekit",
},
serverName: "example.org",
});
});
it("throws if no options are available", async () => {
await expect(
getLocalTransport({
getDomain: () => "example.org",
_unstable_getRTCTransports: async () => Promise.resolve([]),
}),
).rejects.toThrow(new MatrixRTCTransportMissingError("example.org"));
});
});
});
src/state/CallViewModel/localMember/LocalTransport.ts
-41
View File
@@ -1,41 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-IdFentifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type MatrixClient } from "matrix-js-sdk";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import { Config } from "../../../config/Config.ts";
import { MatrixRTCTransportMissingError } from "../../../utils/errors.ts";
import { customTransport as customTransportSetting } from "../../../settings/settings.ts";
import { RtcTransportAutoDiscovery } from "./RtcTransportAutoDiscovery.ts";
import { type TransportLocator } from "../../../livekit/auth/types.ts";
/**
* Determines the transport to advertise in our MatrixRTC membership and publish
* media on.
*/
export async function getLocalTransport(
client: Pick<MatrixClient, "getDomain" | "_unstable_getRTCTransports">,
): Promise<TransportLocator> {
const serverName = client.getDomain();
if (serverName === null) throw new Error("No server name");
const discovery = new RtcTransportAutoDiscovery({
client,
resolvedConfig: Config.get(),
logger: rootLogger.getChild("[LocalTransport]"),
});
// Respect the user's custom transport, if set
const transport =
customTransportSetting.value$.value ??
(await discovery.discoverPreferredTransport());
if (transport === null) throw new MatrixRTCTransportMissingError(serverName);
return { transport, serverName };
}
src/state/CallViewModel/localMember/Publisher.test.ts
-402
View File
@@ -1,402 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { afterEach, beforeEach, describe, expect, it, test, vi } from "vitest";
import {
ConnectionState as LivekitConnectionState,
LocalParticipant,
type LocalTrack,
type LocalTrackPublication,
ParticipantEvent,
Track,
} from "livekit-client";
import { BehaviorSubject } from "rxjs";
import { logger } from "matrix-js-sdk/lib/logger";
import { ObservableScope } from "../../ObservableScope";
import { constant } from "../../Behavior";
import {
flushPromises,
mockLivekitRoom,
mockMediaDevices,
} from "../../../utils/test";
import { Publisher } from "./Publisher";
import { type Connection } from "../remoteMembers/Connection";
import { type MuteStates } from "../../MuteStates";
let scope: ObservableScope;
beforeEach(() => {
scope = new ObservableScope();
});
afterEach(() => scope.end());
function createMockLocalTrack(source: Track.Source): LocalTrack {
const track = {
source,
isMuted: false,
isUpstreamPaused: false,
} as Partial<LocalTrack> as LocalTrack;
vi.mocked(track).mute = vi.fn().mockImplementation(() => {
track.isMuted = true;
});
vi.mocked(track).unmute = vi.fn().mockImplementation(() => {
track.isMuted = false;
});
vi.mocked(track).pauseUpstream = vi.fn().mockImplementation(() => {
// @ts-expect-error - for that test we want to set isUpstreamPaused directly
track.isUpstreamPaused = true;
});
vi.mocked(track).resumeUpstream = vi.fn().mockImplementation(() => {
// @ts-expect-error - for that test we want to set isUpstreamPaused directly
track.isUpstreamPaused = false;
});
return track;
}
function createMockMuteState(enabled$: BehaviorSubject<boolean>): {
enabled$: BehaviorSubject<boolean>;
syncing$: BehaviorSubject<boolean>;
setHandler: (h: (enabled: boolean) => void) => void;
unsetHandler: () => void;
} {
let currentHandler = (enabled: boolean): void => {};
const ms = {
enabled$,
syncing$: new BehaviorSubject(false),
setHandler: vi.fn().mockImplementation((h: (enabled: boolean) => void) => {
currentHandler = h;
}),
unsetHandler: vi.fn().mockImplementation(() => {
currentHandler = (enabled: boolean): void => {};
}),
};
// forward enabled$ emissions to the current handler
enabled$.subscribe((enabled) => {
logger.info(`MockMuteState: enabled changed to ${enabled}`);
currentHandler(enabled);
});
return ms;
}
let connection: Connection;
let muteStates: MuteStates;
let localParticipant: LocalParticipant;
let audioEnabled$: BehaviorSubject<boolean>;
let videoEnabled$: BehaviorSubject<boolean>;
let trackPublications: LocalTrackPublication[];
// use it to control when track creation resolves, default to resolved
let createTrackLock: Promise<void>;
beforeEach(() => {
trackPublications = [];
audioEnabled$ = new BehaviorSubject(false);
videoEnabled$ = new BehaviorSubject(false);
createTrackLock = Promise.resolve();
muteStates = {
audio: createMockMuteState(audioEnabled$),
video: createMockMuteState(videoEnabled$),
} as unknown as MuteStates;
const mockSendDataPacket = vi.fn();
const mockEngine = {
client: {
sendUpdateLocalMetadata: vi.fn(),
},
on: vi.fn().mockReturnThis(),
sendDataPacket: mockSendDataPacket,
};
localParticipant = new LocalParticipant(
"local-sid",
"local-identity",
// @ts-expect-error - for that test we want a real LocalParticipant to have the pending publications logic
mockEngine,
{
adaptiveStream: true,
dynacase: false,
audioCaptureDefaults: {},
videoCaptureDefaults: {},
stopLocalTrackOnUnpublish: true,
reconnectPolicy: "always",
disconnectOnPageLeave: true,
},
new Map(),
{},
{},
{},
);
vi.mocked(localParticipant).createTracks = vi
.fn()
.mockImplementation(async (opts) => {
const tracks: LocalTrack[] = [];
if (opts.audio) {
tracks.push(createMockLocalTrack(Track.Source.Microphone));
}
if (opts.video) {
tracks.push(createMockLocalTrack(Track.Source.Camera));
}
await createTrackLock;
return tracks;
});
vi.mocked(localParticipant).publishTrack = vi
.fn()
.mockImplementation(async (track: LocalTrack) => {
const pub = {
track,
source: track.source,
mute: track.mute,
unmute: track.unmute,
} as Partial<LocalTrackPublication> as LocalTrackPublication;
trackPublications.push(pub);
localParticipant.emit(ParticipantEvent.LocalTrackPublished, pub);
return Promise.resolve(pub);
});
vi.mocked(localParticipant).getTrackPublication = vi
.fn()
.mockImplementation((source: Track.Source) => {
return trackPublications.find((pub) => pub.track?.source === source);
});
connection = {
state$: constant({
state: "ConnectedToLkRoom",
livekitConnectionState$: constant(LivekitConnectionState.Connected),
}),
livekitRoom: mockLivekitRoom({
localParticipant: localParticipant,
}),
} as unknown as Connection;
});
describe("Publisher", () => {
let publisher: Publisher;
beforeEach(() => {
publisher = new Publisher(
connection,
mockMediaDevices({}),
muteStates,
constant({ supported: false, processor: undefined }),
logger,
false,
);
});
afterEach(async () => {
await publisher.destroy();
});
it("Should not create tracks if started muted to avoid unneeded permission requests", async () => {
const createTracksSpy = vi.spyOn(
connection.livekitRoom.localParticipant,
"createTracks",
);
audioEnabled$.next(false);
videoEnabled$.next(false);
await publisher.createAndSetupTracks();
expect(createTracksSpy).not.toHaveBeenCalled();
});
it("should unsetHandler and stop tracks on destroy", async () => {
// setup all spies
const unsetVideoSpy = vi.spyOn(
(
publisher as unknown as {
muteStates: { video: { unsetHandler: () => void } };
}
).muteStates.video,
"unsetHandler",
);
const unsetAudioSpy = vi.spyOn(
(
publisher as unknown as {
muteStates: { audio: { unsetHandler: () => void } };
}
).muteStates.audio,
"unsetHandler",
);
const scopeEndSpy = vi.spyOn(
(publisher as unknown as { scope: { end: () => void } }).scope,
"end",
);
const stopTracksSpy = vi.spyOn(publisher, "stopTracks");
// destroy publisher
await publisher.destroy();
expect(stopTracksSpy).toHaveBeenCalledOnce();
expect(unsetVideoSpy).toHaveBeenCalledOnce();
expect(unsetAudioSpy).toHaveBeenCalledOnce();
expect(scopeEndSpy).toHaveBeenCalled();
});
it("Should minimize permission request by querying create at once", async () => {
const enableCameraAndMicrophoneSpy = vi.spyOn(
localParticipant,
"enableCameraAndMicrophone",
);
const createTracksSpy = vi.spyOn(localParticipant, "createTracks");
audioEnabled$.next(true);
videoEnabled$.next(true);
await publisher.createAndSetupTracks();
await flushPromises();
expect(enableCameraAndMicrophoneSpy).toHaveBeenCalled();
// It should create both at once
expect(createTracksSpy).toHaveBeenCalledWith({
audio: true,
video: true,
});
});
it("Ensure no data is streamed until publish has been called", async () => {
audioEnabled$.next(true);
await publisher.createAndSetupTracks();
// The track should be created and paused
expect(localParticipant.createTracks).toHaveBeenCalledWith({
audio: true,
video: undefined,
});
await flushPromises();
expect(localParticipant.publishTrack).toHaveBeenCalled();
await flushPromises();
const track = localParticipant.getTrackPublication(
Track.Source.Microphone,
)?.track;
expect(track).toBeDefined();
expect(track!.pauseUpstream).toHaveBeenCalled();
expect(track!.isUpstreamPaused).toBe(true);
});
it("Ensure resume upstream when published is called", async () => {
videoEnabled$.next(true);
await publisher.createAndSetupTracks();
// await flushPromises();
await publisher.startPublishing();
const track = localParticipant.getTrackPublication(
Track.Source.Camera,
)?.track;
expect(track).toBeDefined();
// expect(track.pauseUpstream).toHaveBeenCalled();
expect(track!.isUpstreamPaused).toBe(false);
});
describe("Mute states", () => {
let publisher: Publisher;
beforeEach(() => {
publisher = new Publisher(
connection,
mockMediaDevices({}),
muteStates,
constant({ supported: false, processor: undefined }),
logger,
false,
);
});
afterEach(async () => {
await publisher.destroy();
});
test.each([
{ mutes: { audioEnabled: true, videoEnabled: false } },
{ mutes: { audioEnabled: true, videoEnabled: false } },
])("only create the tracks that are unmuted $mutes", async ({ mutes }) => {
// Ensure all muted
audioEnabled$.next(mutes.audioEnabled);
videoEnabled$.next(mutes.videoEnabled);
vi.mocked(connection.livekitRoom.localParticipant).createTracks = vi
.fn()
.mockResolvedValue([]);
await publisher.createAndSetupTracks();
expect(
connection.livekitRoom.localParticipant.createTracks,
).toHaveBeenCalledOnce();
expect(
connection.livekitRoom.localParticipant.createTracks,
).toHaveBeenCalledWith({
audio: mutes.audioEnabled ? true : undefined,
video: mutes.videoEnabled ? true : undefined,
});
});
});
it("does mute unmute audio", async () => {});
});
describe("Bug fix", () => {
// There is a race condition when creating and publishing tracks while the mute state changes.
// This race condition could cause tracks to be published even though they are muted at the
// beginning of a call coming from lobby.
// This is caused by our stack using manually the low level API to create and publish tracks,
// but also using the higher level setMicrophoneEnabled and setCameraEnabled functions that also create
// and publish tracks, and managing pending publications.
// Race is as follow, on creation of the Publisher we create the tracks then publish them.
// If in the middle of that process the mute state changes:
// - the `setMicrophoneEnabled` will be no-op because it is not aware of our created track and can't see any pending publication
// - If start publication is requested it will publish the track even though there was a mute request.
it("wrongly publish tracks while muted", async () => {
// setLogLevel(`debug`);
const publisher = new Publisher(
connection,
mockMediaDevices({}),
muteStates,
constant({ supported: false, processor: undefined }),
logger,
false,
);
audioEnabled$.next(true);
const resolvers = Promise.withResolvers<void>();
createTrackLock = resolvers.promise;
// Initially the audio is unmuted, so creating tracks should publish the audio track
const createTracks = publisher.createAndSetupTracks();
void publisher.startPublishing();
void createTracks.then(() => {
void publisher.startPublishing();
});
// now mute the audio before allowing track creation to complete
audioEnabled$.next(false);
resolvers.resolve(undefined);
await createTracks;
await flushPromises();
const track = localParticipant.getTrackPublication(
Track.Source.Microphone,
)?.track;
expect(track).toBeDefined();
try {
expect(localParticipant.publishTrack).not.toHaveBeenCalled();
} catch {
expect(track!.mute).toHaveBeenCalled();
expect(track!.isMuted).toBe(true);
}
await publisher.destroy();
});
});
src/state/CallViewModel/localMember/Publisher.ts
-445
View File
@@ -1,445 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
Copyright 2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
ConnectionState as LivekitConnectionState,
type LocalTrackPublication,
LocalVideoTrack,
ParticipantEvent,
type Room as LivekitRoom,
Track,
} from "livekit-client";
import {
map,
NEVER,
type Observable,
type Subscription,
switchMap,
} from "rxjs";
import { type Logger } from "matrix-js-sdk/lib/logger";
import type { Behavior } from "../../Behavior.ts";
import type { MediaDevices, SelectedDevice } from "../../MediaDevices.ts";
import type { MuteStates } from "../../MuteStates.ts";
import {
type ProcessorState,
trackProcessorSync,
} from "../../../livekit/TrackProcessorContext.tsx";
import { observeTrackReference$ } from "../../observeTrackReference";
import { type Connection } from "../remoteMembers/Connection.ts";
import { ObservableScope } from "../../ObservableScope.ts";
/**
* A wrapper for a Connection object.
* This wrapper will manage the connection used to publish to the LiveKit room.
* The Publisher is also responsible for creating the media tracks.
*/
export class Publisher {
/**
* By default, livekit will start publishing tracks as soon as they are created.
* In the matrix RTC world, we want to control when tracks are published based
* on whether the user is part of the RTC session or not.
*/
public shouldPublish = false;
private readonly scope = new ObservableScope();
/**
* Creates a new Publisher.
* @param connection - The connection to use for publishing.
* @param devices - The media devices to use for audio and video input.
* @param muteStates - The mute states for audio and video.
* @param trackerProcessorState$ - The processor state for the video track processor (e.g. background blur).
* @param logger - The logger to use for logging :D.
* @param controlledAudioDevices - Whether the app hosting Element Call
* controls the audio output devices, rather than the browser.
*/
public constructor(
private connection: Pick<Connection, "livekitRoom" | "state$">, //setE2EEEnabled,
devices: MediaDevices,
private readonly muteStates: MuteStates,
trackerProcessorState$: Behavior<ProcessorState>,
private logger: Logger,
controlledAudioDevices: boolean,
) {
const room = connection.livekitRoom;
room.setE2EEEnabled(room.options.e2ee !== undefined)?.catch((e: Error) => {
this.logger.error("Failed to set E2EE enabled on room", e);
});
// Setup track processor syncing (blur)
this.observeTrackProcessors(this.scope, room, trackerProcessorState$);
// Observe media device changes and update LiveKit active devices accordingly
this.observeMediaDevices(this.scope, devices, controlledAudioDevices);
this.workaroundRestartAudioInputTrackChrome(devices, this.scope);
this.connection.livekitRoom.localParticipant.on(
ParticipantEvent.LocalTrackPublished,
this.onLocalTrackPublished.bind(this),
);
}
public async destroy(): Promise<void> {
this.scope.end();
this.logger.info("Scope ended -> unset handler");
this.muteStates.audio.unsetHandler();
this.muteStates.video.unsetHandler();
this.logger.info(`Start to stop tracks`);
try {
await this.stopTracks();
this.logger.info(`Done to stop tracks`);
} catch (e) {
this.logger.error(`Failed to stop tracks: ${e}`);
}
}
// LiveKit will publish the tracks as soon as they are created
// but we want to control when tracks are published.
// We cannot just mute the tracks, even if this will effectively stop the publishing,
// it would also prevent the user from seeing their own video/audio preview.
// So for that we use pauseUpStream(): Stops sending media to the server by replacing
// the sender track with null, but keeps the local MediaStreamTrack active.
// The user can still see/hear themselves locally, but remote participants see nothing.
private onLocalTrackPublished(
localTrackPublication: LocalTrackPublication,
): void {
this.logger.info("Local track published", localTrackPublication);
const lkRoom = this.connection.livekitRoom;
if (!this.shouldPublish) {
this.logger.debug("Not publishing, pausing upstream");
this.pauseUpstreams(lkRoom, [localTrackPublication.source]).catch((e) => {
this.logger.error(`Failed to pause upstreams`, e);
});
}
if (localTrackPublication.source === Track.Source.Microphone) {
const muteState = this.muteStates.audio;
// skip this if a sync is in progress: enabled$ still reflects the old
// state while the handler is mid-flight, so the handler itself will apply
// the correct mute state once it completes.
if (!muteState.syncing$.value) {
const enabled = muteState.enabled$.value;
if (!enabled) {
this.logger.info(
"Local audio track just published but muted meanwhile, setting enabled to false",
);
lkRoom.localParticipant.setMicrophoneEnabled(false).catch((e) => {
this.logger.error(
`Failed to enable microphone track, enabled:${enabled}`,
e,
);
});
}
}
} else if (localTrackPublication.source === Track.Source.Camera) {
const muteState = this.muteStates.video;
// skip this if a sync is in progress: enabled$ still reflects the old
// state while the handler is mid-flight, so the handler itself will apply
// the correct mute state once it completes.
if (!muteState.syncing$.value) {
const enabled = muteState.enabled$.value;
if (!enabled) {
this.logger.info(
"Local video track just published but muted meanwhile, setting enabled to false",
);
lkRoom.localParticipant.setCameraEnabled(false).catch((e) => {
this.logger.error(
`Failed to enable camera track, enabled:${enabled}`,
e,
);
});
}
}
}
}
/**
* Create and setup local audio and video tracks based on the current mute states.
* It creates the tracks only if audio and/or video is enabled, to avoid unnecessary
* permission prompts.
*
* It also observes mute state changes to update LiveKit microphone/camera states accordingly.
* If a track is not created initially because disabled, it will be created when unmuting.
*
* This call is not blocking anymore, instead callers can listen to the
* `RoomEvent.MediaDevicesError` event in the LiveKit room to be notified of any errors.
*
*/
public async createAndSetupTracks(): Promise<void> {
this.logger.debug("createAndSetupTracks called");
const lkRoom = this.connection.livekitRoom;
// Observe mute state changes and update LiveKit microphone/camera states accordingly
this.observeMuteStates();
// Check if audio and/or video is enabled. We only create tracks if enabled,
// because it could prompt for permission, and we don't want to do that unnecessarily.
const audio = this.muteStates.audio.enabled$.value;
const video = this.muteStates.video.enabled$.value;
// We don't await the creation, because livekit could block until the tracks
// are fully published, and not only that they are created.
// We don't have control on that, localParticipant creates and publishes the tracks
// asap.
// We are using the `ParticipantEvent.LocalTrackPublished` to be notified
// when tracks are actually published, and at that point
// we can pause upstream if needed (depending on if startPublishing has been called).
if (audio && video) {
// Enable both at once in order to have a single permission prompt!
void lkRoom.localParticipant.enableCameraAndMicrophone();
} else if (audio) {
void lkRoom.localParticipant.setMicrophoneEnabled(true);
} else if (video) {
void lkRoom.localParticipant.setCameraEnabled(true);
}
return Promise.resolve();
}
private async pauseUpstreams(
lkRoom: LivekitRoom,
sources: Track.Source[],
): Promise<void> {
for (const source of sources) {
const track = lkRoom.localParticipant.getTrackPublication(source)?.track;
if (track) {
await track.pauseUpstream();
} else {
this.logger.warn(
`No track found for source ${source} to pause upstream`,
);
}
}
}
private async resumeUpstreams(
lkRoom: LivekitRoom,
sources: Track.Source[],
): Promise<void> {
for (const source of sources) {
const track = lkRoom.localParticipant.getTrackPublication(source)?.track;
if (track) {
await track.resumeUpstream();
} else {
this.logger.warn(
`No track found for source ${source} to resume upstream`,
);
}
}
}
/**
*
* Request to publish local tracks to the LiveKit room.
* This will wait for the connection to be ready before publishing.
* Livekit also have some local retry logic for publishing tracks.
* Can be called multiple times, localparticipant manages the state of published tracks (or pending publications).
*
* @returns
*/
public async startPublishing(): Promise<void> {
if (this.shouldPublish) {
this.logger.debug(`Already publishing, ignoring startPublishing call`);
return;
}
this.shouldPublish = true;
this.logger.debug("startPublishing called");
const lkRoom = this.connection.livekitRoom;
// Resume upstream for both audio and video tracks
// We need to call it explicitly because call setTrackEnabled does not always
// resume upstream. It will only if you switch the track from disabled to enabled,
// but if the track is already enabled but upstream is paused, it won't resume it.
// TODO what about screen share?
try {
await this.resumeUpstreams(lkRoom, [
Track.Source.Microphone,
Track.Source.Camera,
]);
} catch (e) {
this.logger.error(`Failed to resume upstreams`, e);
}
}
public async stopPublishing(): Promise<void> {
this.logger.debug("stopPublishing called");
this.shouldPublish = false;
// Pause upstream will stop sending media to the server, while keeping
// the local MediaStreamTrack active, so the user can still see themselves.
await this.pauseUpstreams(this.connection.livekitRoom, [
Track.Source.Microphone,
Track.Source.Camera,
Track.Source.ScreenShare,
]);
}
public async stopTracks(): Promise<void> {
const lkRoom = this.connection.livekitRoom;
for (const source of [
Track.Source.Microphone,
Track.Source.Camera,
Track.Source.ScreenShare,
]) {
const localPub = lkRoom.localParticipant.getTrackPublication(source);
if (localPub?.track) {
// stops and unpublishes the track
await lkRoom.localParticipant.unpublishTrack(localPub!.track, true);
}
}
}
/// Private methods
// Restart the audio input track whenever we detect that the active media
// device has changed to refer to a different hardware device. We do this
// for the sake of Chrome, which provides a "default" device that is meant
// to match the system's default audio input, whatever that may be.
// This is special-cased for only audio inputs because we need to dig around
// in the LocalParticipant object for the track object and there's not a nice
// way to do that generically. There is usually no OS-level default video capture
// device anyway, and audio outputs work differently.
private workaroundRestartAudioInputTrackChrome(
devices: MediaDevices,
scope: ObservableScope,
): void {
const lkRoom = this.connection.livekitRoom;
devices.audioInput.selected$
.pipe(
switchMap((device) => device?.hardwareDeviceChange$ ?? NEVER),
scope.bind(),
)
.subscribe(() => {
if (lkRoom.state != LivekitConnectionState.Connected) return;
const activeMicTrack = Array.from(
lkRoom.localParticipant.audioTrackPublications.values(),
).find((d) => d.source === Track.Source.Microphone)?.track;
if (
activeMicTrack &&
// only restart if the stream is still running: LiveKit will detect
// when a track stops & restart appropriately, so this is not our job.
// Plus, we need to avoid restarting again if the track is already in
// the process of being restarted.
activeMicTrack.mediaStreamTrack.readyState !== "ended"
) {
this.logger?.info(
"Restarting audio device track due to active media device changed (workaroundRestartAudioInputTrackChrome)",
);
// Restart the track, which will cause Livekit to do another
// getUserMedia() call with deviceId: default to get the *new* default device.
// Note that room.switchActiveDevice() won't work: Livekit will ignore it because
// the deviceId hasn't changed (was & still is default).
lkRoom.localParticipant
.getTrackPublication(Track.Source.Microphone)
?.audioTrack?.restartTrack()
.catch((e) => {
this.logger.error(`Failed to restart audio device track`, e);
});
}
});
}
// Observe changes in the selected media devices and update the LiveKit room accordingly.
private observeMediaDevices(
scope: ObservableScope,
devices: MediaDevices,
controlledAudioDevices: boolean,
): void {
const lkRoom = this.connection.livekitRoom;
const syncDevice = (
kind: MediaDeviceKind,
selected$: Observable<SelectedDevice | undefined>,
): Subscription =>
selected$.pipe(scope.bind()).subscribe((device) => {
if (lkRoom.state != LivekitConnectionState.Connected) return;
// if (this.connectionState$.value !== ConnectionState.Connected) return;
this.logger.info(
"[LivekitRoom] syncDevice room.getActiveDevice(kind) !== d.id :",
lkRoom.getActiveDevice(kind),
" !== ",
device?.id,
);
if (
device !== undefined &&
lkRoom.getActiveDevice(kind) !== device.id
) {
lkRoom
.switchActiveDevice(kind, device.id)
.catch((e: Error) =>
this.logger.error(
`Failed to sync ${kind} device with LiveKit`,
e,
),
);
}
});
syncDevice("audioinput", devices.audioInput.selected$);
if (!controlledAudioDevices)
syncDevice("audiooutput", devices.audioOutput.selected$);
syncDevice("videoinput", devices.videoInput.selected$);
}
/**
* Observe changes in the mute states and update the LiveKit room accordingly.
* @private
*/
private observeMuteStates(): void {
const lkRoom = this.connection.livekitRoom;
this.muteStates.audio.setHandler(async (enable) => {
try {
this.logger.debug(
`handler: Setting LiveKit microphone enabled: ${enable}`,
);
await lkRoom.localParticipant.setMicrophoneEnabled(enable);
// Unmute will restart the track if it was paused upstream,
// but until explicitly requested, we want to keep it paused.
if (!this.shouldPublish && enable) {
await this.pauseUpstreams(lkRoom, [Track.Source.Microphone]);
}
return enable;
} catch (e) {
this.logger.error("Failed to update LiveKit audio input mute state", e);
return lkRoom.localParticipant.isMicrophoneEnabled;
}
});
this.muteStates.video.setHandler(async (enable) => {
try {
this.logger.debug(`handler: Setting LiveKit camera enabled: ${enable}`);
await lkRoom.localParticipant.setCameraEnabled(enable);
// Unmute will restart the track if it was paused upstream,
// but until explicitly requested, we want to keep it paused.
if (!this.shouldPublish && enable) {
await this.pauseUpstreams(lkRoom, [Track.Source.Camera]);
}
return enable;
} catch (e) {
this.logger.error("Failed to update LiveKit video input mute state", e);
return lkRoom.localParticipant.isCameraEnabled;
}
});
}
private observeTrackProcessors(
scope: ObservableScope,
room: LivekitRoom,
trackerProcessorState$: Behavior<ProcessorState>,
): void {
const track$ = scope.behavior(
observeTrackReference$(room.localParticipant, Track.Source.Camera).pipe(
map((trackRef) => {
const track = trackRef?.publication.track;
return track instanceof LocalVideoTrack ? track : null;
}),
),
null,
);
trackProcessorSync(scope, track$, trackerProcessorState$);
}
}
src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.test.ts
-153
View File
@@ -1,153 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
beforeEach,
describe,
expect,
it,
type MockedObject,
vi,
} from "vitest";
import { MatrixError } from "matrix-js-sdk";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import {
type UnstableLivekitTransport,
type Transport,
} from "matrix-js-sdk/lib/matrixrtc";
import type { ResolvedConfigOptions } from "../../../config/ConfigOptions.ts";
import {
RtcTransportAutoDiscovery,
type RtcTransportAutoDiscoveryProps,
} from "./RtcTransportAutoDiscovery.ts";
type DiscoveryClient = RtcTransportAutoDiscoveryProps["client"];
const backendTransport: UnstableLivekitTransport = {
type: "livekit",
livekit_service_url: "https://backend.example.org",
};
const configTransport: UnstableLivekitTransport = {
type: "livekit",
livekit_service_url: "https://config.example.org",
};
function makeClient(): MockedObject<DiscoveryClient> {
return {
getDomain: vi.fn().mockReturnValue("example.org"),
baseUrl: "https://matrix.example.org",
_unstable_getRTCTransports: vi.fn().mockResolvedValue([]),
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
} as unknown as MockedObject<DiscoveryClient>;
}
function makeResolvedConfig(livekitServiceUrl?: string): ResolvedConfigOptions {
return {
livekit: livekitServiceUrl
? {
livekit_service_url: livekitServiceUrl,
}
: undefined,
} as ResolvedConfigOptions;
}
describe("RtcTransportAutoDiscovery", () => {
beforeEach(() => {
vi.clearAllMocks();
});
const VALID_TEST_CASES: Array<{ transports: Transport[] }> = [
{ transports: [backendTransport] },
// will pick the first livekit transport in the list, even if there are other non-livekit transports
{ transports: [{ type: "not_livekit" }, backendTransport] },
];
it.each(VALID_TEST_CASES)(
"prefers backend transport other app config $transports",
async ({ transports }) => {
const client = makeClient();
client._unstable_getRTCTransports.mockResolvedValue(transports);
const discovery = new RtcTransportAutoDiscovery({
client,
resolvedConfig: makeResolvedConfig(configTransport.livekit_service_url),
logger: rootLogger,
});
const discoveredTransport = await discovery.discoverPreferredTransport();
expect(discoveredTransport).toStrictEqual(backendTransport);
expect(discoveredTransport).not.toStrictEqual(configTransport);
expect(client._unstable_getRTCTransports).toHaveBeenCalledTimes(1);
},
);
it("Retries limit_exceeded backend transport", async () => {
const client = makeClient();
client._unstable_getRTCTransports
.mockRejectedValueOnce(
new MatrixError(
{
errcode: "M_LIMIT_EXCEEDED",
error: "Too many requests",
retry_after_ms: 100,
},
429,
),
)
.mockResolvedValue([backendTransport]);
const discovery = new RtcTransportAutoDiscovery({
client,
resolvedConfig: makeResolvedConfig("https://config.example.org"),
logger: rootLogger,
});
await expect(discovery.discoverPreferredTransport()).resolves.toStrictEqual(
backendTransport,
);
expect(client._unstable_getRTCTransports).toHaveBeenCalledTimes(2);
});
const INVALID_TEST_CASES: Array<{ transports: Transport[] }> = [
{ transports: [] },
{ transports: [{ type: "not_livekit" }] },
];
it.each(INVALID_TEST_CASES)(
"falls back to config when backend has no (valid) livekit transports $transports",
async ({ transports }) => {
const client = makeClient();
client._unstable_getRTCTransports.mockResolvedValue(transports);
const discovery = new RtcTransportAutoDiscovery({
client,
resolvedConfig: makeResolvedConfig(configTransport.livekit_service_url),
logger: rootLogger,
});
const discoveredTransport = await discovery.discoverPreferredTransport();
expect(discoveredTransport).not.toStrictEqual(backendTransport);
expect(discoveredTransport).toStrictEqual(configTransport);
},
);
it("returns null when backend and config are all unavailable", async () => {
const client = makeClient();
client._unstable_getRTCTransports.mockResolvedValue([]);
const discovery = new RtcTransportAutoDiscovery({
client,
resolvedConfig: makeResolvedConfig(undefined),
logger: rootLogger,
});
await expect(discovery.discoverPreferredTransport()).resolves.toBeNull();
});
});
src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.ts
-103
View File
@@ -1,103 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-IdFentifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
isUnstableLivekitTransport,
type UnstableLivekitTransport,
} from "matrix-js-sdk/lib/matrixrtc";
import { type MatrixClient } from "matrix-js-sdk";
import { type Logger } from "matrix-js-sdk/lib/logger";
import type { ResolvedConfigOptions } from "../../../config/ConfigOptions.ts";
import { doNetworkOperationWithRetry } from "../../../utils/matrix.ts";
type TransportDiscoveryClient = Pick<
MatrixClient,
"getDomain" | "_unstable_getRTCTransports"
>;
export interface RtcTransportAutoDiscoveryProps {
client: TransportDiscoveryClient;
resolvedConfig: ResolvedConfigOptions;
logger: Logger;
}
export class RtcTransportAutoDiscovery {
private readonly client: TransportDiscoveryClient;
private readonly resolvedConfig: ResolvedConfigOptions;
private readonly logger: Logger;
public constructor({
client,
resolvedConfig,
logger,
}: RtcTransportAutoDiscoveryProps) {
this.client = client;
this.resolvedConfig = resolvedConfig;
this.logger = logger.getChild("[RtcTransportAutoDiscovery]");
}
public async discoverPreferredTransport(): Promise<UnstableLivekitTransport | null> {
// 1) backend transports
const backendTransport = await this.tryBackendTransports();
if (backendTransport) {
this.logger.info(
`Found backend transport: ${JSON.stringify(backendTransport)}`,
);
return backendTransport;
}
// 2) app config URL
const configTransport = this.tryConfigTransport();
if (configTransport) {
this.logger.info(
`Found app config transport: ${JSON.stringify(configTransport)}`,
);
return configTransport;
}
return null;
}
/**
* Fetches the first rtc_foci from the backend.
* This will not throw errors, but instead just log them and return null if the expected config is not found or malformed.
* @private
*/
private async tryBackendTransports(): Promise<UnstableLivekitTransport | null> {
const client = this.client;
// MSC4143: Attempt to fetch transports from backend.
this.logger.info("First try to use getRTCTransports end point ...");
try {
const transportList = await doNetworkOperationWithRetry(async () =>
client._unstable_getRTCTransports(),
);
const first = transportList.find(isUnstableLivekitTransport);
if (first) {
return first;
} else {
this.logger.info(
`No livekit transport found in getRTCTransports end point`,
transportList,
);
}
} catch (ex) {
this.logger.info(`Failed to use getRTCTransports end point: ${ex}`);
}
return null;
}
private tryConfigTransport(): UnstableLivekitTransport | null {
const url = this.resolvedConfig.livekit?.livekit_service_url;
if (url) {
return {
type: "livekit",
livekit_service_url: url,
};
}
return null;
}
}
src/state/CallViewModel/remoteMembers/Connection.test.ts
-749
View File
@@ -1,749 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
Copyright 2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
afterEach,
beforeEach,
describe,
expect,
it,
type MockedObject,
onTestFinished,
vi,
} from "vitest";
import {
type LocalParticipant,
type RemoteParticipant,
type RemoteTrack,
type RemoteTrackPublication,
type Room as LivekitRoom,
RoomEvent,
Track,
ConnectionState as LivekitConnectionState,
} from "livekit-client";
import fetchMock from "fetch-mock";
import EventEmitter from "events";
import { MatrixError } from "matrix-js-sdk";
import { logger, type Logger } from "matrix-js-sdk/lib/logger";
import {
type LivekitGetTokenResponse,
type UnstableLivekitTransport,
} from "matrix-js-sdk/lib/matrixrtc";
import {
Connection,
type ConnectionState,
type ConnectionOpts,
} from "./Connection.ts";
import { ObservableScope } from "../../ObservableScope.ts";
import { FailToGetOpenIdToken } from "../../../utils/errors.ts";
import { testJWTToken } from "../../../utils/test-fixtures.ts";
import { mockRemoteParticipant, ownMemberMock } from "../../../utils/test.ts";
import {
type ClientGetTokenParts,
type ClientOpenIDParts,
} from "../../../livekit/auth";
let testScope: ObservableScope;
let client: MockedObject<ClientGetTokenParts & ClientOpenIDParts>;
let fakeLivekitRoom: MockedObject<LivekitRoom>;
let localParticipantEventEmiter: EventEmitter;
let fakeLocalParticipant: MockedObject<LocalParticipant>;
const ROOM_ID = "!roomID:example.org";
const SLOT_ID = "m.call#ROOM";
const transport: UnstableLivekitTransport = {
type: "livekit",
url: "wss://matrix-rtc.m.localhost/livekit/sfu",
};
function setupTest(): void {
testScope = new ObservableScope();
client = vi.mocked<ClientGetTokenParts & ClientOpenIDParts>({
getOpenIdToken: vi.fn().mockResolvedValue({
access_token: "rYsmGUEwNjKgJYyeNUkZseJN",
token_type: "Bearer",
matrix_server_name: "example.org",
expires_in: 3600,
}),
_unstable_getLivekitToken: vi.fn().mockResolvedValue({ jwt: testJWTToken }),
});
localParticipantEventEmiter = new EventEmitter();
fakeLocalParticipant = vi.mocked<LocalParticipant>({
identity: "@me:example.org",
isMicrophoneEnabled: vi.fn().mockReturnValue(true),
getTrackPublication: vi.fn().mockReturnValue(undefined),
on: localParticipantEventEmiter.on.bind(localParticipantEventEmiter),
off: localParticipantEventEmiter.off.bind(localParticipantEventEmiter),
addListener: localParticipantEventEmiter.addListener.bind(
localParticipantEventEmiter,
),
removeListener: localParticipantEventEmiter.removeListener.bind(
localParticipantEventEmiter,
),
removeAllListeners: localParticipantEventEmiter.removeAllListeners.bind(
localParticipantEventEmiter,
),
} as unknown as LocalParticipant);
const fakeRoomEventEmitter = new EventEmitter();
fakeLivekitRoom = vi.mocked<LivekitRoom>({
connect: vi.fn(),
disconnect: vi.fn(),
remoteParticipants: new Map(),
localParticipant: fakeLocalParticipant,
state: LivekitConnectionState.Disconnected,
on: fakeRoomEventEmitter.on.bind(fakeRoomEventEmitter),
off: fakeRoomEventEmitter.off.bind(fakeRoomEventEmitter),
addListener: fakeRoomEventEmitter.addListener.bind(fakeRoomEventEmitter),
removeListener:
fakeRoomEventEmitter.removeListener.bind(fakeRoomEventEmitter),
removeAllListeners:
fakeRoomEventEmitter.removeAllListeners.bind(fakeRoomEventEmitter),
setE2EEEnabled: vi.fn().mockResolvedValue(undefined),
emit: (eventName: string | symbol, ...args: unknown[]) => {
fakeRoomEventEmitter.emit(eventName, ...args);
},
} as unknown as LivekitRoom);
}
function setupRemoteConnection(): Connection {
const opts: ConnectionOpts = {
client,
roomId: ROOM_ID,
slotId: SLOT_ID,
role: "subscriber",
transport,
serverName: "m.localhost",
scope: testScope,
ownMembershipIdentity: ownMemberMock,
livekitRoomFactory: () => fakeLivekitRoom,
};
fakeLivekitRoom.connect.mockImplementation(async (): Promise<void> => {
const changeEv = RoomEvent.ConnectionStateChanged;
fakeLivekitRoom.state = LivekitConnectionState.Connecting;
fakeLivekitRoom.emit(changeEv, fakeLivekitRoom.state);
fakeLivekitRoom.state = LivekitConnectionState.Connected;
fakeLivekitRoom.emit(changeEv, fakeLivekitRoom.state);
return Promise.resolve();
});
return new Connection(opts, logger);
}
afterEach(() => {
vi.useRealTimers();
vi.clearAllMocks();
fetchMock.reset();
});
describe("Start connection states", () => {
beforeEach(() => {
fetchMock.post(
`https://matrix-rtc.example.org/livekit/jwt/get_token`,
() => {
return {
// Return a non-retryable error, if not, the retry logic will
// wait and fail the test with a timeout.
status: 404,
};
},
);
});
it("start in initialized state", () => {
setupTest();
const opts: ConnectionOpts = {
client,
roomId: ROOM_ID,
slotId: SLOT_ID,
role: "subscriber",
transport,
serverName: "m.localhost",
scope: testScope,
ownMembershipIdentity: ownMemberMock,
livekitRoomFactory: () => fakeLivekitRoom,
};
const connection = new Connection(opts, logger);
expect(connection.state$.getValue()).toEqual({ state: "initialized" });
});
it("fail to get SFU token then error state", async () => {
setupTest();
vi.useFakeTimers();
const opts: ConnectionOpts = {
client,
roomId: ROOM_ID,
slotId: SLOT_ID,
role: "subscriber",
transport,
serverName: "m.localhost",
scope: testScope,
ownMembershipIdentity: ownMemberMock,
livekitRoomFactory: () => fakeLivekitRoom,
};
const connection = new Connection(opts, logger);
const capturedStates: ConnectionState[] = [];
const s = connection.state$.subscribe((value) => {
capturedStates.push(value);
});
onTestFinished(() => s.unsubscribe());
const deferred = Promise.withResolvers<LivekitGetTokenResponse>();
client._unstable_getLivekitToken.mockImplementation(
async (): Promise<LivekitGetTokenResponse> => {
return await deferred.promise;
},
);
connection.start().catch(() => {
// expected to throw
});
let capturedState = capturedStates.pop();
expect(capturedState).toBeDefined();
expect(capturedState).toEqual({ state: "authenticating" });
deferred.reject(new FailToGetOpenIdToken(new Error("Failed to get token")));
await vi.runAllTimersAsync();
capturedState = capturedStates.pop();
if (capturedState?.state === "error") {
expect(capturedState.error.message).toEqual("Something went wrong");
expect(connection.transport).toEqual(transport);
} else {
expect.fail(
"Expected FailedToStart state but got " + JSON.stringify(capturedState),
);
}
});
it("fail to get JWT token and error state", async () => {
setupTest();
vi.useFakeTimers();
const opts: ConnectionOpts = {
client,
roomId: ROOM_ID,
slotId: SLOT_ID,
role: "subscriber",
transport,
serverName: "m.localhost",
scope: testScope,
ownMembershipIdentity: ownMemberMock,
livekitRoomFactory: () => fakeLivekitRoom,
};
const connection = new Connection(opts, logger);
const capturedStates: ConnectionState[] = [];
const s = connection.state$.subscribe((value) => {
capturedStates.push(value);
});
onTestFinished(() => s.unsubscribe());
const deferredSFU = Promise.withResolvers<void>();
client._unstable_getLivekitToken.mockImplementation(async () => {
await deferredSFU.promise;
throw new MatrixError(
{
errcode: "M_LOOKUP_FAILED",
error: "Failed to look up user info from homeserver",
},
500,
);
});
connection.start().catch(() => {
// expected to throw
});
let capturedState = capturedStates.pop();
expect(capturedState).toBeDefined();
expect(capturedState).toEqual({ state: "authenticating" });
deferredSFU.resolve();
await vi.runAllTimersAsync();
capturedState = capturedStates.pop();
if (
capturedState?.state === "error" &&
capturedState.error.cause instanceof Error
) {
expect(capturedState.error.cause.message).toContain(
"Failed to look up user info from homeserver",
);
expect(connection.transport).toEqual(transport);
} else {
expect.fail(
"Expected FailedToStart state but got " + JSON.stringify(capturedState),
);
}
});
it("fail to connect to livekit error state", async () => {
setupTest();
vi.useFakeTimers();
const opts: ConnectionOpts = {
client,
roomId: ROOM_ID,
slotId: SLOT_ID,
role: "subscriber",
transport,
serverName: "m.localhost",
scope: testScope,
ownMembershipIdentity: ownMemberMock,
livekitRoomFactory: () => fakeLivekitRoom,
};
const connection = new Connection(opts, logger);
const capturedStates: ConnectionState[] = [];
const s = connection.state$.subscribe((value) => {
capturedStates.push(value);
});
onTestFinished(() => s.unsubscribe());
const deferredSFU = Promise.withResolvers<void>();
fakeLivekitRoom.connect.mockImplementation(async () => {
await deferredSFU.promise;
throw new Error("Failed to connect to livekit");
});
connection.start().catch(() => {
// expected to throw
});
let capturedState = capturedStates.pop();
expect(capturedState).toBeDefined();
expect(capturedState).toEqual({ state: "authenticating" });
deferredSFU.resolve();
await vi.runAllTimersAsync();
capturedState = capturedStates.pop();
if (
capturedState?.state === "error" &&
capturedState.error.cause instanceof Error
) {
expect(capturedState.error.cause.message).toContain(
"Failed to connect to livekit",
);
expect(connection.transport).toEqual(transport);
} else {
expect.fail(
"Expected FailedToStart state but got " + JSON.stringify(capturedState),
);
}
});
it("connection states happy path", async () => {
vi.useFakeTimers();
setupTest();
const connection = setupRemoteConnection();
const capturedStates: ConnectionState[] = [];
const s = connection.state$.subscribe((value) => {
capturedStates.push(value);
});
onTestFinished(() => s.unsubscribe());
await connection.start();
await vi.runAllTimersAsync();
const initialState = capturedStates.shift();
expect(initialState).toEqual({ state: "initialized" });
const fetchingState = capturedStates.shift();
expect(fetchingState).toEqual({ state: "authenticating" });
const disconnectedState = capturedStates.shift();
expect(disconnectedState).toEqual({
state: "authenticated",
livekitState: LivekitConnectionState.Disconnected,
});
const connectingState = capturedStates.shift();
expect(connectingState).toEqual({
state: "authenticated",
livekitState: LivekitConnectionState.Connecting,
});
const connectedState = capturedStates.shift();
expect(connectedState).toEqual({
state: "authenticated",
livekitState: LivekitConnectionState.Connected,
});
});
it("stopping while connecting does not report an error", async () => {
setupTest();
const connection = setupRemoteConnection();
const capturedStates: ConnectionState[] = [];
const s = connection.state$.subscribe((value) => {
capturedStates.push(value);
});
onTestFinished(() => s.unsubscribe());
// livekit-client rejects a pending connect() when disconnect() is called.
const pendingConnect = Promise.withResolvers<void>();
fakeLivekitRoom.connect.mockReturnValue(pendingConnect.promise);
fakeLivekitRoom.disconnect.mockResolvedValue(undefined);
const started = connection.start();
await vi.waitFor(() => expect(fakeLivekitRoom.connect).toHaveBeenCalled());
const stopping = connection.stop();
pendingConnect.reject(new Error("Client initiated disconnect"));
await stopping;
// start() resolves rather than rejecting (it is not awaited by the caller,
// so a rejection would be unhandled).
await expect(started).resolves.toBeUndefined();
expect(capturedStates.at(-1)).toEqual({ state: "stopped" });
expect(capturedStates.some((st) => st instanceof Error)).toBe(false);
});
it("shutting down the scope should stop the connection", async () => {
setupTest();
vi.useFakeTimers();
const connection = setupRemoteConnection();
await connection.start();
const stopSpy = vi.spyOn(connection, "stop");
testScope.end();
expect(stopSpy).toHaveBeenCalled();
expect(fakeLivekitRoom.disconnect).toHaveBeenCalled();
});
});
describe("remote participants", () => {
it("emits the list of remote participants", () => {
setupTest();
const connection = setupRemoteConnection();
const observedParticipants: RemoteParticipant[][] = [];
const s = connection.remoteParticipants$.subscribe((participants) => {
observedParticipants.push(participants);
});
onTestFinished(() => s.unsubscribe());
// The remoteParticipants$ observable is derived from the current members of the
// livekitRoom and the rtc membership in order to publish the members that are publishing
// on this connection.
let participants: RemoteParticipant[] = [
mockRemoteParticipant({ identity: "@alice:example.org:DEV000" }),
mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }),
mockRemoteParticipant({ identity: "@carol:example.org:DEV222" }),
// Mock Dan to have no published tracks. We want him to still show show up
// in the participants list.
mockRemoteParticipant({
identity: "@dan:example.org:DEV333",
getTrackPublication: () => undefined,
getTrackPublications: () => [],
}),
];
// Let's simulate 3 members on the livekitRoom
vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockImplementation(
() => new Map(participants.map((p) => [p.identity, p])),
);
participants.forEach((p) =>
fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, p),
);
// At this point there should be ~~no~~ publishers
// We do have publisher now, since we do not filter for publishers anymore (to also have participants with only data tracks)
// The filtering we do is just based on the matrixRTC member events.
expect(observedParticipants.pop()!.length).toEqual(4);
participants = [
mockRemoteParticipant({ identity: "@alice:example.org:DEV000" }),
mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }),
mockRemoteParticipant({ identity: "@carol:example.org:DEV222" }),
mockRemoteParticipant({ identity: "@dan:example.org:DEV333" }),
];
participants.forEach((p) =>
fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, p),
);
// At this point there should be no publishers
expect(observedParticipants.pop()!.length).toEqual(4);
});
it("should be scoped to parent scope", (): void => {
setupTest();
const connection = setupRemoteConnection();
let observedParticipants: RemoteParticipant[][] = [];
const s = connection.remoteParticipants$.subscribe((participants) => {
observedParticipants.push(participants);
});
onTestFinished(() => s.unsubscribe());
let participants: RemoteParticipant[] = [
mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }),
];
// Let's simulate 3 members on the livekitRoom
vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockImplementation(
() => new Map(participants.map((p) => [p.identity, p])),
);
for (const participant of participants) {
fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, participant);
}
// We should have bob as a participant now
const ps = observedParticipants.pop();
expect(ps?.length).toEqual(1);
expect(ps?.[0]?.identity).toEqual("@bob:example.org:DEV111");
// end the parent scope
testScope.end();
observedParticipants = [];
// SHOULD NOT emit any more participants as the scope is ended
participants = participants.filter(
(p) => p.identity !== "@bob:example.org:DEV111",
);
fakeLivekitRoom.emit(
RoomEvent.ParticipantDisconnected,
mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }),
);
expect(observedParticipants.length).toEqual(0);
});
});
describe("remote track logging", () => {
it("logs remote participant and track events on the connection logger", () => {
setupTest();
const info = vi.fn();
const warn = vi.fn();
const testLogger = {
getChild: (): unknown => testLogger,
info,
debug: vi.fn(),
warn,
error: vi.fn(),
} as unknown as Logger;
new Connection(
{
client,
roomId: ROOM_ID,
slotId: SLOT_ID,
role: "subscriber",
transport,
serverName: "m.localhost",
scope: testScope,
ownMembershipIdentity: ownMemberMock,
livekitRoomFactory: () => fakeLivekitRoom,
},
testLogger,
);
info.mockClear(); // drop the constructor log line
const bob = mockRemoteParticipant({
identity: "@bob:example.org:DEV111",
sid: "PA_bob",
});
const pub = {
kind: "audio",
source: "microphone",
trackSid: "TR_mic",
isMuted: false,
isEncrypted: true,
} as unknown as RemoteTrackPublication;
const messages = (): string[] => info.mock.calls.map((c) => c[0] as string);
fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, bob);
fakeLivekitRoom.emit(
RoomEvent.TrackSubscribed,
{} as RemoteTrack,
pub,
bob,
);
fakeLivekitRoom.emit(RoomEvent.TrackMuted, pub, bob);
fakeLivekitRoom.emit(
RoomEvent.TrackStreamStateChanged,
pub,
Track.StreamState.Paused,
bob,
);
expect(messages()).toEqual([
"Participant connected: @bob:example.org:DEV111 (PA_bob)",
"Subscribed: audio microphone TR_mic of @bob:example.org:DEV111 encrypted=true muted=false",
"Muted: audio microphone TR_mic of @bob:example.org:DEV111 encrypted=true",
"Stream paused: audio microphone TR_mic of @bob:example.org:DEV111 encrypted=true",
]);
// Encryption status changes are logged; cryptor errors are warnings
fakeLivekitRoom.emit(
RoomEvent.ParticipantEncryptionStatusChanged,
false,
bob,
);
expect(messages().at(-1)).toBe(
"Encryption status of @bob:example.org:DEV111: encrypted=false",
);
const cryptorError = new Error("missing key at index 3");
fakeLivekitRoom.emit(RoomEvent.EncryptionError, cryptorError, bob);
expect(warn).toHaveBeenCalledWith(
"Encryption error for @bob:example.org:DEV111:",
cryptorError,
);
// Local mute events are already logged by the Publisher
fakeLivekitRoom.emit(RoomEvent.TrackMuted, pub, {
...fakeLocalParticipant,
isLocal: true,
} as unknown as LocalParticipant);
expect(messages().filter((m) => m.startsWith("Muted"))).toHaveLength(1);
// Listeners are removed when the scope ends
testScope.end();
fakeLivekitRoom.emit(RoomEvent.ParticipantDisconnected, bob);
expect(
messages().filter((m) => m.startsWith("Participant disconnected")),
).toHaveLength(0);
});
});
//
// NOT USED ANYMORE ?
//
// This setup look like sth for the Publisher. Not a connection.
// describe("PublishConnection", () => {
// // let fakeBlurProcessor: ProcessorWrapper<BackgroundOptions>;
// let roomFactoryMock: Mock<() => LivekitRoom>;
// let muteStates: MockedObject<MuteStates>;
// function setUpPublishConnection(): void {
// setupTest();
// roomFactoryMock = vi.fn().mockReturnValue(fakeLivekitRoom);
// muteStates = mockMuteStates();
// // fakeBlurProcessor = vi.mocked<ProcessorWrapper<BackgroundOptions>>({
// // name: "BackgroundBlur",
// // restart: vi.fn().mockResolvedValue(undefined),
// // setOptions: vi.fn().mockResolvedValue(undefined),
// // getOptions: vi.fn().mockReturnValue({ strength: 0.5 }),
// // isRunning: vi.fn().mockReturnValue(false)
// // });
// }
// describe("Livekit room creation", () => {
// function createSetup(): void {
// setUpPublishConnection();
// const fakeTrackProcessorSubject$ = new BehaviorSubject<ProcessorState>({
// supported: true,
// processor: undefined,
// });
// const opts: ConnectionOpts = {
// client: client,
// transport: livekitFocus,
// scope: testScope,
// livekitRoomFactory: roomFactoryMock,
// };
// const audioInput = {
// available$: of(new Map([["mic1", { id: "mic1" }]])),
// selected$: new BehaviorSubject({ id: "mic1" }),
// select(): void {},
// };
// const videoInput = {
// available$: of(new Map([["cam1", { id: "cam1" }]])),
// selected$: new BehaviorSubject({ id: "cam1" }),
// select(): void {},
// };
// const audioOutput = {
// available$: of(new Map([["speaker", { id: "speaker" }]])),
// selected$: new BehaviorSubject({ id: "speaker" }),
// select(): void {},
// };
// // TODO understand what is wrong with our mocking that requires ts-expect-error
// const fakeDevices = mockMediaDevices({
// // @ts-expect-error Mocking only
// audioInput,
// // @ts-expect-error Mocking only
// videoInput,
// // @ts-expect-error Mocking only
// audioOutput,
// });
// new Connection(
// opts,
// fakeDevices,
// muteStates,
// undefined,
// fakeTrackProcessorSubject$,
// );
// }
// it("should create room with proper initial audio and video settings", () => {
// createSetup();
// expect(roomFactoryMock).toHaveBeenCalled();
// const lastCallArgs =
// roomFactoryMock.mock.calls[roomFactoryMock.mock.calls.length - 1];
// const roomOptions = lastCallArgs.pop() as unknown as RoomOptions;
// expect(roomOptions).toBeDefined();
// expect(roomOptions!.videoCaptureDefaults?.deviceId).toEqual("cam1");
// expect(roomOptions!.audioCaptureDefaults?.deviceId).toEqual("mic1");
// expect(roomOptions!.audioOutput?.deviceId).toEqual("speaker");
// });
// it("respect controlledAudioDevices", () => {
// // TODO: Refactor the code to make it testable.
// // The UrlParams module is a singleton has a cache and is very hard to test.
// // This breaks other tests as well if not handled properly.
// // vi.mock(import("./../UrlParams"), () => {
// // return {
// // getUrlParams: vi.fn().mockReturnValue({
// // controlledAudioDevices: true
// // })
// // };
// // });
// });
// });
// });
src/state/CallViewModel/remoteMembers/Connection.ts
-417
View File
@@ -1,417 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
Copyright 2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
connectedParticipantsObserver,
connectionStateObserver,
} from "@livekit/components-core";
import {
ConnectionError,
ConnectionErrorReason,
type ConnectionState as LivekitConnectionState,
type Participant,
type RemoteParticipant,
type RemoteTrackPublication,
type Room as LivekitRoom,
RoomEvent,
type SubscriptionError,
type Track,
type TrackPublication,
} from "livekit-client";
import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import { BehaviorSubject } from "rxjs";
import { type Logger } from "matrix-js-sdk/lib/logger";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import { type Behavior } from "../../Behavior.ts";
import { type ObservableScope } from "../../ObservableScope.ts";
import {
ElementCallError,
InsufficientCapacityError,
LivekitConnectionError,
PeerConnectionTimeoutError,
SFURoomCreationRestrictedError,
UnknownCallError,
} from "../../../utils/errors.ts";
import {
type ClientGetTokenParts,
getSFUConfig,
type ClientOpenIDParts,
type SFUConfig,
} from "../../../livekit/auth";
export interface ConnectionOpts {
/** The identity parts to use on this connection */
ownMembershipIdentity: CallMembershipIdentityParts;
/** Whether we want to publish or only subscribe on this connection. */
role: "publisher" | "subscriber";
/** The media transport to connect to. */
transport: UnstableLivekitTransport;
/** The name of the homeserver to which the {@link transport} belongs. */
serverName: string;
/** The Matrix client to use for OpenID and SFU config requests. */
client: ClientGetTokenParts & ClientOpenIDParts;
/** The ID of the Matrix room in which the session takes place. */
roomId: string;
/** The ID of the MatrixRTC slot in which the session takes place. */
slotId: string;
/** The observable scope to use for this connection. */
scope: ObservableScope;
/** Optional factory to create the LiveKit room, mainly for testing purposes. */
livekitRoomFactory: () => LivekitRoom;
}
export type ConnectionState =
// The start state of a connection. It has been created but nothing has loaded yet.
| { state: "initialized" }
// `start` has been called on the connection. It acquires the JWT token to connect to the LK room.
| { state: "authenticating" }
| { state: "authenticated"; livekitState: LivekitConnectionState }
| { state: "error"; error: ElementCallError }
| { state: "stopped" };
/**
* A connection to a Matrix RTC LiveKit backend.
*
* Expose observables for participants and connection state.
*/
export class Connection {
// Private Behavior
private readonly _state$ = new BehaviorSubject<ConnectionState>({
state: "initialized",
});
/**
* The current state of the connection to the media transport.
*/
public readonly state$: Behavior<ConnectionState> = this._state$;
/**
* Whether we want to publish or only subscribe on this connection.
*/
public readonly role: "publisher" | "subscriber";
/**
* The media transport to connect to.
*/
public readonly transport: UnstableLivekitTransport;
/**
* The name of the homeserver to which the {@link transport} belongs.
*/
public readonly serverName: string;
public readonly livekitRoom: LivekitRoom;
private scope: ObservableScope;
/**
* The remote LiveKit participants that are visible on this connection.
*
* Note that this may include participants that are connected only to
* subscribe, or publishers that are otherwise unattested in MatrixRTC state.
* It is therefore more low-level than what should be presented to the user.
*/
public readonly remoteParticipants$: Behavior<RemoteParticipant[]>;
/**
* The alias of the LiveKit room.
*/
public get livekitAlias(): string | undefined {
return this._livekitAlias;
}
private _livekitAlias?: string;
/**
* Whether the connection has been stopped.
* @see Connection.stop
* */
protected stopped = false;
// TODO: can we just keep the ConnectionOpts object instead of spreading?
private readonly client: ClientGetTokenParts & ClientOpenIDParts;
private readonly roomId: string;
private readonly slotId: string;
private readonly logger: Logger;
private readonly ownMembershipIdentity: CallMembershipIdentityParts;
/**
* Creates a new connection to a matrix RTC LiveKit backend.
*
* @param opts - Connection options {@link ConnectionOpts}.
*
* @param logger - The logger to use.
*/
public constructor(opts: ConnectionOpts, logger: Logger) {
this.ownMembershipIdentity = opts.ownMembershipIdentity;
this.roomId = opts.roomId;
this.slotId = opts.slotId;
this.logger = logger.getChild(
`[Connection ${JSON.stringify(opts.transport)}]`,
);
this.logger.debug(`constructor called`);
const { role, transport, serverName, client, scope } = opts;
this.scope = scope;
this.livekitRoom = opts.livekitRoomFactory();
this.role = role;
this.transport = transport;
this.serverName = serverName;
this.client = client;
this.remoteParticipants$ = scope.behavior(
// Only tracks remote participants
connectedParticipantsObserver(this.livekitRoom),
);
this.logRemoteTrackEvents();
scope.onEnd(() => {
this.logger.info(`Connection scope ended, stopping connection`);
void this.stop();
});
}
/**
* Logs the lifecycle of remote participants and their tracks as seen by
* livekit-client, so that rageshakes show what a tile should have been
* rendering (published, subscribed, muted, paused) for each remote member.
*/
private logRemoteTrackEvents(): void {
const room = this.livekitRoom;
const log = this.logger.getChild("[RemoteTracks]");
// The encryption flag matters: if the publisher encrypts but this client
// believes the track is unencrypted, the cryptor is bypassed and raw
// ciphertext reaches the decoder (audible as loud noise bursts).
const track = (pub: TrackPublication, p: Participant): string =>
`${pub.kind} ${pub.source} ${pub.trackSid} of ${p.identity} encrypted=${pub.isEncrypted}`;
const onParticipantConnected = (p: RemoteParticipant): void =>
log.info(`Participant connected: ${p.identity} (${p.sid})`);
const onParticipantDisconnected = (p: RemoteParticipant): void =>
log.info(`Participant disconnected: ${p.identity} (${p.sid})`);
const onTrackPublished = (
pub: RemoteTrackPublication,
p: RemoteParticipant,
): void => log.info(`Published: ${track(pub, p)} muted=${pub.isMuted}`);
const onTrackUnpublished = (
pub: RemoteTrackPublication,
p: RemoteParticipant,
): void => log.info(`Unpublished: ${track(pub, p)}`);
const onTrackSubscribed = (
_t: Track,
pub: RemoteTrackPublication,
p: RemoteParticipant,
): void => log.info(`Subscribed: ${track(pub, p)} muted=${pub.isMuted}`);
const onTrackUnsubscribed = (
_t: Track,
pub: RemoteTrackPublication,
p: RemoteParticipant,
): void => log.info(`Unsubscribed: ${track(pub, p)}`);
const onTrackSubscriptionFailed = (
trackSid: string,
p: RemoteParticipant,
reason?: SubscriptionError,
): void =>
log.warn(
`Subscription failed: ${trackSid} of ${p.identity} reason=${reason}`,
);
// Mute events also fire for the local participant, which the Publisher
// already logs.
const onTrackMuted = (pub: TrackPublication, p: Participant): void => {
if (!p.isLocal) log.info(`Muted: ${track(pub, p)}`);
};
const onTrackUnmuted = (pub: TrackPublication, p: Participant): void => {
if (!p.isLocal) log.info(`Unmuted: ${track(pub, p)}`);
};
const onTrackStreamStateChanged = (
pub: RemoteTrackPublication,
state: Track.StreamState,
p: RemoteParticipant,
): void => log.info(`Stream ${state}: ${track(pub, p)}`);
const onEncryptionStatusChanged = (
encrypted: boolean,
p?: Participant,
): void =>
log.info(
`Encryption status of ${p?.identity ?? "unknown participant"}: encrypted=${encrypted}`,
);
// livekit-client throttles these per cryptor; they indicate frames being
// dropped (missing/invalid key).
const onEncryptionError = (error: Error, p?: Participant): void =>
log.warn(
`Encryption error for ${p?.identity ?? "unknown participant"}:`,
error,
);
room
.on(RoomEvent.ParticipantConnected, onParticipantConnected)
.on(RoomEvent.ParticipantDisconnected, onParticipantDisconnected)
.on(RoomEvent.TrackPublished, onTrackPublished)
.on(RoomEvent.TrackUnpublished, onTrackUnpublished)
.on(RoomEvent.TrackSubscribed, onTrackSubscribed)
.on(RoomEvent.TrackUnsubscribed, onTrackUnsubscribed)
.on(RoomEvent.TrackSubscriptionFailed, onTrackSubscriptionFailed)
.on(RoomEvent.TrackMuted, onTrackMuted)
.on(RoomEvent.TrackUnmuted, onTrackUnmuted)
.on(RoomEvent.TrackStreamStateChanged, onTrackStreamStateChanged)
.on(
RoomEvent.ParticipantEncryptionStatusChanged,
onEncryptionStatusChanged,
)
.on(RoomEvent.EncryptionError, onEncryptionError);
this.scope.onEnd(() => {
room
.off(RoomEvent.ParticipantConnected, onParticipantConnected)
.off(RoomEvent.ParticipantDisconnected, onParticipantDisconnected)
.off(RoomEvent.TrackPublished, onTrackPublished)
.off(RoomEvent.TrackUnpublished, onTrackUnpublished)
.off(RoomEvent.TrackSubscribed, onTrackSubscribed)
.off(RoomEvent.TrackUnsubscribed, onTrackUnsubscribed)
.off(RoomEvent.TrackSubscriptionFailed, onTrackSubscriptionFailed)
.off(RoomEvent.TrackMuted, onTrackMuted)
.off(RoomEvent.TrackUnmuted, onTrackUnmuted)
.off(RoomEvent.TrackStreamStateChanged, onTrackStreamStateChanged)
.off(
RoomEvent.ParticipantEncryptionStatusChanged,
onEncryptionStatusChanged,
)
.off(RoomEvent.EncryptionError, onEncryptionError);
});
}
/**
* Starts the connection.
*
* This will:
* 1. Request an OpenId token `request_token` (allows matrix users to verify their identity with a third-party service.)
* 2. Use this token to request the SFU config to the MatrixRtc authentication service.
* 3. Connect to the configured LiveKit room.
*
* The errors are also represented as a state in the `state$` observable.
* It is safe to ignore those errors and handle them accordingly via the `state$` observable.
* @throws {InsufficientCapacityError} if the LiveKit server indicates that it has insufficient capacity to accept the connection.
* @throws {SFURoomCreationRestrictedError} if the LiveKit server indicates that the room does not exist and cannot be created.
*/
// TODO consider an autostart pattern...
public async start(): Promise<void> {
this.logger.debug("Starting Connection");
this.stopped = false;
try {
this._state$.next({ state: "authenticating" });
const { url, jwt, livekitAlias } = await this.getSFUConfig();
this.logger.debug(
"Starting Connection - jwt: ",
jwt,
"wss: ",
url,
"livekitAlias: ",
livekitAlias,
);
this._livekitAlias = livekitAlias;
// If we were stopped while fetching the config, don't proceed to connect
if (this.stopped) return;
// Setup observer once we are done with getSFUConfig
connectionStateObserver(this.livekitRoom)
.pipe(this.scope.bind())
.subscribe((livekitState) =>
this._state$.next({ state: "authenticated", livekitState }),
);
try {
this.logger.info(`livekitRoom.connect ${url}`);
await this.livekitRoom.connect(url, jwt);
this.logger.info(`livekitRoom.connect SUCCESS ${url}`);
} catch (e) {
this.logger.info(`livekitRoom.connect FAILED ${url}`, e);
// LiveKit uses 503 to indicate that the server has hit its track limits.
// https://github.com/livekit/livekit/blob/fcb05e97c5a31812ecf0ca6f7efa57c485cea9fb/pkg/service/rtcservice.go#L171
// It also errors with a status code of 200 (yes, really) for room
// participant limits.
// LiveKit Cloud uses 429 for connection limits.
// Either way, all these errors can be explained as "insufficient capacity".
if (e instanceof ConnectionError) {
if (e.status === 503 || e.status === 200 || e.status === 429) {
throw new InsufficientCapacityError();
}
if (e.status === 404) {
// error msg is "Failed to create call"
// error description is "Call creation might be restricted to authorized users only. Try again later, or contact your server admin if the problem persists."
// The room does not exist. There are two different modes of operation for the SFU:
// - the room is created on the fly when connecting (livekit `auto_create` option)
// - Only authorized users can create rooms, so the room must exist before connecting (done by the auth jwt service)
// In the first case there will not be a 404, so we are in the second case.
throw new SFURoomCreationRestrictedError();
}
if (e.reason === ConnectionErrorReason.Timeout) {
// Unabled to establish peer connection within the timeout
throw new PeerConnectionTimeoutError();
}
throw new LivekitConnectionError(e);
}
throw e;
}
// If we were stopped while connecting, don't proceed to update state.
if (this.stopped) return;
} catch (error) {
if (this.stopped) {
// stop() was called while we were connecting, which makes the pending
// connect reject. That is the abort we asked for, not a failure, so
// don't record an error state on a stopped connection or rethrow it
// (start() is not necessarily awaited by the caller, so a throw here
// becomes an unhandled promise rejection).
this.logger.debug(`Connect aborted because the connection was stopped`);
return;
}
this.logger.debug(`Failed to connect to LiveKit room: ${error}`);
this._state$.next({
state: "error",
error:
error instanceof ElementCallError
? error
: error instanceof Error
? new UnknownCallError(error)
: new UnknownCallError(new Error(`${error}`)),
});
// Its okay to ignore the throw. The error is part of the state.
throw error;
}
}
protected async getSFUConfig(): Promise<SFUConfig> {
return await getSFUConfig({
client: this.client,
membership: this.ownMembershipIdentity,
role: this.role,
transport: this.transport,
serverName: this.serverName,
roomId: this.roomId,
slotId: this.slotId,
logger: this.logger,
});
}
/**
* Stops the connection.
*
* This will disconnect from the LiveKit room.
* If the connection is already stopped, this is a no-op.
*/
public async stop(): Promise<void> {
this.logger.debug("stop: disconnecing from lk room");
if (this.stopped) return;
// Mark as stopped before disconnecting so that a connect() aborted by the
// disconnect sees the flag and does not report the abort as an error.
this.stopped = true;
await this.livekitRoom.disconnect();
this._state$.next({ state: "stopped" });
this.logger.debug("stop: DONE disconnecing from lk room");
}
}
src/state/CallViewModel/remoteMembers/ConnectionFactory.ts
-201
View File
@@ -1,201 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
Room as LivekitRoom,
type RoomOptions,
type BaseKeyProvider,
type E2EEManagerOptions,
type BaseE2EEManager,
} from "livekit-client";
import { logger, type Logger } from "matrix-js-sdk/lib/logger";
// imported as inline to support worker when loaded from a cdn (cross domain)
import E2EEWorker from "livekit-client/e2ee-worker?worker&inline";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import { type ObservableScope } from "../../ObservableScope.ts";
import { Connection } from "./Connection.ts";
import type { MediaDevices } from "../../MediaDevices.ts";
import type { Behavior } from "../../Behavior.ts";
import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx";
import { getLiveKitOptions } from "../../../livekit/options.ts";
import {
advancedCamera,
cameraResolution,
cameraFramerate,
cameraBitrate,
cameraCodec,
parseResolution,
echoCancellationSetting,
noiseSuppressionSetting,
autoGainControlSetting,
} from "../../../settings/settings.ts";
import {
type ClientGetTokenParts,
type ClientOpenIDParts,
} from "../../../livekit/auth";
// TODO evaluate if this should be done like the Publisher Factory
export interface ConnectionFactory {
createConnection(
scope: ObservableScope,
role: "publisher" | "subscriber",
transport: UnstableLivekitTransport,
serverName: string,
ownMembershipIdentity: CallMembershipIdentityParts,
logger: Logger,
): Connection;
}
export class ECConnectionFactory implements ConnectionFactory {
private readonly livekitRoomFactory: () => LivekitRoom;
/**
* Creates a ConnectionFactory for LiveKit connections.
*
* @param client - The OpenID client parts for authentication, needed to get openID and JWT tokens.
* @param roomId - The ID of the Matrix room in which the session takes place.
* @param slotId - The ID of the MatrixRTC slot in which the session takes place.
* @param devices - Used for video/audio out/in capture options.
* @param processorState$ - Effects like background blur (only for publishing connection?)
* @param livekitKeyProvider - Optional key provider for end-to-end encryption.
* @param controlledAudioDevices - Option to indicate whether audio output device is controlled externally (native mobile app).
* @param livekitRoomFactory - Optional factory function (for testing) to create LivekitRoom instances. If not provided, a default factory is used.
*/
public constructor(
private client: ClientGetTokenParts & ClientOpenIDParts,
private readonly roomId: string,
private readonly slotId: string,
private devices: MediaDevices,
private processorState$: Behavior<ProcessorState>,
livekitKeyProvider: BaseKeyProvider | undefined,
private controlledAudioDevices: boolean,
livekitRoomFactory?: () => LivekitRoom,
) {
const defaultFactory = (): LivekitRoom => {
const roomOptions = generateRoomOption({
devices: this.devices,
processorState: this.processorState$.value,
e2eeLivekitOptions: livekitKeyProvider && {
keyProvider: livekitKeyProvider,
// It's important that every room use a separate E2EE worker.
// They get confused if given streams from multiple rooms.
worker: new E2EEWorker(),
},
controlledAudioDevices: this.controlledAudioDevices,
});
logger.info("[ECConnectionFactory] livekit room options: ", roomOptions);
return new LivekitRoom(roomOptions);
};
this.livekitRoomFactory = livekitRoomFactory ?? defaultFactory;
}
/**
*
* @param scope The observable scope (used for clean-up)
* @param transport The transport to use for this connection.
* @param serverName The name of the homeserver to which the {@link transport} belongs.
* @param ownMembershipIdentity required to connect (using the jwt service) with the SFU.
* @param logger The logger instance to use for this connection.
* @param sfuConfig optional config in case we already have a token for this connection.
* @returns
*/
public createConnection(
scope: ObservableScope,
role: "publisher" | "subscriber",
transport: UnstableLivekitTransport,
serverName: string,
ownMembershipIdentity: CallMembershipIdentityParts,
logger: Logger,
): Connection {
return new Connection(
{
roomId: this.roomId,
slotId: this.slotId,
role,
transport,
serverName,
client: this.client,
scope: scope,
livekitRoomFactory: this.livekitRoomFactory,
ownMembershipIdentity,
},
logger,
);
}
}
/**
* Generate the initial LiveKit RoomOptions based on the current media devices and processor state.
* Reads audio processing and camera quality settings directly from Settings.
*/
function generateRoomOption({
devices,
processorState,
e2eeLivekitOptions,
controlledAudioDevices,
}: {
devices: MediaDevices;
processorState: ProcessorState;
e2eeLivekitOptions:
| E2EEManagerOptions
| { e2eeManager: BaseE2EEManager }
| undefined;
controlledAudioDevices: boolean;
}): RoomOptions {
const liveKitOptions = getLiveKitOptions();
// Apply advanced camera settings if enabled
let videoCaptureDefaults = {
...liveKitOptions.videoCaptureDefaults,
deviceId: devices.videoInput.selected$.value?.id,
processor: processorState.processor,
};
let publishDefaults = liveKitOptions.publishDefaults;
if (advancedCamera.getValue()) {
const { width, height } = parseResolution(cameraResolution.getValue());
const fps = cameraFramerate.getValue();
const bps = cameraBitrate.getValue();
const codec = cameraCodec.getValue();
videoCaptureDefaults = {
...videoCaptureDefaults,
resolution: { width, height, frameRate: fps },
};
publishDefaults = {
...publishDefaults,
videoEncoding: { maxBitrate: bps, maxFramerate: fps },
videoCodec: codec,
};
}
return {
...liveKitOptions,
videoCaptureDefaults,
publishDefaults,
audioCaptureDefaults: {
...liveKitOptions.audioCaptureDefaults,
deviceId: devices.audioInput.selected$.value?.id,
echoCancellation: echoCancellationSetting.getValue(),
noiseSuppression: noiseSuppressionSetting.getValue(),
autoGainControl: autoGainControlSetting.getValue(),
},
audioOutput: {
// When using controlled audio devices, we don't want to set the
// deviceId here, because it will be set by the native app.
// (also the id does not need to match a browser device id)
deviceId: controlledAudioDevices
? undefined
: devices.audioOutput.selected$.value?.id,
},
e2ee: e2eeLivekitOptions,
// TODO test and consider this:
// webAudioMix: true,
};
}
src/state/CallViewModel/remoteMembers/ConnectionMap.test.ts
-453
View File
@@ -1,453 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
import { BehaviorSubject } from "rxjs";
import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import { type RemoteParticipant } from "livekit-client";
import { logger } from "matrix-js-sdk/lib/logger";
import { Epoch, mapEpoch, ObservableScope } from "../../ObservableScope.ts";
import {
createConnectionMap$,
ConnectionMap,
keyFor,
} from "./ConnectionMap.ts";
import { type ConnectionFactory } from "./ConnectionFactory.ts";
import { type Connection } from "./Connection.ts";
import { ownMemberMock, withTestScheduler } from "../../../utils/test.ts";
import { type Behavior } from "../../Behavior.ts";
import { type TransportLocator } from "../../../livekit/auth";
// Some test constants
const TRANSPORT_1: TransportLocator = {
transport: {
type: "livekit",
livekit_service_url: "https://lk.example.org",
},
serverName: "example.org",
};
const TRANSPORT_2: TransportLocator = {
transport: {
type: "livekit",
livekit_service_url: "https://lk.sample.com",
},
serverName: "sample.com",
};
function findConnection(
transport: TransportLocator,
connections: Connection[],
): Connection | undefined {
const key = keyFor(transport);
return connections.find((c) => keyFor(c) === key);
}
let fakeConnectionFactory: ConnectionFactory;
let testScope: ObservableScope;
// Can be useful to track all created connections in tests, even the disposed ones
let allCreatedConnections: Connection[];
beforeEach(() => {
testScope = new ObservableScope();
allCreatedConnections = [];
fakeConnectionFactory = {} as unknown as ConnectionFactory;
vi.mocked(fakeConnectionFactory).createConnection = vi
.fn()
.mockImplementation(
(
scope: ObservableScope,
_role: unknown,
transport: UnstableLivekitTransport,
serverName: string,
) => {
const mockConnection = {
transport,
serverName,
remoteParticipants$: new BehaviorSubject([]),
} as unknown as Connection;
vi.mocked(mockConnection).start = vi.fn();
vi.mocked(mockConnection).stop = vi.fn();
// Tie the connection's lifecycle to the scope to test scope lifecycle management
scope.onEnd(() => {
void mockConnection.stop();
});
allCreatedConnections.push(mockConnection);
return mockConnection;
},
);
});
afterEach(() => {
testScope.end();
});
describe("connections$ stream", () => {
test("Should create and start new connections for each transports", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const connectionMap$ = createConnectionMap$({
scope: testScope,
connectionFactory: fakeConnectionFactory,
localTransport: TRANSPORT_1,
remoteTransports$: behavior("a", {
a: new Epoch([TRANSPORT_1, TRANSPORT_2], 0),
}),
logger: logger,
ownMembershipIdentity: ownMemberMock,
});
expectObservable(
connectionMap$.pipe(mapEpoch((d) => d.getConnections())),
).toBe("a", {
a: expect.toSatisfy((e: Epoch<Connection[]>) => {
const connections = e.value;
expect(connections.length).toBe(2);
expect(
vi.mocked(fakeConnectionFactory).createConnection,
).toHaveBeenCalledTimes(2);
const conn1 = findConnection(TRANSPORT_1, connections);
expect(conn1).toBeDefined();
expect(conn1!.start).toHaveBeenCalled();
const conn2 = findConnection(TRANSPORT_2, connections);
expect(conn2).toBeDefined();
expect(conn2!.start).toHaveBeenCalled();
return true;
}),
});
});
});
test("Shares one connection between homeservers advertising the same JWT service", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const sameServiceElsewhere: TransportLocator = {
transport: TRANSPORT_1.transport,
serverName: "elsewhere.org",
};
const connectionMap$ = createConnectionMap$({
scope: testScope,
connectionFactory: fakeConnectionFactory,
localTransport: TRANSPORT_1,
remoteTransports$: behavior("a", {
a: new Epoch([sameServiceElsewhere], 0),
}),
logger: logger,
ownMembershipIdentity: ownMemberMock,
});
expectObservable(
connectionMap$.pipe(mapEpoch((d) => d.getConnections())),
).toBe("a", {
a: expect.toSatisfy((e: Epoch<Connection[]>) => {
expect(e.value.length).toBe(1);
// The JWT service is reached directly, so the server name plays no
// part and the local locator, which came first, is the one used
expect(
vi.mocked(fakeConnectionFactory).createConnection,
).toHaveBeenCalledExactlyOnceWith(
expect.anything(),
"publisher",
TRANSPORT_1.transport,
"example.org",
expect.anything(),
expect.anything(),
);
return true;
}),
});
});
});
test("Keeps one connection per homeserver for transports with a url", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const urlTransport: TransportLocator = {
transport: { type: "livekit", url: "wss://sfu.example.org" },
serverName: "example.org",
};
const sameUrlElsewhere: TransportLocator = {
transport: urlTransport.transport,
serverName: "elsewhere.org",
};
const connectionMap$ = createConnectionMap$({
scope: testScope,
connectionFactory: fakeConnectionFactory,
localTransport: urlTransport,
remoteTransports$: behavior("a", {
a: new Epoch([sameUrlElsewhere], 0),
}),
logger: logger,
ownMembershipIdentity: ownMemberMock,
});
expectObservable(
connectionMap$.pipe(mapEpoch((d) => d.getConnections())),
).toBe("a", {
a: expect.toSatisfy((e: Epoch<Connection[]>) => {
// Our homeserver asks the named server for the token, so each
// server name needs its own connection
expect(e.value.length).toBe(2);
expect(findConnection(urlTransport, e.value)).toBeDefined();
expect(findConnection(sameUrlElsewhere, e.value)).toBeDefined();
return true;
}),
});
});
});
test("Should start connection only once", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const connectionMap$ = createConnectionMap$({
scope: testScope,
connectionFactory: fakeConnectionFactory,
localTransport: TRANSPORT_1,
remoteTransports$: behavior("abcdef", {
a: new Epoch([TRANSPORT_1], 0),
b: new Epoch([TRANSPORT_1], 1),
c: new Epoch([TRANSPORT_1], 2),
d: new Epoch([TRANSPORT_1], 3),
e: new Epoch([TRANSPORT_1], 4),
f: new Epoch([TRANSPORT_1, TRANSPORT_2], 5),
}),
logger: logger,
ownMembershipIdentity: ownMemberMock,
});
expectObservable(
connectionMap$.pipe(mapEpoch((d) => d.getConnections())),
).toBe("xxxxxa", {
x: expect.anything(),
a: expect.toSatisfy((e: Epoch<Connection[]>) => {
const connections = e.value;
expect(connections.length).toBe(2);
expect(
vi.mocked(fakeConnectionFactory).createConnection,
).toHaveBeenCalledTimes(2);
const conn2 = findConnection(TRANSPORT_2, connections);
expect(conn2).toBeDefined();
const conn1 = findConnection(TRANSPORT_1, connections);
expect(conn1).toBeDefined();
expect(conn1!.start).toHaveBeenCalledOnce();
return true;
}),
});
});
});
test("Should cleanup connections when not needed anymore", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const connectionMap$ = createConnectionMap$({
scope: testScope,
connectionFactory: fakeConnectionFactory,
localTransport: TRANSPORT_1,
remoteTransports$: behavior("abc", {
a: new Epoch([TRANSPORT_1], 0),
b: new Epoch([TRANSPORT_1, TRANSPORT_2], 1),
c: new Epoch([TRANSPORT_1], 2),
}),
logger: logger,
ownMembershipIdentity: ownMemberMock,
});
expectObservable(
connectionMap$.pipe(mapEpoch((d) => d.getConnections())),
).toBe("xab", {
x: expect.anything(),
a: expect.toSatisfy((e: Epoch<Connection[]>) => {
const connections = e.value;
expect(connections.length).toBe(2);
return true;
}),
b: expect.toSatisfy((e: Epoch<Connection[]>) => {
const connections = e.value;
expect(connections.length).toBe(1);
// The second connection should have been stopped has it is no longer needed.
const connection2 = findConnection(
TRANSPORT_2,
allCreatedConnections,
);
expect(connection2).toBeDefined();
expect(connection2!.stop).toHaveBeenCalled();
// The first connection should still be active
const conn1 = connections[0];
expect(conn1.stop).not.toHaveBeenCalledOnce();
return true;
}),
});
});
});
});
describe("ConnectionMap", () => {
test("warns when a second connection to the same URL is merged", () => {
const warn = vi.spyOn(logger, "warn").mockImplementation(() => {});
const data = new ConnectionMap(logger);
const connection = TRANSPORT_1 as unknown as Connection;
const p = (identity: string): RemoteParticipant =>
({ identity }) as unknown as RemoteParticipant;
data.add(connection, [p("a")]);
data.add({ ...connection } as Connection, [p("b")]);
expect(warn).toHaveBeenCalledWith(
expect.stringContaining(
'second connection to [null,"https://lk.example.org",null]: existing [a], adding [b]',
),
);
expect(data.getParticipantsForTransport(TRANSPORT_1)).toHaveLength(2);
});
});
describe("connectionMap$", () => {
// Used in test to control fake connections' remoteParticipants$ streams
let fakeRemoteParticipantsStreams: Map<string, Behavior<RemoteParticipant[]>>;
beforeEach(() => {
fakeRemoteParticipantsStreams = new Map();
function getRemoteParticipantsFor(
transport: TransportLocator,
): Behavior<RemoteParticipant[]> {
return (
fakeRemoteParticipantsStreams.get(keyFor(transport)) ??
new BehaviorSubject([])
);
}
// need a more advanced fake connection factory
vi.mocked(fakeConnectionFactory).createConnection = vi
.fn()
.mockImplementation(
(
scope: ObservableScope,
_role: unknown,
transport: UnstableLivekitTransport,
serverName: string,
) => {
const fakeRemoteParticipants$ = new BehaviorSubject<
RemoteParticipant[]
>([]);
const mockConnection = {
transport,
serverName,
remoteParticipants$: getRemoteParticipantsFor({
transport,
serverName,
}),
} as unknown as Connection;
vi.mocked(mockConnection).start = vi.fn();
vi.mocked(mockConnection).stop = vi.fn();
// Tie the connection's lifecycle to the scope to test scope lifecycle management
scope.onEnd(() => {
void mockConnection.stop();
});
fakeRemoteParticipantsStreams.set(
keyFor({ transport, serverName }),
fakeRemoteParticipants$,
);
return mockConnection;
},
);
});
test("Should report connections with the remote participants", () => {
withTestScheduler(({ expectObservable, schedule, behavior }) => {
// Setup the fake participants streams behavior
// ==============================
fakeRemoteParticipantsStreams.set(
keyFor(TRANSPORT_1),
behavior("oa-b", {
o: [],
a: [{ identity: "user1A" } as RemoteParticipant],
b: [
{ identity: "user1A" } as RemoteParticipant,
{ identity: "user1B" } as RemoteParticipant,
],
}),
);
fakeRemoteParticipantsStreams.set(
keyFor(TRANSPORT_2),
behavior("o-a", {
o: [],
a: [{ identity: "user2A" } as RemoteParticipant],
}),
);
// ==============================
const connectionMap$ = createConnectionMap$({
scope: testScope,
connectionFactory: fakeConnectionFactory,
localTransport: TRANSPORT_1,
remoteTransports$: behavior("a", {
a: new Epoch([TRANSPORT_1, TRANSPORT_2], 0),
}),
logger,
ownMembershipIdentity: ownMemberMock,
});
expectObservable(connectionMap$).toBe("abcd", {
a: expect.toSatisfy((e) => {
const data: ConnectionMap = e.value;
expect(data.getConnections().length).toBe(2);
expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(0);
expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(0);
return true;
}),
b: expect.toSatisfy((e) => {
const data: ConnectionMap = e.value;
expect(data.getConnections().length).toBe(2);
expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(1);
expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(0);
expect(
data.getParticipantsForTransport(TRANSPORT_1)[0].identity,
).toBe("user1A");
return true;
}),
c: expect.toSatisfy((e) => {
const data: ConnectionMap = e.value;
expect(data.getConnections().length).toBe(2);
expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(1);
expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(1);
expect(
data.getParticipantsForTransport(TRANSPORT_1)[0].identity,
).toBe("user1A");
expect(
data.getParticipantsForTransport(TRANSPORT_2)[0].identity,
).toBe("user2A");
return true;
}),
d: expect.toSatisfy((e) => {
const data: ConnectionMap = e.value;
expect(data.getConnections().length).toBe(2);
expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(2);
expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(1);
expect(
data.getParticipantsForTransport(TRANSPORT_1)[0].identity,
).toBe("user1A");
expect(
data.getParticipantsForTransport(TRANSPORT_1)[1].identity,
).toBe("user1B");
expect(
data.getParticipantsForTransport(TRANSPORT_2)[0].identity,
).toBe("user2A");
return true;
}),
});
});
});
});
src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts
-287
View File
@@ -1,287 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
import { Room as LivekitRoom } from "livekit-client";
import { BehaviorSubject } from "rxjs";
import fetchMock from "fetch-mock";
import { logger } from "matrix-js-sdk/lib/logger";
import EventEmitter from "events";
import { ObservableScope } from "../../ObservableScope.ts";
import { ECConnectionFactory } from "./ConnectionFactory.ts";
import {
type ClientGetTokenParts,
type ClientOpenIDParts,
} from "../../../livekit/auth";
import {
exampleTransport,
mockMediaDevices,
ownMemberMock,
} from "../../../utils/test.ts";
import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx";
import { constant } from "../../Behavior";
import {
echoCancellationSetting,
noiseSuppressionSetting,
autoGainControlSetting,
advancedCamera,
cameraResolution,
cameraFramerate,
cameraBitrate,
cameraCodec,
} from "../../../settings/settings.ts";
import { testJWTToken } from "../../../utils/test-fixtures.ts";
// At the top of your test file, after imports
vi.mock("livekit-client", async (importOriginal) => {
return {
...(await importOriginal()),
Room: vi.fn().mockImplementation(function (this: LivekitRoom, options) {
const emitter = new EventEmitter();
return {
on: emitter.on.bind(emitter),
off: emitter.off.bind(emitter),
emit: emitter.emit.bind(emitter),
disconnect: vi.fn(),
remoteParticipants: new Map(),
} as unknown as LivekitRoom;
}),
};
});
let testScope: ObservableScope;
let mockClient: ClientGetTokenParts & ClientOpenIDParts;
beforeEach(() => {
testScope = new ObservableScope();
mockClient = {
getOpenIdToken: vi.fn().mockReturnValue(""),
_unstable_getLivekitToken: vi.fn().mockResolvedValue({ jwt: testJWTToken }),
};
});
describe("ECConnectionFactory - Audio inputs options", () => {
test.each([
{ echo: true, noise: true },
{ echo: true, noise: false },
{ echo: false, noise: true },
{ echo: false, noise: false },
])(
"it sets echoCancellation=$echo and noiseSuppression=$noise based on settings",
({ echo, noise }) => {
const RoomConstructor = vi.mocked(LivekitRoom);
// Set audio processing settings
echoCancellationSetting.setValue(echo);
noiseSuppressionSetting.setValue(noise);
const ecConnectionFactory = new ECConnectionFactory(
mockClient,
"!roomid:example.org",
"m.call#ROOM",
mockMediaDevices({}),
new BehaviorSubject<ProcessorState>({
supported: true,
processor: undefined,
}),
undefined,
false,
);
ecConnectionFactory.createConnection(
testScope,
"subscriber",
exampleTransport,
"example.org",
ownMemberMock,
logger,
);
// Check if Room was constructed with expected options
expect(RoomConstructor).toHaveBeenCalledWith(
expect.objectContaining({
audioCaptureDefaults: expect.objectContaining({
echoCancellation: echo,
noiseSuppression: noise,
}),
}),
);
},
);
});
describe("ECConnectionFactory - ControlledAudioDevice", () => {
test.each([{ controlled: true }, { controlled: false }])(
"it sets controlledAudioDevice=$controlled then uses deviceId accordingly",
({ controlled }) => {
const RoomConstructor = vi.mocked(LivekitRoom);
// Explicitly set audio settings so the test doesn't depend on defaults
echoCancellationSetting.setValue(true);
noiseSuppressionSetting.setValue(true);
autoGainControlSetting.setValue(true);
const ecConnectionFactory = new ECConnectionFactory(
mockClient,
"!roomid:example.org",
"m.call#ROOM",
mockMediaDevices({
audioOutput: {
available$: constant(new Map<never, never>()),
selected$: constant({ id: "DEV00", virtualEarpiece: false }),
select: () => {},
},
}),
new BehaviorSubject<ProcessorState>({
supported: true,
processor: undefined,
}),
undefined,
controlled,
);
ecConnectionFactory.createConnection(
testScope,
"subscriber",
exampleTransport,
"example.org",
ownMemberMock,
logger,
);
// Check if Room was constructed with expected options
expect(RoomConstructor).toHaveBeenCalledWith(
expect.objectContaining({
audioOutput: expect.objectContaining({
deviceId: controlled ? undefined : "DEV00",
}),
}),
);
},
);
});
describe("ECConnectionFactory - Camera quality settings", () => {
test("it uses default video options when advancedCamera is disabled", () => {
const RoomConstructor = vi.mocked(LivekitRoom);
advancedCamera.setValue(false);
const ecConnectionFactory = new ECConnectionFactory(
mockClient,
"!roomid:example.org",
"m.call#ROOM",
mockMediaDevices({}),
new BehaviorSubject<ProcessorState>({
supported: true,
processor: undefined,
}),
undefined,
false,
);
ecConnectionFactory.createConnection(
testScope,
"subscriber",
exampleTransport,
"example.org",
ownMemberMock,
logger,
);
// publishDefaults should use config defaults (vp8), not custom settings
expect(RoomConstructor).toHaveBeenCalledWith(
expect.objectContaining({
publishDefaults: expect.objectContaining({
videoCodec: "vp8",
}),
}),
);
});
test("it applies custom camera resolution, encoding, and codec when advancedCamera is enabled", () => {
const RoomConstructor = vi.mocked(LivekitRoom);
advancedCamera.setValue(true);
cameraResolution.setValue("1920x1080");
cameraFramerate.setValue(60);
cameraBitrate.setValue(4_000_000);
cameraCodec.setValue("vp9");
const ecConnectionFactory = new ECConnectionFactory(
mockClient,
"!roomid:example.org",
"m.call#ROOM",
mockMediaDevices({}),
new BehaviorSubject<ProcessorState>({
supported: true,
processor: undefined,
}),
undefined,
false,
);
ecConnectionFactory.createConnection(
testScope,
"subscriber",
exampleTransport,
"example.org",
ownMemberMock,
logger,
);
expect(RoomConstructor).toHaveBeenCalledWith(
expect.objectContaining({
videoCaptureDefaults: expect.objectContaining({
resolution: { width: 1920, height: 1080, frameRate: 60 },
}),
publishDefaults: expect.objectContaining({
videoEncoding: { maxBitrate: 4_000_000, maxFramerate: 60 },
videoCodec: "vp9",
}),
}),
);
});
test("it applies autoGainControl from settings", () => {
const RoomConstructor = vi.mocked(LivekitRoom);
autoGainControlSetting.setValue(false);
echoCancellationSetting.setValue(true);
noiseSuppressionSetting.setValue(true);
const ecConnectionFactory = new ECConnectionFactory(
mockClient,
"!roomid:example.org",
"m.call#ROOM",
mockMediaDevices({}),
new BehaviorSubject<ProcessorState>({
supported: true,
processor: undefined,
}),
undefined,
false,
);
ecConnectionFactory.createConnection(
testScope,
"subscriber",
exampleTransport,
"example.org",
ownMemberMock,
logger,
);
expect(RoomConstructor).toHaveBeenCalledWith(
expect.objectContaining({
audioCaptureDefaults: expect.objectContaining({
autoGainControl: false,
}),
}),
);
});
});
afterEach(() => {
testScope.end();
fetchMock.reset();
});
src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts
-400
View File
@@ -1,400 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { describe, test, expect, beforeEach, afterEach, vi } from "vitest";
import { logger } from "matrix-js-sdk/lib/logger";
import {
type CallMembership,
type UnstableLivekitTransport,
} from "matrix-js-sdk/lib/matrixrtc";
import { BehaviorSubject, combineLatest, map, type Observable } from "rxjs";
import {
type RemoteMatrixLivekitMember,
createRemoteMatrixLivekitMembers$,
} from "./MatrixLivekitMembers.ts";
import { Epoch, ObservableScope, trackEpoch } from "../../ObservableScope.ts";
import { ConnectionMap } from "./ConnectionMap.ts";
import {
flushPromises,
mockRtcMembership,
mockRemoteParticipant,
} from "../../../utils/test.ts";
import { type Connection } from "./Connection.ts";
import { constant } from "../../Behavior.ts";
import { localRtcMember } from "../../../utils/test-fixtures.ts";
import { type TransportLocator } from "../../../livekit/auth";
import { membershipsAndTransports$ } from "../../SessionBehaviors.ts";
let testScope: ObservableScope;
const fallbackMemberId = (userId: string, deviceId: string): string =>
`${userId}:${deviceId}`;
const transportA: UnstableLivekitTransport = {
type: "livekit",
livekit_service_url: "https://lk.example.org",
};
const transportB: UnstableLivekitTransport = {
type: "livekit",
livekit_service_url: "https://lk.sample.com",
};
const bobMembership = mockRtcMembership("@bob:example.org", "DEV000", {
fociPreferred: [transportA],
});
const carlMembership = mockRtcMembership("@carl:sample.com", "DEV111", {
fociPreferred: [transportB],
});
beforeEach(() => {
testScope = new ObservableScope();
});
afterEach(() => {
testScope.end();
});
function epochMeWith$<T, U>(
source$: Observable<Epoch<U>>,
me$: Observable<T>,
): Observable<Epoch<T>> {
return combineLatest([source$, me$]).pipe(
map(([ep, cd]) => {
return new Epoch(cd, ep.epoch);
}),
);
}
test("should signal participant not yet connected to livekit", async () => {
const info = vi.spyOn(logger, "info");
const mockedMemberships$ = new BehaviorSubject([bobMembership]);
const mockConnectionMap$ = new BehaviorSubject(new ConnectionMap());
const { memberships$, membershipsWithTransport$ } =
createEpochedMemberships$(mockedMemberships$);
const connectionMap$ = testScope.behavior(
epochMeWith$(memberships$, mockConnectionMap$),
);
const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionMap$,
localUser: localRtcMember,
});
await flushPromises();
expect(remoteMatrixLivekitMembers$.value.value).toSatisfy(
(data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expect(data[0].membership$.value).toBe(bobMembership);
expect(data[0].participant.value$.value).toBe(null);
expect(data[0].connection$.value).toBe(null);
return true;
},
);
expect(info).toHaveBeenCalledWith(
"[RemoteMatrixLivekitMembers] @bob:example.org:DEV000: LiveKit participant missing on no connection",
);
});
// Helper to create epoch'ed memberships$ and membershipsWithTransport$ from memberships observable.
function createEpochedMemberships$(m$: Observable<CallMembership[]>): {
memberships$: Observable<Epoch<CallMembership[]>>;
membershipsWithTransport$: Observable<
Epoch<
{ membership: CallMembership; transport: TransportLocator | undefined }[]
>
>;
} {
const memberships$ = testScope.behavior(m$.pipe(trackEpoch()));
const { membershipsWithTransport$ } = membershipsAndTransports$(
testScope,
memberships$,
);
return {
memberships$,
membershipsWithTransport$,
};
}
test("should signal participant on a connection that is publishing", async () => {
const bobParticipantId = fallbackMemberId(
bobMembership.userId,
bobMembership.deviceId,
);
const { memberships$, membershipsWithTransport$ } = createEpochedMemberships$(
constant([bobMembership]),
);
const connection = {
transport: bobMembership.getTransport(),
serverName: "example.org",
} as unknown as Connection;
const dataWithPublisher = new ConnectionMap();
dataWithPublisher.add(connection, [
mockRemoteParticipant({ identity: bobParticipantId }),
]);
const connectionMap$ = testScope.behavior(
epochMeWith$(memberships$, constant(dataWithPublisher)),
);
const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionMap$,
localUser: localRtcMember,
});
await flushPromises();
expect(remoteMatrixLivekitMembers$.value.value).toSatisfy(
(data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expect(data[0].membership$.value).toBe(bobMembership);
expect(data[0].participant.value$.value).toSatisfy((participant) => {
expect(participant).toBeDefined();
expect(participant!.identity).toEqual(bobParticipantId);
return true;
});
expect(data[0].connection$.value).toBe(connection);
return true;
},
);
});
// A member advertising both a `url` and a `livekit_service_url` may have
// authenticated through either flow, so it has two candidate identities.
const bothTransport: UnstableLivekitTransport = {
type: "livekit",
url: "wss://sfu.example.org",
livekit_service_url: "https://lk.example.org",
};
const bobLegacyId = fallbackMemberId("@bob:example.org", "DEV000");
const bobHashedId = "2qkXfWz1S7jQmU0bJ6yN8aVtR3pLcDgHe9oZxKwMiA4";
const bobWithBothIdentities = mockRtcMembership("@bob:example.org", "DEV000", {
fociPreferred: [bothTransport],
backendIdentities: [bobHashedId, bobLegacyId],
});
test.each([
["legacy", bobLegacyId],
["hashed", bobHashedId],
])(
"matches a member with two candidate identities publishing under the %s one",
async (_name, identity) => {
const { memberships$, membershipsWithTransport$ } =
createEpochedMemberships$(constant([bobWithBothIdentities]));
const connection = {
transport: bothTransport,
serverName: "example.org",
} as unknown as Connection;
const dataWithPublisher = new ConnectionMap();
dataWithPublisher.add(connection, [mockRemoteParticipant({ identity })]);
const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionMap$: testScope.behavior(
epochMeWith$(memberships$, constant(dataWithPublisher)),
),
localUser: localRtcMember,
});
await flushPromises();
expect(remoteMatrixLivekitMembers$.value.value).toSatisfy(
(data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expect(data[0].participant.value$.value?.identity).toEqual(identity);
expect(data[0].connection$.value).toBe(connection);
return true;
},
);
},
);
test("does not match a member with two candidate identities to anyone else", async () => {
const { memberships$, membershipsWithTransport$ } = createEpochedMemberships$(
constant([bobWithBothIdentities]),
);
const connection = {
transport: bothTransport,
serverName: "example.org",
} as unknown as Connection;
const dataWithStranger = new ConnectionMap();
dataWithStranger.add(connection, [
mockRemoteParticipant({ identity: "@stranger:example.org:DEV999" }),
]);
const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionMap$: testScope.behavior(
epochMeWith$(memberships$, constant(dataWithStranger)),
),
localUser: localRtcMember,
});
await flushPromises();
expect(remoteMatrixLivekitMembers$.value.value).toSatisfy(
(data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expect(data[0].participant.value$.value).toBe(null);
return true;
},
);
});
test("should signal participant on a connection that is not publishing", async () => {
const { memberships$, membershipsWithTransport$ } = createEpochedMemberships$(
constant([bobMembership]),
);
const connection = {
transport: bobMembership.getTransport(),
serverName: "example.org",
} as unknown as Connection;
const dataWithPublisher = new ConnectionMap();
dataWithPublisher.add(connection, []);
const connectionMap$ = testScope.behavior(
epochMeWith$(memberships$, constant(dataWithPublisher)),
);
const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionMap$,
localUser: localRtcMember,
});
await flushPromises();
expect(remoteMatrixLivekitMembers$.value.value).toSatisfy(
(data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expect(data[0].membership$.value).toBe(bobMembership);
expect(data[0].participant.value$.value).toBe(null);
expect(data[0].connection$.value).toBe(connection);
return true;
},
);
});
describe("Publication edge case", () => {
test("bob is publishing in several connections", async () => {
const { memberships$, membershipsWithTransport$ } =
createEpochedMemberships$(constant([bobMembership, carlMembership]));
const connectionWithPublisher = new ConnectionMap();
const bobParticipantId = fallbackMemberId(
bobMembership.userId,
bobMembership.deviceId,
);
const connectionA = {
transport: transportA,
serverName: "example.org",
} as unknown as Connection;
const connectionB = {
transport: transportB,
serverName: "sample.com",
} as unknown as Connection;
connectionWithPublisher.add(connectionA, [
mockRemoteParticipant({ identity: bobParticipantId }),
]);
connectionWithPublisher.add(connectionB, [
mockRemoteParticipant({ identity: bobParticipantId }),
]);
const connectionMap$ = testScope.behavior(
epochMeWith$(memberships$, constant(connectionWithPublisher)),
);
const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionMap$,
localUser: localRtcMember,
});
await flushPromises();
expect(remoteMatrixLivekitMembers$.value.value).toSatisfy(
(data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(2);
expect(data[0].membership$.value).toBe(bobMembership);
expect(data[0].connection$.value).toBe(connectionA);
expect(data[0].participant.value$.value).toSatisfy((participant) => {
expect(participant).toBeDefined();
expect(participant!.identity).toEqual(bobParticipantId);
return true;
});
return true;
},
);
});
});
test("bob is publishing in the wrong connection", async () => {
const mockedMemberships$ = new BehaviorSubject([
bobMembership,
carlMembership,
]);
const { memberships$, membershipsWithTransport$ } =
createEpochedMemberships$(mockedMemberships$);
const connectionWithPublisher = new ConnectionMap();
const bobParticipantId = fallbackMemberId(
bobMembership.userId,
bobMembership.deviceId,
);
const connectionA = {
transport: transportA,
serverName: "example.org",
} as unknown as Connection;
const connectionB = {
transport: transportB,
serverName: "sample.com",
} as unknown as Connection;
// Bob is not publishing on A
connectionWithPublisher.add(connectionA, []);
// Bob is publishing on B but his membership says A
connectionWithPublisher.add(connectionB, [
mockRemoteParticipant({ identity: bobParticipantId }),
]);
const connectionsWithPublisher$ = new BehaviorSubject(
connectionWithPublisher,
);
const connectionMap$ = testScope.behavior(
epochMeWith$(memberships$, connectionsWithPublisher$),
);
const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionMap$,
localUser: localRtcMember,
});
await flushPromises();
expect(remoteMatrixLivekitMembers$.value.value).toSatisfy(
(data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(2);
expect(data[0].membership$.value).toBe(bobMembership);
expect(data[0].connection$.value).toBe(connectionA);
expect(data[0].participant.value$.value).toBe(null);
return true;
},
);
});
src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts
-162
View File
@@ -1,162 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type LocalParticipant, type RemoteParticipant } from "livekit-client";
import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
import { combineLatest, filter, map } from "rxjs";
import { logger } from "matrix-js-sdk/lib/logger";
import { type Behavior } from "../../Behavior";
import { type ConnectionMap } from "./ConnectionMap";
import { Epoch, type ObservableScope } from "../../ObservableScope";
import { type Connection } from "./Connection";
import { generateItemsWithEpoch } from "../../../utils/observable";
import { type TransportLocator } from "../../../livekit/auth";
interface LocalTaggedParticipant {
type: "local";
value$: Behavior<LocalParticipant | null>;
}
interface RemoteTaggedParticipant {
type: "remote";
value$: Behavior<RemoteParticipant | null>;
}
export type TaggedParticipant =
| LocalTaggedParticipant
| RemoteTaggedParticipant;
export interface MatrixLivekitMember {
membership$: Behavior<CallMembership>;
connection$: Behavior<Connection | null>;
// participantId: string; We do not want a participantId here since it will be generated by the jwt
// TODO decide if we can also drop the userId. Its in the matrix membership anyways.
userId: string;
}
/**
* Represents the local Matrix call member and their associated LiveKit participation.
* `livekitParticipant` can be null if the member is not yet connected to the livekit room
* or if it has no livekit transport at all.
*/
export interface LocalMatrixLivekitMember extends MatrixLivekitMember {
participant: LocalTaggedParticipant;
}
/**
* Represents a remote Matrix call member and their associated LiveKit participation.
* `livekitParticipant` can be null if the member is not yet connected to the livekit room
* or if it has no livekit transport at all.
*/
export interface RemoteMatrixLivekitMember extends MatrixLivekitMember {
participant: RemoteTaggedParticipant;
}
interface Props {
scope: ObservableScope;
membershipsWithTransport$: Behavior<
Epoch<
{ membership: CallMembership; transport: TransportLocator | undefined }[]
>
>;
connectionMap$: Behavior<Epoch<ConnectionMap>>;
localUser: { deviceId: string; userId: string };
}
/**
* Combines MatrixRTC and Livekit worlds.
*
* It has a small public interface:
* - in (via constructor):
* - an observable of CallMembership[] to track the call members (The matrix side)
* - a `ConnectionMap` for the lk rooms (The livekit side)
* - out (via public Observable):
* - `remoteMatrixLivekitMember` an observable of MatrixLivekitMember[] to track the remote members and associated livekit data.
*/
export function createRemoteMatrixLivekitMembers$({
scope,
membershipsWithTransport$,
connectionMap$,
localUser,
}: Props): Behavior<Epoch<RemoteMatrixLivekitMember[]>> {
/**
* Behavior of all the remote call members and their associated livekit data (if available).
*/
return scope.behavior(
combineLatest([membershipsWithTransport$, connectionMap$]).pipe(
filter((values) =>
values.every((value) => value.epoch === values[0].epoch),
),
map(([ms, data]) => new Epoch([ms.value, data.value] as const, ms.epoch)),
generateItemsWithEpoch(
"RemoteMatrixLivekitMembers",
// Generator function.
// creates an array of `{key, data}[]`
// Each change in the keys (new key) will result in a call to the factory function.
function* ([membershipsWithTransport, managerData]) {
for (const { membership, transport } of membershipsWithTransport) {
// Exclude the local membership
if (
membership.userId === localUser.userId &&
membership.deviceId === localUser.deviceId
)
continue;
const participants = transport
? managerData.getParticipantsForTransport(transport)
: [];
const matches = participants.filter((p) =>
membership.backendIdentities.includes(p.identity),
);
const participant = matches[0] ?? null;
const connection = transport
? managerData.getConnectionForTransport(transport)
: null;
if (matches.length > 1)
logger.warn(
`[RemoteMatrixLivekitMembers] ${membership.rtcBackendIdentity}: ${matches.length} LiveKit participants match (sids ${matches.map((p) => p.sid).join(", ")}), using ${participant?.sid}`,
);
yield {
// This could just be the backend identity without the other keys.
// The user ID, device ID, and member ID are included however so
// they show up in debug logs.
keys: [
membership.userId,
membership.deviceId,
membership.memberId,
membership.rtcBackendIdentity,
],
data: { membership, participant, connection },
};
}
},
// Each update where the key of the generator array do not change will result in updates to the `data$` behavior.
(scope, data$, userId, _deviceId, _memberId, rtcBackendIdentity) => {
const { participant$, ...rest } = scope.splitBehavior(data$);
// Log whether the member could be matched to a LiveKit participant,
// since a tile shows "waiting for media" for as long as it cannot.
participant$.pipe(scope.bind()).subscribe((p) => {
const transport = data$.value.connection?.transport;
logger.info(
`[RemoteMatrixLivekitMembers] ${rtcBackendIdentity}: LiveKit participant ${p ? `matched (${p.sid})` : "missing"} on ${transport ? JSON.stringify(transport) : "no connection"}`,
);
});
// will only get called once per backend identity.
// updates to data$ and as a result to displayName$ and mxcAvatarUrl$ are more frequent.
return {
userId,
participant: { type: "remote" as const, value$: participant$ },
...rest,
};
},
),
),
new Epoch([], -1),
);
}
// TODO add back in the callviewmodel pauseWhen(this.pretendToBeDisconnected$)
src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.test.ts
-632
View File
@@ -1,632 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { afterEach, beforeEach, describe, vi } from "vitest";
import {
type MatrixEvent,
type RoomMember,
type RoomState,
RoomStateEvent,
} from "matrix-js-sdk";
import EventEmitter from "events";
import { it } from "vitest";
import { ObservableScope } from "../../ObservableScope.ts";
import type { Room as MatrixRoom } from "matrix-js-sdk/lib/models/room";
import {
mockRtcMembership,
mockMatrixRoomMember,
withTestScheduler,
} from "../../../utils/test.ts";
import {
createMatrixMemberMetadata$,
createRoomMembers$,
} from "./MatrixMemberMetadata.ts";
let testScope: ObservableScope;
let mockMatrixRoom: MatrixRoom;
describe("MatrixMemberMetadata", () => {
/*
* To be populated in the test setup.
* Maps userId to a partial/mock RoomMember object.
*/
let fakeMembersMap: Map<string, Partial<RoomMember>>;
beforeEach(() => {
testScope = new ObservableScope();
fakeMembersMap = new Map<string, Partial<RoomMember>>();
const roomEmitter = new EventEmitter();
mockMatrixRoom = {
on: roomEmitter.on.bind(roomEmitter),
off: roomEmitter.off.bind(roomEmitter),
emit: roomEmitter.emit.bind(roomEmitter),
// addListener: roomEmitter.addListener.bind(roomEmitter),
// removeListener: roomEmitter.removeListener.bind(roomEmitter),
getMember: vi.fn().mockImplementation((userId: string) => {
const member = fakeMembersMap.get(userId);
if (member) {
return member as RoomMember;
}
return null;
}),
getMembers: vi.fn().mockImplementation(() => {
const members = Array.from(fakeMembersMap.values());
return members;
}),
getMembersWithMembership: vi.fn().mockImplementation(() => {
const members = Array.from(fakeMembersMap.values());
return members;
}),
} as unknown as MatrixRoom;
});
function fakeMemberWith(data: Partial<RoomMember>): void {
const userId = data.userId || "@alice:example.com";
const member: Partial<RoomMember> = {
userId: userId,
rawDisplayName: data.rawDisplayName ?? userId,
getMxcAvatarUrl:
data.getMxcAvatarUrl ||
vi.fn().mockImplementation(() => {
return `mxc://example.com/${userId}`;
}),
...data,
} as unknown as RoomMember;
fakeMembersMap.set(userId, member);
}
afterEach(() => {
fakeMembersMap.clear();
});
describe("displayname", () => {
function updateDisplayName(
userId: `@${string}:${string}`,
newDisplayName: string,
): void {
const member = fakeMembersMap.get(userId);
if (member) {
member.rawDisplayName = newDisplayName;
// Emit the event to notify listeners
mockMatrixRoom.emit(
RoomStateEvent.Members,
{} as unknown as MatrixEvent,
{} as unknown as RoomState,
member as RoomMember,
);
} else {
throw new Error(`No member found with userId: ${userId}`);
}
}
it("should show our own user if present in rtc session and room", () => {
withTestScheduler(({ scope, behavior, expectObservable }) => {
fakeMemberWith({
userId: "@local:example.com",
rawDisplayName: "it's a me",
});
const memberships$ = behavior("a", {
a: [mockRtcMembership("@local:example.com", "DEVICE1")],
});
const metadataStore = createMatrixMemberMetadata$(
testScope,
memberships$,
createRoomMembers$(testScope, mockMatrixRoom),
);
const dn$ = metadataStore.createDisplayNameBehavior$(
scope,
"@local:example.com",
);
expectObservable(dn$).toBe("a", {
a: "it's a me",
});
expectObservable(metadataStore.displaynameMap$).toBe("a", {
a: new Map<string, string>([["@local:example.com", "it's a me"]]),
});
});
});
function setUpBasicRoom(): void {
fakeMemberWith({
userId: "@local:example.com",
rawDisplayName: "it's a me",
});
fakeMemberWith({ userId: "@alice:example.com", rawDisplayName: "Alice" });
fakeMemberWith({ userId: "@bob:example.com", rawDisplayName: "Bob" });
fakeMemberWith({ userId: "@carl:example.com", rawDisplayName: "Carl" });
fakeMemberWith({ userId: "@evil:example.com", rawDisplayName: "Carl" });
fakeMemberWith({ userId: "@bob:foo.bar", rawDisplayName: "Bob" });
fakeMemberWith({ userId: "@no-name:foo.bar" });
}
it("should get displayName for users", () => {
setUpBasicRoom();
withTestScheduler(({ scope, behavior, expectObservable }) => {
const memberships$ = behavior("a", {
a: [
mockRtcMembership("@alice:example.com", "DEVICE1"),
mockRtcMembership("@bob:example.com", "DEVICE1"),
],
});
const metadataStore = createMatrixMemberMetadata$(
testScope,
memberships$,
createRoomMembers$(testScope, mockMatrixRoom),
);
const aliceDispName$ = metadataStore.createDisplayNameBehavior$(
scope,
"@alice:example.com",
);
expectObservable(aliceDispName$).toBe("a", {
a: "Alice",
});
expectObservable(metadataStore.displaynameMap$).toBe("a", {
a: new Map<string, string>([
["@alice:example.com", "Alice"],
["@bob:example.com", "Bob"],
]),
});
});
});
it("should use userId if no display name", () => {
withTestScheduler(({ behavior, expectObservable }) => {
setUpBasicRoom();
const memberships$ = behavior("a", {
a: [mockRtcMembership("@no-name:foo.bar", "D000")],
});
const metadataStore = createMatrixMemberMetadata$(
testScope,
memberships$,
createRoomMembers$(testScope, mockMatrixRoom),
);
expectObservable(metadataStore.displaynameMap$).toBe("a", {
a: new Map<string, string>([
["@no-name:foo.bar", "@no-name:foo.bar"],
]),
});
});
});
it("should disambiguate users with same display name", () => {
withTestScheduler(({ behavior, expectObservable }) => {
setUpBasicRoom();
const memberships$ = behavior("a", {
a: [
mockRtcMembership("@bob:example.com", "DEVICE1"),
mockRtcMembership("@bob:example.com", "DEVICE2"),
mockRtcMembership("@bob:foo.bar", "BOB000"),
mockRtcMembership("@carl:example.com", "C000"),
mockRtcMembership("@evil:example.com", "E000"),
],
});
const metadataStore = createMatrixMemberMetadata$(
testScope,
memberships$,
createRoomMembers$(testScope, mockMatrixRoom),
);
expectObservable(metadataStore.displaynameMap$).toBe("a", {
a: new Map<string, string>([
// ["@local:example.com", "it's a me"],
["@bob:example.com", "Bob (@bob:example.com)"],
["@bob:example.com", "Bob (@bob:example.com)"],
["@bob:foo.bar", "Bob (@bob:foo.bar)"],
["@carl:example.com", "Carl (@carl:example.com)"],
["@evil:example.com", "Carl (@evil:example.com)"],
]),
});
});
});
it("should start to disambiguate reactivly when needed", () => {
withTestScheduler(({ behavior, expectObservable }) => {
setUpBasicRoom();
const memberships$ = behavior("ab", {
a: [mockRtcMembership("@bob:example.com", "DEVICE1")],
b: [
mockRtcMembership("@bob:example.com", "DEVICE1"),
mockRtcMembership("@bob:foo.bar", "BOB000"),
],
});
const metadataStore = createMatrixMemberMetadata$(
testScope,
memberships$,
createRoomMembers$(testScope, mockMatrixRoom),
);
expectObservable(metadataStore.displaynameMap$).toBe("ab", {
a: new Map<string, string>([["@bob:example.com", "Bob"]]),
b: new Map<string, string>([
["@bob:example.com", "Bob (@bob:example.com)"],
["@bob:foo.bar", "Bob (@bob:foo.bar)"],
]),
});
});
});
it("should keep disambiguated name when other leave", () => {
withTestScheduler(({ behavior, expectObservable }) => {
setUpBasicRoom();
const memberships$ = behavior("ab", {
a: [
mockRtcMembership("@bob:example.com", "DEVICE1"),
mockRtcMembership("@bob:foo.bar", "BOB000"),
],
b: [mockRtcMembership("@bob:example.com", "DEVICE1")],
});
const metadataStore = createMatrixMemberMetadata$(
testScope,
memberships$,
createRoomMembers$(testScope, mockMatrixRoom),
);
expectObservable(metadataStore.displaynameMap$).toBe("ab", {
a: new Map<string, string>([
["@bob:example.com", "Bob (@bob:example.com)"],
["@bob:foo.bar", "Bob (@bob:foo.bar)"],
]),
b: new Map<string, string>([
["@bob:example.com", "Bob (@bob:example.com)"],
]),
});
});
});
it("should disambiguate on name change", () => {
withTestScheduler(({ behavior, schedule, expectObservable }) => {
setUpBasicRoom();
const memberships$ = behavior("a", {
a: [
mockRtcMembership("@bob:example.com", "B000"),
mockRtcMembership("@carl:example.com", "C000"),
],
});
const metadataStore = createMatrixMemberMetadata$(
testScope,
memberships$,
createRoomMembers$(testScope, mockMatrixRoom),
);
schedule("-a", {
a: () => {
updateDisplayName("@carl:example.com", "Bob");
},
});
expectObservable(metadataStore.displaynameMap$).toBe("ab", {
a: new Map<string, string>([
["@bob:example.com", "Bob"],
["@carl:example.com", "Carl"],
]),
b: new Map<string, string>([
["@bob:example.com", "Bob (@bob:example.com)"],
["@carl:example.com", "Bob (@carl:example.com)"],
]),
});
});
});
it("should track individual member id with createDisplayNameBehavior", () => {
withTestScheduler(({ scope, behavior, schedule, expectObservable }) => {
setUpBasicRoom();
const BOB = "@bob:example.com";
const CARL = "@carl:example.com";
// for this test we build a mock environment that does all possible changes:
// - memberships join/leave
// - room join/leave
// - disambiguate
const memberships$ = behavior("ab-d", {
a: [mockRtcMembership(CARL, "C000")],
b: [
mockRtcMembership(CARL, "C000"),
// bob joins
mockRtcMembership(BOB, "B000"),
],
// c carl gets renamed to BOB
d: [
// carl leaves
mockRtcMembership(BOB, "B000"),
],
});
schedule("--a-", {
a: () => {
// carl renames
updateDisplayName(CARL, "Bob");
},
});
const metadataStore = createMatrixMemberMetadata$(
testScope,
memberships$,
createRoomMembers$(testScope, mockMatrixRoom),
);
const bob$ = metadataStore.createDisplayNameBehavior$(scope, BOB);
const carl$ = metadataStore.createDisplayNameBehavior$(scope, CARL);
expectObservable(bob$).toBe("abc-", {
a: undefined,
b: "Bob",
c: "Bob (@bob:example.com)",
// bob stays disambiguate even though carl left
// d: "Bob (@bob:example.com)",
});
expectObservable(carl$).toBe("a-cd", {
a: "Carl",
// b: "Carl",
// carl gets renamed and disambiguate
c: "Bob (@carl:example.com)",
d: undefined,
});
});
});
it("should disambiguate users with invisible characters", () => {
withTestScheduler(({ scope, behavior, expectObservable }) => {
const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB");
const bobZeroWidthSpaceRtcMember = mockRtcMembership(
"@bob2:example.org",
"BBBB",
);
const bob = mockMatrixRoomMember(bobRtcMember, {
rawDisplayName: "Bob",
});
const bobZeroWidthSpace = mockMatrixRoomMember(
bobZeroWidthSpaceRtcMember,
{
rawDisplayName: "Bo\u200bb",
},
);
fakeMemberWith(bob);
fakeMemberWith(bobZeroWidthSpace);
fakeMemberWith({ userId: "@carol:example.org" });
const memberships$ = behavior("ab", {
a: [mockRtcMembership("@carol:example.org", "1111"), bobRtcMember],
b: [
mockRtcMembership("@carol:example.org", "1111"),
bobRtcMember,
bobZeroWidthSpaceRtcMember,
],
});
const metadataStore = createMatrixMemberMetadata$(
testScope,
memberships$,
createRoomMembers$(testScope, mockMatrixRoom),
);
const bob$ = metadataStore.createDisplayNameBehavior$(
scope,
"@bob:example.org",
);
const bob2$ = metadataStore.createDisplayNameBehavior$(
scope,
"@bob2:example.org",
);
const carol$ = metadataStore.createDisplayNameBehavior$(
scope,
"@carol:example.org",
);
expectObservable(bob$).toBe("ab", {
a: "Bob",
b: "Bob (@bob:example.org)",
});
expectObservable(bob2$).toBe("ab", {
a: undefined,
b: "Bo\u200bb (@bob2:example.org)",
});
expectObservable(carol$).toBe("a-", {
a: "@carol:example.org",
});
expectObservable(metadataStore.displaynameMap$).toBe("ab", {
// Carol has no displayname - So userId is used.
a: new Map([
["@carol:example.org", "@carol:example.org"],
["@bob:example.org", "Bob"],
]),
// Other Bob joins, and should handle zero width hacks.
b: new Map([
["@carol:example.org", "@carol:example.org"],
[bobRtcMember.userId, `Bob (@bob:example.org)`],
[
bobZeroWidthSpace.userId,
`${bobZeroWidthSpace.rawDisplayName} (${bobZeroWidthSpace.userId})`,
],
]),
});
});
});
it("should strip RTL characters from displayname", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const daveRtcMember = mockRtcMembership("@dave:example.org", "DDDD");
const daveRTLRtcMember = mockRtcMembership(
"@dave2:example.org",
"DDDD",
);
const dave = mockMatrixRoomMember(daveRtcMember, {
rawDisplayName: "Dave",
});
const daveRTL = mockMatrixRoomMember(daveRTLRtcMember, {
rawDisplayName: "\u202eevaD",
});
fakeMemberWith({ userId: "@carol:example.org" });
fakeMemberWith(daveRTL);
fakeMemberWith(dave);
const memberships$ = behavior("ab", {
a: [mockRtcMembership("@carol:example.org", "DDDD")],
b: [
mockRtcMembership("@carol:example.org", "DDDD"),
daveRtcMember,
daveRTLRtcMember,
],
});
const metadataStore = createMatrixMemberMetadata$(
testScope,
memberships$,
createRoomMembers$(testScope, mockMatrixRoom),
);
expectObservable(metadataStore.displaynameMap$).toBe("ab", {
// Carol has no displayname - So userId is used.
a: new Map([["@carol:example.org", "@carol:example.org"]]),
// Both Dave's join. Since after stripping
b: new Map([
["@carol:example.org", "@carol:example.org"],
// Not disambiguated
["@dave:example.org", "Dave"],
// This one is, since it's using RTL.
["@dave2:example.org", "evaD (@dave2:example.org)"],
]),
});
});
});
});
describe("avatarUrl", () => {
function updateAvatarUrl(
userId: `@${string}:${string}`,
avatarUrl: string,
): void {
const member = fakeMembersMap.get(userId);
if (member) {
member.getMxcAvatarUrl = vi.fn().mockReturnValue(avatarUrl);
// Emit the event to notify listeners
mockMatrixRoom.emit(
RoomStateEvent.Members,
{} as unknown as MatrixEvent,
{} as unknown as RoomState,
member as RoomMember,
);
} else {
throw new Error(`No member found with userId: ${userId}`);
}
}
it("should use avatar url from room members", () => {
withTestScheduler(({ scope, behavior, expectObservable }) => {
fakeMemberWith({
userId: "@local:example.com",
});
fakeMemberWith({
userId: "@alice:example.com",
getMxcAvatarUrl: vi.fn().mockReturnValue("mxc://custom.url/avatar"),
});
const memberships$ = behavior("a", {
a: [
mockRtcMembership("@local:example.com", "DEVICE1"),
mockRtcMembership("@alice:example.com", "DEVICE1"),
],
});
const metadataStore = createMatrixMemberMetadata$(
testScope,
memberships$,
createRoomMembers$(testScope, mockMatrixRoom),
);
const local$ = metadataStore.createAvatarUrlBehavior$(
scope,
"@local:example.com",
);
const alice$ = metadataStore.createAvatarUrlBehavior$(
scope,
"@alice:example.com",
);
expectObservable(local$).toBe("a", {
a: "mxc://example.com/@local:example.com",
});
expectObservable(alice$).toBe("a", {
a: "mxc://custom.url/avatar",
});
expectObservable(metadataStore.avatarMap$).toBe("a", {
a: new Map<string, string>([
["@local:example.com", "mxc://example.com/@local:example.com"],
["@alice:example.com", "mxc://custom.url/avatar"],
]),
});
});
});
it("should update on avatar change and user join/leave", () => {
withTestScheduler(({ scope, behavior, schedule, expectObservable }) => {
fakeMemberWith({ userId: "@carl:example.com" });
fakeMemberWith({ userId: "@bob:example.com" });
const memberships$ = behavior("ab-d", {
a: [mockRtcMembership("@bob:example.com", "B000")],
b: [
mockRtcMembership("@bob:example.com", "B000"),
mockRtcMembership("@carl:example.com", "C000"),
],
d: [mockRtcMembership("@carl:example.com", "C000")],
});
const metadataStore = createMatrixMemberMetadata$(
testScope,
memberships$,
createRoomMembers$(testScope, mockMatrixRoom),
);
schedule("--c-", {
c: () => {
updateAvatarUrl(
"@carl:example.com",
"mxc://updated.me/updatedAvatar",
);
},
});
const bob$ = metadataStore.createAvatarUrlBehavior$(
scope,
"@bob:example.com",
);
const carl$ = metadataStore.createAvatarUrlBehavior$(
scope,
"@carl:example.com",
);
expectObservable(bob$).toBe("a---", {
a: "mxc://example.com/@bob:example.com",
});
expectObservable(carl$).toBe("a-c-", {
a: "mxc://example.com/@carl:example.com",
c: "mxc://updated.me/updatedAvatar",
});
expectObservable(metadataStore.avatarMap$).toBe("a-c-", {
a: new Map<string, string>([
["@bob:example.com", "mxc://example.com/@bob:example.com"],
["@carl:example.com", "mxc://example.com/@carl:example.com"],
]),
// expect an update once we update the avatar URL
c: new Map<string, string>([
["@bob:example.com", "mxc://example.com/@bob:example.com"],
["@carl:example.com", "mxc://updated.me/updatedAvatar"],
]),
});
});
});
});
});
src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.ts
-160
View File
@@ -1,160 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type RoomMember, RoomStateEvent } from "matrix-js-sdk";
import { combineLatest, fromEvent, map } from "rxjs";
import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import {
KnownMembership,
type Room as MatrixRoom,
} from "matrix-js-sdk/lib/matrix";
// eslint-disable-next-line rxjs/no-internal
import { type ObservableScope } from "../../ObservableScope";
import {
calculateDisplayName,
shouldDisambiguate,
} from "../../../utils/displayname";
import { type Behavior } from "../../Behavior";
export type RoomMemberMap = Map<
string,
Pick<RoomMember, "userId" | "getMxcAvatarUrl" | "rawDisplayName">
>;
export function roomToMembersMap(matrixRoom: MatrixRoom): RoomMemberMap {
const members = matrixRoom
.getMembersWithMembership(KnownMembership.Join)
.concat(matrixRoom.getMembersWithMembership(KnownMembership.Invite));
return members.reduce((acc, member) => {
acc.set(member.userId, {
userId: member.userId,
getMxcAvatarUrl: member.getMxcAvatarUrl.bind(member),
rawDisplayName: member.rawDisplayName,
});
return acc;
}, new Map());
}
export function createRoomMembers$(
scope: ObservableScope,
matrixRoom: MatrixRoom,
): Behavior<RoomMemberMap> {
return scope.behavior(
fromEvent(matrixRoom, RoomStateEvent.Members).pipe(
map(() => roomToMembersMap(matrixRoom)),
),
roomToMembersMap(matrixRoom),
);
}
/**
* Displayname for each member of the call. This will disambiguate
* any displayname that clashes with another member. Only members
* joined to the call are considered here.
*
* @returns Map<userId, displayname> uses the Matrix user ID as the key.
*/
// don't do this work more times than we need to. This is achieved by converting to a behavior:
export const memberDisplaynames$ = (
scope: ObservableScope,
memberships$: Behavior<Pick<CallMembership, "userId">[]>,
roomMembers$: Behavior<RoomMemberMap>,
): Behavior<Map<string, string>> => {
const logger = rootLogger.getChild("[MatrixMemberMetadata]");
// This map tracks userIds that at some point needed disambiguation.
// This is a memory leak bound to the number of participants.
// A call application will always increase the memory if there have been more members in a call.
// Its capped by room member participants.
const shouldDisambiguateTrackerMap = new Set<string>();
return scope.behavior(
combineLatest([
// Handle call membership changes
memberships$,
// Additionally handle display name changes (implicitly reacting to them)
roomMembers$,
// TODO: do we need: pauseWhen(this.pretendToBeDisconnected$),
]).pipe(
map(([memberships, roomMembers]) => {
const displaynameMap = new Map<string, string>();
// We only consider RTC members for disambiguation as they are the only visible members.
for (const rtcMember of memberships) {
const member = roomMembers.get(rtcMember.userId);
if (!member) {
logger.error(`Could not find member for user ${rtcMember.userId}`);
continue;
}
const disambiguateComputed = shouldDisambiguate(
member,
memberships,
roomMembers,
);
const disambiguate =
shouldDisambiguateTrackerMap.has(rtcMember.userId) ||
disambiguateComputed;
if (disambiguate) shouldDisambiguateTrackerMap.add(rtcMember.userId);
displaynameMap.set(
rtcMember.userId,
calculateDisplayName(member, disambiguate),
);
}
return displaynameMap;
}),
),
);
};
export const createMatrixMemberMetadata$ = (
scope: ObservableScope,
memberships$: Behavior<Pick<CallMembership, "userId">[]>,
roomMembers$: Behavior<RoomMemberMap>,
): {
createDisplayNameBehavior$: (
scope: ObservableScope,
userId: string,
) => Behavior<string | undefined>;
createAvatarUrlBehavior$: (
scope: ObservableScope,
userId: string,
) => Behavior<string | undefined>;
displaynameMap$: Behavior<Map<string, string>>;
avatarMap$: Behavior<Map<string, string | undefined>>;
} => {
const displaynameMap$ = memberDisplaynames$(
scope,
memberships$,
roomMembers$,
);
const avatarMap$ = scope.behavior(
roomMembers$.pipe(
map((roomMembers) =>
Array.from(roomMembers.keys()).reduce((acc, key) => {
acc.set(key, roomMembers.get(key)?.getMxcAvatarUrl());
return acc;
}, new Map<string, string | undefined>()),
),
),
);
return {
createDisplayNameBehavior$: (scope: ObservableScope, userId: string) =>
scope.behavior(
displaynameMap$.pipe(
map((displaynameMap) => displaynameMap.get(userId)),
),
),
createAvatarUrlBehavior$: (scope: ObservableScope, userId: string) =>
scope.behavior(
roomMembers$.pipe(
map((roomMembers) => roomMembers.get(userId)?.getMxcAvatarUrl()),
),
),
// mostly for testing purposes
displaynameMap$,
avatarMap$,
};
};
src/state/CallViewModel/remoteMembers/integration.test.ts
-223
View File
@@ -1,223 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { test, vi, expect, beforeEach, afterEach } from "vitest";
import { BehaviorSubject } from "rxjs";
import { type Room as LivekitRoom } from "livekit-client";
import EventEmitter from "events";
import fetchMock from "fetch-mock";
import { logger } from "matrix-js-sdk/lib/logger";
import {
type Epoch,
ObservableScope,
trackEpoch,
} from "../../ObservableScope.ts";
import { ECConnectionFactory } from "./ConnectionFactory.ts";
import {
type ClientGetTokenParts,
type ClientOpenIDParts,
} from "../../../livekit/auth";
import {
exampleTransportLocator,
mockMediaDevices,
mockRtcMembership,
ownMemberMock,
withTestScheduler,
} from "../../../utils/test.ts";
import { type ProcessorState } from "../../../livekit/TrackProcessorContext.tsx";
import {
createRemoteMatrixLivekitMembers$,
type RemoteMatrixLivekitMember,
} from "./MatrixLivekitMembers.ts";
import { createConnectionMap$ } from "./ConnectionMap.ts";
import { membershipsAndTransports$ } from "../../SessionBehaviors.ts";
import { localRtcMember, testJWTToken } from "../../../utils/test-fixtures.ts";
import { deepCompare } from "matrix-js-sdk/lib/utils";
// Test the integration of ConnectionMap and MatrixLivekitMembers
let testScope: ObservableScope;
let ecConnectionFactory: ECConnectionFactory;
let mockClient: ClientGetTokenParts & ClientOpenIDParts;
let lkRoomFactory: () => LivekitRoom;
const createdMockLivekitRooms: Map<string, LivekitRoom> = new Map();
beforeEach(() => {
testScope = new ObservableScope();
mockClient = {
getOpenIdToken: vi.fn().mockReturnValue(""),
_unstable_getLivekitToken: vi.fn().mockResolvedValue({ jwt: testJWTToken }),
};
lkRoomFactory = vi.fn().mockImplementation(() => {
const emitter = new EventEmitter();
const base = {
on: emitter.on.bind(emitter),
off: emitter.off.bind(emitter),
emit: emitter.emit.bind(emitter),
disconnect: vi.fn(),
remoteParticipants: new Map(),
} as unknown as LivekitRoom;
vi.mocked(base).connect = vi.fn().mockImplementation(({ url }) => {
createdMockLivekitRooms.set(url, base);
});
return base;
});
ecConnectionFactory = new ECConnectionFactory(
mockClient,
"!roomid:example.org",
"m.call#ROOM",
mockMediaDevices({}),
new BehaviorSubject<ProcessorState>({
supported: true,
processor: undefined,
}),
undefined,
false,
lkRoomFactory,
);
//TODO a bit annoying to have to do a http mock?
fetchMock.post(`path:/sfu/get`, (url) => {
const domain = new URL(url).hostname; // Extract the domain from the URL
return {
status: 200,
body: {
url: `wss://${domain}/livekit/sfu`,
jwt: testJWTToken,
},
};
});
});
afterEach(() => {
testScope.end();
fetchMock.reset();
});
test("bob, carl, then bob joining no tracks yet", () => {
withTestScheduler(({ expectObservable, behavior, scope }) => {
const bobMembership = mockRtcMembership("@bob:example.com", "BDEV000");
const carlMembership = mockRtcMembership("@carl:example.com", "CDEV000");
const daveMembership = mockRtcMembership("@dave:foo.bar", "DDEV000");
const eMarble = "abc";
const vMarble = "abc";
const memberships$ = scope.behavior(
behavior(eMarble, {
a: [bobMembership],
b: [bobMembership, carlMembership],
c: [bobMembership, carlMembership, daveMembership],
}).pipe(trackEpoch()),
);
const membershipsAndTransports = membershipsAndTransports$(
testScope,
memberships$,
);
const connectionMap$ = createConnectionMap$({
scope: testScope,
connectionFactory: ecConnectionFactory,
localTransport: exampleTransportLocator,
remoteTransports$: membershipsAndTransports.transports$,
logger: logger,
ownMembershipIdentity: ownMemberMock,
});
const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$:
membershipsAndTransports.membershipsWithTransport$,
connectionMap$,
localUser: localRtcMember,
});
expectObservable(remoteMatrixLivekitMembers$).toBe(vMarble, {
a: expect.toSatisfy((e: Epoch<RemoteMatrixLivekitMember[]>) => {
const items = e.value;
expect(items.length).toBe(1);
const item = items[0]!;
expectObservable(item.membership$).toBe("a", {
a: bobMembership,
});
expectObservable(item.connection$).toBe("a", {
a: expect.toSatisfy((co) =>
deepCompare(co.transport, bobMembership.transports[0]),
),
});
expectObservable(item.participant.value$).toBe("a", {
a: null,
});
return true;
}),
b: expect.toSatisfy((e: Epoch<RemoteMatrixLivekitMember[]>) => {
const items = e.value;
expect(items.length).toBe(2);
{
const item = items[0]!;
expectObservable(item.membership$).toBe("a", {
a: bobMembership,
});
expectObservable(item.participant.value$).toBe("a", {
a: null,
});
}
{
const item = items[1]!;
expectObservable(item.membership$).toBe("a", {
a: carlMembership,
});
expectObservable(item.participant.value$).toBe("a", {
a: null,
});
expectObservable(item.connection$).toBe("a", {
a: expect.toSatisfy((connection) =>
deepCompare(connection.transport, carlMembership.transports[0]),
),
});
}
return true;
}),
c: expect.toSatisfy((e: Epoch<RemoteMatrixLivekitMember[]>) => {
const items = e.value;
expect(items.length).toBe(3);
expectObservable(items[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(items[1].membership$).toBe("b", {
a: carlMembership,
});
{
const item = items[2]!;
expectObservable(item.membership$).toBe("a", {
a: daveMembership,
});
expectObservable(item.connection$).toBe("a", {
a: expect.toSatisfy((connection) =>
deepCompare(connection.transport, daveMembership.transports[0]),
),
});
expectObservable(item.participant.value$).toBe("a", {
a: null,
});
}
return true;
}),
x: expect.anything(),
});
});
});
src/state/CallViewModelWidget.test.ts
-3
View File
@@ -8,7 +8,6 @@ Please see LICENSE in the repository root for full details.
import { it, vi, expect } from "vitest";
import { Subject } from "rxjs";
// import * as ComponentsCore from "@livekit/components-core";
import { withCallViewModel } from "./CallViewModel/CallViewModelTestUtils.ts";
import { type CallViewModel } from "./CallViewModel/CallViewModel.ts";
import { constant } from "./Behavior.ts";
@@ -21,8 +20,6 @@ import {
import { E2eeType } from "../e2ee/e2eeType.ts";
import { MatrixRTCMode } from "../config/ConfigOptions.ts";
vi.mock("@livekit/components-core", { spy: true });
it.each([[MatrixRTCMode.Compatibility], [MatrixRTCMode.Matrix_2_0]])(
"expect leave when the host asks us to hang up (%s mode)",
async (mode) => {
src/state/ObservableScope.test.ts
-293
View File
@@ -1,293 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { BehaviorSubject, combineLatest, Subject } from "rxjs";
import { logger } from "matrix-js-sdk/lib/logger";
import { sleep } from "matrix-js-sdk/lib/utils";
import {
Epoch,
mapEpoch,
ObservableScope,
trackEpoch,
} from "./ObservableScope";
import { testScope, withTestScheduler } from "../utils/test";
describe("Epoch", () => {
it("should map the value correctly", () => {
const epoch = new Epoch(1);
const mappedEpoch = epoch.mapInner((v) => v + 1);
expect(mappedEpoch.value).toBe(2);
expect(mappedEpoch.epoch).toBe(0);
});
it("should be tracked from an observable", () => {
withTestScheduler(({ expectObservable, behavior }) => {
const observable$ = behavior("abc", {
a: 1,
b: 2,
c: 3,
});
const epochObservable$ = observable$.pipe(trackEpoch());
expectObservable(epochObservable$).toBe("abc", {
a: expect.toSatisfy((e) => e.epoch === 0 && e.value === 1),
b: expect.toSatisfy((e) => e.epoch === 1 && e.value === 2),
c: expect.toSatisfy((e) => e.epoch === 2 && e.value === 3),
});
});
});
it("can be mapped without loosing epoch information", () => {
withTestScheduler(({ expectObservable, behavior }) => {
const observable$ = behavior("abc", {
a: "A",
b: "B",
c: "C",
});
const epochObservable$ = observable$.pipe(trackEpoch());
const derivedEpoch$ = epochObservable$.pipe(
mapEpoch((e) => e + "-mapped"),
);
expectObservable(derivedEpoch$).toBe("abc", {
a: new Epoch("A-mapped", 0),
b: new Epoch("B-mapped", 1),
c: new Epoch("C-mapped", 2),
});
});
});
it("diamonds emits in a predictable order", () => {
const sb$ = new BehaviorSubject("initial");
const root$ = sb$.pipe(trackEpoch());
const derivedA$ = root$.pipe(mapEpoch((e) => e + "-A"));
const derivedB$ = root$.pipe(mapEpoch((e) => e + "-B"));
combineLatest([root$, derivedB$, derivedA$]).subscribe(
([root, derivedA, derivedB]) => {
logger.log(
"combined" +
root.epoch +
root.value +
"\n" +
derivedA.epoch +
derivedA.value +
"\n" +
derivedB.epoch +
derivedB.value,
);
},
);
sb$.next("updated");
sb$.next("ANOTERUPDATE");
});
it("behavior test", () => {
const scope = testScope();
const s$ = new Subject();
const behavior$ = scope.behavior(s$, 0);
behavior$.subscribe((value) => {
logger.log(`Received value: ${value}`);
});
s$.next(1);
s$.next(2);
s$.next(3);
s$.next(3);
s$.next(3);
s$.next(3);
s$.next(3);
s$.complete();
});
});
describe("Reconcile", () => {
beforeEach(() => {
vi.useFakeTimers();
});
afterEach(() => {
vi.useRealTimers();
});
it("should wait clean up before processing next", async () => {
vi.useFakeTimers();
const scope = testScope();
const behavior$ = new BehaviorSubject<number>(0);
const setup = vi.fn().mockImplementation(async () => await sleep(100));
const cleanup = vi
.fn()
.mockImplementation(async (n: number) => await sleep(100));
scope.reconcile(behavior$, async (value) => {
await setup();
return async (): Promise<void> => {
await cleanup(value);
};
});
// Let the initial setup process
await vi.advanceTimersByTimeAsync(120);
expect(setup).toHaveBeenCalledTimes(1);
expect(cleanup).toHaveBeenCalledTimes(0);
// Send next value
behavior$.next(1);
await vi.advanceTimersByTimeAsync(50);
// Should not have started setup for 1 yet
expect(setup).toHaveBeenCalledTimes(1);
expect(cleanup).toHaveBeenCalledTimes(1);
expect(cleanup).toHaveBeenCalledWith(0);
// Let cleanup finish
await vi.advanceTimersByTimeAsync(50);
// Now setup for 1 should have started
expect(setup).toHaveBeenCalledTimes(2);
});
it("should skip intermediates values that are not setup", async () => {
vi.useFakeTimers();
const scope = testScope();
const behavior$ = new BehaviorSubject<number>(0);
const setup = vi
.fn()
.mockImplementation(async (n: number) => await sleep(100));
const cleanupLock = Promise.withResolvers();
const cleanup = vi
.fn()
.mockImplementation(async (n: number) => await cleanupLock.promise);
scope.reconcile(behavior$, async (value) => {
await setup(value);
return async (): Promise<void> => {
await cleanup(value);
};
});
// Let the initial setup process (0)
await vi.advanceTimersByTimeAsync(120);
// Send 4 next values quickly
behavior$.next(1);
behavior$.next(2);
behavior$.next(3);
behavior$.next(4);
await vi.advanceTimersByTimeAsync(3000);
// should have only called cleanup for 0
expect(cleanup).toHaveBeenCalledTimes(1);
expect(cleanup).toHaveBeenCalledWith(0);
// Let cleanup finish
cleanupLock.resolve(undefined);
await vi.advanceTimersByTimeAsync(120);
// Now setup for 4 should have started, skipping 1,2,3
expect(setup).toHaveBeenCalledTimes(2);
expect(setup).toHaveBeenCalledWith(4);
expect(setup).not.toHaveBeenCalledWith(1);
expect(setup).not.toHaveBeenCalledWith(2);
expect(setup).not.toHaveBeenCalledWith(3);
});
it("should wait for setup to complete before starting cleanup", async () => {
vi.useFakeTimers();
const scope = testScope();
const behavior$ = new BehaviorSubject<number>(0);
const setup = vi
.fn()
.mockImplementation(async (n: number) => await sleep(3000));
const cleanupLock = Promise.withResolvers();
const cleanup = vi
.fn()
.mockImplementation(async (n: number) => await cleanupLock.promise);
scope.reconcile(behavior$, async (value) => {
await setup(value);
return async (): Promise<void> => {
await cleanup(value);
};
});
await vi.advanceTimersByTimeAsync(500);
// Setup for 0 should be in progress
expect(setup).toHaveBeenCalledTimes(1);
behavior$.next(1);
await vi.advanceTimersByTimeAsync(500);
// Should not have started setup for 1 yet
expect(setup).not.toHaveBeenCalledWith(1);
// Should not have called cleanup yet, because the setup for 0 is not done
expect(cleanup).toHaveBeenCalledTimes(0);
// Let setup for 0 finish
await vi.advanceTimersByTimeAsync(2500 + 100);
// Now cleanup for 0 should have started
expect(cleanup).toHaveBeenCalledTimes(1);
expect(cleanup).toHaveBeenCalledWith(0);
cleanupLock.resolve(undefined);
await vi.advanceTimersByTimeAsync(100);
// Now setup for 1 should have started
expect(setup).toHaveBeenCalledWith(1);
});
});
describe("behavior", () => {
it("warns with the tag and nesting depth, logging each new depth once", () => {
const warn = vi.spyOn(logger, "warn").mockImplementation(() => {});
const scope = testScope();
const source$ = new Subject<number>();
const behavior$ = scope.behavior(source$, 0, "tagged");
// Re-enter once on value 1 (depth 2), then a nested re-entry on value 2
// (depth 3), which is the shape needed to strand a middle subscriber.
behavior$.subscribe((v) => {
if (v === 1) source$.next(2);
});
behavior$.subscribe((v) => {
if (v === 2) source$.next(3);
});
const seen: number[] = [];
behavior$.subscribe((v) => seen.push(v));
source$.next(1);
const messages = warn.mock.calls.map((c) => c[0] as string);
expect(messages).toEqual([
expect.stringContaining("Behavior (tagged) re-entered at depth 2"),
expect.stringContaining("Behavior (tagged) re-entered at depth 3"),
]);
warn.mock.calls.forEach((c) => expect(c[1]).toEqual(expect.any(String)));
// The behavior settles on the newest value while the last subscriber is
// left stranded on the oldest.
expect(behavior$.value).toBe(3);
expect(seen.at(-1)).toBe(1);
});
it("warns when the scope is ended while a behavior is mid-delivery", () => {
const warn = vi.spyOn(logger, "warn").mockImplementation(() => {});
const scope = new ObservableScope();
const source$ = new Subject<number>();
const behavior$ = scope.behavior(source$, 0);
behavior$.subscribe((v) => {
if (v === 1) scope.end();
});
// A derived behavior is bound to the scope, so ending the scope
// mid-delivery unsubscribes it before the value reaches it.
const derived$ = scope.behavior(behavior$);
source$.next(1);
expect(warn).toHaveBeenCalledWith(
expect.stringContaining("Scope ended while 1 of its behaviors"),
expect.any(String),
);
expect(behavior$.value).toBe(1);
expect(derived$.value).toBe(0);
});
});
src/state/ObservableScope.ts
+8 -322
View File
@@ -1,334 +1,20 @@
/*
Copyright 2024 New Vector Ltd.
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
BehaviorSubject,
catchError,
distinctUntilChanged,
EMPTY,
endWith,
filter,
map,
type Observable,
type OperatorFunction,
share,
take,
takeUntil,
} from "rxjs";
import { ObservableScope } from "@element-hq/matrixrtc-sdk";
import { logger } from "matrix-js-sdk/lib/logger";
import { type Behavior } from "./Behavior";
type MonoTypeOperator = <T>(o: Observable<T>) => Observable<T>;
type SplitBehavior<T> = keyof T extends string | number
? { [K in keyof T as `${K}$`]: Behavior<T[K]> }
: never;
const nothing = Symbol("nothing");
/**
* A scope which limits the execution lifetime of its bound Observables.
*/
export class ObservableScope {
/**
* Number of this scope's behaviors currently mid-delivery. Used to detect a
* scope being torn down synchronously from within one of its own emissions.
*/
private delivering = 0;
private readonly ended$ = new BehaviorSubject(false);
private readonly bindImpl: MonoTypeOperator = takeUntil(
this.ended$.pipe(filter((ended) => ended)),
);
/**
* Binds an Observable to this scope, so that it completes when the scope
* ends.
*/
public bind(): MonoTypeOperator {
return this.bindImpl;
}
private readonly shareImpl: MonoTypeOperator = share({
resetOnError: false,
resetOnComplete: false,
resetOnRefCountZero: false,
});
/**
* Shares (multicasts) the Observable as a hot Observable.
*/
public readonly share: MonoTypeOperator = (input$) =>
input$.pipe(this.bindImpl, this.shareImpl);
/**
* Converts an Observable to a Behavior. If no initial value is specified, the
* Observable must synchronously emit an initial value.
*/
public behavior<T>(
setValue$: Observable<T>,
initialValue: T | typeof nothing = nothing,
// An optional label used only in the re-entrancy warning below, so that a
// torn behavior can be identified from a rageshake.
tag?: string,
): Behavior<T> {
const subject$ = new BehaviorSubject(initialValue);
// Push values from the Observable into the BehaviorSubject.
// BehaviorSubjects have an undesirable feature where if you call 'complete',
// they will no longer re-emit their current value upon subscription. We want
// to support Observables that complete (for example `of({})`), so we have to
// take care to not propagate the completion event.
// If a subscriber synchronously causes this same behavior to emit again,
// rxjs delivers the nested value to every subscriber first and then
// resumes delivering the outer (older) value to the remaining subscribers,
// leaving them permanently out of sync with the others. A single such
// re-entry can only strand a contiguous run of subscribers; stranding a
// subscriber in the middle of the list needs a nested (deeper) re-entry, so
// we report the depth and log each new depth (with a stack trace) rather
// than only the first occurrence, to make a nested re-entry visible.
let depth = 0;
let maxReportedDepth = 1;
setValue$.pipe(this.bind(), distinctUntilChanged()).subscribe({
next: (value) => {
if (depth > 0 && depth + 1 > maxReportedDepth) {
maxReportedDepth = depth + 1;
logger.warn(
`Behavior${tag ? ` (${tag})` : ""} re-entered at depth ${
depth + 1
} while delivering a value; later subscribers will be left with a stale value`,
new Error().stack,
);
}
depth++;
this.delivering++;
try {
subject$.next(value);
} finally {
depth--;
this.delivering--;
}
},
error: (err: unknown) => {
subject$.error(err);
},
});
if (subject$.value === nothing)
throw new Error("Behavior failed to synchronously emit an initial value");
return subject$ as Behavior<T>;
}
/**
* Ends the scope, causing any bound Observables to complete.
*/
public end(): void {
if (this.delivering > 0)
logger.warn(
`Scope ended while ${this.delivering} of its behaviors were still delivering a value; later subscribers will be left with a stale value`,
new Error().stack,
);
this.ended$.next(true);
}
/**
* Register a callback to be executed when the scope is ended.
*/
public onEnd(callback: () => void): void {
this.ended$
.pipe(
filter((ended) => ended),
take(1),
)
.subscribe(callback);
}
/**
* For the duration of the scope, sync some external state with the value of
* the provided Behavior by way of an async function which attempts to update
* (reconcile) the external state. The reconciliation function may return a
* clean-up callback which will be called and awaited before the next change
* in value (or the end of the scope).
*
* All calls to the function and its clean-up callbacks are serialized. If the
* value changes faster than the handlers can keep up with, intermediate
* values may be skipped.
*
* Basically, this is like React's useEffect but async and for Behaviors.
*/
public reconcile<T>(
value$: Behavior<T>,
callback: (value: T) => Promise<(() => Promise<void>) | void>,
): void {
let latestValue: T | typeof nothing = nothing;
let reconcilePromise: Promise<void> | undefined = undefined;
let cleanUp: (() => Promise<void>) | void = undefined;
let prevVal: T | typeof nothing = nothing;
// While this loop runs it will process the latest from `value$` until it caught up with the updates.
// It might skip updates from `value$` and only process the newest value after callback has resolved.
const reconcileLoop = async (): Promise<void> => {
while (latestValue !== prevVal) {
await cleanUp?.(); // Call the previous value's clean-up handler
prevVal = latestValue;
if (latestValue !== nothing) cleanUp = await callback(latestValue); // Sync current value...
// `latestValue` might have gotten updated during the `await callback`. That is why we loop here
}
};
value$
.pipe(
catchError(() => EMPTY), // Ignore errors
this.bind(), // Limit to the duration of the scope
endWith(nothing), // Clean up when the scope ends
)
.subscribe((value) => {
// Always track the latest value! The `reconcileLoop` will run until it "processed" the "last" `latestValue`.
latestValue = value;
// There's already an instance of the below 'reconcileLoop' loop running
// concurrently. So lets let the loop handle it. NEVER instanciate two `reconcileLoop`s.
if (reconcilePromise) return;
reconcilePromise = reconcileLoop().finally(() => {
reconcilePromise = undefined;
});
});
}
/**
* Splits a Behavior of objects with static properties into an object with
* Behavior properties.
*
* For example, splitting a `Behavior<{ name: string; age: number }>` results
* in an object of type `{ name$: Behavior<string>; age$: Behavior<number> }`.
*/
public splitBehavior<T extends object>(
input$: Behavior<T>,
): SplitBehavior<T> {
return Object.fromEntries(
Object.keys(input$.value).map((key) => [
`${key}$`,
this.behavior(
input$.pipe(map((input) => input[key as keyof T])),
nothing,
key,
),
]),
) as SplitBehavior<T>;
}
}
export {
Epoch,
mapEpoch,
ObservableScope,
trackEpoch,
} from "@element-hq/matrixrtc-sdk";
/**
* The global scope, a scope which never ends.
*/
export const globalScope = new ObservableScope();
/**
* `Epoch`'s can be used to create `Behavior`s and `Observable`s which derivitives can be merged
* with `combinedLatest` without duplicated emissions.
*
* This is useful in the following example:
* ```
* const rootObs$ = of("red","green","blue");
* const derivedObs$ = rootObs$.pipe(
* map((v)=> {red:"fire", green:"grass", blue:"water"}[v])
* );
* const otherDerivedObs$ = rootObs$.pipe(
* map((v)=> {red:"tomatoes", green:"leaves", blue:"sky"}[v])
* );
* const mergedObs$ = combineLatest([rootObs$, derivedObs$, otherDerivedObs$]).pipe(
* map(([color, a,b]) => color + " like " + a + " and " + b)
* );
*
* ```
* will result in 6 emissions with mismatching items like "red like fire and leaves"
*
* # Use Epoch
* ```
* const ancestorObs$ = of(1,2,3).pipe(trackEpoch());
* const derivedObs$ = ancestorObs$.pipe(
* mapEpoch((v)=> "this number: " + v)
* );
* const otherDerivedObs$ = ancestorObs$.pipe(
* mapEpoch((v)=> "multiplied by: " + v)
* );
* const mergedObs$ = combineLatest([derivedObs$, otherDerivedObs$]).pipe(
* filter((values) => values.every((v) => v.epoch === values[0].v)),
* map(([color, a, b]) => color + " like " + a + " and " + b)
* );
*
* ```
* will result in 3 emissions all matching (e.g. "blue like water and sky")
*/
export class Epoch<T> {
public readonly epoch: number;
public readonly value: T;
public constructor(value: T, epoch?: number) {
this.value = value;
this.epoch = epoch ?? 0;
}
/**
* Maps the value inside the epoch to a new value while keeping the epoch number.
* # usage
* ```
* const myEpoch$ = myObservable$.pipe(
* map(trackEpoch()),
* // this is the preferred way using mapEpoch
* mapEpoch((v)=> v+1)
* // This is how inner map can be used:
* map((epoch) => epoch.innerMap((v)=> v+1))
* // It is equivalent to:
* map((epoch) => new Epoch(epoch.value + 1, epoch.epoch))
* )
* ```
* See also `Epoch<T>`
*/
public mapInner<U>(map: (value: T) => U): Epoch<U> {
return new Epoch<U>(map(this.value), this.epoch);
}
}
/**
* A `pipe` compatible map oparator that keeps the epoch in tact but allows mapping the value.
* # usage
* ```
* const myEpoch$ = myObservable$.pipe(
* map(trackEpoch()),
* // this is the preferred way using mapEpoch
* mapEpoch((v)=> v+1)
* // This is how inner map can be used:
* map((epoch) => epoch.innerMap((v)=> v+1))
* // It is equivalent to:
* map((epoch) => new Epoch(epoch.value + 1, epoch.epoch))
* )
* ```
* See also `Epoch<T>`
*/
export function mapEpoch<T, U>(
mapFn: (value: T) => U,
): OperatorFunction<Epoch<T>, Epoch<U>> {
return map((e) => e.mapInner(mapFn));
}
/**
* # usage
* ```
* const myEpoch$ = myObservable$.pipe(
* map(trackEpoch()),
* map((epoch) => epoch.innerMap((v)=> v+1))
* )
* const derived = myEpoch$.pipe(
* mapEpoch((v)=>v^2)
* )
* ```
* See also `Epoch<T>`
*/
export function trackEpoch<T>(): OperatorFunction<T, Epoch<T>> {
return map<T, Epoch<T>>((value, number) => new Epoch(value, number));
}
src/state/SessionBehaviors.test.ts
-100
View File
@@ -1,100 +0,0 @@
/*
Copyright 2025, 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { describe, expect, it, vi } from "vitest";
import {
MatrixRTCSessionEvent,
type Transport,
type UnstableLivekitTransport,
} from "matrix-js-sdk/lib/matrixrtc";
import { EventEmitter } from "events";
import {
createKeyRotationSuppressed$,
membershipsAndTransports$,
} from "./SessionBehaviors";
import { Epoch } from "./ObservableScope";
import { constant } from "./Behavior";
import { mockRtcMembership, testScope } from "../utils/test";
describe("SessionBehaviors", () => {
describe("createKeyRotationSuppressed$", () => {
it("emits initial value from isKeyRotationSuppressed and updates when KeyRotationSuppressedChanged event is emitted", () => {
const scope = testScope();
const emitter = new EventEmitter();
const mockSession = Object.assign(emitter, {
on: vi.fn(),
off: vi.fn(),
isKeyRotationSuppressed: false,
});
const keyRotationSuppressed$ = createKeyRotationSuppressed$(
scope,
mockSession as any,
);
expect(keyRotationSuppressed$.value).toBe(false);
emitter.emit(MatrixRTCSessionEvent.KeyRotationSuppressedChanged, true);
expect(keyRotationSuppressed$.value).toBe(true);
emitter.emit(MatrixRTCSessionEvent.KeyRotationSuppressedChanged, false);
expect(keyRotationSuppressed$.value).toBe(false);
});
});
describe("membershipsAndTransports$", () => {
const serviceTransport: UnstableLivekitTransport = {
type: "livekit",
livekit_service_url: "https://lk.example.org",
};
const urlTransport: UnstableLivekitTransport = {
type: "livekit",
url: "wss://sfu.sample.com",
};
it("locates each member's transport on the member's homeserver", () => {
const bob = mockRtcMembership("@bob:example.org", "DEV000", {
fociPreferred: [serviceTransport],
});
const carl = mockRtcMembership("@carl:sample.com", "DEV111", {
fociPreferred: [urlTransport],
});
const { membershipsWithTransport$, transports$ } =
membershipsAndTransports$(
testScope(),
constant(new Epoch([bob, carl])),
);
expect(transports$.value.value).toEqual([
{ transport: serviceTransport, serverName: "example.org" },
{ transport: urlTransport, serverName: "sample.com" },
]);
expect(
membershipsWithTransport$.value.value.map(
(m) => m.transport?.serverName,
),
).toEqual(["example.org", "sample.com"]);
});
it("leaves out members without a LiveKit transport", () => {
const dave = mockRtcMembership("@dave:example.org", "DEV222", {
fociPreferred: [{ type: "something-else" } as Transport],
});
const { membershipsWithTransport$, transports$ } =
membershipsAndTransports$(testScope(), constant(new Epoch([dave])));
expect(transports$.value.value).toEqual([]);
expect(membershipsWithTransport$.value.value).toEqual([
{ membership: dave, transport: undefined },
]);
});
});
});
src/state/SessionBehaviors.ts
-106
View File
@@ -1,106 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type CallMembership,
type MatrixRTCSession,
MatrixRTCSessionEvent,
isUnstableLivekitTransport,
} from "matrix-js-sdk/lib/matrixrtc";
import { fromEvent } from "rxjs";
import {
Epoch,
mapEpoch,
trackEpoch,
type ObservableScope,
} from "./ObservableScope";
import { type Behavior } from "./Behavior";
import { type TransportLocator } from "../livekit/auth";
/**
* Tracks the transports used by ourselves, plus all other MatrixRTC session
* members.
*/
export const membershipsAndTransports$ = (
scope: ObservableScope,
memberships$: Behavior<Epoch<CallMembership[]>>,
): {
membershipsWithTransport$: Behavior<
Epoch<
{ membership: CallMembership; transport: TransportLocator | undefined }[]
>
>;
transports$: Behavior<Epoch<TransportLocator[]>>;
} => {
const membershipsWithTransport$: Behavior<
Epoch<
{ membership: CallMembership; transport: TransportLocator | undefined }[]
>
> = scope.behavior(
memberships$.pipe(
mapEpoch((memberships) => {
return memberships.map((membership) => {
const transport = membership.getTransport();
return {
membership,
transport: isUnstableLivekitTransport(transport)
? {
transport,
serverName: membership.userId.replace(/^.*?:/, ""),
}
: undefined,
};
});
}),
),
);
const transports$: Behavior<Epoch<TransportLocator[]>> = scope.behavior(
membershipsWithTransport$.pipe(
mapEpoch((mts) => mts.flatMap(({ transport: t }) => (t ? [t] : []))),
),
);
return {
membershipsWithTransport$,
transports$,
};
};
export const createMemberships$ = (
scope: ObservableScope,
matrixRTCSession: MatrixRTCSession,
): Behavior<Epoch<CallMembership[]>> => {
return scope.behavior(
fromEvent(
matrixRTCSession,
MatrixRTCSessionEvent.MembershipsChanged,
(_, memberships: CallMembership[]) => memberships,
).pipe(trackEpoch()),
new Epoch(matrixRTCSession.memberships),
);
};
/**
* Whether the session has grown large enough that MatrixRTC has stopped rotating the media
* encryption key. While this is true the key in use is still shared with new joiners, but no new
* key is generated when someone joins or leaves.
*/
export const createKeyRotationSuppressed$ = (
scope: ObservableScope,
matrixRTCSession: MatrixRTCSession,
): Behavior<boolean> => {
return scope.behavior(
fromEvent(
matrixRTCSession,
MatrixRTCSessionEvent.KeyRotationSuppressedChanged,
(suppressed: boolean) => suppressed,
),
matrixRTCSession.isKeyRotationSuppressed,
);
};
src/state/media/LocalScreenShareViewModel.ts
+2 -6
View File
@@ -6,23 +6,19 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type LocalParticipant } from "livekit-client";
import { type ObservableScope } from "@element-hq/matrixrtc-sdk";
import { type Behavior } from "../Behavior";
import {
type BaseScreenShareInputs,
type BaseScreenShareViewModel,
createBaseScreenShare,
} from "./ScreenShareViewModel";
import { type ObservableScope } from "../ObservableScope";
export interface LocalScreenShareViewModel extends BaseScreenShareViewModel {
local: true;
}
export interface LocalScreenShareInputs extends BaseScreenShareInputs {
participant$: Behavior<LocalParticipant | null>;
}
export type LocalScreenShareInputs = BaseScreenShareInputs;
export function createLocalScreenShare(
scope: ObservableScope,
src/state/media/LocalUserMediaViewModel.ts
+22 -66
View File
@@ -7,29 +7,18 @@ Please see LICENSE in the repository root for full details.
*/
import {
facingModeFromLocalTrack,
type LocalParticipant,
LocalVideoTrack,
TrackEvent,
} from "livekit-client";
import {
fromEvent,
map,
merge,
type Observable,
of,
startWith,
switchMap,
} from "rxjs";
type Behavior,
type LocalMemberMedia,
type ObservableScope,
} from "@element-hq/matrixrtc-sdk";
import { map, of, switchMap } from "rxjs";
import { logger } from "matrix-js-sdk/lib/logger";
import { type Behavior } from "../Behavior";
import {
type BaseUserMediaInputs,
type BaseUserMediaViewModel,
createBaseUserMedia,
} from "./UserMediaViewModel";
import { type ObservableScope } from "../ObservableScope";
import { alwaysShowSelf } from "../../settings/settings";
import { platform } from "../../Platform";
import { type MediaDevices } from "../MediaDevices";
@@ -51,9 +40,9 @@ export interface LocalUserMediaViewModel extends BaseUserMediaViewModel {
export interface LocalUserMediaInputs extends Omit<
BaseUserMediaInputs,
"statsType"
"media$"
> {
participant$: Behavior<LocalParticipant | null>;
media$: Behavior<LocalMemberMedia | null>;
mediaDevices: MediaDevices;
}
@@ -61,76 +50,43 @@ export function createLocalUserMedia(
scope: ObservableScope,
{ mediaDevices, ...inputs }: LocalUserMediaInputs,
): LocalUserMediaViewModel {
const baseUserMedia = createBaseUserMedia(scope, {
...inputs,
statsType: "outbound-rtp",
});
/**
* The local video track as an observable that emits whenever the track
* changes, the camera is switched, or the track is muted.
*/
const videoTrack$: Observable<LocalVideoTrack | null> =
const baseUserMedia = createBaseUserMedia(scope, inputs);
const facingMode$ = scope.behavior(
baseUserMedia.video$.pipe(
switchMap((v) => {
const track = v?.publication.track;
if (!(track instanceof LocalVideoTrack)) return of(null);
return merge(
// Watch for track restarts because they indicate a camera switch.
// This event is also emitted when unmuting the track object.
fromEvent(track, TrackEvent.Restarted).pipe(
startWith(null),
map(() => track),
switchMap((video) => video?.facingMode$ ?? of(undefined)),
),
// When the track object is muted, reset it to null.
fromEvent(track, TrackEvent.Muted).pipe(map(() => null)),
);
}),
);
return {
...baseUserMedia,
local: true,
mirror$: scope.behavior(
videoTrack$.pipe(
// Mirror only front-facing cameras (those that face the user)
map(
(track) =>
track !== null &&
facingModeFromLocalTrack(track).facingMode === "user",
),
),
mirror$: scope.behavior(
facingMode$.pipe(map((facingMode) => facingMode === "user")),
),
alwaysShow$: alwaysShowSelf.value$,
setAlwaysShow: alwaysShowSelf.setValue,
switchCamera$: scope.behavior(
platform === "desktop"
? of(null)
: videoTrack$.pipe(
map((track) => {
if (track === null) return null;
const facingMode = facingModeFromLocalTrack(track).facingMode;
: facingMode$.pipe(
// If the camera isn't front or back-facing, don't provide a switch
// camera shortcut at all
if (facingMode !== "user" && facingMode !== "environment")
return null;
// Restart the track with a camera facing the opposite direction
return (): void =>
void track
.restartTrack({
facingMode: facingMode === "user" ? "environment" : "user",
})
.then(() => {
map((facingMode) =>
facingMode === undefined
? null
: (): void =>
void inputs.media$.value
?.switchCamera()
.then((deviceId) => {
// Inform the MediaDevices which camera was chosen
const deviceId =
track.mediaStreamTrack.getSettings().deviceId;
if (deviceId !== undefined)
mediaDevices.videoInput.select(deviceId);
})
.catch((e) =>
logger.error("Failed to switch camera", facingMode, e),
);
}),
),
),
),
),
};
src/state/media/MediaViewModel.test.ts
+97 -145
View File
@@ -7,35 +7,26 @@ Please see LICENSE in the repository root for full details.
import { expect, onTestFinished, test, vi } from "vitest";
import {
type LocalTrackPublication,
LocalVideoTrack,
Track,
TrackEvent,
} from "livekit-client";
type Behavior,
constant,
type MediaStreamStats,
} from "@element-hq/matrixrtc-sdk";
import { waitFor } from "@testing-library/dom";
import { BehaviorSubject } from "rxjs";
import {
mockLocalParticipant,
mockMediaDevices,
mockRtcMembership,
mockAudioTrack,
mockLocalMedia,
mockMediaDevices,
mockMemberMedia,
mockRemoteMedia,
withTestScheduler,
mockRemoteParticipant,
mockRemoteScreenShare,
mockRtcMembership,
mockVideoTrack,
withTestScheduler,
} from "../../utils/test";
import { constant } from "../Behavior";
import { showConnectionStats } from "../../settings/settings";
global.MediaStreamTrack = class {} as unknown as {
new (): MediaStreamTrack;
prototype: MediaStreamTrack;
};
global.MediaStream = class {} as unknown as {
new (): MediaStream;
prototype: MediaStream;
};
const platformMock = vi.hoisted(() => vi.fn(() => "desktop"));
vi.mock("../../Platform", () => ({
get platform(): string {
@@ -43,49 +34,45 @@ vi.mock("../../Platform", () => ({
},
}));
const observeRtpStreamStatsMock = vi.hoisted(() => vi.fn());
vi.mock("./observeRtpStreamStats", async (importOriginal) => ({
...(await importOriginal<Record<string, unknown>>()),
observeRtpStreamStats$: observeRtpStreamStatsMock,
}));
const rtcMembership = mockRtcMembership("@alice:example.org", "AAAA");
test("control a participant's volume", () => {
const setVolumeSpy = vi.fn();
const setVolume = vi.fn();
const vm = mockRemoteMedia(
rtcMembership,
{},
mockRemoteParticipant({ setVolume: setVolumeSpy }),
mockMemberMedia({
microphone$: constant(mockAudioTrack({ setVolume })),
}),
);
withTestScheduler(({ expectObservable, schedule }) => {
schedule("-ab---c---d|", {
a() {
// Try muting by toggling
vm.togglePlaybackMuted();
expect(setVolumeSpy).toHaveBeenLastCalledWith(0);
expect(setVolume).toHaveBeenLastCalledWith(0);
},
b() {
// Try unmuting by dragging the slider back up
vm.adjustPlaybackVolume(0.6);
vm.adjustPlaybackVolume(0.8);
vm.commitPlaybackVolume();
expect(setVolumeSpy).toHaveBeenCalledWith(0.6);
expect(setVolumeSpy).toHaveBeenLastCalledWith(0.8);
expect(setVolume).toHaveBeenCalledWith(0.6);
expect(setVolume).toHaveBeenLastCalledWith(0.8);
},
c() {
// Try muting by dragging the slider back down
vm.adjustPlaybackVolume(0.2);
vm.adjustPlaybackVolume(0);
vm.commitPlaybackVolume();
expect(setVolumeSpy).toHaveBeenCalledWith(0.2);
expect(setVolumeSpy).toHaveBeenLastCalledWith(0);
expect(setVolume).toHaveBeenCalledWith(0.2);
expect(setVolume).toHaveBeenLastCalledWith(0);
},
d() {
// Try unmuting by toggling
vm.togglePlaybackMuted();
// The volume should return to the last non-zero committed volume
expect(setVolumeSpy).toHaveBeenLastCalledWith(0.8);
expect(setVolume).toHaveBeenLastCalledWith(0.8);
},
});
expectObservable(vm.playbackVolume$).toBe("ab(cd)(ef)g", {
@@ -101,58 +88,39 @@ test("control a participant's volume", () => {
});
test("control a participant's screen share volume", () => {
const setVolumeSpy = vi.fn();
const setVolume = vi.fn();
const vm = mockRemoteScreenShare(
rtcMembership,
{},
mockRemoteParticipant({ setVolume: setVolumeSpy }),
mockMemberMedia({
screenShareAudio$: constant(
mockAudioTrack({ source: "screenShareAudio", setVolume }),
),
}),
);
withTestScheduler(({ expectObservable, schedule }) => {
schedule("-ab---c---d|", {
a() {
// Try muting by toggling
vm.togglePlaybackMuted();
expect(setVolumeSpy).toHaveBeenLastCalledWith(
0,
Track.Source.ScreenShareAudio,
);
expect(setVolume).toHaveBeenLastCalledWith(0);
},
b() {
// Try unmuting by dragging the slider back up
vm.adjustPlaybackVolume(0.6);
vm.adjustPlaybackVolume(0.8);
vm.commitPlaybackVolume();
expect(setVolumeSpy).toHaveBeenCalledWith(
0.6,
Track.Source.ScreenShareAudio,
);
expect(setVolumeSpy).toHaveBeenLastCalledWith(
0.8,
Track.Source.ScreenShareAudio,
);
expect(setVolume).toHaveBeenCalledWith(0.6);
expect(setVolume).toHaveBeenLastCalledWith(0.8);
},
c() {
// Try muting by dragging the slider back down
vm.adjustPlaybackVolume(0.2);
vm.adjustPlaybackVolume(0);
vm.commitPlaybackVolume();
expect(setVolumeSpy).toHaveBeenCalledWith(
0.2,
Track.Source.ScreenShareAudio,
);
expect(setVolumeSpy).toHaveBeenLastCalledWith(
0,
Track.Source.ScreenShareAudio,
);
expect(setVolume).toHaveBeenCalledWith(0.2);
expect(setVolume).toHaveBeenLastCalledWith(0);
},
d() {
// Try unmuting by toggling
vm.togglePlaybackMuted();
// The volume should return to the last non-zero committed volume
expect(setVolumeSpy).toHaveBeenLastCalledWith(
0.8,
Track.Source.ScreenShareAudio,
);
expect(setVolume).toHaveBeenLastCalledWith(0.8);
},
});
expectObservable(vm.playbackVolume$).toBe("ab(cd)(ef)g", {
@@ -165,25 +133,25 @@ test("control a participant's screen share volume", () => {
g: 0.8,
});
});
expect(vm.audioEnabled$.value).toBe(true);
});
test("local media remembers whether it should always be shown", () => {
const vm1 = mockLocalMedia(
rtcMembership,
{},
mockLocalParticipant({}),
mockMemberMedia({ local: true }),
mockMediaDevices({}),
);
withTestScheduler(({ expectObservable, schedule }) => {
schedule("-a|", { a: () => vm1.setAlwaysShow(false) });
expectObservable(vm1.alwaysShow$).toBe("ab", { a: true, b: false });
});
// Next local media should start out *not* always shown
const vm2 = mockLocalMedia(
rtcMembership,
{},
mockLocalParticipant({}),
mockMemberMedia({ local: true }),
mockMediaDevices({}),
);
withTestScheduler(({ expectObservable, schedule }) => {
@@ -196,43 +164,24 @@ test("switch cameras", async () => {
// Camera switching is only available on mobile
platformMock.mockReturnValue("android");
onTestFinished(() => void platformMock.mockReset());
// Construct a mock video track which knows how to be restarted
const track = new LocalVideoTrack({
getConstraints() {},
addEventListener() {},
removeEventListener() {},
} as unknown as MediaStreamTrack);
let deviceId = "front camera";
const restartTrack = vi.fn(async ({ facingMode }) => {
deviceId = facingMode === "user" ? "front camera" : "back camera";
track.emit(TrackEvent.Restarted);
return Promise.resolve();
const facingMode$ = new BehaviorSubject<"user" | "environment" | undefined>(
"user",
);
// The SDK restarts the camera the other way round and says which device it
// ended up on
const switchCamera = vi.fn(async (): Promise<string> => {
const back = facingMode$.value === "user";
facingMode$.next(back ? "environment" : "user");
return Promise.resolve(back ? "back camera" : "front camera");
});
track.restartTrack = restartTrack;
Object.defineProperty(track, "mediaStreamTrack", {
get() {
return {
label: "Video",
getSettings: (): object => ({
deviceId,
facingMode: deviceId === "front camera" ? "user" : "environment",
}),
};
},
});
const selectVideoInput = vi.fn();
const vm = mockLocalMedia(
rtcMembership,
{},
mockLocalParticipant({
getTrackPublication() {
return { track } as unknown as LocalTrackPublication;
},
mockMemberMedia({
local: true,
camera$: constant(mockVideoTrack({ facingMode$ })),
switchCamera,
}),
mockMediaDevices({
videoInput: {
@@ -242,69 +191,82 @@ test("switch cameras", async () => {
},
}),
);
expect(vm.mirror$.value).toBe(true);
// Switch to back camera
vm.switchCamera$.value!();
expect(restartTrack).toHaveBeenCalledExactlyOnceWith({
facingMode: "environment",
});
expect(switchCamera).toHaveBeenCalledTimes(1);
await waitFor(() => {
expect(selectVideoInput).toHaveBeenCalledTimes(1);
expect(selectVideoInput).toHaveBeenCalledWith("back camera");
});
expect(deviceId).toBe("back camera");
expect(vm.mirror$.value).toBe(false);
// Switch to front camera
vm.switchCamera$.value!();
expect(restartTrack).toHaveBeenCalledTimes(2);
expect(restartTrack).toHaveBeenLastCalledWith({ facingMode: "user" });
expect(switchCamera).toHaveBeenCalledTimes(2);
await waitFor(() => {
expect(selectVideoInput).toHaveBeenCalledTimes(2);
expect(selectVideoInput).toHaveBeenLastCalledWith("front camera");
});
expect(deviceId).toBe("front camera");
expect(vm.mirror$.value).toBe(true);
});
test("remote media is in waiting state when participant has not yet connected", () => {
const vm = mockRemoteMedia(rtcMembership, {}, null); // null participant
test("no camera switch where the facing mode is unknown", () => {
platformMock.mockReturnValue("android");
onTestFinished(() => void platformMock.mockReset());
const vm = mockLocalMedia(
rtcMembership,
{},
mockMemberMedia({ local: true, camera$: constant(mockVideoTrack()) }),
mockMediaDevices({}),
);
expect(vm.switchCamera$.value).toBeNull();
});
test("remote media is in waiting state while its media has not arrived", () => {
const vm = mockRemoteMedia(rtcMembership, {}, null);
expect(vm.waitingForMedia$.value).toBe(true);
});
test("remote media is not in waiting state when participant is connected", () => {
const vm = mockRemoteMedia(rtcMembership, {}, mockRemoteParticipant({}));
expect(vm.waitingForMedia$.value).toBe(false);
});
test("remote media is not in waiting state when participant is connected with no publications", () => {
const vm = mockRemoteMedia(
rtcMembership,
{},
mockRemoteParticipant({
getTrackPublication: () => undefined,
getTrackPublications: () => [],
}),
);
test("remote media is not in waiting state once its media is there", () => {
const vm = mockRemoteMedia(rtcMembership, {}, mockMemberMedia());
expect(vm.waitingForMedia$.value).toBe(false);
});
test("remote media is not in waiting state when user does not intend to publish anywhere", () => {
const vm = mockRemoteMedia(
rtcMembership,
{},
mockRemoteParticipant({}),
undefined, // No room (no advertised transport)
);
const vm = mockRemoteMedia(rtcMembership, {}, null, {});
expect(vm.waitingForMedia$.value).toBe(false);
});
test("audio and video follow the tracks' mute state", () => {
const vm = mockRemoteMedia(
rtcMembership,
{},
mockMemberMedia({
microphone$: constant(mockAudioTrack({ muted$: constant(false) })),
camera$: constant(mockVideoTrack({ muted$: constant(true) })),
}),
);
expect(vm.audioEnabled$.value).toBe(true);
expect(vm.videoEnabled$.value).toBe(false);
});
test("user media polls stream stats only while the setting is on", () => {
const participant = mockRemoteParticipant({});
const vm = mockRemoteMedia(rtcMembership, {}, participant);
onTestFinished(() => showConnectionStats.setValue(false));
withTestScheduler(({ cold, expectObservable, schedule }) => {
const stats = { type: "inbound-rtp" } as RTCInboundRtpStreamStats;
observeRtpStreamStatsMock.mockImplementation(() =>
cold("-s", { s: stats }),
// A behavior that only runs its source while subscribed, as the SDK's does
const vm = mockRemoteMedia(
rtcMembership,
{},
mockMemberMedia({
microphone$: constant(
mockAudioTrack({
stats$: cold("-s", {
s: stats,
}) as unknown as Behavior<MediaStreamStats>,
}),
),
}),
);
schedule("-a-b", {
a() {
@@ -319,14 +281,4 @@ test("user media polls stream stats only while the setting is on", () => {
s: stats,
});
});
expect(observeRtpStreamStatsMock).toHaveBeenCalledWith(
participant,
Track.Source.Microphone,
"inbound-rtp",
);
expect(observeRtpStreamStatsMock).toHaveBeenCalledWith(
participant,
Track.Source.Camera,
"inbound-rtp",
);
});
src/state/media/MemberMediaViewModel.ts
+94 -169
View File
@@ -7,18 +7,16 @@ Please see LICENSE in the repository root for full details.
*/
import {
type Room as LivekitRoom,
RoomEvent as LivekitRoomEvent,
type Participant,
type Track,
} from "livekit-client";
import {
type AudioSource,
roomEventSelector,
type TrackReference,
type VideoSource,
} from "@livekit/components-core";
import { type LocalParticipant, type RemoteParticipant } from "livekit-client";
type AudioMediaTrack,
type Behavior,
E2eeType,
type EncryptionError,
type EncryptionSystem,
type MediaTrack,
type MemberMedia,
type ObservableScope,
type VideoMediaTrack,
} from "@element-hq/matrixrtc-sdk";
import {
combineLatest,
distinctUntilChanged,
@@ -31,13 +29,7 @@ import {
throttleTime,
} from "rxjs";
import { type Behavior } from "../Behavior";
import { type BaseMediaViewModel, createBaseMedia } from "./MediaViewModel";
import { type EncryptionSystem } from "../../e2ee/sharedKeyManagement";
import { type ObservableScope } from "../ObservableScope";
import { observeTrackReference$ } from "../observeTrackReference";
import { E2eeType } from "../../e2ee/e2eeType";
import { observeInboundRtpStreamStats$ } from "./observeRtpStreamStats";
import { type UserMediaViewModel } from "./UserMediaViewModel";
import { type ScreenShareViewModel } from "./ScreenShareViewModel";
import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
@@ -56,12 +48,12 @@ export enum EncryptionStatus {
*/
export interface BaseMemberMediaViewModel extends BaseMediaViewModel {
/**
* The LiveKit video track for this media.
* The video track for this media.
*/
video$: Behavior<TrackReference | undefined>;
video$: Behavior<VideoMediaTrack | undefined>;
/**
* The LiveKit transport on which this member should be publishing. Exposed
* for debugging.
* The URL of the transport on which this member should be publishing.
* Exposed for debugging.
*/
transport$: Behavior<UnstableLivekitTransport | undefined>;
/**
@@ -72,19 +64,31 @@ export interface BaseMemberMediaViewModel extends BaseMediaViewModel {
}
export interface MemberMediaInputs extends BaseMediaViewModel {
participant$: Behavior<LocalParticipant | RemoteParticipant | null>;
livekitRoom$: Behavior<LivekitRoom | undefined>;
audioSource: AudioSource;
videoSource: VideoSource;
transport$: Behavior<UnstableLivekitTransport | undefined>;
media$: Behavior<MemberMedia | null>;
audioSource: "microphone" | "screenShareAudio";
videoSource: "camera" | "screenShare";
focusUrl$: Behavior<string | undefined>;
encryptionSystem: EncryptionSystem;
}
/** The member's track for a source, undefined while there is none. */
export function memberTrack$<T extends MediaTrack>(
media$: Behavior<MemberMedia | null>,
source: "microphone" | "camera" | "screenShare" | "screenShareAudio",
): Observable<T | undefined> {
return media$.pipe(
switchMap((media) =>
media === null
? of(undefined)
: (media[`${source}$`] as Behavior<T | undefined>),
),
);
}
export function createMemberMedia(
scope: ObservableScope,
{
participant$,
livekitRoom$,
media$,
audioSource,
videoSource,
transport$,
@@ -92,183 +96,104 @@ export function createMemberMedia(
...inputs
}: MemberMediaInputs,
): BaseMemberMediaViewModel {
const trackBehavior$ = (
scope: ObservableScope,
source: Track.Source,
): Behavior<TrackReference | undefined> =>
scope.behavior(
participant$.pipe(
switchMap((p) =>
!p ? of(undefined) : observeTrackReference$(p, source),
),
const audio$ = scope.behavior(
memberTrack$<AudioMediaTrack>(media$, audioSource),
);
const video$ = scope.behavior(
memberTrack$<VideoMediaTrack>(media$, videoSource),
);
const unencrypted$ = (
track$: Behavior<MediaTrack | undefined>,
): Observable<boolean> =>
track$.pipe(
switchMap((track) =>
track === undefined
? of(false)
: track.encrypted$.pipe(map((encrypted) => !encrypted)),
),
);
const audio$ = trackBehavior$(scope, audioSource);
const video$ = trackBehavior$(scope, videoSource);
return {
...createBaseMedia(inputs),
video$,
transport$,
unencryptedWarning$: scope.behavior(
combineLatest(
[audio$, video$],
(a, v) =>
encryptionSystem.kind !== E2eeType.NONE &&
(a?.publication.isEncrypted === false ||
v?.publication.isEncrypted === false),
[unencrypted$(audio$), unencrypted$(video$)],
(a, v) => encryptionSystem.kind !== E2eeType.NONE && (a || v),
),
),
encryptionStatus$: scope.behavior(
participant$.pipe(
switchMap((participant): Observable<EncryptionStatus> => {
if (!participant) {
return of(EncryptionStatus.Connecting);
} else if (
participant.isLocal ||
encryptionSystem.kind === E2eeType.NONE
) {
media$.pipe(
switchMap((media): Observable<EncryptionStatus> => {
if (media === null) return of(EncryptionStatus.Connecting);
if (media.local || encryptionSystem.kind === E2eeType.NONE)
return of(EncryptionStatus.Okay);
} else if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) {
const perParticipant =
encryptionSystem.kind === E2eeType.PER_PARTICIPANT;
return combineLatest([
encryptionErrorObservable$(
livekitRoom$,
participant,
encryptionSystem,
"MissingKey",
),
encryptionErrorObservable$(
livekitRoom$,
participant,
encryptionSystem,
"InvalidKey",
),
observeRemoteTrackReceivingOkay$(participant, audioSource),
observeRemoteTrackReceivingOkay$(participant, videoSource),
encryptionError$(media, "MissingKey"),
encryptionError$(media, "InvalidKey"),
receivingOkay$(audio$),
receivingOkay$(video$),
]).pipe(
map(([keyMissing, keyInvalid, audioOkay, videoOkay]) => {
if (keyMissing) return EncryptionStatus.KeyMissing;
if (keyInvalid) return EncryptionStatus.KeyInvalid;
if (perParticipant && keyMissing)
return EncryptionStatus.KeyMissing;
if (keyInvalid)
return perParticipant
? EncryptionStatus.KeyInvalid
: EncryptionStatus.PasswordInvalid;
if (audioOkay || videoOkay) return EncryptionStatus.Okay;
return undefined; // no change
}),
filter((x) => !!x),
filter((x) => x !== undefined),
startWith(EncryptionStatus.Connecting),
);
} else {
return combineLatest([
encryptionErrorObservable$(
livekitRoom$,
participant,
encryptionSystem,
"InvalidKey",
),
observeRemoteTrackReceivingOkay$(participant, audioSource),
observeRemoteTrackReceivingOkay$(participant, videoSource),
]).pipe(
map(
([keyInvalid, audioOkay, videoOkay]):
| EncryptionStatus
| undefined => {
if (keyInvalid) return EncryptionStatus.PasswordInvalid;
if (audioOkay || videoOkay) return EncryptionStatus.Okay;
return undefined; // no change
},
),
filter((x) => !!x),
startWith(EncryptionStatus.Connecting),
);
}
}),
),
),
};
}
function encryptionErrorObservable$(
room$: Behavior<LivekitRoom | undefined>,
participant: Participant,
encryptionSystem: EncryptionSystem,
criteria: string,
function encryptionError$(
media: MemberMedia,
criteria: EncryptionError,
): Observable<boolean> {
return room$.pipe(
switchMap((room) => {
if (room === undefined) return of(false);
return roomEventSelector(room, LivekitRoomEvent.EncryptionError).pipe(
map((e) => {
const [err] = e;
if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) {
return (
// Ideally we would pull the participant identity from the field on the error.
// However, it gets lost in the serialization process between workers.
// So, instead we do a string match
(err?.message.includes(participant.identity) &&
err?.message.includes(criteria)) ??
false
);
} else if (encryptionSystem.kind === E2eeType.SHARED_KEY) {
return !!err?.message.includes(criteria);
}
return false;
}),
);
}),
return media.encryptionError$.pipe(
map((error) => error === criteria),
distinctUntilChanged(),
throttleTime(1000), // Throttle to avoid spamming the UI
startWith(false),
);
}
function observeRemoteTrackReceivingOkay$(
participant: Participant,
source: Track.Source,
/**
* Whether frames arrive and decode, which is the only sign that the key in
* use is the right one.
*/
function receivingOkay$(
track$: Behavior<MediaTrack | undefined>,
): Observable<boolean | undefined> {
let lastStats: {
framesDecoded: number | undefined;
framesDropped: number | undefined;
framesReceived: number | undefined;
} = {
framesDecoded: undefined,
framesDropped: undefined,
framesReceived: undefined,
};
return observeInboundRtpStreamStats$(participant, source).pipe(
map((stats) => {
if (!stats) return undefined;
const { framesDecoded, framesDropped, framesReceived } = stats;
return {
framesDecoded,
framesDropped,
framesReceived,
};
}),
filter((newStats) => !!newStats),
map((newStats): boolean | undefined => {
const oldStats = lastStats;
lastStats = newStats;
let last: { framesDecoded?: number; framesReceived?: number } = {};
return track$.pipe(
switchMap((track) => track?.stats$ ?? of(undefined)),
map((stats): boolean | undefined => {
if (stats === undefined || stats.type !== "inbound-rtp") return undefined;
const { framesDecoded, framesReceived } =
stats as RTCInboundRtpStreamStats;
const previous = last;
last = { framesDecoded, framesReceived };
if (
typeof newStats.framesReceived === "number" &&
typeof oldStats.framesReceived === "number" &&
typeof newStats.framesDecoded === "number" &&
typeof oldStats.framesDecoded === "number"
) {
const framesReceivedDelta =
newStats.framesReceived - oldStats.framesReceived;
const framesDecodedDelta =
newStats.framesDecoded - oldStats.framesDecoded;
// if we received >0 frames and managed to decode >0 frames then we treat that as success
if (framesReceivedDelta > 0) {
return framesDecodedDelta > 0;
}
}
// no change
framesReceived === undefined ||
previous.framesReceived === undefined ||
framesDecoded === undefined ||
previous.framesDecoded === undefined
)
return undefined;
const received = framesReceived - previous.framesReceived;
if (received > 0) return framesDecoded - previous.framesDecoded > 0;
return undefined; // no change
}),
filter((x) => typeof x === "boolean"),
startWith(undefined),
src/state/media/RemoteScreenShareViewModel.ts
+13 -19
View File
@@ -6,18 +6,20 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { Track, type RemoteParticipant } from "livekit-client";
import { map, of, switchMap } from "rxjs";
import {
type AudioMediaTrack,
type Behavior,
type ObservableScope,
} from "@element-hq/matrixrtc-sdk";
import { map } from "rxjs";
import { type Behavior } from "../Behavior";
import {
type BaseScreenShareInputs,
type BaseScreenShareViewModel,
createBaseScreenShare,
} from "./ScreenShareViewModel";
import { type ObservableScope } from "../ObservableScope";
import { createVolumeControls, type VolumeControls } from "../VolumeControls";
import { observeTrackReference$ } from "../observeTrackReference";
import { memberTrack$ } from "./MemberMediaViewModel";
export interface RemoteScreenShareViewModel
extends BaseScreenShareViewModel, VolumeControls {
@@ -33,7 +35,6 @@ export interface RemoteScreenShareViewModel
}
export interface RemoteScreenShareInputs extends BaseScreenShareInputs {
participant$: Behavior<RemoteParticipant | null>;
pretendToBeDisconnected$: Behavior<boolean>;
}
@@ -41,16 +42,16 @@ export function createRemoteScreenShare(
scope: ObservableScope,
{ pretendToBeDisconnected$, ...inputs }: RemoteScreenShareInputs,
): RemoteScreenShareViewModel {
const audio$ = scope.behavior(
memberTrack$<AudioMediaTrack>(inputs.media$, "screenShareAudio"),
);
return {
...createBaseScreenShare(scope, inputs),
...createVolumeControls(scope, {
pretendToBeDisconnected$,
sink$: scope.behavior(
inputs.participant$.pipe(
map(
(p) => (volume) =>
p?.setVolume(volume, Track.Source.ScreenShareAudio),
),
audio$.pipe(
map((track) => (volume: number) => track?.setVolume(volume)),
),
),
}),
@@ -59,14 +60,7 @@ export function createRemoteScreenShare(
pretendToBeDisconnected$.pipe(map((disconnected) => !disconnected)),
),
audioEnabled$: scope.behavior(
inputs.participant$.pipe(
switchMap((p) =>
p
? observeTrackReference$(p, Track.Source.ScreenShareAudio)
: of(null),
),
map(Boolean),
),
audio$.pipe(map((track) => track !== undefined)),
),
};
}
src/state/media/RemoteUserMediaViewModel.ts
+18 -21
View File
@@ -6,34 +6,33 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type RemoteParticipant } from "livekit-client";
import {
type AudioMediaTrack,
type Behavior,
type ObservableScope,
} from "@element-hq/matrixrtc-sdk";
import { combineLatest, map, of, switchMap } from "rxjs";
import { logger } from "matrix-js-sdk/lib/logger";
import { type Behavior } from "../Behavior";
import { createVolumeControls, type VolumeControls } from "../VolumeControls";
import {
type BaseUserMediaInputs,
type BaseUserMediaViewModel,
createBaseUserMedia,
} from "./UserMediaViewModel";
import { type ObservableScope } from "../ObservableScope";
import { memberTrack$ } from "./MemberMediaViewModel";
export interface RemoteUserMediaViewModel
extends BaseUserMediaViewModel, VolumeControls {
local: false;
/**
* Whether we are waiting for this user's LiveKit participant to exist. This
* could be because either we or the remote party are still connecting.
* Whether we are waiting for this user's media to arrive. This could be
* because either we or the remote party are still connecting.
*/
waitingForMedia$: Behavior<boolean>;
}
export interface RemoteUserMediaInputs extends Omit<
BaseUserMediaInputs,
"statsType"
> {
participant$: Behavior<RemoteParticipant | null>;
export interface RemoteUserMediaInputs extends BaseUserMediaInputs {
pretendToBeDisconnected$: Behavior<boolean>;
}
@@ -41,19 +40,15 @@ export function createRemoteUserMedia(
scope: ObservableScope,
{ pretendToBeDisconnected$, ...inputs }: RemoteUserMediaInputs,
): RemoteUserMediaViewModel {
const baseUserMedia = createBaseUserMedia(scope, {
...inputs,
statsType: "inbound-rtp",
});
const baseUserMedia = createBaseUserMedia(scope, inputs);
const waitingForMedia$ = scope.behavior(
combineLatest(
[inputs.livekitRoom$, inputs.participant$],
(livekitRoom, participant) =>
// If livekitRoom is undefined, the user is not attempting to publish on
// any transport and so we shouldn't expect a participant. (They might
// be a subscribe-only bot for example.)
livekitRoom !== undefined && participant === null,
[inputs.focusUrl$, inputs.media$],
// Without a transport the user is not attempting to publish anywhere
// and so we shouldn't expect media. (They might be a subscribe-only bot
// for example.)
(focusUrl, media) => focusUrl !== undefined && media === null,
),
);
waitingForMedia$.pipe(scope.bind()).subscribe((waiting) => {
@@ -65,7 +60,9 @@ export function createRemoteUserMedia(
...createVolumeControls(scope, {
pretendToBeDisconnected$,
sink$: scope.behavior(
inputs.participant$.pipe(map((p) => (volume) => p?.setVolume(volume))),
memberTrack$<AudioMediaTrack>(inputs.media$, "microphone").pipe(
map((track) => (volume: number) => track?.setVolume(volume)),
),
),
}),
local: false,
src/state/media/ScreenShareViewModel.ts
+3 -4
View File
@@ -6,9 +6,8 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { Track } from "livekit-client";
import { type ObservableScope } from "@element-hq/matrixrtc-sdk";
import { type ObservableScope } from "../ObservableScope";
import { type LocalScreenShareViewModel } from "./LocalScreenShareViewModel";
import {
createMemberMedia,
@@ -43,8 +42,8 @@ export function createBaseScreenShare(
return {
...createMemberMedia(scope, {
...inputs,
audioSource: Track.Source.ScreenShareAudio,
videoSource: Track.Source.ScreenShare,
audioSource: "screenShareAudio",
videoSource: "screenShare",
}),
type: "screen share",
};
src/state/media/UserMediaViewModel.ts
+41 -51
View File
@@ -6,6 +6,12 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type Behavior,
type MediaStreamStats,
type MediaTrack,
type ObservableScope,
} from "@element-hq/matrixrtc-sdk";
import {
BehaviorSubject,
combineLatest,
@@ -14,24 +20,17 @@ import {
Subject,
switchMap,
} from "rxjs";
import {
observeParticipantEvents,
observeParticipantMedia,
} from "@livekit/components-core";
import { ParticipantEvent, Track } from "livekit-client";
import { type ReactionOption } from "../../reactions";
import { type Behavior } from "../Behavior";
import { type LocalUserMediaViewModel } from "./LocalUserMediaViewModel";
import {
createMemberMedia,
type MemberMediaInputs,
memberTrack$,
type BaseMemberMediaViewModel,
} from "./MemberMediaViewModel";
import { type RemoteUserMediaViewModel } from "./RemoteUserMediaViewModel";
import { type ObservableScope } from "../ObservableScope";
import { showConnectionStats } from "../../settings/settings";
import { observeRtpStreamStats$ } from "./observeRtpStreamStats";
/**
* A participant's user media (i.e. their microphone and camera feed).
@@ -48,17 +47,13 @@ export interface BaseUserMediaViewModel extends BaseMemberMediaViewModel {
videoOrientation$: Behavior<"landscape" | "portrait">;
toggleCropVideo: () => void;
/**
* The expected identity of the LiveKit participant. Exposed for debugging.
* The expected identity of the member on the media backend. Exposed for debugging.
*/
rtcBackendIdentity: string;
handRaised$: Behavior<Date | null>;
reaction$: Behavior<ReactionOption | null>;
audioStreamStats$: Behavior<
RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined
>;
videoStreamStats$: Behavior<
RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined
>;
audioStreamStats$: Behavior<MediaStreamStats>;
videoStreamStats$: Behavior<MediaStreamStats>;
/**
* Set the aspect ratio of the video track to determine the orientation.
*/
@@ -72,7 +67,6 @@ export interface BaseUserMediaInputs extends Omit<
rtcBackendIdentity: string;
handRaised$: Behavior<Date | null>;
reaction$: Behavior<ReactionOption | null>;
statsType: "inbound-rtp" | "outbound-rtp";
}
export function createBaseUserMedia(
@@ -81,30 +75,37 @@ export function createBaseUserMedia(
rtcBackendIdentity,
handRaised$,
reaction$,
statsType,
...inputs
}: BaseUserMediaInputs,
): BaseUserMediaViewModel {
const { participant$ } = inputs;
const media$ = scope.behavior(
participant$.pipe(
switchMap((p) => (p && observeParticipantMedia(p)) ?? of(undefined)),
),
);
const { media$ } = inputs;
const toggleCropVideo$ = new Subject<void>();
const videoAspectRatio$ = new BehaviorSubject(NaN);
const enabled$ = (
scope: ObservableScope,
source: "microphone" | "camera",
): Behavior<boolean> =>
scope.behavior(
memberTrack$<MediaTrack>(media$, source).pipe(
switchMap((track) =>
track === undefined
? of(false)
: track.muted$.pipe(map((muted) => !muted)),
),
),
);
// The statistics are only polled while the setting asks for them
const streamStats$ = (
scope: ObservableScope,
source: Track.Source,
): Behavior<
RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined
> =>
source: "microphone" | "camera",
): Behavior<MediaStreamStats> =>
scope.behavior(
combineLatest([participant$, showConnectionStats.value$]).pipe(
switchMap(([p, showConnectionStats]) =>
p && showConnectionStats
? observeRtpStreamStats$(p, source, statsType)
: of(undefined),
combineLatest([
memberTrack$<MediaTrack>(media$, source),
showConnectionStats.value$,
]).pipe(
switchMap(([track, show]) =>
track !== undefined && show ? track.stats$ : of(undefined),
),
),
);
@@ -112,28 +113,17 @@ export function createBaseUserMedia(
return {
...createMemberMedia(scope, {
...inputs,
audioSource: Track.Source.Microphone,
videoSource: Track.Source.Camera,
audioSource: "microphone",
videoSource: "camera",
}),
type: "user",
speaking$: scope.behavior(
participant$.pipe(
switchMap((p) =>
p
? observeParticipantEvents(
p,
ParticipantEvent.IsSpeakingChanged,
).pipe(map((p) => p.isSpeaking))
: of(false),
media$.pipe(
switchMap((media) => (media === null ? of(false) : media.speaking$)),
),
),
),
audioEnabled$: scope.behavior(
media$.pipe(map((m) => m?.microphoneTrack?.isMuted === false)),
),
videoEnabled$: scope.behavior(
media$.pipe(map((m) => m?.cameraTrack?.isMuted === false)),
),
audioEnabled$: enabled$(scope, "microphone"),
videoEnabled$: enabled$(scope, "camera"),
videoOrientation$: scope.behavior(
videoAspectRatio$.pipe(
map((aspect) => (aspect > 1 ? "landscape" : "portrait")),
@@ -144,8 +134,8 @@ export function createBaseUserMedia(
rtcBackendIdentity,
handRaised$,
reaction$,
audioStreamStats$: streamStats$(scope, Track.Source.Microphone),
videoStreamStats$: streamStats$(scope, Track.Source.Camera),
audioStreamStats$: streamStats$(scope, "microphone"),
videoStreamStats$: streamStats$(scope, "camera"),
setVideoAspectRatio: (ratio) => videoAspectRatio$.next(ratio),
};
}
src/state/media/WrappedUserMediaViewModel.ts
+29 -57
View File
@@ -5,20 +5,16 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { combineLatest, map, of, switchMap } from "rxjs";
import {
type LocalParticipant,
ParticipantEvent,
type RemoteParticipant,
} from "livekit-client";
import { observeParticipantEvents } from "@livekit/components-core";
type Behavior,
generateItems,
type LocalMemberMedia,
type MemberMedia,
type ObservableScope,
} from "@element-hq/matrixrtc-sdk";
import { combineLatest, map, of, switchMap } from "rxjs";
import { type ObservableScope } from "../ObservableScope.ts";
import type { Behavior } from "../Behavior.ts";
import type { MediaDevices } from "../MediaDevices.ts";
import { observeSpeaker$ } from "./observeSpeaker.ts";
import { generateItems } from "../../utils/observable.ts";
import { type TaggedParticipant } from "../CallViewModel/remoteMembers/MatrixLivekitMembers.ts";
import { type UserMediaViewModel } from "./UserMediaViewModel.ts";
import { type ScreenShareViewModel } from "./ScreenShareViewModel.ts";
import {
@@ -82,84 +78,60 @@ export type WrappedUserMediaViewModel = UserMediaViewModel & {
bin$: Behavior<SortingBin>;
};
interface WrappedUserMediaInputs extends Omit<
type WrappedUserMediaInputs = Omit<
LocalUserMediaInputs & RemoteUserMediaInputs,
"participant$"
> {
participant: TaggedParticipant;
mediaDevices: MediaDevices;
pretendToBeDisconnected$: Behavior<boolean>;
}
"media$"
> &
(
| { local: true; media$: Behavior<LocalMemberMedia | null> }
| { local: false; media$: Behavior<MemberMedia | null> }
);
export function createWrappedUserMedia(
scope: ObservableScope,
{
participant,
mediaDevices,
pretendToBeDisconnected$,
...inputs
}: WrappedUserMediaInputs,
{ mediaDevices, pretendToBeDisconnected$, ...rest }: WrappedUserMediaInputs,
): WrappedUserMediaViewModel {
const userMedia =
participant.type === "local"
const { local, media$: _media$, ...inputs } = rest;
const userMedia = rest.local
? createLocalUserMedia(scope, {
participant$: participant.value$,
media$: rest.media$,
mediaDevices,
...inputs,
})
: createRemoteUserMedia(scope, {
participant$: participant.value$,
media$: rest.media$,
pretendToBeDisconnected$,
...inputs,
});
// TypeScript needs this widening of the type to happen in a separate statement
const participant$: Behavior<LocalParticipant | RemoteParticipant | null> =
participant.value$;
const media$: Behavior<MemberMedia | null> = _media$;
const screenShares$ = scope.behavior(
participant$.pipe(
switchMap((p) =>
p === null
? of([])
: observeParticipantEvents(
p,
ParticipantEvent.TrackPublished,
ParticipantEvent.TrackUnpublished,
ParticipantEvent.LocalTrackPublished,
ParticipantEvent.LocalTrackUnpublished,
).pipe(
media$.pipe(
switchMap((media) =>
media === null ? of(false) : media.screenShareEnabled$,
),
// Technically more than one screen share might be possible... our
// MediaViewModels don't support it though since they look for a unique
// track for the given source. So generateItems here is a bit overkill.
generateItems(
`${inputs.id} screenShares$`,
function* (p) {
if (p.isScreenShareEnabled)
yield {
keys: ["screen-share"],
data: undefined,
};
function* (enabled) {
if (enabled) yield { keys: ["screen-share"], data: undefined };
},
(scope, _data$, key) => {
const id = `${inputs.id}:${key}`;
return participant.type === "local"
? createLocalScreenShare(scope, {
...inputs,
id,
participant$: participant.value$,
})
return local
? createLocalScreenShare(scope, { ...inputs, id, media$ })
: createRemoteScreenShare(scope, {
...inputs,
id,
participant$: participant.value$,
media$,
pretendToBeDisconnected$,
});
},
),
),
),
),
);
const speaker$ = scope.behavior(observeSpeaker$(userMedia.speaking$));
src/state/media/observeRtpStreamStats.ts
-74
View File
@@ -1,74 +0,0 @@
/*
Copyright 2023, 2024 New Vector Ltd.
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
LocalTrack,
type Participant,
RemoteTrack,
type Track,
} from "livekit-client";
import {
combineLatest,
interval,
type Observable,
startWith,
switchMap,
map,
share,
} from "rxjs";
import { observeTrackReference$ } from "../observeTrackReference";
// Use a shared timer for all the stats observers so that we don't clog up the
// event loop with hundreds of timers in large calls
const refreshStats$ = interval(1000).pipe(share());
export function observeRtpStreamStats$(
participant: Participant,
source: Track.Source,
type: "inbound-rtp" | "outbound-rtp",
): Observable<
RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined
> {
return combineLatest([
observeTrackReference$(participant, source),
refreshStats$.pipe(startWith(0)),
]).pipe(
switchMap(async ([trackReference]) => {
const track = trackReference?.publication?.track;
if (
!track ||
!(track instanceof RemoteTrack || track instanceof LocalTrack)
) {
return undefined;
}
const report = await track.getRTCStatsReport();
if (!report) {
return undefined;
}
for (const v of report.values()) {
if (v.type === type) {
return v;
}
}
return undefined;
}),
startWith(undefined),
);
}
export function observeInboundRtpStreamStats$(
participant: Participant,
source: Track.Source,
): Observable<RTCInboundRtpStreamStats | undefined> {
return observeRtpStreamStats$(participant, source, "inbound-rtp").pipe(
map((x) => x as RTCInboundRtpStreamStats | undefined),
);
}
src/state/observeTrackReference.ts
-28
View File
@@ -1,28 +0,0 @@
/*
Copyright 2023, 2024 New Vector Ltd.
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
observeParticipantMedia,
type TrackReference,
} from "@livekit/components-core";
import { type Participant, type Track } from "livekit-client";
import { distinctUntilChanged, map, type Observable } from "rxjs";
/**
* Reactively reads a participant's track reference for a given media source.
*/
export function observeTrackReference$(
participant: Participant,
source: Track.Source,
): Observable<TrackReference | undefined> {
return observeParticipantMedia(participant).pipe(
map(() => participant.getTrackPublication(source)),
distinctUntilChanged(),
map((publication) => publication && { participant, publication, source }),
);
}
src/tile/GridTile.test.tsx
+3 -15
View File
@@ -5,10 +5,6 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type LocalTrackPublication,
type RemoteTrackPublication,
} from "livekit-client";
import { test, expect } from "vitest";
import { act, render, screen } from "@testing-library/react";
import { axe } from "vitest-axe";
@@ -19,10 +15,9 @@ import { GridTile } from "./GridTile";
import {
mockRtcMembership,
mockRemoteMedia,
mockRemoteParticipant,
mockLocalMedia,
mockLocalParticipant,
mockMediaDevices,
mockMemberMedia,
} from "../utils/test";
import { GridTileViewModel } from "../state/TileViewModel";
import { ReactionsSenderProvider } from "../reactions/useReactionsSender";
@@ -67,11 +62,7 @@ test("GridTile displays remote media", async () => {
rawDisplayName: "Alice",
getMxcAvatarUrl: () => "mxc://adfsg",
},
mockRemoteParticipant({
setVolume() {},
getTrackPublication: () =>
({}) as Partial<RemoteTrackPublication> as RemoteTrackPublication,
}),
mockMemberMedia(),
);
const { container } = render(
@@ -101,10 +92,7 @@ test("GridTile displays local media", async () => {
rawDisplayName: "Alice",
getMxcAvatarUrl: () => "mxc://adfsg",
},
mockLocalParticipant({
getTrackPublication: () =>
({}) as Partial<LocalTrackPublication> as LocalTrackPublication,
}),
mockMemberMedia({ local: true }),
mockMediaDevices({}),
);
src/tile/MediaView.test.tsx
+3 -26
View File
@@ -9,26 +9,13 @@ import { describe, expect, it, test } from "vitest";
import { render, screen } from "@testing-library/react";
import { axe } from "vitest-axe";
import { TooltipProvider } from "@vector-im/compound-web";
import {
type TrackReference,
type TrackReferencePlaceholder,
} from "@livekit/components-core";
import { type LocalTrackPublication, Track } from "livekit-client";
import { type ComponentProps } from "react";
import { MediaView } from "./MediaView";
import { mockLocalParticipant } from "../utils/test";
import { mockVideoTrack } from "../utils/test";
describe("MediaView", () => {
const participant = mockLocalParticipant({});
const trackReferencePlaceholder: TrackReferencePlaceholder = {
participant,
source: Track.Source.Camera,
};
const trackReference: TrackReference = {
...trackReferencePlaceholder,
publication: {} as Partial<LocalTrackPublication> as LocalTrackPublication,
};
const track = mockVideoTrack({ source: "camera" });
const baseProps: ComponentProps<typeof MediaView> = {
displayName: "some name",
@@ -38,7 +25,7 @@ describe("MediaView", () => {
mirror: false,
unencryptedWarning: false,
showNameTags: true,
video: trackReference,
video: track,
userId: "@alice:example.com",
mxcAvatarUrl: undefined,
focusable: true,
@@ -49,16 +36,6 @@ describe("MediaView", () => {
expect(await axe(container)).toHaveNoViolations();
});
describe("placeholder track", () => {
test("neither video nor avatar are shown", () => {
render(<MediaView {...baseProps} video={trackReferencePlaceholder} />);
expect(screen.queryByTestId("video")).toBeNull();
expect(
screen.queryAllByRole("img", { name: "@alice:example.com" }).length,
).toBe(0);
});
});
describe("with no video", () => {
it("shows avatar", () => {
render(<MediaView {...baseProps} video={undefined} />);
src/tile/MediaView.tsx
+19 -7
View File
@@ -5,18 +5,19 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type TrackReferenceOrPlaceholder } from "@livekit/components-core";
import { type VideoMediaTrack } from "@element-hq/matrixrtc-sdk";
import { animated } from "@react-spring/web";
import {
type FC,
type ComponentProps,
type FC,
type ReactNode,
type SyntheticEvent,
useEffect,
useRef,
useState,
} from "react";
import { useTranslation } from "react-i18next";
import classNames from "classnames";
import { VideoTrack } from "@livekit/components-react";
import { Text, Tooltip } from "@vector-im/compound-web";
import { ErrorSolidIcon } from "@vector-im/compound-design-tokens/assets/web/icons";
@@ -40,7 +41,7 @@ interface Props extends ComponentProps<typeof animated.div> {
style?: ComponentProps<typeof animated.div>["style"];
targetWidth: number;
targetHeight: number;
video: TrackReferenceOrPlaceholder | undefined;
video: VideoMediaTrack | undefined;
/**
* How to fit the video content inside the tile. When undefined, MediaView
* chooses a smart default based on the aspect ratios of the tile and video.
@@ -128,13 +129,23 @@ export const MediaView: FC<Props> = ({
setOurVideoAspectRatio(ratio);
setTheirVideoAspectRatio?.(ratio);
};
const videoElement = useRef<HTMLVideoElement | null>(null);
const videoRef = (el: HTMLVideoElement | null) => {
videoElement.current = el;
if (el !== null) setVideoAspectRatio(el.videoWidth / el.videoHeight);
};
const onResize = (ev: SyntheticEvent<HTMLVideoElement>) =>
setVideoAspectRatio(
ev.currentTarget.videoWidth / ev.currentTarget.videoHeight,
);
// The SDK plays the track on the element and watches its size, so the
// element has to be handed back before it leaves the DOM
useEffect(() => {
const el = videoElement.current;
if (video === undefined || el === null) return;
video.attach(el);
return (): void => video.detach(el);
}, [video]);
const warnings = unencryptedWarning && (
<Tooltip
@@ -186,9 +197,10 @@ export const MediaView: FC<Props> = ({
className={styles.avatar}
style={{ display: video && videoEnabled ? "none" : "initial" }}
/>
{video?.publication !== undefined && (
<VideoTrack
trackRef={video}
{video !== undefined && (
// A live camera has no captions to offer
// eslint-disable-next-line jsx-a11y/media-has-caption
<video
// There's no reason for this to be focusable
tabIndex={-1}
disablePictureInPicture
src/tile/SpotlightTile.test.tsx
+5 -6
View File
@@ -14,13 +14,12 @@ import { BehaviorSubject } from "rxjs";
import { SpotlightTile } from "./SpotlightTile";
import {
mockLocalParticipant,
mockMediaDevices,
mockRtcMembership,
mockLocalMedia,
mockRemoteMedia,
mockRemoteParticipant,
mockRemoteScreenShare,
mockMemberMedia,
} from "../utils/test";
import { SpotlightTileViewModel } from "../state/TileViewModel";
import { constant } from "../state/Behavior";
@@ -42,7 +41,7 @@ test("SpotlightTile is accessible", async () => {
rawDisplayName: "Alice",
getMxcAvatarUrl: () => "mxc://adfsg",
},
mockRemoteParticipant({}),
mockMemberMedia(),
);
const vm2 = mockLocalMedia(
@@ -51,7 +50,7 @@ test("SpotlightTile is accessible", async () => {
rawDisplayName: "Bob",
getMxcAvatarUrl: () => "mxc://dlskf",
},
mockLocalParticipant({}),
mockMemberMedia({ local: true }),
mockMediaDevices({}),
);
@@ -99,7 +98,7 @@ test("Screen share volume UI is shown when screen share has audio", async () =>
const vm = mockRemoteScreenShare(
mockRtcMembership("@alice:example.org", "AAAA"),
{},
mockRemoteParticipant({}),
mockMemberMedia(),
);
vi.spyOn(vm, "audioEnabled$", "get").mockReturnValue(constant(true));
@@ -137,7 +136,7 @@ test("Screen share volume UI is hidden when screen share has no audio", async ()
const vm = mockRemoteScreenShare(
mockRtcMembership("@alice:example.org", "AAAA"),
{},
mockRemoteParticipant({}),
mockMemberMedia(),
);
vi.spyOn(vm, "audioEnabled$", "get").mockReturnValue(constant(false));
src/tile/SpotlightTile.tsx
+2 -2
View File
@@ -29,7 +29,7 @@ import { animated } from "@react-spring/web";
import { BehaviorSubject, type Observable, map } from "rxjs";
import { useTranslation } from "react-i18next";
import classNames from "classnames";
import { type TrackReferenceOrPlaceholder } from "@livekit/components-core";
import { type VideoMediaTrack } from "@element-hq/matrixrtc-sdk";
import { Menu, MenuItem, Text } from "@vector-im/compound-web";
import FullScreenMaximiseIcon from "../icons/FullScreenMaximise.svg?react";
@@ -73,7 +73,7 @@ interface SpotlightItemBaseProps {
}
interface SpotlightMemberMediaItemBaseProps extends SpotlightItemBaseProps {
video: TrackReferenceOrPlaceholder | undefined;
video: VideoMediaTrack | undefined;
unencryptedWarning: boolean;
transport: UnstableLivekitTransport | undefined;
}
src/livekit/BlurBackgroundTransformer.ts → src/tracks/BlurBackgroundTransformer.ts
View File
src/tracks/MemberAudioRenderer.tsx
+177
View File
@@ -0,0 +1,177 @@
/*
Copyright 2025 New Vector Ltd.
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type AudioMediaTrack,
type MemberMedia,
type RemoteRTCMember,
} from "@element-hq/matrixrtc-sdk";
import { type FC, useEffect, useMemo, useState } from "react";
import { useEarpieceAudioConfig } from "../MediaDevicesContext";
import { useBehavior } from "../useBehavior";
import * as controls from "../controls";
export interface MemberAudioRendererProps {
/** The remote members whose audio should be heard; never the local one. */
members: RemoteRTCMember[];
/**
* If set to `true`, mutes every audio track.
* @remarks
* If set to `true`, the server will stop sending audio track data to the client.
*/
muted?: boolean;
}
/**
* Plays every remote member's microphone and screen share audio, one audio
* element per track. Only members of the session reach here, so anything
* else that publishes on the transport stays inaudible.
*
* It also takes care of the earpiece audio configuration for iOS devices.
* This is done by using the WebAudio API to create a stereo pan effect that
* mimics the earpiece audio.
*/
export const MemberAudioRenderer: FC<MemberAudioRendererProps> = ({
members,
muted,
}) => {
// This component is also (in addition to the "only play audio for session
// members" logic above) responsible for mimicking earpiece audio on iPhones.
// The Safari audio devices enumeration does not expose an earpiece audio device.
// We alternatively use the audioContext pan node to only use one of the stereo channels.
// This component does get additionally complicated because of a Safari bug.
// (see: https://bugs.webkit.org/show_bug.cgi?id=251532
// and the related issues: https://bugs.webkit.org/show_bug.cgi?id=237878
// and https://bugs.webkit.org/show_bug.cgi?id=231105)
//
// AudioContext gets stopped if the webview gets moved into the background.
// Once the phone is in standby audio playback will stop.
// So we can only use the pan trick only works is the phone is not in standby.
// If earpiece mode is not used we do not use audioContext to allow standby playback.
// shouldUseAudioContext is set to false if stereoPan === 0 to allow standby bluetooth playback.
const { pan: stereoPan, volume: volumeFactor } = useEarpieceAudioConfig();
const shouldUseAudioContext = stereoPan !== 0;
// initialize the potentially used audio context.
const [audioContext, setAudioContext] = useState<AudioContext | undefined>(
undefined,
);
useEffect(() => {
const ctx = new AudioContext();
setAudioContext(ctx);
return (): void => {
void ctx.close();
};
}, []);
const audioNodes = useMemo(
() => ({
gain: audioContext?.createGain(),
pan: audioContext?.createStereoPanner(),
}),
[audioContext],
);
// Simple effects to update the gain and pan node based on the props
useEffect(() => {
if (audioNodes.pan) audioNodes.pan.pan.value = stereoPan;
}, [audioNodes.pan, stereoPan]);
useEffect(() => {
if (audioNodes.gain) audioNodes.gain.gain.value = volumeFactor;
}, [audioNodes.gain, volumeFactor]);
const plugins = useMemo(
() =>
shouldUseAudioContext && audioNodes.gain && audioNodes.pan
? { context: audioContext, nodes: [audioNodes.gain, audioNodes.pan] }
: { context: undefined, nodes: [] },
[shouldUseAudioContext, audioContext, audioNodes],
);
return (
// We add all audio elements into one <div> for the browser developer tool experience/tidyness.
<div style={{ display: "none" }}>
{members.map((member) => (
<MemberAudio
key={member.id}
media$={member.media$}
muted={muted}
plugins={plugins}
/>
))}
</div>
);
};
interface AudioPlugins {
context: AudioContext | undefined;
nodes: AudioNode[];
}
const MemberAudio: FC<{
media$: RemoteRTCMember["media$"];
muted?: boolean;
plugins: AudioPlugins;
}> = ({ media$, muted, plugins }) => {
const media = useBehavior(media$);
return (
media !== null && (
<>
<AudioTrack
track$={media.microphone$}
muted={muted}
plugins={plugins}
/>
<AudioTrack
track$={media.screenShareAudio$}
muted={muted}
plugins={plugins}
/>
</>
)
);
};
const AudioTrack: FC<{
track$: MemberMedia["microphone$"];
muted?: boolean;
plugins: AudioPlugins;
}> = ({ track$, muted, plugins }) => {
const track = useBehavior(track$);
return (
track !== undefined && (
<TrackAudio track={track} muted={muted} plugins={plugins} />
)
);
};
/**
* One audio element, handed to the SDK to play on. The Web Audio graph for the
* earpiece is applied before the element so that playback starts through it.
*/
const TrackAudio: FC<{
track: AudioMediaTrack;
muted?: boolean;
plugins: AudioPlugins;
}> = ({ track, muted, plugins }) => {
const [element, setElement] = useState<HTMLAudioElement | null>(null);
useEffect(() => {
track.setAudioContext(plugins.context, plugins.nodes);
}, [track, plugins]);
useEffect(() => {
if (element === null) return;
track.attach(element);
controls.setPlaybackStarted();
return (): void => track.detach(element);
}, [track, element]);
// A live microphone has no captions to offer
// eslint-disable-next-line jsx-a11y/media-has-caption
return <audio ref={setElement} muted={muted} data-testid="audio" />;
};
src/livekit/TrackProcessorContext.test.ts → src/tracks/TrackProcessorContext.test.ts
View File
src/livekit/TrackProcessorContext.tsx → src/tracks/TrackProcessorContext.tsx
View File
src/utils/displayname-integration.test.ts
-40
View File
@@ -1,40 +0,0 @@
/*
Copyright 2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { afterEach, beforeAll, describe, expect, test, vi } from "vitest";
import { type RoomMember } from "matrix-js-sdk";
import { shouldDisambiguate } from "./displayname";
import { alice } from "./test-fixtures";
// Ideally these tests would be in ./displayname.test.ts but I can't figure out how to
// just spy on the removeHiddenChars() function without impacting the other tests.
// So, these tests are in this separate test file.
vi.mock("matrix-js-sdk/lib/utils");
describe("shouldDisambiguate", () => {
// eslint-disable-next-line @typescript-eslint/consistent-type-imports
let jsUtils: typeof import("matrix-js-sdk/lib/utils");
beforeAll(async () => {
jsUtils = await import("matrix-js-sdk/lib/utils");
vi.spyOn(jsUtils, "removeHiddenChars").mockImplementation((str) => str);
});
afterEach(() => {
vi.clearAllMocks();
});
test("should only call removeHiddenChars once for a single displayname", () => {
const room: Map<string, Pick<RoomMember, "userId">> = new Map([]);
shouldDisambiguate(alice, [], room);
expect(jsUtils.removeHiddenChars).toHaveBeenCalledTimes(1);
for (let i = 0; i < 10; i++) {
shouldDisambiguate(alice, [], room);
}
expect(jsUtils.removeHiddenChars).toHaveBeenCalledTimes(1);
});
});
src/utils/displayname.test.ts
-133
View File
@@ -1,133 +0,0 @@
/*
Copyright 2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { describe, expect, test } from "vitest";
import { calculateDisplayName, shouldDisambiguate } from "./displayname";
import {
alice,
aliceDoppelganger,
aliceDoppelgangerRtcMember,
aliceRtcMember,
bob,
bobRtcMember,
bobZeroWidthSpace,
bobZeroWidthSpaceRtcMember,
daveRTL,
} from "./test-fixtures";
import { mockMatrixRoom } from "./test";
import { roomToMembersMap } from "../state/CallViewModel/remoteMembers/MatrixMemberMetadata";
describe("shouldDisambiguate", () => {
test("should not disambiguate a solo member", () => {
const room = mockMatrixRoom({
getMembersWithMembership: () => [],
});
expect(shouldDisambiguate(alice, [], roomToMembersMap(room))).toEqual(
false,
);
});
test("should not disambiguate a member with an empty displayname", () => {
const room = mockMatrixRoom({
getMembersWithMembership: () => [alice, aliceDoppelganger],
});
expect(
shouldDisambiguate(
{ rawDisplayName: "", userId: alice.userId },
[aliceRtcMember, aliceDoppelgangerRtcMember],
roomToMembersMap(room),
),
).toEqual(false);
});
test("should disambiguate a member with RTL characters", () => {
const room = mockMatrixRoom({ getMembersWithMembership: () => [] });
expect(shouldDisambiguate(daveRTL, [], roomToMembersMap(room))).toEqual(
true,
);
});
test("should disambiguate a member with a matching displayname", () => {
const room = mockMatrixRoom({
getMembersWithMembership: () => [alice, aliceDoppelganger],
});
expect(
shouldDisambiguate(
alice,
[aliceRtcMember, aliceDoppelgangerRtcMember],
roomToMembersMap(room),
),
).toEqual(true);
expect(
shouldDisambiguate(
aliceDoppelganger,
[aliceRtcMember, aliceDoppelgangerRtcMember],
roomToMembersMap(room),
),
).toEqual(true);
});
test("should disambiguate a member with a matching displayname with hidden spaces", () => {
const room = mockMatrixRoom({
getMembersWithMembership: () => [bob, bobZeroWidthSpace],
});
expect(
shouldDisambiguate(
bob,
[bobRtcMember, bobZeroWidthSpaceRtcMember],
roomToMembersMap(room),
),
).toEqual(true);
expect(
shouldDisambiguate(
bobZeroWidthSpace,
[bobRtcMember, bobZeroWidthSpaceRtcMember],
roomToMembersMap(room),
),
).toEqual(true);
});
test.for(["Alice @foo:bar", "@foo:b", "A@foo:lice", "A @f oo: ba r"])(
"should disambiguate a member with a displayname containing a mxid-like string '%s'",
(rawDisplayName) => {
const room = mockMatrixRoom({
getMembersWithMembership: () => [alice, aliceDoppelganger],
});
expect(
shouldDisambiguate(
{ rawDisplayName, userId: alice.userId },
[],
roomToMembersMap(room),
),
).toEqual(true);
},
);
});
describe("calculateDisplayName", () => {
test.for<[{ rawDisplayName?: string; userId: string }, boolean, string]>([
[alice, false, alice.rawDisplayName],
[alice, true, `${alice.rawDisplayName} (${alice.userId})`],
// Empty strings and zero width strings that are effectively empty are resolved as userIds
[{ rawDisplayName: "", userId: alice.userId }, false, alice.userId],
[
{ rawDisplayName: "\u200b\u200b\u200b", userId: alice.userId },
false,
alice.userId,
],
[
{ rawDisplayName: alice.userId, userId: alice.userId },
false,
alice.userId,
],
// Zero width strings are kept intact
[bobZeroWidthSpace, false, bobZeroWidthSpace.rawDisplayName],
// Directional characters are stripped.
[daveRTL, false, daveRTL.rawDisplayName.slice(1)],
[daveRTL, true, `${daveRTL.rawDisplayName.slice(1)} (${daveRTL.userId})`],
// Ensure we do NOT unhomoglyth
[{ ...alice, rawDisplayName: "alice m" }, false, "alice m"],
])("correctly calculates displayname", ([member, disambiguate, result]) =>
expect(calculateDisplayName(member, disambiguate)).toEqual(result),
);
});
src/utils/displayname.ts
-114
View File
@@ -1,114 +0,0 @@
/*
Copyright 2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
removeDirectionOverrideChars,
removeHiddenChars as removeHiddenCharsUncached,
} from "matrix-js-sdk/lib/utils";
import type { RoomMember } from "matrix-js-sdk";
import type { CallMembership } from "matrix-js-sdk/lib/matrixrtc";
// Calling removeHiddenChars() can be slow on Safari, so we cache the results.
// To illustrate a simple benchmark:
// Chrome: 10,000 calls took 2.599ms
// Safari: 10,000 calls took 242ms
// See: https://github.com/element-hq/element-call/issues/3065
const removeHiddenCharsCache = new Map<string, string>();
/**
* Calls removeHiddenCharsUncached and caches the result
*/
function removeHiddenChars(str: string): string {
if (removeHiddenCharsCache.has(str)) {
return removeHiddenCharsCache.get(str)!;
}
const result = removeHiddenCharsUncached(str);
// this is naive but should be good enough for our purposes
if (removeHiddenCharsCache.size > 500) {
removeHiddenCharsCache.clear();
}
removeHiddenCharsCache.set(str, result);
return result;
}
// Borrowed from https://github.com/matrix-org/matrix-js-sdk/blob/f10deb5ef2e8f061ff005af0476034382ea128ca/src/models/room-member.ts#L409
export function shouldDisambiguate(
member: { rawDisplayName?: string; userId: string },
memberships: Pick<CallMembership, "userId">[],
roomMembers: Map<string, Pick<RoomMember, "userId">>,
): boolean {
const { rawDisplayName: displayName, userId } = member;
if (!displayName || displayName === userId) return false;
// First check if the displayname is something we consider truthy
// after stripping it of zero width characters and padding spaces
const strippedDisplayName = removeHiddenChars(displayName);
if (!strippedDisplayName) return false;
// Next check if the name contains something that look like a mxid
// If it does, it may be someone trying to impersonate someone else
// Show full mxid in this case
if (/@.+:.+/.test(displayName)) return true;
// Also show mxid if the display name contains any LTR/RTL characters as these
// make it very difficult for us to find similar *looking* display names
// E.g "Mark" could be cloned by writing "kraM" but in RTL.
if (/[\u200E\u200F\u202A-\u202F]/.test(displayName)) return true;
// Also show mxid if there are other people with the same or similar
// displayname, after hidden character removal.
return (
memberships
.map((m) => m.userId && roomMembers.get(m.userId))
// NOTE: We *should* have a room member for everyone.
.filter((m) => !!m)
.filter((m) => m.userId !== userId)
.some(
(m) =>
removeHiddenChars(calculateDisplayName(m, false)) ===
strippedDisplayName,
)
);
}
/**
* Calculates a display name for a member, optionally disambiguating it.
* @param member - The member to calculate the display name for.
* @param member.rawDisplayName - The raw display name of the member
* @param member.userId - The user ID of the member
* @param disambiguate - Whether to disambiguate the display name.
*/
export function calculateDisplayName(
member: { rawDisplayName?: string; userId: string },
disambiguate: boolean,
): string {
const { rawDisplayName: displayName, userId } = member;
if (!displayName || displayName === userId) return userId;
const resultDisplayname = removeDirectionOverrideChars(displayName);
if (disambiguate) return resultDisplayname + " (" + userId + ")";
// First check if the displayname is something we consider truthy
// after stripping it of zero width characters and padding spaces
if (!removeHiddenChars(displayName)) return userId;
// We always strip the direction override characters (LRO and RLO).
// These override the text direction for all subsequent characters
// in the paragraph so if display names contained these, they'd
// need to be wrapped in something to prevent this from leaking out
// (which we can do in HTML but not text) or we'd need to add
// control characters to the string to reset any overrides (eg.
// adding PDF characters at the end). As far as we can see,
// there should be no reason these would be necessary - rtl display
// names should flip into the correct direction automatically based on
// the characters, and you can still embed rtl in ltr or vice versa
// with the embed chars or marker chars.
return resultDisplayname;
}
src/utils/errors.ts
+45
View File
@@ -6,6 +6,19 @@ Please see LICENSE in the repository root for full details.
*/
import { type ConnectionError } from "livekit-client";
import {
ErrorCode as RTCErrorCode,
FailToGetOpenIdToken as RTCFailToGetOpenIdToken,
FailToStartLivekitConnection as RTCFailToStartLivekitConnection,
InsufficientCapacityError as RTCInsufficientCapacityError,
LivekitConnectionError as RTCLivekitConnectionError,
type MatrixRTCError,
MatrixRTCTransportMissingError as RTCTransportMissingError,
MembershipManagerError as RTCMembershipManagerError,
NoMatrix2AuthorizationService as RTCNoMatrix2AuthorizationService,
PeerConnectionTimeoutError as RTCPeerConnectionTimeoutError,
SFURoomCreationRestrictedError as RTCSFURoomCreationRestrictedError,
} from "@element-hq/matrixrtc-sdk";
import { i18n, i18nKey } from "./i18n";
@@ -292,3 +305,35 @@ export class LivekitConnectionError extends ElementCallError {
this.localisedMessageValues = { reason: cause.reasonName };
}
}
/**
* The SDK's error as the one Element Call shows, with its translation. The
* SDK speaks in codes and classes; the title and description come from here.
*/
export function fromMatrixRTCError(error: MatrixRTCError): ElementCallError {
if (error instanceof RTCTransportMissingError)
return new MatrixRTCTransportMissingError(error.domain);
if (error instanceof RTCMembershipManagerError)
return new MembershipManagerError(causeOf(error));
if (error instanceof RTCFailToGetOpenIdToken)
return new FailToGetOpenIdToken(causeOf(error));
if (error instanceof RTCNoMatrix2AuthorizationService)
return new NoMatrix2AuthorizationService(causeOf(error));
if (error instanceof RTCFailToStartLivekitConnection)
return new FailToStartLivekitConnection(error.message);
if (error instanceof RTCInsufficientCapacityError)
return new InsufficientCapacityError();
if (error instanceof RTCSFURoomCreationRestrictedError)
return new SFURoomCreationRestrictedError();
if (error instanceof RTCPeerConnectionTimeoutError)
return new PeerConnectionTimeoutError();
if (error instanceof RTCLivekitConnectionError)
return new LivekitConnectionError(error.cause as ConnectionError);
if (error.code === RTCErrorCode.CONNECTION_LOST_ERROR)
return new ConnectionLostError();
return new UnknownCallError(causeOf(error));
}
function causeOf(error: MatrixRTCError): Error {
return error.cause instanceof Error ? error.cause : error;
}
src/utils/matrix.ts
-35
View File
@@ -7,7 +7,6 @@ Please see LICENSE in the repository root for full details.
import {
ClientEvent,
calculateRetryBackoff,
createClient,
IndexedDBStore,
MemoryStore,
@@ -17,7 +16,6 @@ import {
import { type ISyncStateData, type SyncState } from "matrix-js-sdk/lib/sync";
import { logger } from "matrix-js-sdk/lib/logger";
import { secureRandomBase64Url } from "matrix-js-sdk/lib/randomstring";
import { sleep } from "matrix-js-sdk/lib/utils";
import type { ICreateClientOpts, MatrixClient, Room } from "matrix-js-sdk";
import IndexedDBWorker from "../IndexedDBWorker?worker";
@@ -333,36 +331,3 @@ export function getRelativeRoomUrl(
: "";
return `/room/#${roomPart}?${generateUrlSearchParams(roomId, encryptionSystem, viaServers).toString()}`;
}
/**
* Perform a network operation with retries on ConnectionError.
* If the error is not retryable, or the max number of retries is reached, the error is rethrown.
* Supports handling of matrix quotas.
* @param operation The operation to perform.
* @param shouldRetry Rules out errors that retrying cannot fix, on top of the
* standard backoff rule. Retries everything by default.
*/
export async function doNetworkOperationWithRetry<T>(
operation: () => Promise<T>,
shouldRetry: (e: unknown) => boolean = () => true,
): Promise<T> {
let currentRetryCount = 0;
// eslint-disable-next-line no-constant-condition
while (true) {
try {
return await operation();
} catch (e) {
currentRetryCount++;
const backoff = shouldRetry(e)
? calculateRetryBackoff(e, currentRetryCount, true)
: -1;
if (backoff < 0) {
// Max number of retries reached, or error is not retryable. rethrow the error
throw e;
}
// wait for the specified time and then retry the request
await sleep(backoff);
}
}
}
src/utils/observable.test.ts
-104
View File
@@ -1,104 +0,0 @@
/*
Copyright 2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { expect, test } from "vitest";
import { type Observable, of, Subject, switchMap } from "rxjs";
import { withTestScheduler } from "./test";
import { filterBehavior, generateItems, pauseWhen } from "./observable";
import { type Behavior } from "../state/Behavior";
test("pauseWhen", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const inputMarbles = " abcdefgh-i-jk-";
const pauseMarbles = " n-y--n-yn-y--n";
const outputMarbles = "abc--fgh-i---k";
expectObservable(
behavior(inputMarbles).pipe(
pauseWhen(behavior(pauseMarbles, { y: true, n: false })),
),
).toBe(outputMarbles);
});
});
test("generateItems", () => {
const scope1$ = new Subject<string>();
const scope2$ = new Subject<string>();
const scope3$ = new Subject<string>();
const scope4$ = new Subject<string>();
const scopeSubjects = [scope1$, scope2$, scope3$, scope4$];
withTestScheduler(({ hot, expectObservable }) => {
// Each scope should start when the input number reaches or surpasses their
// number and end when the input number drops back below their number.
// At the very end, unsubscribing should end all remaining scopes.
const inputMarbles = " 123242";
const outputMarbles = " abcbdb";
const subscriptionMarbles = "^-----!";
const scope1Marbles = " y-----n";
const scope2Marbles = " -y----n";
const scope3Marbles = " --ynyn";
const scope4Marbles = " ----yn";
expectObservable(
hot<string>(inputMarbles).pipe(
generateItems(
"test items",
function* (input) {
for (let i = 1; i <= +input; i++) {
yield { keys: [i], data: undefined };
}
},
(scope, data$, i) => {
scopeSubjects[i - 1].next("y");
scope.onEnd(() => scopeSubjects[i - 1].next("n"));
return i.toString();
},
),
),
subscriptionMarbles,
).toBe(outputMarbles, {
a: ["1"],
b: ["1", "2"],
c: ["1", "2", "3"],
d: ["1", "2", "3", "4"],
});
expectObservable(scope1$).toBe(scope1Marbles);
expectObservable(scope2$).toBe(scope2Marbles);
expectObservable(scope3$).toBe(scope3Marbles);
expectObservable(scope4$).toBe(scope4Marbles);
});
});
test("filterBehavior", () => {
withTestScheduler(({ behavior, expectObservable }) => {
// Filtering the input should segment it into 2 modes of non-null behavior.
const inputMarbles = " abcxabx";
const filteredMarbles = "a--xa-x";
const input$ = behavior(inputMarbles, {
a: "a",
b: "b",
c: "c",
x: null,
});
const filtered$: Observable<Behavior<string> | null> = input$.pipe(
filterBehavior((value) => typeof value === "string"),
);
expectObservable(filtered$).toBe(filteredMarbles, {
a: expect.any(Object),
x: null,
});
expectObservable(
filtered$.pipe(
switchMap((value$) => (value$ === null ? of(null) : value$)),
),
).toBe(inputMarbles, { a: "a", b: "b", c: "c", x: null });
});
});
src/utils/observable.ts
+12 -281
View File
@@ -5,53 +5,22 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type Observable,
audit,
concat,
defer,
filter,
finalize,
map,
of,
scan,
startWith,
takeWhile,
tap,
withLatestFrom,
BehaviorSubject,
type OperatorFunction,
distinctUntilChanged,
} from "rxjs";
import { logger } from "matrix-js-sdk/lib/logger";
import { type Behavior, type ObservableScope } from "@element-hq/matrixrtc-sdk";
import { type Observable, concat, map, scan, startWith, takeWhile } from "rxjs";
import { type Behavior } from "../state/Behavior";
import { Epoch, ObservableScope } from "../state/ObservableScope";
const nothing = Symbol("nothing");
/**
* RxJS operator that invokes a callback when the Observable is finalized,
* passing the most recently emitted value. If no value was emitted, the
* callback will not be invoked.
*/
export function finalizeValue<T>(callback: (finalValue: T) => void) {
return (source$: Observable<T>): Observable<T> =>
defer(() => {
let finalValue: T | typeof nothing = nothing;
return source$.pipe(
tap((value) => (finalValue = value)),
finalize(() => {
if (finalValue !== nothing) callback(finalValue);
}),
);
});
}
export {
filterBehavior,
finalizeValue,
generateItems,
generateItemsWithEpoch,
pauseWhen,
} from "@element-hq/matrixrtc-sdk";
/**
* RxJS operator that accumulates a state from a source of events. This is like
* scan, except it emits an initial value immediately before any events arrive.
*/
export function accumulate<State, Event>(
initial: State,
update: (state: State, event: Event) => State,
@@ -105,249 +74,11 @@ export function switchWhen<A, B>(
* React, where an initial value is needed. You should never use it to create an
* Observable derived from another Observable; use reactive operators instead.
*/
const nothing = Symbol("nothing");
export function getValue<T>(state$: Observable<T>): T {
let value: T | typeof nothing = nothing;
state$.subscribe((x) => (value = x)).unsubscribe();
if (value === nothing) throw new Error("Not a state Observable");
return value;
}
/**
* RxJS operator that pauses all changes in the input value whenever a Behavior
* is true. When the Behavior returns to being false, the most recently
* suppressed change is emitted as the most recent value.
*/
export function pauseWhen<T>(pause$: Behavior<boolean>) {
return (value$: Observable<T>): Observable<T> =>
value$.pipe(
withLatestFrom(pause$),
audit(([, pause]) =>
pause ? pause$.pipe(filter((pause) => !pause)) : of(null),
),
map(([value]) => value),
);
}
interface ItemHandle<Keys extends unknown[], Data, Item> {
scope: ObservableScope;
keys: readonly [...Keys];
data$: BehaviorSubject<Data>;
item: Item;
}
/**
* Maps a changing input value to a collection of items that each capture some
* dynamic data and are tied to a key. Items will be automatically created when
* their key is requested for the first time, reused when the same key is
* requested at a later time, and destroyed (have their scope ended) when the
* key is no longer requested.
*
* @param name A name for this collection to use in debug logs.
* @param generator A generator function yielding a tuple of keys and the
* currently associated data for each item that it wants to exist.
* @param factory A function constructing an individual item, given the item's key,
* dynamic data, and an automatically managed ObservableScope for the item.
*/
export function generateItems<
Input,
Keys extends [unknown, ...unknown[]],
Data,
Item,
>(
name: string,
generator: (
input: Input,
) => Iterable<{ keys: readonly [...Keys]; data: Data }, void, void>,
factory: (
scope: ObservableScope,
data$: Behavior<Data>,
...keys: Keys
) => Item,
): OperatorFunction<Input, Item[]> {
return generateItemsInternal(name, generator, factory, (items) => items);
}
/**
* Same as generateItems, but preserves epoch data.
*/
export function generateItemsWithEpoch<
Input,
Keys extends [unknown, ...unknown[]],
Data,
Item,
>(
name: string,
generator: (
input: Input,
) => Iterable<{ keys: readonly [...Keys]; data: Data }, void, void>,
factory: (
scope: ObservableScope,
data$: Behavior<Data>,
...keys: Keys
) => Item,
): OperatorFunction<Epoch<Input>, Epoch<Item[]>> {
return generateItemsInternal(
name,
function* (input) {
yield* generator(input.value);
},
factory,
(items, input) => new Epoch(items, input.epoch),
);
}
/**
* Segments a behavior into periods during which its value matches the filter
* (outputting a behavior with a narrowed type) and periods during which it does
* not match (outputting null).
*/
export function filterBehavior<T, S extends T>(
predicate: (value: T) => value is S,
): OperatorFunction<T, Behavior<S> | null> {
return (input$) =>
input$.pipe(
scan<T, BehaviorSubject<S> | null>((acc$, input) => {
if (predicate(input)) {
const output$ = acc$ ?? new BehaviorSubject(input);
output$.next(input);
return output$;
}
return null;
}, null),
distinctUntilChanged(),
);
}
/**
* Applies a given projection to each value emitted by the source Observable.
* The `project` function additionally receives an {@link ObservableScope}.
*/
export function mapScoped<A, B>(
name: string,
project: (a: A, scope: ObservableScope) => B,
): OperatorFunction<A, B> {
return (a$) =>
a$.pipe(
generateItemsInternal(
name,
function* (a) {
yield { keys: [a], data: undefined };
},
(scope, _data$, a) => project(a, scope),
(items) => items,
),
map(([item]) => item),
);
}
function generateItemsInternal<
Input,
Keys extends [unknown, ...unknown[]],
Data,
Item,
Output,
>(
name: string,
generator: (
input: Input,
) => Iterable<{ keys: readonly [...Keys]; data: Data }, void, void>,
factory: (
scope: ObservableScope,
data$: Behavior<Data>,
...keys: Keys
) => Item,
project: (items: Item[], input: Input) => Output,
): OperatorFunction<Input, Output> {
/* eslint-disable @typescript-eslint/no-explicit-any */
return (input$) =>
input$.pipe(
// Keep track of the existing items over time, so they can persist
scan<
Input,
{
map: Map<any, any>;
items: Set<ItemHandle<Keys, Data, Item>>;
input: Input;
},
{ map: Map<any, any>; items: Set<ItemHandle<Keys, Data, Item>> }
>(
({ map: prevMap, items: prevItems }, input) => {
const nextMap = new Map();
const nextItems = new Set<ItemHandle<Keys, Data, Item>>();
for (const { keys, data } of generator(input)) {
// Disable type checks for a second to grab the item out of a nested map
let i: any = prevMap;
for (const key of keys) i = i?.get(key);
let item = i as ItemHandle<Keys, Data, Item> | undefined;
if (item === undefined) {
// First time requesting the key; create the item
const scope = new ObservableScope();
const data$ = new BehaviorSubject(data);
logger.debug(
`[${name}] Creating item with keys ${keys.join(", ")}`,
);
item = {
scope,
keys,
data$,
item: factory(scope, data$, ...keys),
};
} else {
item.data$.next(data);
}
// Likewise, disable type checks to insert the item in the nested map
let m: Map<any, any> = nextMap;
for (let i = 0; i < keys.length - 1; i++) {
let inner = m.get(keys[i]);
if (inner === undefined) {
inner = new Map();
m.set(keys[i], inner);
}
m = inner;
}
const finalKey = keys[keys.length - 1];
if (m.has(finalKey))
throw new Error(
`Keys must be unique (tried to generate multiple items for key ${keys.join(", ")})`,
);
m.set(keys[keys.length - 1], item);
nextItems.add(item);
}
// Destroy all items that are no longer being requested
for (const item of prevItems)
if (!nextItems.has(item)) {
logger.debug(
`[${name}] Destroying item with keys ${item.keys.join(", ")}`,
);
item.scope.end();
}
return { map: nextMap, items: nextItems, input };
},
{ map: new Map(), items: new Set() },
),
finalizeValue(({ items }) => {
// Destroy all remaining items when no longer subscribed
logger.debug(
`[${name}] End of scope, destroying all ${items.size} items…`,
);
for (const item of items) {
logger.debug(
`[${name}] Destroying item with keys ${item.keys.join(", ")}`,
);
item.scope.end();
}
}),
map(({ items, input }) =>
project(
[...items].map(({ item }) => item),
input,
),
),
);
/* eslint-enable @typescript-eslint/no-explicit-any */
}
src/utils/test-fixtures.ts
-21
View File
@@ -63,24 +63,3 @@ export const daveRTLRtcMember = mockRtcMembership("@dave2:example.org", "DDDD");
export const daveRTL = mockMatrixRoomMember(daveRTLRtcMember, {
rawDisplayName: "\u202eevaD",
});
export function createJWTToken(sub: string, room: string): string {
return [
{}, // header
{
// payload
sub,
video: {
room,
},
},
{}, // signature
]
.map((d) => global.btoa(JSON.stringify(d)))
.join(".");
}
export const testJWTToken = createJWTToken(
"@me:example.org:ABCDEF",
"!example_room_id",
);
src/utils/test-viewmodel.ts
+47 -17
View File
@@ -16,9 +16,8 @@ import {
type Room,
SyncState,
} from "matrix-js-sdk";
import { ConnectionState, type Room as LivekitRoom } from "livekit-client";
import { E2eeType } from "../e2ee/e2eeType";
import { E2eeType } from "@element-hq/matrixrtc-sdk";
import {
type CallViewModel,
createCallViewModel$,
@@ -26,20 +25,22 @@ import {
} from "../state/CallViewModel/CallViewModel";
import {
mockConfig,
mockLivekitRoom,
mockLocalParticipant,
mockMatrixRoom,
mockMatrixRTCClient,
mockMediaDevices,
mockMemberMedia,
mockMuteStates,
MockRTCSession,
mockRTCMember,
testScope,
} from "./test";
import { type MediaDevices } from "../state/MediaDevices";
import { aliceRtcMember, localRtcMember } from "./test-fixtures";
import { type RaisedHandInfo, type ReactionInfo } from "../reactions";
import { constant } from "../state/Behavior";
import { MatrixRTCMode } from "../config/ConfigOptions";
import { constant } from "@element-hq/matrixrtc-sdk";
import { map } from "rxjs";
import { createCallFooterViewModel } from "../components/CallFooterViewModel";
import { createSentCallNotification$ } from "../state/CallViewModel/CallNotificationLifecycle";
import { HeaderStyle } from "../UrlParams";
import { type FooterSnapshot } from "../components/CallFooter";
import { type ViewModel } from "../state/ViewModel";
@@ -159,29 +160,58 @@ export function getBasicCallViewModelEnvironment(
const scope = testScope();
const muteStates = mockMuteStates();
const mediaDevices = mediaDevicesOverride ?? mockMediaDevices({});
// Every membership is a member whose media has arrived
const rtcClient = mockMatrixRTCClient(scope, {
localMember$: scope.behavior(
rtcMemberships$.pipe(
map((memberships) =>
memberships.some((m) => m.userId === localRtcMember.userId)
? mockRTCMember(true, {
membership: localRtcMember,
roomMember: members.find(
(m) => m.userId === localRtcMember.userId,
),
transportUrl: "https://example.com",
media$: constant(mockMemberMedia({ local: true })),
})
: null,
),
),
),
remoteMembers$: scope.behavior(
rtcMemberships$.pipe(
map((memberships) =>
memberships
.filter((m) => m.userId !== localRtcMember.userId)
.map((membership) =>
mockRTCMember(false, {
membership,
roomMember: members.find((m) => m.userId === membership.userId),
transportUrl: "https://example.com",
media$: constant(mockMemberMedia()),
}),
),
),
),
),
});
const vm = createCallViewModel$(
scope,
rtcSession.asMockedSession(),
rtcClient,
matrixRoom,
mediaDevices,
muteStates,
{
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
livekitRoomFactory: (): LivekitRoom =>
mockLivekitRoom({
localParticipant: mockLocalParticipant({ identity: "" }),
remoteParticipants: new Map(),
disconnect: async () => Promise.resolve(),
setE2EEEnabled: async () => Promise.resolve(),
}),
connectionState$: constant(ConnectionState.Connected),
matrixRTCMode: MatrixRTCMode.Compatibility,
sentCallNotification$: createSentCallNotification$(
scope,
rtcSession.asMockedSession(),
),
windowSize$: constant({ width: 1000, height: 800 }),
...callViewModelOptions,
},
handRaisedSubject$,
reactionsSubject$,
constant({ processor: undefined, supported: false }),
);
const footerVm = createCallFooterViewModel(
testScope(),
src/utils/test.ts
+198 -135
View File
@@ -4,7 +4,14 @@ Copyright 2023, 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { map, type Observable, of, type SchedulerLike } from "rxjs";
import {
combineLatest,
map,
NEVER,
type Observable,
of,
type SchedulerLike,
} from "rxjs";
import { type RunHelpers, TestScheduler } from "rxjs/testing";
import {
expect,
@@ -36,23 +43,13 @@ import {
type Transport,
} from "matrix-js-sdk/lib/matrixrtc";
import { type MembershipManagerEventHandlerMap } from "matrix-js-sdk/lib/matrixrtc/IMembershipManager";
import {
type LocalParticipant,
type LocalTrackPublication,
type Participant,
type RemoteParticipant,
type RemoteTrackPublication,
type Room as LivekitRoom,
Track,
} from "livekit-client";
import { type RemoteParticipant } from "livekit-client";
import { randomUUID } from "crypto";
import { type TrackReference } from "@livekit/components-core";
import EventEmitter from "events";
import {
type KeyTransportEvents,
type KeyTransportEventsHandlerMap,
} from "matrix-js-sdk/lib/matrixrtc/IKeyTransport";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import { E2eeType } from "../e2ee/e2eeType";
import {
@@ -61,7 +58,20 @@ import {
} from "../config/ConfigOptions";
import { Config } from "../config/Config";
import { type MediaDevices } from "../state/MediaDevices";
import { type Behavior, constant } from "../state/Behavior";
import {
type AudioMediaTrack,
type Behavior,
constant,
type DisconnectReason,
type LocalMemberMedia,
type LocalRTCMember,
type MatrixRTCClient,
type MatrixRTCError,
type MemberMedia,
type RemoteRTCMember,
type TransportMetadata,
type VideoMediaTrack,
} from "@element-hq/matrixrtc-sdk";
import { ObservableScope } from "../state/ObservableScope";
import { MuteStates } from "../state/MuteStates";
import { nullHostBridge } from "../HostBridge";
@@ -77,8 +87,6 @@ import {
createRemoteScreenShare,
type RemoteScreenShareViewModel,
} from "../state/media/RemoteScreenShareViewModel";
import { Connection } from "../state/CallViewModel/remoteMembers/Connection";
import { type TransportLocator } from "../livekit/auth";
export function withFakeTimers(continuation: () => void): void {
vi.useFakeTimers();
@@ -93,13 +101,6 @@ export async function flushPromises(): Promise<void> {
await new Promise<void>((resolve) => window.setTimeout(resolve));
}
export type NodeEventHandler = (...args: unknown[]) => void;
export interface NodeStyleEventEmitter {
addListener(eventName: string | symbol, handler: NodeEventHandler): this;
removeListener(eventName: string | symbol, handler: NodeEventHandler): this;
}
export interface OurRunHelpers extends RunHelpers {
/**
* Schedules a sequence of actions to happen, as described by a marble
@@ -214,11 +215,6 @@ export const exampleTransport: UnstableLivekitTransport = {
livekit_service_url: "https://lk.example.org",
};
export const exampleTransportLocator: TransportLocator = {
transport: exampleTransport,
serverName: "example.org",
};
export function mockRtcMembership(
user: string | RoomMember,
deviceId: string,
@@ -273,11 +269,6 @@ export function mockRtcMembership(
return cms;
}
export const ownMemberMock: CallMembershipIdentityParts = {
userId: "@alice:example.org",
deviceId: "DEVICE",
memberId: "@alice:example.org:DEVICE",
};
// Maybe it'd be good to move this to matrix-js-sdk? Our testing needs are
// rather simple, but if one util to mock a member is good enough for us, maybe
// it's useful for matrix-js-sdk consumers in general.
@@ -300,54 +291,76 @@ export function mockMatrixRoom(room: Partial<MatrixRoom>): MatrixRoom {
return { ...mockEmitter(), ...room } as Partial<MatrixRoom> as MatrixRoom;
}
export function mockLivekitRoom(
room: Partial<LivekitRoom>,
{
remoteParticipants$,
}: { remoteParticipants$?: Observable<RemoteParticipant[]> } = {},
): LivekitRoom {
const livekitRoom = {
options: {},
setE2EEEnabled: vi.fn(),
/** A track as the SDK hands it out, with spies where a test may look. */
export type MockMediaTrack = Omit<AudioMediaTrack, "kind"> &
Omit<VideoMediaTrack, "kind"> & { kind: "audio" | "video" };
...mockEmitter(),
...room,
} as Partial<LivekitRoom> as LivekitRoom;
if (remoteParticipants$) {
livekitRoom.remoteParticipants = new Map();
remoteParticipants$.subscribe((newRemoteParticipants) => {
livekitRoom.remoteParticipants.clear();
newRemoteParticipants.forEach((p) => {
livekitRoom.remoteParticipants.set(p.identity, p);
});
});
}
return livekitRoom;
export function mockMediaTrack(
track: Partial<MockMediaTrack> = {},
): MockMediaTrack {
return {
source: "camera",
kind: "video",
id: "track",
muted$: constant(false),
encrypted$: constant(true),
stats$: constant(undefined),
attach: vi.fn(),
detach: vi.fn(),
setAudioContext: vi.fn(),
setVolume: vi.fn(),
facingMode$: constant(undefined),
...track,
};
}
export function mockLocalParticipant(
participant: Partial<LocalParticipant>,
): LocalParticipant {
export const mockVideoTrack = (
track: Partial<MockMediaTrack> = {},
): VideoMediaTrack =>
mockMediaTrack({ kind: "video", ...track }) as VideoMediaTrack;
export const mockAudioTrack = (
track: Partial<MockMediaTrack> = {},
): AudioMediaTrack =>
mockMediaTrack({
kind: "audio",
source: "microphone",
...track,
}) as AudioMediaTrack;
/** A member's media as the SDK hands it out, with nothing published unless given. */
export function mockMemberMedia(
media: Partial<Omit<LocalMemberMedia, "local">> & { local?: boolean } = {},
): LocalMemberMedia {
return {
isLocal: true,
trackPublications: new Map(),
publishTrack: vi.fn(),
unpublishTracks: vi.fn().mockResolvedValue([]),
createTracks: vi.fn(),
setMicrophoneEnabled: vi.fn(),
setCameraEnabled: vi.fn(),
getTrackPublication: () =>
({}) as Partial<LocalTrackPublication> as LocalTrackPublication,
local: false,
speaking$: constant(false),
screenShareEnabled$: constant(false),
microphone$: constant(undefined),
camera$: constant(undefined),
screenShare$: constant(undefined),
screenShareAudio$: constant(undefined),
encryptionError$: NEVER,
switchCamera: vi.fn().mockResolvedValue(undefined),
...media,
} as LocalMemberMedia;
}
/** Whoever publishes media, known by identity only. */
export function mockRemoteParticipant(
participant: Partial<RemoteParticipant>,
): RemoteParticipant {
return {
isLocal: false,
...mockEmitter(),
...participant,
} as Partial<LocalParticipant> as LocalParticipant;
} as RemoteParticipant;
}
export function mockLocalMedia(
rtcMember: CallMembership,
roomMember: Partial<RoomMember>,
localParticipant: LocalParticipant,
media: LocalMemberMedia | null,
mediaDevices: MediaDevices,
): LocalUserMediaViewModel {
const member = mockMatrixRoomMember(rtcMember, roomMember);
@@ -355,10 +368,9 @@ export function mockLocalMedia(
id: "local",
userId: member.userId,
rtcBackendIdentity: rtcMember.rtcBackendIdentity,
participant$: constant(localParticipant),
media$: constant(media),
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
livekitRoom$: constant(mockLivekitRoom({ localParticipant })),
transport$: constant(exampleTransport),
focusUrl$: constant("https://rtc-example.org"),
mediaDevices,
displayName$: constant(member.rawDisplayName ?? "nodisplayname"),
mxcAvatarUrl$: constant(member.getMxcAvatarUrl()),
@@ -367,41 +379,20 @@ export function mockLocalMedia(
});
}
export function mockRemoteParticipant(
participant: Partial<RemoteParticipant>,
): RemoteParticipant {
return {
isLocal: false,
setVolume() {},
getTrackPublication: () =>
({}) as Partial<RemoteTrackPublication> as RemoteTrackPublication,
// this will only get used for `getTrackPublications().length`
getTrackPublications: () => [0],
...mockEmitter(),
...participant,
} as RemoteParticipant;
}
export function mockRemoteMedia(
rtcMember: CallMembership,
roomMember: Partial<RoomMember>,
participant: RemoteParticipant | null,
livekitRoom: LivekitRoom | undefined = mockLivekitRoom(
{},
{
remoteParticipants$: of(participant ? [participant] : []),
},
),
media: MemberMedia | null,
{ focusUrl }: { focusUrl?: string } = { focusUrl: "https://rtc-example.org" },
): RemoteUserMediaViewModel {
const member = mockMatrixRoomMember(rtcMember, roomMember);
return createRemoteUserMedia(testScope(), {
id: "remote",
userId: member.userId,
rtcBackendIdentity: rtcMember.rtcBackendIdentity,
participant$: constant(participant),
media$: constant(media),
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
livekitRoom$: constant(livekitRoom),
transport$: constant(exampleTransport),
focusUrl$: constant(focusUrl),
pretendToBeDisconnected$: constant(false),
displayName$: constant(member.rawDisplayName ?? "nodisplayname"),
mxcAvatarUrl$: constant(member.getMxcAvatarUrl()),
@@ -413,28 +404,128 @@ export function mockRemoteMedia(
export function mockRemoteScreenShare(
rtcMember: CallMembership,
roomMember: Partial<RoomMember>,
participant: RemoteParticipant | null,
livekitRoom: LivekitRoom | undefined = mockLivekitRoom(
{},
{
remoteParticipants$: of(participant ? [participant] : []),
},
),
media: MemberMedia | null,
): RemoteScreenShareViewModel {
const member = mockMatrixRoomMember(rtcMember, roomMember);
return createRemoteScreenShare(testScope(), {
id: "screenshare",
userId: member.userId,
participant$: constant(participant),
media$: constant(media),
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
livekitRoom$: constant(livekitRoom),
transport$: constant(exampleTransport),
focusUrl$: constant("https://rtc-example.org"),
pretendToBeDisconnected$: constant(false),
displayName$: constant(member.rawDisplayName ?? "nodisplayname"),
mxcAvatarUrl$: constant(member.getMxcAvatarUrl()),
});
}
export interface MockMemberInputs {
membership: CallMembership;
roomMember?: Partial<RoomMember>;
media$?: Behavior<LocalMemberMedia | null>;
transportUrl?: string | undefined;
}
/** A member as the client hands it out. */
export function mockRTCMember(
local: true,
inputs: MockMemberInputs,
): LocalRTCMember;
export function mockRTCMember(
local: false,
inputs: MockMemberInputs,
): RemoteRTCMember;
export function mockRTCMember(
local: boolean,
{ membership, roomMember, media$, transportUrl }: MockMemberInputs,
): LocalRTCMember | RemoteRTCMember {
const member = mockMatrixRoomMember(membership, roomMember);
const transport: TransportMetadata | undefined =
transportUrl === undefined
? undefined
: {
type: "livekit",
id: transportUrl,
raw: { type: "livekit", livekit_service_url: transportUrl },
resolved$: constant(undefined),
};
const base = {
id: membership.rtcBackendIdentity,
userId: membership.userId,
deviceId: membership.deviceId,
membership$: constant(membership),
displayName$: constant(member.rawDisplayName ?? membership.userId),
avatarUrl$: constant(member.getMxcAvatarUrl()),
transport$: constant(transport),
media$: media$ ?? constant(null),
};
if (!local) return { ...base, local: false };
return {
...base,
local: true,
sharingScreen$: constant(false),
toggleScreenSharing: null,
screenShareError$: constant(null),
dismissScreenShareError: vi.fn(),
};
}
export interface MockClientInputs {
localMember$?: Behavior<LocalRTCMember | null>;
remoteMembers$?: Behavior<RemoteRTCMember[]>;
connected$?: Behavior<boolean>;
reconnecting$?: Behavior<boolean>;
disconnectReason$?: Behavior<DisconnectReason | null>;
fatalError$?: Behavior<MatrixRTCError | null>;
keyRotationSuppressed$?: Behavior<boolean>;
connectedTransports$?: Behavior<TransportMetadata[]>;
}
/** A client that does nothing but hold the state a test hands it. */
export function mockMatrixRTCClient(
scope: ObservableScope,
{
localMember$ = constant(null),
remoteMembers$ = constant([]),
connected$ = constant(true),
reconnecting$ = constant(false),
disconnectReason$ = constant(null),
fatalError$ = constant(null),
keyRotationSuppressed$ = constant(false),
connectedTransports$ = constant([]),
}: MockClientInputs = {},
): MatrixRTCClient {
return {
join: vi.fn(),
leave: vi.fn(),
status$: scope.behavior(
connected$.pipe(map((c) => (c ? "connected" : "connecting"))),
),
connected$,
reconnecting$,
disconnectReason$,
setMicrophoneEnabled: vi.fn(async (enabled: boolean) =>
Promise.resolve(enabled),
),
setCameraEnabled: vi.fn(async (enabled: boolean) =>
Promise.resolve(enabled),
),
fatalError$,
localMember$,
remoteMembers$,
memberCount$: scope.behavior(
combineLatest(
[localMember$, remoteMembers$],
(local, remote) => remote.length + (local === null ? 0 : 1),
),
),
keyRotationSuppressed$,
connectedTransports$,
sendData: vi.fn(async () => Promise.resolve()),
data$: NEVER,
};
}
export function mockConfig(
config: Partial<ResolvedConfigOptions> = {},
): MockInstance<() => ResolvedConfigOptions> {
@@ -537,29 +628,6 @@ export class MockRTCSession extends TypedEventEmitter<
}
}
export const mockTrack = (
participant: Participant,
kind?: Track.Kind,
source?: Track.Source,
): TrackReference =>
({
participant,
publication: {
kind: kind ?? Track.Kind.Audio,
source: source ?? Track.Source.Microphone,
trackSid: `123##${participant.identity}`,
track: {
attach: vi.fn(),
detach: vi.fn(),
setAudioContext: vi.fn(),
setWebAudioPlugins: vi.fn(),
setVolume: vi.fn(),
},
},
track: {},
source: {},
}) as unknown as TrackReference;
export const deviceStub = {
available$: of(new Map<never, never>()),
selected$: of(undefined),
@@ -587,11 +655,6 @@ export function mockMuteStates(
);
}
export class MockConnection extends Connection {
public async start(): Promise<void> {}
public async stop(): Promise<void> {}
}
export interface StubbedCapture {
getUserMedia: Mock;
/** Grants the microphone, as the browser does once permission is given. */
vite-component-dev.config.ts
+3 -1
View File
@@ -33,8 +33,9 @@ export default defineConfig(({ mode }) => {
} catch {}
}
const base = vitePluginsConfig({ mode, html: false });
return {
...vitePluginsConfig({ mode, html: false }),
...base,
root: "component/dev",
// The same scoping the library build applies, so the harness shows what a
// host will get — including whether its own page is left alone
@@ -60,6 +61,7 @@ export default defineConfig(({ mode }) => {
},
resolve: {
alias: {
...base.resolve?.alias,
// matrix-widget-api has its transpiled lib/index.js as its entry point,
// which Vite for some reason refuses to work with, so we point it to
// src/index.ts instead
vite-sdk-target-based-on-call-view-model.config.ts
+14 -3
View File
@@ -5,14 +5,25 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { fileURLToPath } from "node:url";
import { defineConfig } from "vite";
import nodePolyfills from "vite-plugin-node-stdlib-browser";
// Config for the SDK demo that predates the sdk/ package: a single bundle of
// Element Call's CallViewModel for a plain web page or widget. Kept building
// until the harness in sdk/dev replaces it (see sdk-plan.md).
// Config for the legacy SDK bundle that predates the sdk/ package: a single
// bundle of Element Call's CallViewModel for a plain web page or widget. It is
// in production use, so it keeps building on top of the sdk/ package until its
// consumers have moved to that package directly.
export default defineConfig(() => ({
worker: { format: "es" as const },
resolve: {
alias: {
// The SDK by its package name, resolved to its source, like the app does
"@element-hq/matrixrtc-sdk": fileURLToPath(
new URL("./sdk/index.ts", import.meta.url),
),
},
},
// Relative URLs, so the bundle can be hosted under any path
base: "./",
// Otherwise the developer's own config.json in `public` is copied into the output
vitest.config.ts
+2
View File
@@ -38,6 +38,8 @@ export default defineConfig((configEnv) =>
}),
...vitePluginsConfig(configEnv).plugins!,
],
// The stories import the SDK by its package name like the app does
resolve: vitePluginsConfig(configEnv).resolve,
test: {
name: "storybook",
browser: {