Compare commits

..
75 changed files with 915 additions and 6988 deletions
+2 -2
View File
@@ -19,7 +19,7 @@ services:
- ./backend/playwright_homeserver-othersite.yaml:/data/cfg/homeserver.yaml:Z
element-web:
# Pin to a SHA so that upstream cannot break our tests. Renovate handles regular updates.
image: ghcr.io/element-hq/element-web:develop@sha256:79e47d8d5a93848403cb83042a405d8ed233a7727c499fa53b4b39a80f090be4
image: ghcr.io/element-hq/element-web:develop@sha256:e464b2aca73e248195b24b8c6f62cd331100302284d60cc735405807fb2730be
element-web-1:
# Pin to a SHA so that upstream cannot break our tests. Renovate handles regular updates.
image: ghcr.io/element-hq/element-web:develop@sha256:79e47d8d5a93848403cb83042a405d8ed233a7727c499fa53b4b39a80f090be4
image: ghcr.io/element-hq/element-web:develop@sha256:e464b2aca73e248195b24b8c6f62cd331100302284d60cc735405807fb2730be
+2 -4
View File
@@ -71,9 +71,6 @@
"device_id": "Device ID: {{id}}",
"duplicate_tiles_label": "Number of additional tile copies per participant",
"environment_variables": "Environment variables",
"full_mesh_peer_connections": "Full mesh peer connections",
"full_mesh_publishing": "Publish on full mesh (MSC4575) instead of LiveKit",
"full_mesh_transport": "Full mesh: {{device}}",
"hostname": "Hostname: {{hostname}}",
"livekit_server_info": "LiveKit Server Info",
"livekit_sfu": "LiveKit SFU: {{url}}",
@@ -91,6 +88,7 @@
},
"mute_all_audio": "Mute all audio (participants, reactions, join sounds)",
"show_connection_stats": "Show connection statistics",
"throw_error_button": "Throw test error",
"url_params": "URL parameters"
},
"disconnected_banner": "Connectivity to the server has been lost.",
@@ -109,7 +107,7 @@
"insufficient_capacity_description": "The server has reached its maximum capacity and you cannot join the call at this time. Try again later, or contact your server admin if the problem persists.",
"livekit_connection_error": "Failed to connect to Livekit server",
"livekit_connection_error_description": "An error occurred while connecting to the Livekit server (<1>Reason:</1> <2>{{ reason }}</2>).",
"matrix_rtc_transport_missing": "The server is not configured to work with {{brand}} and no other call participant provides a usable transport. Please contact your server admin (Domain: {{domain}}, Error Code: {{ errorCode }}).",
"matrix_rtc_transport_missing": "The server is not configured to work with {{brand}}. Please contact your server admin (Domain: {{domain}}, Error Code: {{ errorCode }}).",
"membership_manager": "Membership Manager Error",
"membership_manager_description": "The Membership Manager had to shut down. This is caused by many consecutive failed network requests.",
"no_matrix_2_authorization_service": "The authorization service for your media server (SFU) is out of date.",
+1 -1
View File
@@ -107,7 +107,7 @@
"livekit-client": "^2.18.1",
"lodash-es": "^4.17.21",
"loglevel": "^1.9.1",
"matrix-js-sdk": "github:matrix-org/matrix-js-sdk#3f05dc4d104a3dfc753c5e3549a18c74fab4b076",
"matrix-js-sdk": "github:matrix-org/matrix-js-sdk#develop",
"matrix-widget-api": "^1.18.0",
"node-stdlib-browser": "^1.3.1",
"normalize.css": "^8.0.1",
@@ -0,0 +1,64 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { expect, test } from "@playwright/test";
import { widgetTest } from "../fixtures/widget-user.ts";
import { TestHelpers } from "./test-helpers.ts";
widgetTest.skip(
({ browserName }) => browserName === "firefox",
"Widget specs only run on Chromium, see simple-create.spec.ts",
);
widgetTest(
"Switching rooms away from a crashed call dismisses it for good",
async ({ asWidget }) => {
test.slow();
const { brooks } = asWidget;
const page = brooks.page;
const call = page.locator('iframe[title="Element Call"]');
const callFrame = call.contentFrame();
// Somewhere to switch to once the call has crashed
await TestHelpers.createRoom("Other Room", page);
await TestHelpers.switchToRoomNamed(page, "Welcome Room");
await TestHelpers.startCallInCurrentRoom(page);
// A dev server compiles the widget on first load
await expect(callFrame.getByTestId("lobby_joinCall")).toBeVisible({
timeout: 30000,
});
await TestHelpers.joinCallFromLobby(page);
await expect(
callFrame.getByRole("button", { name: "End call" }),
).toBeVisible();
// Crash the call from its developer settings
await callFrame.getByRole("button", { name: "Settings" }).click();
await callFrame.getByRole("tab", { name: "Preferences" }).click();
await callFrame.getByText("Developer mode", { exact: true }).check();
await callFrame.getByRole("tab", { name: "Developer" }).click();
await callFrame.getByRole("button", { name: "Throw test error" }).click();
await expect(callFrame.getByText("Something went wrong")).toBeVisible();
// Showing the call collapses the room list
await TestHelpers.expandRoomList(page);
await TestHelpers.switchToRoomNamed(page, "Other Room");
await expect(
page.getByRole("banner").getByRole("heading", { name: "Other Room" }),
).toBeVisible();
// The crashed call is gone and stays gone: no restart, no reconnect
await expect(call).toHaveCount(0);
await page.waitForTimeout(5000);
await expect(call).toHaveCount(0);
await expect(
page.getByRole("banner").getByRole("heading", { name: "Other Room" }),
).toBeVisible();
},
);
-167
View File
@@ -1,167 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { expect, type Page, test } from "@playwright/test";
import { widgetTest } from "../fixtures/widget-user";
import { HOST1, TestHelpers } from "./test-helpers";
/**
* Records the LiveKit signalling websockets a page opens: the SFU is reached
* with an access token in the url, which full mesh never needs.
*/
function recordSfuSockets(page: Page): string[] {
const urls: string[] = [];
page.on("websocket", (ws) => {
if (ws.url().includes("access_token")) urls.push(ws.url());
});
return urls;
}
/**
* Starts recording the types of the to-device messages the widget on a page
* asks its host to send, and returns a function reading them. On the wire
* they are encrypted, so the type is only visible in the widget API.
*/
async function recordWidgetToDevice(
page: Page,
): Promise<() => Promise<string[]>> {
await page.evaluate(() => {
const types: string[] = [];
Object.assign(window, { widgetToDeviceTypes: types });
window.addEventListener("message", ({ data }) => {
const {
action,
response,
data: request,
} = (data ?? {}) as {
action?: string;
response?: unknown;
data?: { type?: string };
};
if (action === "send_to_device" && response === undefined)
types.push(request?.type ?? "");
});
});
return async () =>
page.evaluate(
() =>
(window as unknown as { widgetToDeviceTypes: string[] })
.widgetToDeviceTypes,
);
}
const cases: { name: string; fullMesh: [boolean, boolean] }[] = [
// Both publish on full mesh: the media flows without any SFU
{ name: "both on full mesh", fullMesh: [true, true] },
// One on each: Florian receives Valere over LiveKit without publishing there
{ name: "full mesh and LiveKit mixed", fullMesh: [true, false] },
];
cases.forEach(({ name, fullMesh: [florianMesh, valereMesh] }) => {
widgetTest(`Full mesh call: ${name}`, async ({ addUser, browserName }) => {
test.skip(
browserName === "firefox",
"The is test is not working on firefox CI environment. No mic/audio device inputs so cam/mic are disabled",
);
test.slow();
// Distinct per case: the cases run in parallel and register at once
const suffix = `${Number(florianMesh)}${Number(valereMesh)}`;
const [florian, valere] = await Promise.all([
addUser(`florian${suffix}`, HOST1),
addUser(`valere${suffix}`, HOST1),
]);
// Element Web doesn't auto-approve the full mesh signalling for Element
// Call yet, so it asks for it whenever a widget with full mesh opens
for (const { page } of [florian, valere])
await page.addLocatorHandler(
page.getByRole("heading", { name: "Approve widget permissions" }),
async () => page.getByRole("button", { name: "Approve" }).click(),
);
const sockets = {
florian: recordSfuSockets(florian.page),
valere: recordSfuSockets(valere.page),
};
const roomName = "Full Mesh Room";
await TestHelpers.createRoom(roomName, florian.page, [valere.mxId]);
await TestHelpers.acceptRoomInvite(roomName, valere.page);
await TestHelpers.openWidgetSetEmbeddedElementCallRtcModeCloseWidget(
florian.page,
"2_0",
{ fullMesh: florianMesh },
);
await TestHelpers.openWidgetSetEmbeddedElementCallRtcModeCloseWidget(
valere.page,
"2_0",
{ fullMesh: valereMesh },
);
await TestHelpers.startCallInCurrentRoom(florian.page, false);
await TestHelpers.joinCallFromLobby(florian.page);
await TestHelpers.joinCallInCurrentRoom(valere.page);
// A widget only asks for the full mesh signalling with full mesh turned
// on, so a LiveKit publisher in Element Web can't receive full mesh yet:
// the other's media arrives if it is on LiveKit or we have full mesh
const receives = {
florian: !valereMesh || florianMesh,
valere: !florianMesh || valereMesh,
};
// In parallel: assertions on a page also run its permission handler
await Promise.all(
(["florian", "valere"] as const).map(async (name) => {
const frame = { florian, valere }[name].page
.locator('iframe[title="Element Call"]')
.contentFrame();
await expect(frame.getByTestId("videoTile")).toHaveCount(2, {
timeout: 15000,
});
const waiting = frame.getByText("Waiting for media...");
if (receives[name]) {
// Over whichever transport carries it
await expect(waiting).not.toBeVisible({ timeout: 20000 });
await TestHelpers.expectVisibleVideoCount(frame, 2);
} else {
await expect(waiting).toBeVisible({ timeout: 20000 });
await TestHelpers.expectVisibleVideoCount(frame, 1);
}
}),
);
if (florianMesh && valereMesh) {
// Nobody published on LiveKit, so nobody connected to an SFU
expect(sockets.florian).toEqual([]);
expect(sockets.valere).toEqual([]);
} else {
// Florian receives Valere's media from the SFU without publishing there
expect(sockets.florian.length).toBeGreaterThan(0);
}
// Florian sees Valere's camera go off: over the track metadata for full
// mesh, without renegotiating
const toDevice = await recordWidgetToDevice(valere.page);
const valereFrame = valere.page
.locator('iframe[title="Element Call"]')
.contentFrame();
await valereFrame.getByRole("switch", { name: "Stop video" }).click();
const florianFrame = florian.page
.locator('iframe[title="Element Call"]')
.contentFrame();
await TestHelpers.expectVisibleVideoCount(florianFrame, 1);
// On the connection's control channel, not through the homeserver.
// Candidates may still trickle there until the channel is up
if (florianMesh && valereMesh)
expect(
(await toDevice()).filter(
(type) => type.includes("full_mesh") && !type.endsWith(".candidates"),
),
).toEqual([]);
});
});
+3 -13
View File
@@ -159,7 +159,7 @@ export class TestHelpers {
public static async dismissStartupToasts(page: Page): Promise<void> {
const expectedToasts = [
{ title: "Failed to load service worker", button: "OK" },
{ title: "Back up your chats", button: "Not now" },
{ title: "Back up your chats", button: "Dismiss" },
{ title: "Turn on key storage", button: "Dismiss" },
{ title: "Element does not support this browser", button: "Dismiss" },
];
@@ -178,11 +178,7 @@ export class TestHelpers {
);
if (toastConfig) {
// Bounded so that a renamed button leaves the toast in place and the
// next step reports it, instead of hanging until the test timeout.
await toast
.getByRole("button", { name: toastConfig.button })
.click({ timeout: 5000 });
await toast.getByRole("button", { name: toastConfig.button }).click();
} else {
// Unknown toast. We don't want to act on unknown toasts
break;
@@ -292,7 +288,6 @@ export class TestHelpers {
public static async openWidgetSetEmbeddedElementCallRtcModeCloseWidget(
page: Page,
mode: RtcMode,
{ fullMesh = false }: { fullMesh?: boolean } = {},
): Promise<void> {
await page.getByRole("button", { name: "Video call" }).click({
timeout: 5000,
@@ -301,7 +296,7 @@ export class TestHelpers {
timeout: 10000,
});
await TestHelpers.setEmbeddedElementCallRtcMode(page, mode, { fullMesh });
await TestHelpers.setEmbeddedElementCallRtcMode(page, mode);
await page.getByRole("button", { name: "Close lobby" }).click();
}
@@ -314,7 +309,6 @@ export class TestHelpers {
public static async setEmbeddedElementCallRtcMode(
page: Page,
mode: RtcMode,
{ fullMesh = false }: { fullMesh?: boolean } = {},
): Promise<void> {
const iframe = page.locator('iframe[title="Element Call"]').contentFrame();
@@ -332,10 +326,6 @@ export class TestHelpers {
// compat
await iframe.getByText("Compatibility: state events").click();
}
// Publish on the full mesh transport (MSC4575) instead of LiveKit
await iframe
.getByText("Publish on full mesh", { exact: false })
.setChecked(fullMesh);
await iframe.getByTestId("modal_close").click();
}
+6 -2
View File
@@ -66,9 +66,13 @@ widgetTest(
await TestHelpers.expectVisibleVideoCount(frame, 2);
}
// Element Web used for Playwright does support MSC4195 for widgets.
// Element Web does not expose the MSC4195 endpoints to widgets yet, so
// neither side reached the CS API. The host published through the JWT
// service's legacy endpoint and subscribed to the guest through its
// default one; the guest published the same way and fell back to the
// default endpoint for the host's transport.
expect(hostFlows).toEqual(new Set(["service-legacy", "service-default"]));
expect(guestFlows).toEqual(new Set(["service-legacy", "cs-api"]));
expect(guestFlows).toEqual(new Set(["service-legacy", "service-default"]));
},
);
+270 -541
View File
File diff suppressed because it is too large Load Diff
+5 -15
View File
@@ -66,14 +66,9 @@ media backend: whatever carries the media, behind the `MediaBackend` interface i
`sdk/src/media-backend/api.ts`. The client is the join of the two: a member exists
once its membership does, and gets its media once the backend has something
for that membership on its transport; `state$` is the membership state combined
with the local media connection. Two backends exist, LiveKit (`livekit/`) and
full mesh (`full-mesh/`, MSC4575), and the client always runs both, combined
(`combineBackends.ts`). It publishes on the backend of the type of
`options.transport`, LiveKit without one, and hands every remote member to the
backend of that member's transport, so members on either are received. A
backend nobody is on connects nowhere. Another transport is another folder
under `sdk/src/media-backend/`, added to the combination, and nothing in the
left box changes; `options.backend` replaces the combination altogether.
with the local media connection. LiveKit is the one backend today; a
cascading SFU or full mesh is another folder under `sdk/src/media-backend/`, handed
in through `options.backend`, and nothing in the left box changes.
### How Element Call uses it
@@ -106,8 +101,6 @@ LiveKit participants.
│ create ───▶│ │ │
│ │ GET rtc/transports (session) ────────▶│ │
│ │◀── first transport of the backend's type │
│ │ (none, LiveKit only: a member's SFU │ │
│ │ that lets us publish, get_token) ─▶│ │
│ │ prepareLocalTransport (backend): │ │
│ │ publisher token: CS API get_token, │ │
│ │ or OpenID token then /sfu/get ─────▶│ │
@@ -154,7 +147,7 @@ the sections below still describe `MatrixRTCClient` until that lands.
| publish request | a `PublishRequest`: a source, where to capture it from and how to encode it. Given at the join in `publish`, or later to `publish` on the participation | |
| attach, detach | handing a `<video>` or `<audio>` element to a track and taking it back. The SDK owns the stream and the observers on the element | LiveKit `Track.attach` |
| transport | an `RTCTransport`: where media is exchanged, a LiveKit service url today. Carries the raw js-sdk `Transport` from the membership and, in `resolved$`, what the backend fetched to connect to it | MSC4143 "focus" |
| media backend | a `MediaBackend`: what carries the media, behind `sdk/src/media-backend/api.ts`. One per participation: LiveKit and full mesh combined, publishing on the type of `transport` in the join options, unless `backend` replaces them | "SFU", a LiveKit room, a peer connection |
| media backend | a `MediaBackend`: what carries the media, behind `sdk/src/media-backend/api.ts`. One per participation, chosen with `backend` in the join options; LiveKit under `sdk/src/media-backend/livekit/` is the one that exists | "SFU", a LiveKit room, a peer connection |
Names use `RTC` in capitals, as the js-sdk does: `RTCSlot`, `RTCParticipation`,
`RTCMember`, `matrixRTCMode`.
@@ -563,10 +556,7 @@ export interface MediaBackend {
```
What stays on the client's side: discovery of the local transport from the
homeserver's list (a homeserver endpoint, so not a backend's) with, for the
LiveKit backend only, a last resort of another member's SFU that lets us
publish (`findRemotePublishLivekitTransport`, wired in by `RTCParticipation` rather
than by the backend interface, since it is an MSC4195 matter), the membership
homeserver's list (a homeserver endpoint, so not a backend's), the membership
state machine in `LocalMembership.ts` and its combination with
`local.connectionState$` into `state$`, the choice of delayed leave timings,
mapping a data packet's sender to a member, and the transport registry, which keys on the raw
-8
View File
@@ -61,14 +61,6 @@ export {
createLivekitBackend,
type LivekitBackendOptions,
} from "./src/media-backend/livekit/LivekitBackend";
// Join with `transport: { type: FULL_MESH_TRANSPORT_TYPE }` to publish on full mesh
export {
createFullMeshBackend,
FULL_MESH_TRANSPORT_TYPE,
type FullMeshBackendOptions,
} from "./src/media-backend/full-mesh/FullMeshBackend";
// For a widget host to allow the full mesh signalling
export { signallingEventTypes as fullMeshSignallingEventTypes } from "./src/media-backend/full-mesh/signalling";
// For a host that creates LiveKit tracks itself, such as a preview before joining
export { convertToLivekitProcessor } from "./src/media-backend/livekit/videoProcessor";
// For a developer panel to check a transport before using it
-9
View File
@@ -1,9 +0,0 @@
lockfileVersion: '9.0'
settings:
autoInstallPeers: false
excludeLinksFromLockfile: false
importers:
.: {}
+6 -61
View File
@@ -8,11 +8,9 @@ Please see LICENSE in the repository root for full details.
import { type MatrixClient, type Room, SyncState } from "matrix-js-sdk";
import { logger } from "matrix-js-sdk/lib/logger";
import {
type CallMembership,
MatrixRTCSessionEvent,
MembershipManagerEvent,
type Transport,
type UnstableLivekitTransport,
} from "matrix-js-sdk/lib/matrixrtc";
import { BehaviorSubject, EMPTY, Subject } from "rxjs";
import { describe, expect, it, type Mock, vi } from "vitest";
@@ -40,32 +38,13 @@ import { constant } from "./reactive/Behavior";
import { ObservableScope } from "./reactive/ObservableScope";
import { createRTCParticipation } from "./RTCParticipation";
import { transportId } from "./matrixrtc/Transports";
import {
flushPromises,
mockEmitter,
mockRtcMembership,
MockRTCSession,
} from "./utils/test";
import { flushPromises, mockEmitter, MockRTCSession } from "./utils/test";
const transport: UnstableLivekitTransport = {
const transport: Transport = {
type: "livekit",
livekit_service_url: "https://lk.example.org",
};
const transportWithAlias: Transport = {
livekit_alias: "!room:example.org",
...transport,
};
const userId = "@alice:example.org";
/** A token with the publish grant, as a homeserver issues it for its own SFU. */
const publisherJWT = [
{},
{ sub: "identity", video: { room: "room", canPublish: true } },
{},
]
.map((d) => global.btoa(JSON.stringify(d)))
.join(".");
const deviceId = "AAAA";
const legacyMemberId = `${userId}:${deviceId}`;
@@ -112,13 +91,6 @@ interface FakeBackend {
function setup(
options: Partial<RTCParticipationOptions> = {},
slotOptions: { matrixRTCMode?: MatrixRTCMode } = {},
{
homeserverTransports = [transport],
memberships = [],
}: {
homeserverTransports?: Transport[];
memberships?: CallMembership[];
} = {},
): {
scope: ObservableScope;
participation: ReturnType<typeof createRTCParticipation>["participation"];
@@ -131,7 +103,7 @@ function setup(
} {
const scope = new ObservableScope();
const room = { roomId: "!room:example.org" } as Room;
const session = new MockRTCSession(room, memberships);
const session = new MockRTCSession(room);
const mockedSession = session.asMockedSession();
const join = vi.spyOn(session, "join");
const client = {
@@ -140,8 +112,7 @@ function setup(
getDeviceId: () => deviceId,
getDomain: () => "example.org",
getSyncState: () => SyncState.Syncing,
_unstable_getRTCTransports: vi.fn().mockResolvedValue(homeserverTransports),
_unstable_getLivekitToken: vi.fn().mockResolvedValue({ jwt: publisherJWT }),
_unstable_getRTCTransports: vi.fn().mockResolvedValue([transport]),
} as unknown as MatrixClient;
const members$ = new BehaviorSubject<RTCMember[]>([]);
@@ -230,7 +201,7 @@ describe("createRTCParticipation", () => {
await flushPromises();
expect(join).toHaveBeenCalledWith(
{ userId, deviceId, memberId: legacyMemberId },
[transportWithAlias],
[transport],
expect.objectContaining({
manageMediaKeys: true,
unstableSendStickyEvents: false,
@@ -249,32 +220,6 @@ describe("createRTCParticipation", () => {
});
});
it("publishes on another member's SFU where the homeserver advertises none", async () => {
const bobOnSfu = mockRtcMembership("@bob:example.org", "BOB", {
fociPreferred: [{ type: "livekit", url: "wss://sfu.example.org" }],
});
const { join, fake } = setup(
{},
{},
{ homeserverTransports: [], memberships: [bobOnSfu] },
);
fake.prepared.resolve({ canDelegateDelayedLeave: false });
await flushPromises();
await flushPromises();
expect(join).toHaveBeenCalledWith(
expect.anything(),
[
{
type: "livekit",
url: "wss://sfu.example.org",
server_name: "example.org",
},
],
expect.anything(),
);
});
it("publishes only while joined", async () => {
const { participation, session, fake, scope } = setup();
const ended = vi.fn();
@@ -326,7 +271,7 @@ describe("createRTCParticipation", () => {
await flushPromises();
expect(join).toHaveBeenCalledWith(
expect.anything(),
[transportWithAlias],
[transport],
expect.objectContaining({ delayedLeaveEventDelayMs: 7_200_000 }),
);
expect(fake.delegateDelayedLeave).not.toHaveBeenCalled();
+17 -69
View File
@@ -11,7 +11,6 @@ import {
type MatrixRTCSession as JsSdkRTCSession,
MatrixRTCSessionEvent,
MembershipManagerEvent,
type Transport,
} from "matrix-js-sdk/lib/matrixrtc";
import { type IMembershipManager } from "matrix-js-sdk/lib/matrixrtc/IMembershipManager";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
@@ -50,13 +49,7 @@ import {
type MediaBackendContext,
type MediaKey,
} from "./media-backend/api";
import { combineBackends } from "./media-backend/combineBackends";
import {
completeFullMeshTransport,
createFullMeshBackend,
} from "./media-backend/full-mesh/FullMeshBackend";
import { createLivekitBackend } from "./media-backend/livekit/LivekitBackend";
import { findRemotePublishLivekitTransport } from "./media-backend/livekit/RemotePublishTransport";
import { createHomeserverConnected$ } from "./matrixrtc/HomeserverConnected";
import { joinJsSdkSession } from "./matrixrtc/joinJsSdkSession";
import {
@@ -126,13 +119,6 @@ export function createRTCParticipation(
: `${userId}:${deviceId}`,
};
// The host names full mesh by its type alone; the device is ours
if (options.transport)
options = {
...options,
transport: completeFullMeshTransport(options.transport, deviceId),
};
const backend = createBackend(scope, client, options, {
roomId: room.roomId,
slotId: jsSdkSession.slotId,
@@ -144,34 +130,13 @@ export function createRTCParticipation(
logger,
});
// If the backend is LiveKit, we check it for transports as well, as a
// fallback: another member's SFU may let us publish (MSC4195). LiveKit only,
// so it stays out of the backend interface
const transportFromBackends = async (): Promise<Transport | null> =>
backend.transportType === "livekit"
? findRemotePublishLivekitTransport({
client,
// A snapshot: if nobody with a usable SFU is here yet, the user has
// to come back later rather than wait in a half-working lobby
memberships: jsSdkSession.memberships,
ownMembershipIdentity,
roomId: room.roomId,
slotId: jsSdkSession.slotId,
logger,
})
: null;
const preparedTransport$ = from(
discoverLocalTransport(
client,
backend.transportType,
options,
transportFromBackends,
logger,
).then(async (transport): Promise<PreparedTransport> => ({
transport,
...(await backend.prepareLocalTransport(transport)),
})),
discoverLocalTransport(client, backend.transportType, options, logger).then(
async (transport): Promise<PreparedTransport> => ({
transport,
...(await backend.prepareLocalTransport(transport)),
}),
),
);
const localMembership = createLocalMembership$({
@@ -186,20 +151,14 @@ export function createRTCParticipation(
timings.syncDisconnectGracePeriodMs,
),
joinMatrixRTC: (transport, delayedLeave) =>
joinJsSdkSession(
room.roomId,
jsSdkSession,
ownMembershipIdentity,
transport,
{
encryptMedia: encryptionSystem.kind !== E2eeType.NONE,
matrixRTCMode,
sendNotificationType: options.sendNotificationType,
applicationData: options.applicationData,
timings,
delayedLeave,
},
),
joinJsSdkSession(jsSdkSession, ownMembershipIdentity, transport, {
encryptMedia: encryptionSystem.kind !== E2eeType.NONE,
matrixRTCMode,
sendNotificationType: options.sendNotificationType,
applicationData: options.applicationData,
timings,
delayedLeave,
}),
membershipManagerError$: fromEvent(
jsSdkSession,
MatrixRTCSessionEvent.MembershipManagerError,
@@ -322,26 +281,15 @@ export function createRTCParticipation(
};
}
/**
* The host's backend, or every backend the SDK has, combined: we publish on
* the type of the transport the host named (LiveKit without one) and receive
* from members on any of them.
*/
/** The host's backend, or LiveKit configured from the participation options. */
function createBackend(
scope: ObservableScope,
client: MatrixClient,
{ backend, mediaQuality, transport }: RTCParticipationOptions,
{ backend, mediaQuality }: RTCParticipationOptions,
context: MediaBackendContext,
): MediaBackend {
if (backend) return backend(scope, context);
return combineBackends(
scope,
[
createLivekitBackend(scope, context, { client, mediaQuality }),
createFullMeshBackend(scope, context, { client }),
],
transport?.type ?? "livekit",
);
return createLivekitBackend(scope, context, { client, mediaQuality });
}
/** The slot's per-participant keys: the ones it already holds on subscribe, then each new one. */
+1 -36
View File
@@ -20,11 +20,6 @@ const hostTransport: Transport = {
type: "livekit",
livekit_service_url: "https://host.example.org",
};
const bobsTransport: Transport = {
type: "livekit",
url: "wss://sfu.bob.com",
server_name: "bob.com",
};
function client(
advertised: Transport[],
@@ -35,12 +30,6 @@ function client(
};
}
function fromBackends(
transport: Transport | null = null,
): ReturnType<typeof vi.fn<() => Promise<Transport | null>>> {
return vi.fn(async () => Promise.resolve(transport));
}
describe("discoverLocalTransport", () => {
it("takes the transport the host names without asking the homeserver", async () => {
const c = client([homeserverTransport]);
@@ -50,7 +39,6 @@ describe("discoverLocalTransport", () => {
c,
"livekit",
{ transport: hostTransport },
fromBackends(),
logger,
),
).resolves.toBe(hostTransport);
@@ -63,7 +51,6 @@ describe("discoverLocalTransport", () => {
client([homeserverTransport]),
"livekit",
{ fallbackTransport: hostTransport },
fromBackends(),
logger,
),
).resolves.toEqual(homeserverTransport);
@@ -75,35 +62,13 @@ describe("discoverLocalTransport", () => {
client([{ type: "not_livekit" }]),
"livekit",
{ fallbackTransport: hostTransport },
fromBackends(),
logger,
),
).resolves.toEqual(hostTransport);
});
it("prefers a local transport over the backends' without asking them", async () => {
const backends = fromBackends(bobsTransport);
await expect(
discoverLocalTransport(
client([homeserverTransport]),
"livekit",
{},
backends,
logger,
),
).resolves.toEqual(homeserverTransport);
expect(backends).not.toHaveBeenCalled();
});
it("fails naming the homeserver where there is no transport at all", async () => {
const promise = discoverLocalTransport(
client([]),
"livekit",
{},
fromBackends(),
logger,
);
const promise = discoverLocalTransport(client([]), "livekit", {}, logger);
await expect(promise).rejects.toBeInstanceOf(
MatrixRTCTransportMissingError,
+2 -8
View File
@@ -15,10 +15,7 @@ import { TransportAutoDiscovery } from "./TransportAutoDiscovery";
/**
* The transport the local member advertises: the one the host names, else the
* homeserver's first of the type the backend serves, else the host's fallback,
* else the first the backends can provide, which are asked only then.
*
* @throws {MatrixRTCTransportMissingError} where none of these exists.
* homeserver's first of the type the backend serves, else the host's fallback.
*/
export async function discoverLocalTransport(
client: Pick<MatrixClient, "getDomain" | "_unstable_getRTCTransports">,
@@ -27,7 +24,6 @@ export async function discoverLocalTransport(
transport: override,
fallbackTransport,
}: Pick<RTCParticipationOptions, "transport" | "fallbackTransport">,
transportFromBackends: () => Promise<Transport | null>,
logger: Logger,
): Promise<Transport> {
const transport =
@@ -37,9 +33,7 @@ export async function discoverLocalTransport(
transportType,
fallbackTransport,
logger,
}).discoverPreferredTransport()) ??
(await transportFromBackends()) ??
null;
}).discoverPreferredTransport());
if (transport === null)
throw new MatrixRTCTransportMissingError(client.getDomain() ?? "");
return transport;
+6 -35
View File
@@ -6,7 +6,6 @@ Please see LICENSE in the repository root for full details.
*/
import {
type UnstableLivekitTransport,
type MatrixRTCSession,
type Transport,
} from "matrix-js-sdk/lib/matrixrtc";
@@ -16,16 +15,11 @@ import { defaultSessionTimings, MatrixRTCMode } from "../config";
import { ownMemberMock } from "../utils/test";
import { joinJsSdkSession } from "./joinJsSdkSession";
const transport: UnstableLivekitTransport = {
const transport: Transport = {
type: "livekit",
livekit_service_url: "https://lk.example.org",
};
const transportWithAlias: Transport = {
livekit_alias: "!room:example.org",
...transport,
};
/** A session that only records the join; nothing else of it is touched. */
function session(): MatrixRTCSession & {
join: Mock<MatrixRTCSession["join"]>;
@@ -45,7 +39,7 @@ describe("joinJsSdkSession", () => {
keyRotationParticipantLimit: 20,
};
joinJsSdkSession("!room:example.org", s, ownMemberMock, transport, {
joinJsSdkSession(s, ownMemberMock, transport, {
encryptMedia: true,
matrixRTCMode: MatrixRTCMode.Compatibility,
sendNotificationType: "ring",
@@ -57,7 +51,7 @@ describe("joinJsSdkSession", () => {
},
});
expect(s.join).toHaveBeenCalledWith(ownMemberMock, [transportWithAlias], {
expect(s.join).toHaveBeenCalledWith(ownMemberMock, [transport], {
notificationType: "ring",
callIntent: undefined,
applicationData: {},
@@ -78,7 +72,7 @@ describe("joinJsSdkSession", () => {
it("gives up on the network once the delayed leave has probably been sent", () => {
const s = session();
joinJsSdkSession("!room:example.org", s, ownMemberMock, transport, {
joinJsSdkSession(s, ownMemberMock, transport, {
encryptMedia: false,
matrixRTCMode: MatrixRTCMode.Compatibility,
timings: { ...defaultSessionTimings, networkErrorRetryMs: 4_000 },
@@ -87,7 +81,7 @@ describe("joinJsSdkSession", () => {
expect(s.join).toHaveBeenCalledWith(
ownMemberMock,
[transportWithAlias],
[transport],
expect.objectContaining({
manageMediaKeys: false,
maximumNetworkErrorRetryCount: 4,
@@ -98,30 +92,7 @@ describe("joinJsSdkSession", () => {
it("lifts the call intent out of the application data and sends sticky events in Matrix 2.0 mode", () => {
const s = session();
joinJsSdkSession("!room:example.org", s, ownMemberMock, transport, {
encryptMedia: true,
matrixRTCMode: MatrixRTCMode.Matrix_2_0,
applicationData: { "m.call.intent": "audio", custom: 1 },
timings: defaultSessionTimings,
delayedLeave: defaultSessionTimings.delayedLeave,
});
expect(s.join).toHaveBeenCalledWith(
ownMemberMock,
[transportWithAlias],
expect.objectContaining({
callIntent: "audio",
applicationData: { custom: 1 },
unstableSendStickyEvents: true,
}),
);
});
it("does not add livekit_alias to newer transport format", () => {
const s = session();
const transport = { type: "livekit", url: "wss://lk.example.org" };
joinJsSdkSession("!room:example.org", s, ownMemberMock, transport, {
joinJsSdkSession(s, ownMemberMock, transport, {
encryptMedia: true,
matrixRTCMode: MatrixRTCMode.Matrix_2_0,
applicationData: { "m.call.intent": "audio", custom: 1 },
+16 -34
View File
@@ -35,7 +35,6 @@ interface Options {
* @throws If the join could not be started.
*/
export function joinJsSdkSession(
roomId: string,
session: JsSdkRTCSession,
ownMembershipIdentity: CallMembershipIdentityParts,
transport: Transport,
@@ -59,37 +58,20 @@ export function joinJsSdkSession(
timings.syncDisconnectGracePeriodMs,
delayedLeave.delay_ms,
);
session.join(
ownMembershipIdentity,
// TODO: `livekit_alias` should be unused these days, but some Ruma-based
// clients still expect it to be present. Remove from `publishedTransports`
// as soon as https://github.com/ruma/ruma/pull/2642 and
// https://github.com/matrix-org/matrix-rust-sdk/pull/7335 are widely
// deployed.
[
{
...("livekit_service_url" in transport
? { livekit_alias: roomId }
: {}),
...transport,
},
],
{
notificationType: sendNotificationType,
callIntent,
applicationData: rest,
manageMediaKeys: encryptMedia,
delayedLeaveEventRestartMs: delayedLeave.restart_ms,
delayedLeaveEventDelayMs: delayedLeave.delay_ms,
delayedLeaveEventRestartLocalTimeoutMs: delayedLeave.restart_timeout_ms,
networkErrorRetryMs: timings.networkErrorRetryMs,
makeKeyDelay: timings.waitForKeyRotationMs,
membershipEventExpiryMs: timings.membershipEventExpiryMs,
keyRotationParticipantLimit: timings.keyRotationParticipantLimit,
unstableSendStickyEvents: matrixRTCMode === MatrixRTCMode.Matrix_2_0,
maximumNetworkErrorRetryCount:
Math.ceil(maxWaitMs / timings.networkErrorRetryMs) + 1,
},
);
session.join(ownMembershipIdentity, [transport], {
notificationType: sendNotificationType,
callIntent,
applicationData: rest,
manageMediaKeys: encryptMedia,
delayedLeaveEventRestartMs: delayedLeave.restart_ms,
delayedLeaveEventDelayMs: delayedLeave.delay_ms,
delayedLeaveEventRestartLocalTimeoutMs: delayedLeave.restart_timeout_ms,
networkErrorRetryMs: timings.networkErrorRetryMs,
makeKeyDelay: timings.waitForKeyRotationMs,
membershipEventExpiryMs: timings.membershipEventExpiryMs,
keyRotationParticipantLimit: timings.keyRotationParticipantLimit,
unstableSendStickyEvents: matrixRTCMode === MatrixRTCMode.Matrix_2_0,
maximumNetworkErrorRetryCount:
Math.ceil(maxWaitMs / timings.networkErrorRetryMs) + 1,
});
}
@@ -1,91 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { describe, expect, it, vi } from "vitest";
import { BehaviorSubject, NEVER } from "rxjs";
import { type RTCMember, type TransportMetadata } from "../api";
import { type MemberMedia } from "../media-api";
import { constant } from "../reactive/Behavior";
import { ObservableScope } from "../reactive/ObservableScope";
import { type MediaBackend } from "./api";
import { combineBackends } from "./combineBackends";
function fakeBackend(type: string): MediaBackend & { ended: string[] } {
const ended: string[] = [];
return {
ended,
transportType: type,
prepareLocalTransport: vi.fn(async () =>
Promise.resolve({ canDelegateDelayedLeave: false }),
),
delegateDelayedLeave: vi.fn(),
local: {} as MediaBackend["local"],
mediaFor: (scope, member): MemberMedia => {
scope.onEnd(() => ended.push(member.memberId));
return {
tracks$: constant([{ id: `${type}:${member.memberId}` }] as never),
encryptionError$: NEVER,
};
},
connections$: constant([
{ transport: { type }, state: "connected" as never, resolved: undefined },
]),
setAudioOutputDeviceId: vi.fn(),
sendData: vi.fn(),
data$: NEVER,
};
}
function transport(type: string): TransportMetadata {
return { type, id: type, raw: { type }, resolved$: constant(undefined) };
}
describe("combineBackends", () => {
it("routes each member to the backend of its transport type", () => {
const scope = new ObservableScope();
const livekit = fakeBackend("livekit");
const other = fakeBackend("other");
const combined = combineBackends(scope, [livekit, other], "livekit");
expect(combined.transportType).toBe("livekit");
expect(combined.connections$.value).toHaveLength(2);
const transport$ = new BehaviorSubject<TransportMetadata | undefined>(
transport("other"),
);
const memberScope = new ObservableScope();
const media = combined.mediaFor(memberScope, {
memberId: "m1",
transport$,
} as unknown as RTCMember);
expect(media.tracks$.value).toEqual([{ id: "other:m1" }]);
transport$.next(transport("livekit"));
expect(other.ended).toEqual(["m1"]);
expect(media.tracks$.value).toEqual([{ id: "livekit:m1" }]);
transport$.next(transport("unknown"));
expect(media.tracks$.value).toBeNull();
// The publishing backend keeps every member for as long as it is one
expect(livekit.ended).toEqual([]);
memberScope.end();
expect(livekit.ended).toEqual(["m1"]);
scope.end();
});
it("publishes on the backend of the publish type", () => {
const scope = new ObservableScope();
const livekit = fakeBackend("livekit");
const other = fakeBackend("other");
const combined = combineBackends(scope, [livekit, other], "other");
expect(combined.transportType).toBe("other");
expect(() => combineBackends(scope, [livekit, other], "unknown")).toThrow(
"No media backend for transport unknown",
);
scope.end();
});
});
-80
View File
@@ -1,80 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { combineLatest, map, merge, NEVER, of, switchMap } from "rxjs";
import { type RTCMember } from "../api";
import { MatrixRTCError } from "../errors";
import { type MemberMedia } from "../media-api";
import { type ObservableScope } from "../reactive/ObservableScope";
import { mapScoped } from "../utils/mapScoped";
import { type MediaBackend } from "./api";
/**
* Several backends as one: we publish on the backend of `publishType`, and
* receive from every member on a transport any of the backends serves. A
* member is handed to the backend of its current transport type. A backend
* nobody is on costs nothing: it connects only for the members handed to it.
*/
export function combineBackends(
scope: ObservableScope,
all: MediaBackend[],
publishType: string,
): MediaBackend {
const byType = new Map(all.map((b) => [b.transportType, b]));
const publish = byType.get(publishType);
if (publish === undefined)
throw new MatrixRTCError(`No media backend for transport ${publishType}`);
const mediaFor = (
memberScope: ObservableScope,
member: RTCMember,
): MemberMedia => {
// The publishing backend sees every member, whatever its transport, so
// that it can tell members from strangers when a backend is asked for our
// media by members on other transports. It connects nowhere for them
const publishMedia = publish.mediaFor(memberScope, member);
const type$ = memberScope.behavior(
member.transport$.pipe(map((transport) => transport?.type)),
);
const media$ = mapScoped(memberScope, type$, (s, type) =>
type === publish.transportType
? publishMedia
: byType.get(type)?.mediaFor(s, member),
);
return {
tracks$: memberScope.behavior(
media$.pipe(switchMap((media) => media?.tracks$ ?? of(null))),
),
encryptionError$: media$.pipe(
switchMap((media) => media?.encryptionError$ ?? NEVER),
),
};
};
return {
transportType: publish.transportType,
prepareLocalTransport: async (transport) =>
publish.prepareLocalTransport(transport),
delegateDelayedLeave: async (delayId) =>
publish.delegateDelayedLeave(delayId),
local: publish.local,
mediaFor,
connections$: scope.behavior(
combineLatest(all.map((b) => b.connections$)).pipe(
map((lists) => lists.flat()),
),
),
setAudioOutputDeviceId: async (deviceId) => {
await Promise.all(
all.map(async (b) => b.setAudioOutputDeviceId(deviceId)),
);
},
sendData: async (topic, text) => publish.sendData(topic, text),
data$: merge(...all.map((b) => b.data$)),
};
}
@@ -1,87 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
/**
* The control channel of a full mesh connection (MSC4575): a negotiated data
* channel both sides create, which carries every message of the connection
* once it is open, apart from `close`.
*/
import {
type ConsumptionMetadata,
parseConsumptionMetadata,
parseTrackMetadata,
type TrackMetadata,
} from "./metadata";
export const CONTROL_CHANNEL_LABEL = "org.matrix.msc4575.rtc.full_mesh.control";
const CONTROL_CHANNEL_ID = 0;
export type ControlMessage =
| {
type: "description";
description: { type: "offer" | "answer"; sdp: string };
track_metadata?: TrackMetadata;
}
| { type: "candidates"; candidates: RTCIceCandidateInit[] }
| { type: "track_metadata"; track_metadata: TrackMetadata }
| { type: "consumption_metadata"; consumption_metadata: ConsumptionMetadata };
export function createControlChannel(pc: RTCPeerConnection): RTCDataChannel {
return pc.createDataChannel(CONTROL_CHANNEL_LABEL, {
negotiated: true,
id: CONTROL_CHANNEL_ID,
});
}
/** A received control message, or null for one to ignore: malformed, or of a type we don't know. */
export function parseControlMessage(data: unknown): ControlMessage | null {
if (typeof data !== "string") return null;
let m: Record<string, unknown>;
try {
m = JSON.parse(data) as Record<string, unknown>;
} catch {
return null;
}
if (typeof m !== "object" || m === null) return null;
switch (m.type) {
case "description": {
const d = m.description as { type?: unknown; sdp?: unknown } | undefined;
if (
(d?.type !== "offer" && d?.type !== "answer") ||
typeof d.sdp !== "string"
)
return null;
return {
type: "description",
description: { type: d.type, sdp: d.sdp },
track_metadata: parseTrackMetadata(m.track_metadata) ?? undefined,
};
}
case "candidates":
if (!Array.isArray(m.candidates)) return null;
return {
type: "candidates",
candidates: m.candidates as RTCIceCandidateInit[],
};
case "track_metadata": {
const metadata = parseTrackMetadata(m.track_metadata);
return metadata && { type: "track_metadata", track_metadata: metadata };
}
case "consumption_metadata": {
const metadata = parseConsumptionMetadata(m.consumption_metadata);
return (
metadata && {
type: "consumption_metadata",
consumption_metadata: metadata,
}
);
}
default:
return null;
}
}
@@ -1,488 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { EventEmitter } from "events";
import {
ClientEvent,
EventType,
type ITurnServer,
MatrixEvent,
} from "matrix-js-sdk";
import { logger } from "matrix-js-sdk/lib/logger";
import { type Transport } from "matrix-js-sdk/lib/matrixrtc";
import { BehaviorSubject, NEVER } from "rxjs";
import { type RTCMember, type TransportMetadata } from "../../api";
import { defaultSessionTimings } from "../../config";
import { E2eeType } from "../../encryption";
import { constant } from "../../reactive/Behavior";
import { ObservableScope } from "../../reactive/ObservableScope";
import {
type DataPacket,
type MediaBackend,
MediaConnectionState,
} from "../api";
import {
completeFullMeshTransport,
createFullMeshBackend,
type FullMeshBackendOptions,
} from "./FullMeshBackend";
import { DATA_CHANNEL_LABEL, DATA_PROTOCOL } from "./metadata";
import { FULL_MESH_TRANSPORT_TYPE, SignallingEventType } from "./signalling";
import {
FakePeerConnection,
installWebRtcFakes,
lastPeerConnection,
peer,
settle,
} from "./test";
const roomId = "!room:example.org";
const slotId = "m.call#ROOM";
const own = { userId: "@me:example.org", deviceId: "MYDEVICE", memberId: "me" };
const fullMesh = (deviceId: string): Transport => ({
type: FULL_MESH_TRANSPORT_TYPE,
device_id: deviceId,
});
const livekit: Transport = { type: "livekit", livekit_service_url: "x" };
function metadata(raw: Transport): TransportMetadata {
return {
type: raw.type,
id: JSON.stringify(raw),
raw,
resolved$: constant(undefined),
};
}
describe("createFullMeshBackend", () => {
let scope: ObservableScope;
let client: EventEmitter & FullMeshBackendOptions["client"];
let send: ReturnType<typeof vi.fn>;
let turnServers: ITurnServer[];
let backend: MediaBackend;
const seq = new Map<string, number>();
/** A to-device message from `peer` (or another sender), as the sync would deliver it. */
function deliver(
type: string,
connectionId: string,
content: Record<string, unknown> = {},
from = peer,
): void {
const next = seq.get(from.memberId) ?? 0;
seq.set(from.memberId, next + 1);
const event = new MatrixEvent({
type,
sender: from.userId,
content: {
room_id: roomId,
slot_id: slotId,
member_id: from.memberId,
target_member_id: own.memberId,
device_id: from.deviceId,
connection_id: connectionId,
seq: next,
...content,
},
});
vi.spyOn(event, "getWireType").mockReturnValue(
EventType.RoomMessageEncrypted,
);
client.emit(ClientEvent.ToDeviceEvent, event);
}
/** What we sent, as [type, recipient device, content]. */
function sent(type?: string): [string, string, Record<string, unknown>][] {
return send.mock.calls
.filter(([t]) => type === undefined || t === type)
.map(([t, [target], content]) => [t, target.deviceId, content]);
}
/** A member of the slot, as the session hands it to the backend. */
function addMember(
transport: Transport,
who = peer,
): { scope: ObservableScope; tracks: () => unknown } {
const memberScope = new ObservableScope();
const media = backend.mediaFor(memberScope, {
...who,
rtcBackendIdentity: `${who.userId}:${who.deviceId}`,
transport$: new BehaviorSubject(metadata(transport)),
} as unknown as RTCMember);
return { scope: memberScope, tracks: () => media.tracks$.value };
}
beforeEach(() => {
vi.useFakeTimers();
installWebRtcFakes();
seq.clear();
scope = new ObservableScope();
send = vi.fn(async () => Promise.resolve());
turnServers = [];
client = Object.assign(new EventEmitter(), {
encryptAndSendToDevice: send,
getTurnServers: (): ITurnServer[] => turnServers,
checkTurnServers: vi.fn(async () => {
turnServers = [{ urls: ["turn:x"], username: "u", credential: "p" }];
return Promise.resolve(true);
}),
}) as unknown as EventEmitter & FullMeshBackendOptions["client"];
backend = createFullMeshBackend(
scope,
{
roomId,
slotId,
ownMembershipIdentity: own,
publish: [],
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
mediaKeys$: NEVER,
timings: defaultSessionTimings,
logger,
},
{ client },
);
});
afterEach(() => {
scope.end();
vi.useRealTimers();
vi.unstubAllGlobals();
});
it("only prepares a full mesh transport", async () => {
await expect(backend.prepareLocalTransport(livekit)).rejects.toThrow(
"Not a full mesh transport: livekit",
);
await expect(
backend.prepareLocalTransport(fullMesh(own.deviceId)),
).resolves.toEqual({ canDelegateDelayedLeave: false });
expect(backend.local.connectionState$.value).toBe(
MediaConnectionState.Connected,
);
});
it("fetches TURN servers when the client has none yet", async () => {
await backend.prepareLocalTransport(fullMesh(own.deviceId));
expect(client.checkTurnServers).toHaveBeenCalledOnce();
addMember(fullMesh(peer.deviceId));
await settle();
expect(lastPeerConnection().config.iceServers).toEqual([
{ urls: ["turn:x"], username: "u", credential: "p" },
]);
});
describe("as a publisher", () => {
it("turns subscribers away while it doesn't publish", async () => {
addMember(livekit);
deliver(SignallingEventType.subscribe, "c1");
await settle();
expect(sent()).toEqual([
[
SignallingEventType.close,
peer.deviceId,
expect.objectContaining({
reason: "not_publishing",
connection_id: "c1",
}),
],
]);
});
it("offers to a member on any transport", async () => {
await backend.prepareLocalTransport(fullMesh(own.deviceId));
addMember(livekit);
deliver(SignallingEventType.subscribe, "c1");
await settle();
expect(sent(SignallingEventType.description)).toEqual([
[
SignallingEventType.description,
peer.deviceId,
expect.objectContaining({
connection_id: "c1",
target_member_id: peer.memberId,
description: { type: "offer", sdp: "offer-1" },
}),
],
]);
});
it("waits for the membership of a subscriber that is ahead of it", async () => {
await backend.prepareLocalTransport(fullMesh(own.deviceId));
deliver(SignallingEventType.subscribe, "c1");
await vi.advanceTimersByTimeAsync(5_000);
expect(sent()).toEqual([]);
addMember(livekit);
await settle();
expect(sent(SignallingEventType.description)).toHaveLength(1);
});
it("rejects a subscriber that never becomes a member", async () => {
await backend.prepareLocalTransport(fullMesh(own.deviceId));
deliver(SignallingEventType.subscribe, "c1");
await vi.advanceTimersByTimeAsync(10_000);
expect(sent()).toEqual([
[
SignallingEventType.close,
peer.deviceId,
expect.objectContaining({ reason: "not_a_member" }),
],
]);
expect(FakePeerConnection.instances).toEqual([]);
});
it("rejects a subscriber claiming another device of the member", async () => {
await backend.prepareLocalTransport(fullMesh(own.deviceId));
addMember(livekit);
deliver(
SignallingEventType.subscribe,
"c1",
{},
{
...peer,
deviceId: "OTHERDEVICE",
},
);
await settle();
expect(sent()).toEqual([
[
SignallingEventType.close,
"OTHERDEVICE",
expect.objectContaining({ reason: "not_a_member" }),
],
]);
});
it("only serves the newest of several waiting subscribe requests", async () => {
await backend.prepareLocalTransport(fullMesh(own.deviceId));
deliver(SignallingEventType.subscribe, "c1");
deliver(SignallingEventType.subscribe, "c2");
addMember(livekit);
await settle();
expect(
sent(SignallingEventType.description).map(([, , c]) => c.connection_id),
).toEqual(["c2"]);
});
it("replaces the connection of a member that subscribes again", async () => {
await backend.prepareLocalTransport(fullMesh(own.deviceId));
addMember(livekit);
deliver(SignallingEventType.subscribe, "c1");
await settle();
const first = lastPeerConnection();
deliver(SignallingEventType.subscribe, "c2");
await settle();
expect(first.closed).toBe(true);
expect(lastPeerConnection()).not.toBe(first);
});
it("hands the subscriber's messages to its connection", async () => {
await backend.prepareLocalTransport(fullMesh(own.deviceId));
addMember(livekit);
deliver(SignallingEventType.subscribe, "c1");
await settle();
deliver(SignallingEventType.description, "c1", {
description: { type: "answer", sdp: "answer" },
});
deliver(SignallingEventType.candidates, "c1", {
candidates: [{ candidate: "x" }],
});
// A late message of an earlier connection goes nowhere
deliver(SignallingEventType.candidates, "c0", {
candidates: [{ candidate: "stale" }],
});
await settle();
const pc = lastPeerConnection();
expect(pc.remoteDescription).toEqual({ type: "answer", sdp: "answer" });
expect(pc.addedCandidates).toEqual([{ candidate: "x" }]);
});
it("stops sending to a member that leaves", async () => {
await backend.prepareLocalTransport(fullMesh(own.deviceId));
const member = addMember(livekit);
deliver(SignallingEventType.subscribe, "c1");
await settle();
member.scope.end();
expect(lastPeerConnection().closed).toBe(true);
expect(sent(SignallingEventType.close)).toEqual([
[
SignallingEventType.close,
peer.deviceId,
expect.objectContaining({ connection_id: "c1" }),
],
]);
});
it("lists its subscribers for the developer tools", async () => {
expect(backend.connections$.value).toEqual([]);
await backend.prepareLocalTransport(fullMesh(own.deviceId));
expect(backend.connections$.value).toEqual([
{
transport: fullMesh(own.deviceId),
state: MediaConnectionState.Connected,
resolved: {
type: FULL_MESH_TRANSPORT_TYPE,
role: "publisher",
peerConnections: [],
},
},
]);
addMember(livekit);
deliver(SignallingEventType.subscribe, "c1");
await settle();
expect(backend.connections$.value[0].resolved).toMatchObject({
peerConnections: [
{
memberId: peer.memberId,
connectionId: "c1",
connectionState: "new",
},
],
});
});
});
describe("as a subscriber", () => {
it("subscribes to a member publishing on full mesh", async () => {
const member = addMember(fullMesh(peer.deviceId));
await settle();
expect(sent()).toEqual([
[
SignallingEventType.subscribe,
peer.deviceId,
expect.objectContaining({
room_id: roomId,
slot_id: slotId,
member_id: own.memberId,
device_id: own.deviceId,
target_member_id: peer.memberId,
seq: 0,
}),
],
]);
expect(member.tracks()).toBeNull();
expect(backend.connections$.value).toEqual([
expect.objectContaining({
transport: fullMesh(peer.deviceId),
resolved: expect.objectContaining({ role: "subscriber" }),
}),
]);
});
it("ignores a full mesh transport advertising another device", async () => {
addMember(fullMesh("OTHERDEVICE"));
await settle();
expect(sent()).toEqual([]);
});
it("leaves members on other transports alone", async () => {
const member = addMember(livekit);
await settle();
expect(sent()).toEqual([]);
expect(member.tracks()).toBeNull();
});
it("hands the publisher's messages to the subscription", async () => {
addMember(fullMesh(peer.deviceId));
await settle();
const [[, , subscribe]] = sent();
lastPeerConnection().receive("video", "1");
deliver(
SignallingEventType.description,
subscribe.connection_id as string,
{
description: { type: "offer", sdp: "offer" },
track_metadata: { "1": { purpose: "m.usermedia" } },
},
);
await settle();
expect(sent(SignallingEventType.description)).toEqual([
[
SignallingEventType.description,
peer.deviceId,
expect.objectContaining({
description: { type: "answer", sdp: "answer" },
}),
],
]);
});
it("closes the subscription when the member leaves", async () => {
const member = addMember(fullMesh(peer.deviceId));
await settle();
member.scope.end();
expect(lastPeerConnection().closed).toBe(true);
expect(backend.connections$.value).toEqual([]);
});
});
it("doesn't take over the delayed leave", async () => {
await expect(backend.delegateDelayedLeave("id")).rejects.toThrow();
});
describe("data", () => {
it("can't be sent while not publishing", async () => {
await expect(backend.sendData("topic", "text")).rejects.toThrow();
});
it("is sent to every subscriber on its data channel", async () => {
await backend.prepareLocalTransport(fullMesh(own.deviceId));
addMember(livekit);
deliver(SignallingEventType.subscribe, "c1");
await settle();
const data = lastPeerConnection().dataChannels.find(
(c) => c.label === DATA_CHANNEL_LABEL,
)!;
data.open();
await backend.sendData("greeting", "hello");
expect(data.sent.map((m) => JSON.parse(m) as unknown)).toEqual([
{ topic: "greeting", text: "hello" },
]);
});
it("is received from a publisher, named by its member", async () => {
const received: DataPacket[] = [];
backend.data$.subscribe((p) => received.push(p));
addMember(fullMesh(peer.deviceId));
await settle();
const [[, , subscribe]] = sent();
deliver(
SignallingEventType.description,
subscribe.connection_id as string,
{
description: { type: "offer", sdp: "offer" },
track_metadata: {},
},
);
await settle();
const channel = lastPeerConnection().announceDataChannel(
DATA_CHANNEL_LABEL,
DATA_PROTOCOL,
);
channel.receive({ topic: "greeting", text: "hello" });
channel.receive({ topic: 1 });
expect(received).toEqual([
{
senderId: `${peer.userId}:${peer.deviceId}`,
topic: "greeting",
text: "hello",
},
]);
});
});
});
describe("completeFullMeshTransport", () => {
it("adds our device to a full mesh transport only", () => {
expect(
completeFullMeshTransport({ type: FULL_MESH_TRANSPORT_TYPE }, "DEV"),
).toEqual({ type: FULL_MESH_TRANSPORT_TYPE, device_id: "DEV" });
expect(completeFullMeshTransport(livekit, "DEV")).toBe(livekit);
});
});
@@ -1,424 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
/**
* The full mesh transport (MSC4575): one WebRTC connection per subscriber
* and publisher, signalled over to-device messages, through the TURN server
* of each side's homeserver. Works subscribe-only too: without
* `prepareLocalTransport` it publishes nothing and still receives from every
* member on full mesh.
*/
import { type MatrixClient } from "matrix-js-sdk";
import { type Transport } from "matrix-js-sdk/lib/matrixrtc";
import {
BehaviorSubject,
combineLatest,
distinctUntilChanged,
filter,
firstValueFrom,
map,
NEVER,
type Observable,
of,
share,
Subject,
switchMap,
timeout,
} from "rxjs";
import { type RTCMember } from "../../api";
import { MatrixRTCError } from "../../errors";
import { type MemberMedia } from "../../media-api";
import { type Behavior } from "../../reactive/Behavior";
import { ObservableScope } from "../../reactive/ObservableScope";
import { mapScoped } from "../../utils/mapScoped";
import {
type BackendConnection,
type DataPacket,
type MediaBackend,
type MediaBackendContext,
MediaConnectionState,
} from "../api";
import { createFullMeshLocalMedia } from "./LocalMedia";
import { type PeerConnectionBase } from "./PeerConnection";
import {
createPublisherConnection,
type PublisherConnection,
} from "./PublisherConnection";
import {
createSignalling,
FULL_MESH_TRANSPORT_TYPE,
type Peer,
type SignallingClient,
type SignallingMessage,
} from "./signalling";
import {
createSubscriberConnection,
type SubscriberConnection,
} from "./SubscriberConnection";
export { FULL_MESH_TRANSPORT_TYPE } from "./signalling";
/** How long a subscribe request waits for the subscriber's membership to arrive. */
const SUBSCRIBE_GRACE_MS = 10_000;
export interface FullMeshBackendOptions {
client: SignallingClient &
Pick<MatrixClient, "getTurnServers" | "checkTurnServers">;
}
/**
* The transport object a member publishing on full mesh advertises: a host
* passes `{ type: FULL_MESH_TRANSPORT_TYPE }`, and the device is filled in.
* Any other transport is returned as it is.
*/
export function completeFullMeshTransport(
transport: Transport,
deviceId: string,
): Transport {
return transport.type === FULL_MESH_TRANSPORT_TYPE
? { ...transport, device_id: deviceId }
: transport;
}
export function createFullMeshBackend(
scope: ObservableScope,
{
roomId,
slotId,
ownMembershipIdentity,
publish,
logger: parentLogger,
}: MediaBackendContext,
{ client }: FullMeshBackendOptions,
): MediaBackend {
const logger = parentLogger.getChild("[FullMeshBackend]");
const signalling = createSignalling({
scope,
client,
roomId,
slotId,
own: ownMembershipIdentity,
logger,
});
const iceServers = async (): Promise<RTCIceServer[]> => {
if (client.getTurnServers().length === 0)
await client
.checkTurnServers()
.catch((e: unknown) => logger.warn("No TURN servers", e));
return client.getTurnServers().map(({ urls, username, credential }) => ({
urls,
username,
credential,
}));
};
let localTransport: Transport | null = null;
const prepared$ = new BehaviorSubject(false);
const audioOutputDeviceId$ = new BehaviorSubject<string | undefined>(
undefined,
);
const data$ = new Subject<DataPacket>();
scope.onEnd(() => data$.complete());
const members = new Map<string, RTCMember>();
const memberAdded$ = new Subject<RTCMember>();
const subscribers = new Map<
string,
{ transport: Transport; peer: Peer; connection: SubscriberConnection }
>();
const publishers = new Map<
string,
{ connection: PublisherConnection; peer: Peer; scope: ObservableScope }
>();
scope.onEnd(() => {
for (const { scope: s } of Array.from(publishers.values())) s.end();
});
/** The connection id of the newest subscribe request per member. */
const latestSubscribe = new Map<string, string>();
// For the developer tools: one entry per publisher we subscribe to, and one
// for our own transport with every subscriber we send to. `resolved` lists
// the peer connections, as an SFU's entry lists what it connected to
const connectionsChanged$ = new BehaviorSubject<void>(undefined);
const connections$ = scope.behavior<BackendConnection[]>(
combineLatest([connectionsChanged$, prepared$]).pipe(
switchMap(([, prepared]) => {
const entries: Observable<BackendConnection>[] = [];
for (const { transport, peer, connection } of subscribers.values()) {
const base = connection.current$.value;
if (base)
entries.push(
base.state$.pipe(
map((state) => ({
transport,
state,
resolved: {
type: FULL_MESH_TRANSPORT_TYPE,
role: "subscriber",
peerConnections: [describe(peer, base)],
},
})),
),
);
}
if (localTransport && prepared) {
const transport = localTransport;
const outgoing = [...publishers.values()];
entries.push(
(outgoing.length === 0
? of([])
: combineLatest(outgoing.map((p) => p.connection.state$))
).pipe(
map(() => ({
transport,
// Publishing needs no server, so it is up once prepared
state: MediaConnectionState.Connected,
resolved: {
type: FULL_MESH_TRANSPORT_TYPE,
role: "publisher",
peerConnections: outgoing.map(({ peer, connection }) =>
describe(peer, connection),
),
},
})),
),
);
}
return entries.length === 0 ? of([]) : combineLatest(entries);
}),
),
);
const local = createFullMeshLocalMedia({
scope,
publish,
connectionState$: scope.behavior(
prepared$.pipe(
map((prepared) =>
prepared
? MediaConnectionState.Connected
: MediaConnectionState.Initialized,
),
),
),
peerConnections: () =>
[...publishers.values()].map(({ connection }) => connection.pc),
logger,
});
const acceptSubscribe = async (message: SignallingMessage): Promise<void> => {
const peer: Peer = {
userId: message.userId,
deviceId: message.deviceId,
memberId: message.memberId,
};
// We don't advertise full mesh, so nobody should have asked
if (!prepared$.value) {
signalling.send(peer, message.connectionId, {
type: "close",
reason: "not_publishing",
});
return;
}
latestSubscribe.set(peer.memberId, message.connectionId);
// The request may overtake the subscriber's membership
const member =
members.get(peer.memberId) ??
(await firstValueFrom(
memberAdded$.pipe(
filter((m) => m.memberId === peer.memberId),
timeout(SUBSCRIBE_GRACE_MS),
),
).catch(() => undefined));
if (latestSubscribe.get(peer.memberId) !== message.connectionId) return;
if (
member === undefined ||
member.userId !== peer.userId ||
member.deviceId !== peer.deviceId
) {
logger.warn(`Rejecting a subscribe from ${peer.memberId}`);
signalling.send(peer, message.connectionId, {
type: "close",
reason: "not_a_member",
});
return;
}
// At most one connection per member pair: the new one replaces the old
publishers.get(peer.memberId)?.scope.end();
const pubScope = new ObservableScope();
const connection = createPublisherConnection({
scope: pubScope,
peer,
connectionId: message.connectionId,
signalling,
iceServers: await iceServers(),
local,
logger,
});
publishers.set(peer.memberId, { connection, peer, scope: pubScope });
pubScope.onEnd(() => {
if (publishers.get(peer.memberId)?.connection === connection) {
publishers.delete(peer.memberId);
connectionsChanged$.next();
}
});
connectionsChanged$.next();
};
signalling.messages$.pipe(scope.bind()).subscribe((message) => {
if (message.type === "subscribe") return void acceptSubscribe(message);
const subscriber = subscribers.get(message.memberId)?.connection;
if (subscriber?.current$.value?.connectionId === message.connectionId)
return subscriber.handle(message);
const publisher = publishers.get(message.memberId)?.connection;
if (publisher?.connectionId === message.connectionId)
return publisher.handle(message);
logger.debug(`Dropping ${message.type} of a stale connection`);
});
const mediaFor = (
memberScope: ObservableScope,
member: RTCMember,
): MemberMedia => {
members.set(member.memberId, member);
memberAdded$.next(member);
memberScope.onEnd(() => {
members.delete(member.memberId);
// We stop sending to members that are no longer joined
publishers.get(member.memberId)?.scope.end();
});
// In encrypted rooms the advertised device has to be the one the
// membership came from
const fullMesh$ = memberScope.behavior(
member.transport$.pipe(
map((transport) =>
transport?.type === FULL_MESH_TRANSPORT_TYPE &&
transport.raw.device_id === member.deviceId
? transport.raw
: undefined,
),
distinctUntilChanged((a, b) => a?.device_id === b?.device_id),
),
);
const subscriber$ = mapScoped(memberScope, fullMesh$, (s, transport) => {
const connection = createSubscriberConnection({
scope: s,
peer: {
userId: member.userId,
deviceId: member.deviceId,
memberId: member.memberId,
},
signalling,
iceServers,
audioOutputDeviceId$,
logger,
});
subscribers.set(member.memberId, {
transport,
peer: {
userId: member.userId,
deviceId: member.deviceId,
memberId: member.memberId,
},
connection,
});
connection.current$
.pipe(s.bind())
.subscribe(() => connectionsChanged$.next());
connection.data$.pipe(s.bind()).subscribe((message) => {
const packet = parseDataMessage(message);
if (packet === null)
logger.debug(`Ignoring a data message from ${member.memberId}`);
else data$.next({ senderId: member.rtcBackendIdentity, ...packet });
});
s.onEnd(() => {
subscribers.delete(member.memberId);
connectionsChanged$.next();
});
return connection;
});
return {
tracks$: memberScope.behavior(
subscriber$.pipe(switchMap((s) => s?.tracks$ ?? of(null))),
),
encryptionError$: NEVER,
};
};
return {
transportType: FULL_MESH_TRANSPORT_TYPE,
prepareLocalTransport: async (transport) => {
if (transport.type !== FULL_MESH_TRANSPORT_TYPE)
throw new MatrixRTCError(
`Not a full mesh transport: ${transport.type}`,
);
localTransport = transport;
// Fetched now, so that the first subscriber isn't kept waiting
await iceServers();
prepared$.next(true);
return { canDelegateDelayedLeave: false };
},
delegateDelayedLeave: async () =>
Promise.reject(
new MatrixRTCError("Full mesh cannot take over the delayed leave"),
),
local,
mediaFor,
connections$: connections$ as Behavior<BackendConnection[]>,
setAudioOutputDeviceId: async (deviceId) => {
audioOutputDeviceId$.next(deviceId);
return Promise.resolve();
},
sendData: async (topic, text) => {
if (!prepared$.value)
throw new MatrixRTCError(
"Cannot send data: not publishing on full mesh",
);
const message = JSON.stringify({ topic, text });
for (const { connection } of publishers.values())
connection.sendData(message);
return Promise.resolve();
},
data$: data$.pipe(share()),
};
}
/** A message of the `sendData` data channel, or null for a malformed one. */
function parseDataMessage(
message: string,
): Pick<DataPacket, "topic" | "text"> | null {
try {
const { topic, text } = JSON.parse(message) as Record<string, unknown>;
return typeof topic === "string" && typeof text === "string"
? { topic, text }
: null;
} catch {
return null;
}
}
/** One peer connection as the developer tools show it. */
function describe(
{ userId, deviceId, memberId }: Peer,
{ connectionId, pc }: PeerConnectionBase,
): Record<string, string> {
return {
userId,
deviceId,
memberId,
connectionId,
connectionState: pc.connectionState,
iceConnectionState: pc.iceConnectionState,
signalingState: pc.signalingState,
};
}
@@ -1,425 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type Logger } from "matrix-js-sdk/lib/logger";
import {
BehaviorSubject,
combineLatest,
filter,
firstValueFrom,
map,
NEVER,
} from "rxjs";
import { v4 as uuidv4 } from "uuid";
import { MatrixRTCError, toMatrixRTCError } from "../../errors";
import {
type LocalAudioMediaTrack,
type LocalVideoMediaTrack,
type MediaSource,
type PublishRequest,
type VideoProcessor,
} from "../../media-api";
import { type Behavior } from "../../reactive/Behavior";
import { ObservableScope } from "../../reactive/ObservableScope";
import { type LocalMediaBackend, type MediaConnectionState } from "../api";
import { createFullMeshMediaTrack, rtpStats } from "./MediaTrack";
type PublishedSource = Exclude<MediaSource, "unknown">;
/** One local track as the publisher connections send it. */
export interface LocalPublication {
source: PublishedSource;
kind: "audio" | "video";
/**
* The `MediaStream` it is sent in: one for the user media, one for the
* screen share, so that the subscriber plays each in sync.
*/
stream: MediaStream;
/** What goes on the wire; null while the source is off. */
track$: Behavior<MediaStreamTrack | null>;
mediaTrack: LocalAudioMediaTrack | LocalVideoMediaTrack;
}
export interface FullMeshLocalMedia extends LocalMediaBackend {
/** Every local track, once the first `setPublishing(true)` started the capture. */
publications$: Behavior<LocalPublication[]>;
/** Whether the tracks reach the publisher connections. */
publishing$: Behavior<boolean>;
}
interface Props {
scope: ObservableScope;
publish: PublishRequest[];
connectionState$: Behavior<MediaConnectionState | Error>;
/** The publisher connections, for the outbound statistics of a track. */
peerConnections: () => RTCPeerConnection[];
logger: Logger;
}
interface Capture {
/** What was captured, before any processor. */
raw: MediaStreamTrack;
out: MediaStreamTrack;
}
/** The local member's side: capture, the publication list and the controls. */
export function createFullMeshLocalMedia({
scope,
publish: initialRequests,
connectionState$,
peerConnections,
logger: parentLogger,
}: Props): FullMeshLocalMedia {
const logger = parentLogger.getChild("[LocalMedia]");
// Only their ids matter: the tracks are added per connection
const userMediaStream = new MediaStream();
const screenShareStream = new MediaStream();
const desired = new Map<PublishRequest["source"], PublishRequest>(
initialRequests.map((r) => [r.source, r]),
);
const started$ = new BehaviorSubject(false);
const publishing$ = new BehaviorSubject(false);
const publishError$ = new BehaviorSubject<Error | null>(null);
const publications$ = new BehaviorSubject<LocalPublication[]>([]);
const teardowns = new Map<PublishedSource, () => void>();
const statsFor = async (track: MediaStreamTrack | null) => {
for (const pc of peerConnections())
if (pc.getSenders().some((s) => s.track === track && track !== null))
return rtpStats(pc, track, "outbound-rtp");
return undefined;
};
/** Adds a publication whose track is managed by `control`. */
const addPublication = (
source: PublishedSource,
kind: "audio" | "video",
stream: MediaStream,
initial: MediaStreamTrack,
control: {
setEnabled: (
enabled: boolean,
track$: BehaviorSubject<MediaStreamTrack | null>,
) => Promise<boolean>;
setDevice?: (
deviceId: string,
track$: BehaviorSubject<MediaStreamTrack | null>,
) => Promise<void>;
switchFacingMode?: (
track$: BehaviorSubject<MediaStreamTrack | null>,
) => Promise<string | undefined>;
setProcessor?: (
p: VideoProcessor | undefined,
track$: BehaviorSubject<MediaStreamTrack | null>,
) => Promise<void>;
stop: () => void;
},
): LocalPublication => {
const pubScope = new ObservableScope();
const track$ = new BehaviorSubject<MediaStreamTrack | null>(initial);
const enabled$ = new BehaviorSubject(true);
const base = createFullMeshMediaTrack(pubScope, {
source,
kind,
id: uuidv4(),
track$,
muted$: pubScope.behavior(enabled$.pipe(map((e) => !e))),
stats: async () => statsFor(track$.value),
});
const local = {
setEnabled: async (enabled: boolean): Promise<boolean> => {
const result = await control.setEnabled(enabled, track$);
enabled$.next(result);
return result;
},
setDevice: async (deviceId: string): Promise<void> => {
if (!control.setDevice)
throw new MatrixRTCError(`A ${source} track has no device`);
await control.setDevice(deviceId, track$);
},
};
const mediaTrack =
base.kind === "audio"
? { ...base, ...local }
: {
...base,
...local,
facingMode$: pubScope.behavior(
track$.pipe(map((t) => facingModeOf(t))),
),
switchFacingMode: async () => control.switchFacingMode?.(track$),
setProcessor: async (p: VideoProcessor | undefined) =>
control.setProcessor?.(p, track$),
};
const publication: LocalPublication = {
source,
kind,
stream,
track$,
mediaTrack: mediaTrack as LocalAudioMediaTrack | LocalVideoMediaTrack,
};
publications$.next([...publications$.value, publication]);
teardowns.set(source, () => {
control.stop();
track$.value?.stop();
pubScope.end();
publications$.next(publications$.value.filter((p) => p !== publication));
teardowns.delete(source);
});
return publication;
};
const captureMicrophone = async (
request: Extract<PublishRequest, { source: "microphone" }>,
): Promise<void> => {
const track = await getTrack({
audio: { deviceId: exact(request.deviceId), ...request.capture },
});
addPublication("microphone", "audio", userMediaStream, track, {
// Muting a microphone keeps the device: unmuting is instant
setEnabled: async (enabled, track$) => {
if (track$.value) track$.value.enabled = enabled;
return Promise.resolve(enabled);
},
setDevice: async (deviceId, track$) => {
const next = await getTrack({
audio: { deviceId: { exact: deviceId }, ...request.capture },
});
next.enabled = track$.value?.enabled ?? true;
track$.value?.stop();
track$.next(next);
},
stop: () => {},
});
};
const captureCamera = async (
request: Extract<PublishRequest, { source: "camera" }>,
): Promise<void> => {
let deviceId = request.deviceId;
let facingMode: string | undefined;
let processor = request.processor;
let capture: Capture | null = null;
let processorStarted = false;
const constraints = (): MediaTrackConstraints => ({
deviceId: exact(deviceId),
facingMode,
width: request.capture?.resolution?.width,
height: request.capture?.resolution?.height,
frameRate: request.capture?.resolution?.frameRate,
});
const open = async (): Promise<Capture> => {
const raw = await getTrack({ video: constraints() });
if (processor === undefined) return { raw, out: raw };
const out = processorStarted
? await processor.restart(raw)
: await processor.init(raw);
processorStarted = true;
return { raw, out };
};
const close = (): void => {
capture?.raw.stop();
if (capture?.out !== capture?.raw) capture?.out.stop();
capture = null;
};
// Reopens the camera with the current settings, if it is on
const reopen = async (
track$: BehaviorSubject<MediaStreamTrack | null>,
): Promise<void> => {
if (capture === null) return;
close();
capture = await open();
track$.next(capture.out);
};
capture = await open();
addPublication("camera", "video", userMediaStream, capture.out, {
// Muting a camera releases it, so that its light goes off
setEnabled: async (enabled, track$) => {
if (enabled === (capture !== null)) return enabled;
if (!enabled) {
close();
track$.next(null);
return false;
}
try {
capture = await open();
track$.next(capture.out);
return true;
} catch (e) {
logger.warn("Could not reopen the camera", e);
return false;
}
},
setDevice: async (id, track$) => {
deviceId = id;
facingMode = undefined;
await reopen(track$);
},
switchFacingMode: async (track$) => {
const current = facingModeOf(track$.value);
if (current === undefined) return undefined;
deviceId = undefined;
facingMode = current === "user" ? "environment" : "user";
await reopen(track$);
return capture?.raw.getSettings().deviceId;
},
setProcessor: async (p, track$) => {
await processor?.destroy();
processor = p;
processorStarted = false;
if (capture === null) return;
const out = p ? await p.init(capture.raw) : capture.raw;
processorStarted = p !== undefined;
if (capture.out !== capture.raw) capture.out.stop();
capture = { raw: capture.raw, out };
track$.next(out);
},
stop: () => {
close();
void processor?.destroy();
},
});
};
const captureScreen = async (
request: Extract<PublishRequest, { source: "screenShare" }>,
): Promise<void> => {
const stream = await navigator.mediaDevices.getDisplayMedia({
video: {
width: request.capture?.resolution?.width,
height: request.capture?.resolution?.height,
frameRate: request.capture?.resolution?.frameRate,
},
audio: request.audio ?? true,
});
const [video] = stream.getVideoTracks();
const [audio] = stream.getAudioTracks();
const toggle = async (
enabled: boolean,
track$: BehaviorSubject<MediaStreamTrack | null>,
): Promise<boolean> => {
if (track$.value) track$.value.enabled = enabled;
return Promise.resolve(enabled);
};
addPublication("screenShare", "video", screenShareStream, video, {
setEnabled: toggle,
// The audio goes with its video
stop: () => {
audio?.stop();
teardowns.get("screenShareAudio")?.();
},
});
if (audio)
addPublication("screenShareAudio", "audio", screenShareStream, audio, {
setEnabled: toggle,
stop: () => {},
});
// Stopped from the browser's own controls
video.addEventListener("ended", () => void unpublish("screenShare"));
};
const capture = async (request: PublishRequest): Promise<void> => {
teardowns.get(request.source)?.();
switch (request.source) {
case "microphone":
return captureMicrophone(request);
case "camera":
return captureCamera(request);
case "screenShare":
return captureScreen(request);
}
};
const start = async (): Promise<void> => {
for (const request of desired.values())
await capture(request).catch((e: unknown) => {
logger.warn(`Could not capture ${request.source}`, e);
if (publishError$.value === null)
publishError$.next(toMatrixRTCError(e));
});
};
const setPublishing = (publish: boolean): void => {
publishing$.next(publish);
if (publish && !started$.value) {
started$.next(true);
void start();
}
};
const publish = async (
request: PublishRequest,
): Promise<LocalAudioMediaTrack | LocalVideoMediaTrack> => {
desired.set(request.source, request);
try {
// Before the start, the start captures everything desired itself
if (started$.value) await capture(request);
return await firstValueFrom(
publications$.pipe(
map((ps) => ps.find((p) => p.source === request.source)),
filter((p) => p !== undefined),
map((p) => p.mediaTrack),
),
);
} catch (e) {
desired.delete(request.source);
throw toMatrixRTCError(e);
}
};
const unpublish = async (source: MediaSource): Promise<void> => {
const published = source === "screenShareAudio" ? "screenShare" : source;
if (published === "unknown") return;
desired.delete(published);
teardowns.get(published)?.();
return Promise.resolve();
};
scope.onEnd(() => {
// Each teardown removes itself from the map
for (const teardown of Array.from(teardowns.values())) teardown();
});
const tracks$ = scope.behavior(
combineLatest([started$, publications$]).pipe(
map(([started, ps]) => (started ? ps.map((p) => p.mediaTrack) : null)),
),
);
return {
connectionState$,
tracks$,
encryptionError$: NEVER,
setPublishing,
publishError$,
publish,
unpublish,
publications$,
publishing$,
};
}
function exact(deviceId: string | undefined): ConstrainDOMString | undefined {
return deviceId === undefined ? undefined : { exact: deviceId };
}
async function getTrack(
constraints: MediaStreamConstraints,
): Promise<MediaStreamTrack> {
const stream = await navigator.mediaDevices.getUserMedia(constraints);
const [track] = stream.getTracks();
if (track === undefined) throw new MatrixRTCError("No track was captured");
return track;
}
function facingModeOf(
track: MediaStreamTrack | null,
): "user" | "environment" | undefined {
const mode = track?.getSettings().facingMode;
return mode === "user" || mode === "environment" ? mode : undefined;
}
@@ -1,325 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
BehaviorSubject,
combineLatest,
distinctUntilChanged,
map,
Observable,
of,
scan,
switchMap,
timer,
} from "rxjs";
import { type Behavior, constant } from "../../reactive/Behavior";
import { type ObservableScope } from "../../reactive/ObservableScope";
import {
type AudioMediaTrack,
type MediaSource,
type MediaStreamStats,
type MediaTrack,
type VideoMediaTrack,
} from "../../media-api";
export interface TrackProps {
source: MediaSource;
kind: "audio" | "video";
id: string;
/** The track behind it; may change, as on a device switch, or be null while off. */
track$: Behavior<MediaStreamTrack | null>;
muted$: Behavior<boolean>;
/** Statistics of the RTP stream carrying it, if there is one. */
stats: () => Promise<MediaStreamStats>;
/** Where attached elements play; only for tracks that are played. */
audioOutputDeviceId$?: Behavior<string | undefined>;
/** Whether to follow where attached video elements are shown, for `viewport$`. */
observeViewport?: boolean;
}
/** Where a video is shown: the largest of its visible elements, in device pixels if known. */
export interface Viewport {
width?: number;
height?: number;
}
export type FullMeshMediaTrack = (AudioMediaTrack | VideoMediaTrack) & {
/** Null while no attached element is visible; always null unless observed. */
viewport$: Behavior<Viewport | null>;
};
/** Above this level a track turns active; below `INACTIVE_LEVEL` for `INACTIVE_AFTER_MS` it turns inactive. */
const ACTIVE_LEVEL = 0.04;
const INACTIVE_LEVEL = 0.02;
const LEVEL_INTERVAL_MS = 100;
const INACTIVE_AFTER_MS = 500;
let levelContext: AudioContext | undefined;
/**
* A `MediaStreamTrack` as a `MediaTrack`: the same object for the life of a
* publication, with every attached element following the track behind it.
*/
export function createFullMeshMediaTrack(
scope: ObservableScope,
props: TrackProps,
): FullMeshMediaTrack {
const { source, kind, id, track$, muted$, stats, audioOutputDeviceId$ } =
props;
const attached = new Set<HTMLMediaElement>();
const viewport =
kind === "video" && props.observeViewport
? observeViewport(scope)
: undefined;
let audioContext: AudioContext | undefined;
let audioPlugins: AudioNode[] = [];
let volume = 1;
let webAudio: { nodes: AudioNode[]; gain: GainNode } | undefined;
const routeWebAudio = (track: MediaStreamTrack | null): void => {
for (const node of webAudio?.nodes ?? []) node.disconnect();
webAudio = undefined;
if (audioContext === undefined || track === null) return;
const input = audioContext.createMediaStreamSource(
new MediaStream([track]),
);
const gain = audioContext.createGain();
gain.gain.value = volume;
const chain: AudioNode[] = [input, ...audioPlugins, gain];
for (let i = 0; i < chain.length - 1; i++) chain[i].connect(chain[i + 1]);
gain.connect(audioContext.destination);
webAudio = { nodes: chain, gain };
};
const apply = (element: HTMLMediaElement): void => {
const track = track$.value;
element.srcObject = track ? new MediaStream([track]) : null;
if (kind === "audio") {
// Played through Web Audio instead, where it is routed there
element.muted = audioContext !== undefined;
element.volume = volume;
const sinkId = audioOutputDeviceId$?.value;
if (sinkId !== undefined && "setSinkId" in element)
void element.setSinkId(sinkId).catch(() => {});
}
if (element.paused) void element.play().catch(() => {});
};
track$.pipe(scope.bind()).subscribe((track) => {
for (const element of attached) apply(element);
if (kind === "audio") routeWebAudio(track);
});
audioOutputDeviceId$?.pipe(scope.bind()).subscribe(() => {
for (const element of attached) apply(element);
});
scope.onEnd(() => {
for (const element of attached) element.srcObject = null;
attached.clear();
routeWebAudio(null);
});
const base: MediaTrack = {
source,
kind,
id,
muted$,
// DTLS-SRTP, its fingerprints exchanged over Olm
encrypted: true,
stats$: (intervalMs): Observable<MediaStreamStats> =>
timer(0, intervalMs).pipe(
switchMap(async () => stats()),
scope.bind(),
),
attach: (element) => {
if (attached.has(element)) return;
attached.add(element);
apply(element);
viewport?.observe(element);
},
detach: (element) => {
if (!attached.delete(element)) return;
element.srcObject = null;
viewport?.unobserve(element);
},
};
if (kind === "video")
return {
...base,
kind: "video",
viewport$: viewport?.viewport$ ?? constant(null),
};
const audioLevel$ = scope.behavior(
combineLatest([track$, muted$]).pipe(
switchMap(([track, muted]) =>
track === null || muted ? of(0) : level$(track),
),
),
0,
);
const isActive$ = scope.behavior(
audioLevel$.pipe(
scan(
({ active, quietMs }, level) => {
if (level >= ACTIVE_LEVEL) return { active: true, quietMs: 0 };
if (level >= INACTIVE_LEVEL) return { active, quietMs: 0 };
const quiet = quietMs + LEVEL_INTERVAL_MS;
return {
active: active && quiet < INACTIVE_AFTER_MS,
quietMs: quiet,
};
},
{ active: false, quietMs: 0 },
),
map(({ active }) => active),
distinctUntilChanged(),
),
false,
);
return {
...base,
kind: "audio",
viewport$: constant(null),
audioLevel$,
isActive$,
setAudioContext: (ctx, plugins = []) => {
audioContext = ctx;
audioPlugins = plugins;
routeWebAudio(track$.value);
for (const element of attached) apply(element);
},
setVolume: (v) => {
volume = v;
if (webAudio) webAudio.gain.gain.value = v;
for (const element of attached) element.volume = v;
},
};
}
/**
* Follows the size and visibility of elements. Where the platform has no
* observers, an element counts as visible at an unknown size.
*/
function observeViewport(scope: ObservableScope): {
viewport$: Behavior<Viewport | null>;
observe: (element: Element) => void;
unobserve: (element: Element) => void;
} {
const shown = new Map<
Element,
{ visible: boolean; width?: number; height?: number }
>();
const viewport$ = new BehaviorSubject<Viewport | null>(null);
const update = (): void => {
const visible = [...shown.values()].filter(
({ visible, width, height }) => visible && width !== 0 && height !== 0,
);
// One element of unknown size leaves the size unknown
const next: Viewport | null =
visible.length === 0
? null
: visible.some((e) => e.width === undefined || e.height === undefined)
? {}
: {
width: Math.max(...visible.map((e) => e.width!)),
height: Math.max(...visible.map((e) => e.height!)),
};
const current = viewport$.value;
if (
(current === null) !== (next === null) ||
current?.width !== next?.width ||
current?.height !== next?.height
)
viewport$.next(next);
};
const pixelRatio = globalThis.devicePixelRatio ?? 1;
const resizes =
typeof ResizeObserver === "undefined"
? undefined
: new ResizeObserver((entries) => {
for (const { target, contentRect } of entries) {
const entry = shown.get(target);
if (entry === undefined) continue;
entry.width = Math.ceil(contentRect.width * pixelRatio);
entry.height = Math.ceil(contentRect.height * pixelRatio);
}
update();
});
const intersections =
typeof IntersectionObserver === "undefined"
? undefined
: new IntersectionObserver((entries) => {
for (const { target, isIntersecting } of entries) {
const entry = shown.get(target);
if (entry !== undefined) entry.visible = isIntersecting;
}
update();
});
scope.onEnd(() => {
resizes?.disconnect();
intersections?.disconnect();
});
return {
viewport$: scope.behavior(viewport$),
observe: (element): void => {
// Visible until the observers say otherwise, so that nothing is paused
// before they first report
shown.set(element, { visible: true });
resizes?.observe(element);
intersections?.observe(element);
update();
},
unobserve: (element): void => {
shown.delete(element);
resizes?.unobserve(element);
intersections?.unobserve(element);
update();
},
};
}
/** The RMS level of an audio track, 0 to 1, sampled while subscribed. */
function level$(track: MediaStreamTrack): Observable<number> {
return new Observable<number>((subscriber) => {
levelContext ??= new AudioContext();
const ctx = levelContext;
const input = ctx.createMediaStreamSource(new MediaStream([track]));
const analyser = ctx.createAnalyser();
analyser.fftSize = 512;
input.connect(analyser);
const samples = new Float32Array(analyser.fftSize);
const sub = timer(0, LEVEL_INTERVAL_MS).subscribe(() => {
analyser.getFloatTimeDomainData(samples);
let sum = 0;
for (const s of samples) sum += s * s;
subscriber.next(Math.min(1, Math.sqrt(sum / samples.length) * 4));
});
return () => {
sub.unsubscribe();
input.disconnect();
};
});
}
/** The inbound or outbound RTP statistics of one track on a peer connection. */
export async function rtpStats(
pc: RTCPeerConnection | undefined,
track: MediaStreamTrack | null,
direction: "inbound-rtp" | "outbound-rtp",
): Promise<MediaStreamStats> {
if (pc === undefined || track === null) return undefined;
const report = await pc.getStats(track);
for (const stats of report.values())
if ((stats as RTCStats).type === direction)
return stats as RTCInboundRtpStreamStats & RTCOutboundRtpStreamStats;
return undefined;
}
@@ -1,248 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { logger } from "matrix-js-sdk/lib/logger";
import { ObservableScope } from "../../reactive/ObservableScope";
import { MediaConnectionState } from "../api";
import {
CANDIDATE_BATCH_MS,
createPeerConnectionBase,
type PeerConnectionBase,
} from "./PeerConnection";
import { type ControlMessage, CONTROL_CHANNEL_LABEL } from "./ControlChannel";
import {
type FakeSignalling,
fakeSignalling,
installWebRtcFakes,
lastPeerConnection,
peer,
settle,
} from "./test";
describe("createPeerConnectionBase", () => {
let scope: ObservableScope;
let signalling: FakeSignalling;
let base: PeerConnectionBase;
let received: ControlMessage[];
beforeEach(() => {
vi.useFakeTimers();
installWebRtcFakes();
scope = new ObservableScope();
signalling = fakeSignalling();
received = [];
base = createPeerConnectionBase({
scope,
peer,
connectionId: "c1",
signalling,
iceServers: [{ urls: ["turn:example.org"] }],
onControlMessage: (message) => received.push(message),
logger,
});
});
afterEach(() => {
scope.end();
vi.useRealTimers();
vi.unstubAllGlobals();
});
it("uses the ICE servers it was given", () => {
expect(lastPeerConnection().config.iceServers).toEqual([
{ urls: ["turn:example.org"] },
]);
});
it("sends the candidates gathered within a batch in one message", () => {
const pc = lastPeerConnection();
pc.gather({ candidate: "a", sdpMid: "0", sdpMLineIndex: 0 });
pc.gather({ candidate: "b", sdpMid: "0", sdpMLineIndex: 0 });
pc.gather(null);
expect(signalling.send).not.toHaveBeenCalled();
vi.advanceTimersByTime(CANDIDATE_BATCH_MS);
expect(signalling.sent()).toEqual([
{
type: "candidates",
candidates: [
{ candidate: "a", sdpMid: "0", sdpMLineIndex: 0 },
{ candidate: "b", sdpMid: "0", sdpMLineIndex: 0 },
// The end of the gathering
{ candidate: "", sdpMid: null, sdpMLineIndex: null },
],
},
]);
expect(signalling.sentConnectionIds()).toEqual(["c1"]);
});
it("drops pending candidates when it closes", () => {
lastPeerConnection().gather({ candidate: "a" });
base.close("bye", false);
vi.advanceTimersByTime(CANDIDATE_BATCH_MS);
expect(signalling.send).not.toHaveBeenCalled();
});
it("runs queued operations in order, past a failing one", async () => {
const order: string[] = [];
base.enqueue(async () => {
order.push("first");
return Promise.reject(new Error("failed"));
});
base.enqueue(async () => {
order.push("second");
return Promise.resolve();
});
await settle();
expect(order).toEqual(["first", "second"]);
});
it("applies received candidates, ignoring one that fails", async () => {
base.addCandidates([{ candidate: "bad" }, { candidate: "good" }]);
await settle();
expect(lastPeerConnection().addedCandidates).toEqual([
{ candidate: "good" },
]);
});
it("tells the peer once when it closes", () => {
base.close("done", true);
base.close("again", true);
expect(lastPeerConnection().closed).toBe(true);
expect(signalling.sent()).toEqual([{ type: "close", reason: "done" }]);
});
it("doesn't tell a peer that closed first", () => {
base.close("theirs", false);
expect(lastPeerConnection().closed).toBe(true);
expect(signalling.send).not.toHaveBeenCalled();
});
it("closes when its scope ends", () => {
scope.end();
expect(lastPeerConnection().closed).toBe(true);
expect(signalling.sent()).toEqual([{ type: "close", reason: "ended" }]);
});
it("maps the connection state", () => {
const pc = lastPeerConnection();
expect(base.state$.value).toBe(MediaConnectionState.Initialized);
pc.setConnectionState("connecting");
expect(base.state$.value).toBe(MediaConnectionState.Connecting);
pc.setConnectionState("connected");
expect(base.state$.value).toBe(MediaConnectionState.Connected);
pc.setConnectionState("disconnected");
expect(base.state$.value).toBe(MediaConnectionState.Reconnecting);
pc.setConnectionState("failed");
expect(base.state$.value).toBeInstanceOf(Error);
pc.setConnectionState("closed");
expect(base.state$.value).toBe(MediaConnectionState.Stopped);
});
describe("the control channel", () => {
const offer: ControlMessage = {
type: "description",
description: { type: "offer", sdp: "o" },
};
const metadata = (muted: boolean): ControlMessage => ({
type: "track_metadata",
track_metadata: { "0": { purpose: "m.usermedia", muted } },
});
it("is negotiated, with the reserved label and id", () => {
base.openControlChannel();
base.openControlChannel();
const [channel] = lastPeerConnection().dataChannels;
expect(lastPeerConnection().dataChannels).toHaveLength(1);
expect(channel.label).toBe(CONTROL_CHANNEL_LABEL);
expect(channel.init).toEqual({ negotiated: true, id: 0 });
});
it("carries every message once it is open, apart from close", () => {
base.openControlChannel();
base.send(offer);
expect(signalling.sent()).toEqual([offer]);
const channel = lastPeerConnection().control();
channel.open();
base.answerReceived();
base.send(offer);
lastPeerConnection().gather({ candidate: "a" });
vi.advanceTimersByTime(CANDIDATE_BATCH_MS);
expect(channel.sentMessages()).toEqual([
offer,
{ type: "candidates", candidates: [{ candidate: "a" }] },
]);
base.close("done", true);
expect(signalling.sent()).toEqual([
offer,
{ type: "close", reason: "done" },
]);
});
it("is only switched to once no to-device offer awaits its answer", () => {
base.openControlChannel();
base.send(offer);
base.send(metadata(true));
const channel = lastPeerConnection().control();
channel.open();
// A late to-device offer could otherwise overwrite this metadata
lastPeerConnection().gather({ candidate: "a" });
vi.advanceTimersByTime(CANDIDATE_BATCH_MS);
expect(channel.sentMessages()).toEqual([]);
expect(signalling.sent()).toEqual([
offer,
{ type: "candidates", candidates: [{ candidate: "a" }] },
]);
base.answerReceived();
expect(channel.sentMessages()).toEqual([metadata(true)]);
});
it("holds the latest metadata until it opens", () => {
base.openControlChannel();
base.send(metadata(true));
base.send(metadata(false));
expect(signalling.send).not.toHaveBeenCalled();
const channel = lastPeerConnection().control();
channel.open();
expect(channel.sentMessages()).toEqual([metadata(false)]);
});
it("hands over the messages it receives, ignoring what it doesn't know", () => {
base.openControlChannel();
const channel = lastPeerConnection().control();
channel.open();
channel.receive(metadata(true));
channel.receive({ type: "future_message" });
channel.receive("not json");
channel.receive({ type: "description", description: { type: "x" } });
expect(received).toEqual([metadata(true)]);
});
it("fails the connection when it closes on its own", () => {
base.openControlChannel();
const pc = lastPeerConnection();
pc.setConnectionState("connected");
pc.control().open();
pc.control().close();
expect(base.state$.value).toEqual(
new Error("The control channel closed"),
);
});
it("doesn't fail a connection we close", () => {
base.openControlChannel();
lastPeerConnection().control().open();
base.close("done", true);
expect(base.state$.value).not.toBeInstanceOf(Error);
});
});
});
@@ -1,226 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
/** What the publisher and the subscriber side of a connection share. */
import { type Logger } from "matrix-js-sdk/lib/logger";
import {
BehaviorSubject,
combineLatest,
fromEvent,
map,
startWith,
} from "rxjs";
import { type Behavior } from "../../reactive/Behavior";
import { type ObservableScope } from "../../reactive/ObservableScope";
import { MediaConnectionState } from "../api";
import {
type ControlMessage,
createControlChannel,
parseControlMessage,
} from "./ControlChannel";
import { type Peer, type Signalling } from "./signalling";
/** How long trickled candidates are collected before they are sent in one message. */
export const CANDIDATE_BATCH_MS = 200;
type MetadataMessage = Extract<
ControlMessage,
{ type: "track_metadata" | "consumption_metadata" }
>;
export interface PeerConnectionBase {
readonly connectionId: string;
readonly pc: RTCPeerConnection;
readonly state$: Behavior<MediaConnectionState | Error>;
/** Runs `op` after every operation queued before it, so that SDP and candidates apply in order. */
enqueue(op: () => Promise<void>): void;
/**
* Creates our side of the control channel. The publisher does so before
* its first offer, the subscriber once it applied that offer.
*/
openControlChannel(): void;
/**
* Sends a message to the peer: over the control channel once we switched
* to it, before that as a to-device message. Metadata has no to-device
* message of its own: the latest waits for the switch.
*/
send(message: ControlMessage): void;
/**
* Tells that the answer to our latest offer arrived. We switch to the
* control channel only once no to-device offer is waiting for its answer,
* so that a late offer can't overwrite newer metadata.
*/
answerReceived(): void;
/** Applies a received batch of candidates. */
addCandidates(candidates: RTCIceCandidateInit[]): void;
/** Closes the peer connection, telling the peer unless it closed first. */
close(reason: string | undefined, notifyPeer: boolean): void;
}
export function createPeerConnectionBase({
scope,
peer,
connectionId,
signalling,
iceServers,
onControlMessage,
logger,
}: {
scope: ObservableScope;
peer: Peer;
connectionId: string;
signalling: Signalling;
iceServers: RTCIceServer[];
/** A message the peer sent over the control channel. */
onControlMessage: (message: ControlMessage) => void;
logger: Logger;
}): PeerConnectionBase {
const pc = new RTCPeerConnection({ iceServers });
const closed$ = new BehaviorSubject(false);
const controlFailed$ = new BehaviorSubject(false);
let queue = Promise.resolve();
const enqueue = (op: () => Promise<void>): void => {
queue = queue.then(op).catch((e: unknown) => {
logger.warn("Signalling operation failed", e);
});
};
let control: RTCDataChannel | null = null;
let switched = false;
let awaitingAnswer = false;
const pendingMetadata = new Map<MetadataMessage["type"], MetadataMessage>();
const trySwitch = (): void => {
if (switched || awaitingAnswer || control?.readyState !== "open") return;
switched = true;
logger.debug("Switched to the control channel");
for (const message of pendingMetadata.values())
control.send(JSON.stringify(message));
pendingMetadata.clear();
};
const openControlChannel = (): void => {
if (control !== null) return;
const channel = createControlChannel(pc);
control = channel;
channel.addEventListener("open", trySwitch);
channel.addEventListener("message", ({ data }: MessageEvent) => {
const message = parseControlMessage(data);
if (message === null) logger.debug("Ignoring a control message", data);
else onControlMessage(message);
});
// Ours to close only with the connection: closing otherwise is a failure
channel.addEventListener("close", () => {
if (!closed$.value) controlFailed$.next(true);
});
};
const send = (message: ControlMessage): void => {
if (closed$.value) return;
if (switched) control!.send(JSON.stringify(message));
else if (
message.type === "track_metadata" ||
message.type === "consumption_metadata"
)
pendingMetadata.set(message.type, message);
else {
if (
message.type === "description" &&
message.description.type === "offer"
)
awaitingAnswer = true;
signalling.send(peer, connectionId, message);
}
};
const answerReceived = (): void => {
awaitingAnswer = false;
trySwitch();
};
// Trickled, in batches
let pending: RTCIceCandidateInit[] = [];
let batchTimer: ReturnType<typeof setTimeout> | undefined;
const flushCandidates = (): void => {
batchTimer = undefined;
if (pending.length === 0) return;
send({ type: "candidates", candidates: pending });
pending = [];
};
pc.addEventListener("icecandidate", ({ candidate }) => {
// A null candidate ends the gathering: an empty one says so on the wire
pending.push(
candidate
? candidate.toJSON()
: { candidate: "", sdpMid: null, sdpMLineIndex: null },
);
batchTimer ??= setTimeout(flushCandidates, CANDIDATE_BATCH_MS);
});
const addCandidates: PeerConnectionBase["addCandidates"] = (candidates) =>
enqueue(async () => {
for (const candidate of candidates)
await pc
.addIceCandidate(candidate)
// A candidate of an outdated description, or a bad one: not fatal
.catch((e: unknown) => logger.debug("Ignored a candidate", e));
});
const close: PeerConnectionBase["close"] = (reason, notifyPeer) => {
if (closed$.value) return;
clearTimeout(batchTimer);
// Always to-device: closing the peer connection would race a channel message
if (notifyPeer)
signalling.send(peer, connectionId, { type: "close", reason });
closed$.next(true);
pc.close();
};
scope.onEnd(() => close("ended", true));
const state$ = scope.behavior(
combineLatest([
fromEvent(pc, "connectionstatechange").pipe(startWith(null)),
controlFailed$,
]).pipe(
map(([, controlFailed]) =>
controlFailed && pc.connectionState !== "closed"
? new Error("The control channel closed")
: toState(pc.connectionState),
),
),
);
return {
connectionId,
pc,
state$,
enqueue,
openControlChannel,
send,
answerReceived,
addCandidates,
close,
};
}
function toState(state: RTCPeerConnectionState): MediaConnectionState | Error {
switch (state) {
case "new":
return MediaConnectionState.Initialized;
case "connecting":
return MediaConnectionState.Connecting;
case "connected":
return MediaConnectionState.Connected;
case "disconnected":
return MediaConnectionState.Reconnecting;
case "failed":
return new Error("The peer connection failed");
case "closed":
return MediaConnectionState.Stopped;
}
}
@@ -1,360 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { logger } from "matrix-js-sdk/lib/logger";
import { ObservableScope } from "../../reactive/ObservableScope";
import { CONTROL_CHANNEL_LABEL } from "./ControlChannel";
import { type FullMeshLocalMedia } from "./LocalMedia";
import { DATA_CHANNEL_LABEL, DATA_PROTOCOL } from "./metadata";
import {
createPublisherConnection,
type PublisherConnection,
} from "./PublisherConnection";
import {
type FakeSignalling,
fakeLocalMedia,
fakePublication,
fakeSignalling,
fakeTrack,
fromPeer,
type PeerMessage,
installWebRtcFakes,
lastPeerConnection,
peer,
settle,
} from "./test";
describe("createPublisherConnection", () => {
let scope: ObservableScope;
let signalling: FakeSignalling;
let local: ReturnType<typeof fakeLocalMedia>;
function create(): PublisherConnection {
return createPublisherConnection({
scope,
peer,
connectionId: "c1",
signalling,
iceServers: [],
local: local as unknown as FullMeshLocalMedia,
logger,
});
}
const answer = (): PeerMessage =>
fromPeer("c1", {
type: "description",
description: { type: "answer", sdp: "answer" },
});
beforeEach(() => {
installWebRtcFakes();
scope = new ObservableScope();
signalling = fakeSignalling();
local = fakeLocalMedia();
});
afterEach(() => {
scope.end();
vi.unstubAllGlobals();
});
it("offers right away, even without tracks", async () => {
create();
await settle();
expect(signalling.sent("description")).toEqual([
{
type: "description",
description: { type: "offer", sdp: "offer-1" },
track_metadata: {},
},
]);
});
it("describes each track by the mid it gets with the offer", async () => {
const mic = fakePublication("audio");
const camera = fakePublication("video", { stream: mic.stream });
const screen = fakePublication("video", { source: "screenShare" });
camera.muted$.next(true);
local.publications$.next([mic, camera, screen]);
create();
await settle();
expect(signalling.sent("description")).toEqual([
expect.objectContaining({
track_metadata: {
"0": { purpose: "m.usermedia", muted: false },
"1": { purpose: "m.usermedia", muted: true },
"2": { purpose: "m.screenshare", muted: false },
},
}),
]);
});
it("creates the control channel and the data channel before its first offer", () => {
create();
const [control, data] = lastPeerConnection().dataChannels;
expect(control.label).toBe(CONTROL_CHANNEL_LABEL);
expect(data).toMatchObject({
label: DATA_CHANNEL_LABEL,
protocol: DATA_PROTOCOL,
});
expect(lastPeerConnection().offers).toBe(0);
});
it("sends each publication on a sendonly transceiver in its stream", async () => {
const mic = fakePublication("audio");
const camera = fakePublication("video", { stream: mic.stream });
local.publications$.next([mic, camera]);
create();
await settle();
const [a, v] = lastPeerConnection().transceivers;
expect(a).toMatchObject({ kind: "audio", direction: "sendonly" });
expect(a.streams).toEqual([mic.stream]);
expect(a.sender.track).toBe(mic.track$.value);
expect(v).toMatchObject({ kind: "video", direction: "sendonly" });
expect(v.sender.track).toBe(camera.track$.value);
// One offer for both
expect(signalling.sent("description")).toHaveLength(1);
});
it("follows a track change without renegotiating", async () => {
const camera = fakePublication("video");
local.publications$.next([camera]);
create();
await settle();
const next = fakeTrack("video");
camera.track$.next(next);
expect(lastPeerConnection().transceivers[0].sender.track).toBe(next);
expect(signalling.sent("description")).toHaveLength(1);
});
it("sends nothing while not publishing", async () => {
const camera = fakePublication("video");
local.publications$.next([camera]);
local.publishing$.next(false);
create();
await settle();
const { sender } = lastPeerConnection().transceivers[0];
expect(sender.track).toBeNull();
local.publishing$.next(true);
expect(sender.track).toBe(camera.track$.value);
});
it("holds changes while an offer is unanswered, then offers once for all", async () => {
const connection = create();
const pc = lastPeerConnection();
await settle();
expect(pc.offers).toBe(1);
local.publications$.next([fakePublication("audio")]);
local.publications$.next([
...local.publications$.value,
fakePublication("video"),
]);
await settle();
expect(pc.offers).toBe(1);
connection.handle(answer());
await settle();
expect(pc.offers).toBe(2);
expect(pc.transceivers).toHaveLength(2);
// Nothing changed since: the next answer doesn't lead to another offer
connection.handle(answer());
await settle();
expect(pc.offers).toBe(2);
});
it("stops the transceiver of a removed publication and renegotiates", async () => {
const camera = fakePublication("video");
local.publications$.next([camera]);
const connection = create();
await settle();
connection.handle(answer());
await settle();
local.publications$.next([]);
await settle();
const pc = lastPeerConnection();
expect(pc.transceivers[0].stopped).toBe(true);
expect(pc.offers).toBe(2);
});
it("signals mute changes over the control channel without renegotiating", async () => {
const mic = fakePublication("audio");
local.publications$.next([mic]);
const connection = create();
await settle();
connection.handle(answer());
await settle();
const control = lastPeerConnection().control();
control.open();
mic.muted$.next(true);
expect(control.sentMessages()).toEqual([
{
type: "track_metadata",
track_metadata: { "0": { purpose: "m.usermedia", muted: true } },
},
]);
expect(lastPeerConnection().offers).toBe(1);
});
it("holds a mute change until its to-device offer is answered", async () => {
const mic = fakePublication("audio");
local.publications$.next([mic]);
const connection = create();
await settle();
const control = lastPeerConnection().control();
control.open();
mic.muted$.next(true);
expect(control.sentMessages()).toEqual([]);
connection.handle(answer());
await settle();
expect(control.sentMessages()).toEqual([
{
type: "track_metadata",
track_metadata: { "0": { purpose: "m.usermedia", muted: true } },
},
]);
});
it("renegotiates over the control channel once it is open", async () => {
const connection = create();
await settle();
connection.handle(answer());
await settle();
const control = lastPeerConnection().control();
control.open();
local.publications$.next([fakePublication("audio")]);
await settle();
expect(control.sentMessages()).toContainEqual(
expect.objectContaining({
type: "description",
description: { type: "offer", sdp: "offer-2" },
}),
);
expect(signalling.sent("description")).toHaveLength(1);
// The answer comes back the same way
control.receive({
type: "description",
description: { type: "answer", sdp: "answer-2" },
});
await settle();
expect(lastPeerConnection().remoteDescription).toEqual({
type: "answer",
sdp: "answer-2",
});
});
describe("consumption metadata", () => {
/** A connection sending one camera track, with its control channel open. */
async function sendingCamera(settings: MediaTrackSettings = {}): Promise<{
camera: ReturnType<typeof fakePublication>;
consume: (metadata: unknown) => Promise<void>;
}> {
const camera = fakePublication("video");
camera.track$.next(fakeTrack("video", settings));
local.publications$.next([camera]);
const connection = create();
await settle();
connection.handle(answer());
await settle();
const control = lastPeerConnection().control();
control.open();
return {
camera,
consume: async (metadata): Promise<void> => {
control.receive({
type: "consumption_metadata",
consumption_metadata: metadata,
});
await settle();
},
};
}
it("stops sending a track that isn't in use, without renegotiating", async () => {
const { camera, consume } = await sendingCamera();
const { sender } = lastPeerConnection().transceivers[0];
await consume({ "0": { in_use: false } });
expect(sender.track).toBeNull();
await consume({});
expect(sender.track).toBe(camera.track$.value);
expect(lastPeerConnection().offers).toBe(1);
});
it("scales the video down to the largest size shown", async () => {
const { consume } = await sendingCamera({ width: 1280, height: 720 });
const { sender } = lastPeerConnection().transceivers[0];
await consume({ "0": { max_width: 320, max_height: 320 } });
expect(sender.parameters.encodings).toEqual([
{ scaleResolutionDownBy: 4 },
]);
// Larger than the camera: sent as it is
await consume({ "0": { max_width: 4000, max_height: 4000 } });
expect(sender.parameters.encodings).toEqual([
{ scaleResolutionDownBy: 1 },
]);
});
});
it("sends data once the data channel is open", () => {
const connection = create();
expect(connection.sendData("early")).toBe(false);
const data = lastPeerConnection().dataChannels[1];
data.open();
expect(connection.sendData("hello")).toBe(true);
expect(data.sent).toEqual(["hello"]);
});
it("closes, telling the subscriber, when the control channel fails", async () => {
create();
await settle();
const pc = lastPeerConnection();
pc.control().open();
pc.control().close();
expect(pc.closed).toBe(true);
expect(signalling.sent("close")).toEqual([
{ type: "close", reason: "failed" },
]);
});
it("applies the subscriber's candidates", async () => {
const connection = create();
await settle();
connection.handle(answer());
connection.handle(
fromPeer("c1", {
type: "candidates",
candidates: [{ candidate: "x" }],
}),
);
await settle();
expect(lastPeerConnection().addedCandidates).toEqual([{ candidate: "x" }]);
});
it("ends without answering when the subscriber closes", async () => {
const connection = create();
await settle();
connection.handle(fromPeer("c1", { type: "close", reason: "bye" }));
const pc = lastPeerConnection();
expect(pc.closed).toBe(true);
expect(signalling.sent("close")).toEqual([]);
// Its scope ended: no more offers go out
local.publications$.next([fakePublication("audio")]);
await settle();
expect(pc.offers).toBe(1);
});
});
@@ -1,281 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type Logger } from "matrix-js-sdk/lib/logger";
import {
BehaviorSubject,
combineLatest,
distinctUntilChanged,
filter,
map,
merge,
type Observable,
skip,
type Subscription,
switchMap,
} from "rxjs";
import { type ObservableScope } from "../../reactive/ObservableScope";
import { type FullMeshLocalMedia, type LocalPublication } from "./LocalMedia";
import {
type ConsumptionMetadata,
type ConsumptionMetadataEntry,
DATA_CHANNEL_LABEL,
DATA_PROTOCOL,
type MediaPurpose,
type TrackMetadata,
} from "./metadata";
import {
createPeerConnectionBase,
type PeerConnectionBase,
} from "./PeerConnection";
import {
type ConnectionMessage,
type Peer,
type Signalling,
} from "./signalling";
export interface PublisherConnection extends PeerConnectionBase {
/** A to-device message of this connection from the subscriber. */
handle(message: ConnectionMessage): void;
/** Sends one message on the data channel; false while it isn't open. */
sendData(message: string): boolean;
}
/**
* Our side of a connection a subscriber asked for: we only send, and we are
* the only side that makes offers, so there is no glare.
*/
export function createPublisherConnection({
scope,
peer,
connectionId,
signalling,
iceServers,
local,
logger: parentLogger,
}: {
scope: ObservableScope;
peer: Peer;
connectionId: string;
signalling: Signalling;
iceServers: RTCIceServer[];
local: FullMeshLocalMedia;
logger: Logger;
}): PublisherConnection {
const logger = parentLogger.getChild(
`[Publisher → ${peer.memberId} ${connectionId}]`,
);
const base = createPeerConnectionBase({
scope,
peer,
connectionId,
signalling,
iceServers,
onControlMessage: (message) => handle(message),
logger,
});
const { pc, enqueue } = base;
const consumption$ = new BehaviorSubject<ConsumptionMetadata>({});
// Both before the first offer, which so has the connection's data section
base.openControlChannel();
const data = pc.createDataChannel(DATA_CHANNEL_LABEL, {
protocol: DATA_PROTOCOL,
});
// One sendonly transceiver per publication, for as long as it exists
const transceivers = new Map<
LocalPublication,
{ transceiver: RTCRtpTransceiver; tracks: Subscription }
>();
/** The metadata of every track with a `mid`, which it gets with the offer that adds it. */
const trackMetadata = (): TrackMetadata => {
const metadata: TrackMetadata = {};
for (const [publication, { transceiver }] of transceivers)
if (transceiver.mid !== null)
metadata[transceiver.mid] = {
purpose: purposeOf(publication),
muted: publication.mediaTrack.muted$.value,
};
return metadata;
};
// A new offer has to wait while one is unanswered; changes made meanwhile
// go out together in the next one
let offerPending = false;
let renegotiate = false;
const negotiate = (): void => {
if (offerPending) {
renegotiate = true;
return;
}
offerPending = true;
renegotiate = false;
enqueue(async () => {
const offer = await pc.createOffer();
await pc.setLocalDescription(offer);
base.send({
type: "description",
description: { type: "offer", sdp: pc.localDescription!.sdp },
track_metadata: trackMetadata(),
});
});
};
scope.onEnd(() => {
for (const { tracks } of transceivers.values()) tracks.unsubscribe();
});
local.publications$.pipe(scope.bind()).subscribe((publications) => {
let changed = false;
for (const [publication, { transceiver, tracks }] of transceivers)
if (!publications.includes(publication)) {
tracks.unsubscribe();
if (pc.signalingState !== "closed") transceiver.stop();
transceivers.delete(publication);
changed = true;
}
for (const publication of publications)
if (!transceivers.has(publication)) {
const transceiver = pc.addTransceiver(publication.kind, {
direction: "sendonly",
streams: [publication.stream],
});
const tracks = sendPublication(transceiver, publication);
transceivers.set(publication, { transceiver, tracks });
changed = true;
}
if (changed) negotiate();
});
// Even without tracks: the subscriber learns that we are there
if (transceivers.size === 0) negotiate();
/**
* Sends the track of a publication as the subscriber wants it. None while
* we don't publish or it isn't in use: the local preview stays live, and
* turning it back on needs no renegotiation.
*/
function sendPublication(
transceiver: RTCRtpTransceiver,
publication: LocalPublication,
): Subscription {
const { sender } = transceiver;
return combineLatest([
publication.track$,
local.publishing$,
consumptionOf(transceiver),
]).subscribe(([track, publishing, consumption]) => {
const inUse = consumption.in_use ?? true;
void sender
.replaceTrack(publishing && inUse ? track : null)
.then(async () => {
if (publication.kind === "video" && track !== null)
await constrainVideo(sender, track, consumption);
})
.catch((e: unknown) => logger.warn("Could not send a track", e));
});
}
// The mid is only set with the local description
function consumptionOf(
transceiver: RTCRtpTransceiver,
): Observable<ConsumptionMetadataEntry> {
return consumption$.pipe(
map((c) => (transceiver.mid === null ? {} : (c[transceiver.mid] ?? {}))),
distinctUntilChanged(
(a, b) =>
a.in_use === b.in_use &&
a.max_width === b.max_width &&
a.max_height === b.max_height,
),
);
}
// Mute changes without renegotiating; an offer carries the state at its time
local.publications$
.pipe(
switchMap((ps) =>
merge(...ps.map((p) => p.mediaTrack.muted$.pipe(skip(1)))),
),
scope.bind(),
)
.subscribe(() =>
base.send({ type: "track_metadata", track_metadata: trackMetadata() }),
);
base.state$
.pipe(
filter((s) => s instanceof Error),
scope.bind(),
)
.subscribe((e) => {
// The subscriber retries; a publisher doesn't
logger.info(`Connection failed: ${e.message}`);
base.close("failed", true);
scope.end();
});
function handle(message: ConnectionMessage): void {
switch (message.type) {
case "description":
if (message.description.type !== "answer") return;
enqueue(async () => {
await pc.setRemoteDescription(message.description);
base.answerReceived();
offerPending = false;
if (renegotiate) negotiate();
});
break;
case "candidates":
base.addCandidates(message.candidates);
break;
case "consumption_metadata":
consumption$.next(message.consumption_metadata);
break;
case "close":
logger.info(`Closed by the subscriber: ${message.reason}`);
base.close(message.reason, false);
scope.end();
break;
default:
logger.debug(`Ignoring ${message.type}`);
}
}
const sendData = (message: string): boolean => {
if (data.readyState !== "open") return false;
data.send(message);
return true;
};
return { ...base, handle, sendData };
}
function purposeOf({ source }: LocalPublication): MediaPurpose {
return source === "microphone" || source === "camera"
? "m.usermedia"
: "m.screenshare";
}
/** Scales the video down to the size the subscriber shows it at. */
async function constrainVideo(
sender: RTCRtpSender,
track: MediaStreamTrack,
{ max_width, max_height }: ConsumptionMetadataEntry,
): Promise<void> {
const parameters = sender.getParameters();
const [encoding] = parameters.encodings ?? [];
if (encoding === undefined) return;
const { width, height } = track.getSettings();
encoding.scaleResolutionDownBy = Math.max(
1,
width && max_width ? width / max_width : 1,
height && max_height ? height / max_height : 1,
);
await sender.setParameters(parameters);
}
-21
View File
@@ -1,21 +0,0 @@
# Full mesh media backend
The MatrixRTC full mesh transport, as specified in
[MSC4575: Full mesh transport for MatrixRTC](https://github.com/matrix-org/matrix-spec-proposals/pull/4575).
Every subscriber opens a WebRTC connection to every publisher it receives from. To-device messages set
the connection up; after that, the connection's control channel carries all signalling.
The MSC is not merged, so all identifiers use its unstable prefix `org.matrix.msc4575`.
| File | What it implements |
| ------------------------- | ------------------------------------------------------------------- |
| `FullMeshBackend.ts` | The `MediaBackend`: subscribing to members, accepting subscribers |
| `signalling.ts` | The to-device messages and their ordering by `seq` |
| `ControlChannel.ts` | The control channel and its messages |
| `PeerConnection.ts` | What both sides of a connection share, including the channel switch |
| `PublisherConnection.ts` | Sending our tracks and the data channel to one subscriber |
| `SubscriberConnection.ts` | Receiving one publisher's tracks, with retries |
| `metadata.ts` | Track and consumption metadata, and their validation |
| `LocalMedia.ts` | Capturing the local tracks every publisher connection sends |
| `MediaTrack.ts` | A WebRTC track as the SDK's `MediaTrack` |
@@ -1,307 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { logger } from "matrix-js-sdk/lib/logger";
import { constant } from "../../reactive/Behavior";
import { ObservableScope } from "../../reactive/ObservableScope";
import {
DATA_CHANNEL_LABEL,
DATA_PROTOCOL,
type TrackMetadata,
} from "./metadata";
import {
createSubscriberConnection,
type SubscriberConnection,
} from "./SubscriberConnection";
import {
type FakeSignalling,
fakeSignalling,
fromPeer,
type PeerMessage,
installWebRtcFakes,
lastPeerConnection,
peer,
settle,
} from "./test";
describe("createSubscriberConnection", () => {
let scope: ObservableScope;
let signalling: FakeSignalling;
let subscriber: SubscriberConnection;
beforeEach(async () => {
vi.useFakeTimers();
installWebRtcFakes();
scope = new ObservableScope();
signalling = fakeSignalling();
subscriber = createSubscriberConnection({
scope,
peer,
signalling,
iceServers: async () => Promise.resolve([{ urls: ["turn:x"] }]),
audioOutputDeviceId$: constant(undefined),
logger,
});
await settle();
});
afterEach(() => {
scope.end();
vi.useRealTimers();
vi.unstubAllGlobals();
});
it("asks the publisher for its media on a fresh connection", () => {
expect(signalling.sent()).toEqual([{ type: "subscribe" }]);
expect(signalling.sentConnectionIds()).toEqual([current()]);
expect(lastPeerConnection().config.iceServers).toEqual([
{ urls: ["turn:x"] },
]);
});
it("answers an offer and shows the tracks it brings", async () => {
expect(subscriber.tracks$.value).toBeNull();
const pc = lastPeerConnection();
pc.receive("video", "1");
pc.receive("video", "2");
subscriber.handle(
offer(current(), {
"1": { purpose: "m.usermedia", muted: true },
"2": { purpose: "m.screenshare" },
}),
);
await settle();
expect(pc.remoteDescription).toEqual({ type: "offer", sdp: "offer" });
// How it shows them goes over the control channel once it is open
expect(signalling.sent("description")).toEqual([
{ type: "description", description: { type: "answer", sdp: "answer" } },
]);
const tracks = subscriber.tracks$.value!;
expect(tracks.map((t) => [t.source, t.kind, t.muted$.value])).toEqual([
["camera", "video", true],
["screenShare", "video", false],
]);
});
it("names audio by its purpose", async () => {
const pc = lastPeerConnection();
pc.receive("audio", "0");
pc.receive("audio", "3");
subscriber.handle(
offer(current(), {
"0": { purpose: "m.usermedia" },
"3": { purpose: "m.screenshare" },
}),
);
await settle();
expect(subscriber.tracks$.value!.map((t) => t.source)).toEqual([
"microphone",
"screenShareAudio",
]);
});
it("leaves out tracks without metadata or with an unknown purpose", async () => {
const pc = lastPeerConnection();
pc.receive("video", "0");
pc.receive("video", "1");
subscriber.handle(offer(current(), { "1": { purpose: "m.whiteboard" } }));
await settle();
expect(subscriber.tracks$.value).toEqual([]);
});
it("follows mute changes, keeping the same track", async () => {
lastPeerConnection().receive("video", "1");
subscriber.handle(offer(current(), { "1": { purpose: "m.usermedia" } }));
await settle();
const [track] = subscriber.tracks$.value!;
expect(track.muted$.value).toBe(false);
lastPeerConnection()
.control()
.receive({
type: "track_metadata",
track_metadata: { "1": { purpose: "m.usermedia", muted: true } },
});
await settle();
expect(subscriber.tracks$.value).toEqual([track]);
expect(track.muted$.value).toBe(true);
});
it("drops a track whose transceiver no longer receives", async () => {
const pc = lastPeerConnection();
const transceiver = pc.receive("video", "1");
subscriber.handle(offer(current(), { "1": { purpose: "m.usermedia" } }));
await settle();
expect(subscriber.tracks$.value).toHaveLength(1);
transceiver.currentDirection = "inactive";
subscriber.handle(offer(current(), { "1": { purpose: "m.usermedia" } }));
await settle();
expect(subscriber.tracks$.value).toEqual([]);
});
it("ignores messages of another connection", async () => {
subscriber.handle(offer("other", {}));
await settle();
expect(lastPeerConnection().remoteDescription).toBeNull();
});
it("subscribes anew after the publisher closes, backing off", async () => {
const first = current();
subscriber.handle(fromPeer(first, { type: "close", reason: "nope" }));
expect(subscriber.current$.value).toBeNull();
expect(subscriber.tracks$.value).toBeNull();
// It doesn't echo the publisher's close
expect(signalling.sent("close")).toEqual([]);
await vi.advanceTimersByTimeAsync(1_000);
const second = current();
expect(second).not.toBe(first);
expect(signalling.sentConnectionIds("subscribe")).toEqual([first, second]);
// A failed connection is closed and retried after twice the wait
lastPeerConnection().setConnectionState("failed");
expect(signalling.sent("close")).toEqual([
{ type: "close", reason: "failed" },
]);
await vi.advanceTimersByTimeAsync(1_000);
expect(signalling.sent("subscribe")).toHaveLength(2);
await vi.advanceTimersByTimeAsync(1_000);
expect(signalling.sent("subscribe")).toHaveLength(3);
});
it("stops retrying once its scope ends", async () => {
subscriber.handle(fromPeer(current(), { type: "close", reason: "nope" }));
scope.end();
await vi.advanceTimersByTimeAsync(60_000);
expect(signalling.sent("subscribe")).toHaveLength(1);
});
describe("the control channel", () => {
it("is created once the offer is applied", async () => {
const pc = lastPeerConnection();
expect(pc.dataChannels).toEqual([]);
subscriber.handle(offer(current(), {}));
await settle();
expect(pc.control().init).toEqual({ negotiated: true, id: 0 });
});
it("failing leads to a new subscription", async () => {
const first = current();
subscriber.handle(offer(first, {}));
await settle();
const control = lastPeerConnection().control();
control.open();
control.close();
expect(signalling.sent("close")).toEqual([
{ type: "close", reason: "failed" },
]);
await vi.advanceTimersByTimeAsync(1_000);
expect(current()).not.toBe(first);
});
it("carries the answers to later offers", async () => {
subscriber.handle(offer(current(), {}));
await settle();
const control = lastPeerConnection().control();
control.open();
control.receive({
type: "description",
description: { type: "offer", sdp: "offer-2" },
track_metadata: {},
});
await vi.advanceTimersByTimeAsync(200);
expect(control.sentMessages()).toContainEqual({
type: "description",
description: { type: "answer", sdp: "answer" },
});
expect(signalling.sent("description")).toHaveLength(1);
});
});
describe("consumption metadata", () => {
it("tells the publisher how each video is shown, once the channel is open", async () => {
const pc = lastPeerConnection();
pc.receive("audio", "0");
pc.receive("video", "1");
subscriber.handle(
offer(current(), {
"0": { purpose: "m.usermedia" },
"1": { purpose: "m.usermedia" },
}),
);
await vi.advanceTimersByTimeAsync(200);
const control = pc.control();
control.open();
// Nothing shows the video yet; the audio is always in use
expect(control.sentMessages()).toEqual([
{
type: "consumption_metadata",
consumption_metadata: { "1": { in_use: false } },
},
]);
const video = subscriber.tracks$.value!.find((t) => t.kind === "video")!;
const element = { paused: false } as HTMLMediaElement;
video.attach(element);
await vi.advanceTimersByTimeAsync(200);
// Shown at a size the platform doesn't report: no constraint at all
expect(control.sentMessages().at(-1)).toEqual({
type: "consumption_metadata",
consumption_metadata: {},
});
video.detach(element);
await vi.advanceTimersByTimeAsync(200);
expect(control.sentMessages().at(-1)).toEqual({
type: "consumption_metadata",
consumption_metadata: { "1": { in_use: false } },
});
});
});
describe("the data channel", () => {
it("delivers what the publisher sends on it", async () => {
const received: string[] = [];
subscriber.data$.subscribe((text) => received.push(text));
subscriber.handle(offer(current(), {}));
await settle();
const pc = lastPeerConnection();
pc.announceDataChannel(DATA_CHANNEL_LABEL, DATA_PROTOCOL).receive("hi");
expect(received).toEqual(["hi"]);
});
it("ignores a channel whose protocol it doesn't know, whatever its label", async () => {
const received: string[] = [];
subscriber.data$.subscribe((text) => received.push(text));
subscriber.handle(offer(current(), {}));
await settle();
const pc = lastPeerConnection();
pc.announceDataChannel(DATA_CHANNEL_LABEL, "x.other").receive("no");
pc.announceDataChannel(DATA_CHANNEL_LABEL).receive("no");
expect(received).toEqual([]);
});
});
function offer(connectionId: string, metadata: TrackMetadata): PeerMessage {
return fromPeer(connectionId, {
type: "description",
description: { type: "offer", sdp: "offer" },
track_metadata: metadata,
});
}
/** The connection id of the current attempt. */
function current(): string {
const base = subscriber.current$.value;
if (base === null) throw new Error("No current attempt");
return base.connectionId;
}
});
@@ -1,366 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type Logger } from "matrix-js-sdk/lib/logger";
import {
BehaviorSubject,
combineLatest,
debounceTime,
distinctUntilChanged,
filter,
map,
type Observable,
of,
Subject,
switchMap,
} from "rxjs";
import { v4 as uuidv4 } from "uuid";
import { type MediaSource } from "../../media-api";
import { type Behavior } from "../../reactive/Behavior";
import { ObservableScope } from "../../reactive/ObservableScope";
import { MediaConnectionState } from "../api";
import {
createFullMeshMediaTrack,
type FullMeshMediaTrack,
rtpStats,
type Viewport,
} from "./MediaTrack";
import {
type ConsumptionMetadata,
DATA_PROTOCOL,
isMediaPurpose,
type MediaPurpose,
type TrackMetadata,
} from "./metadata";
import {
createPeerConnectionBase,
type PeerConnectionBase,
} from "./PeerConnection";
import {
type ConnectionMessage,
type Peer,
type Signalling,
} from "./signalling";
const RETRY_MIN_MS = 1_000;
const RETRY_MAX_MS = 30_000;
/** How long the shown size of the tracks settles before the publisher hears of it. */
const CONSUMPTION_DEBOUNCE_MS = 200;
export interface SubscriberConnection {
/** The current attempt; null while waiting to retry. */
current$: Behavior<PeerConnectionBase | null>;
/** The publisher's tracks; null until its first offer is applied. */
tracks$: Behavior<FullMeshMediaTrack[] | null>;
/** What the publisher sent on its data channel, on any attempt. */
data$: Observable<string>;
/** A to-device message from the publisher; dropped unless it is for the current attempt. */
handle(message: ConnectionMessage & { connectionId: string }): void;
}
/**
* Our subscription to one publisher: asks it for its media and answers its
* offers. A failed or closed attempt is retried with a fresh connection id,
* backing off, for as long as `scope` lives.
*/
export function createSubscriberConnection({
scope,
peer,
signalling,
iceServers,
audioOutputDeviceId$,
logger: parentLogger,
}: {
scope: ObservableScope;
peer: Peer;
signalling: Signalling;
iceServers: () => Promise<RTCIceServer[]>;
audioOutputDeviceId$: Behavior<string | undefined>;
logger: Logger;
}): SubscriberConnection {
const logger = parentLogger.getChild(`[Subscriber ← ${peer.memberId}]`);
const attempt$ = new BehaviorSubject<Attempt | null>(null);
const data$ = new Subject<string>();
let retryMs = RETRY_MIN_MS;
let retryTimer: ReturnType<typeof setTimeout> | undefined;
let ended = false;
scope.onEnd(() => (ended = true));
const startAttempt = async (): Promise<void> => {
const servers = await iceServers();
if (ended) return;
const attemptScope = new ObservableScope();
const attempt = createAttempt({
scope: attemptScope,
peer,
signalling,
iceServers: servers,
audioOutputDeviceId$,
onData: (text) => data$.next(text),
onFailed: (reason) => {
logger.info(`Attempt ${attempt.connectionId} ended: ${reason}`);
attemptScope.end();
if (attempt$.value === attempt) attempt$.next(null);
if (ended) return;
retryTimer = setTimeout(() => void startAttempt(), retryMs);
retryMs = Math.min(retryMs * 2, RETRY_MAX_MS);
},
onConnected: () => (retryMs = RETRY_MIN_MS),
logger,
});
attempt$.next(attempt);
};
void startAttempt();
scope.onEnd(() => {
clearTimeout(retryTimer);
attempt$.value?.end();
attempt$.next(null);
data$.complete();
});
return {
current$: scope.behavior(attempt$.pipe(map((a) => a?.base ?? null))),
tracks$: scope.behavior(
attempt$.pipe(switchMap((a) => a?.tracks$ ?? of(null))),
),
data$,
handle: (message) => {
if (message.connectionId === attempt$.value?.connectionId)
attempt$.value.handle(message);
},
};
}
interface Attempt {
connectionId: string;
base: PeerConnectionBase;
tracks$: Behavior<FullMeshMediaTrack[] | null>;
handle(message: ConnectionMessage): void;
end(): void;
}
interface ReceivedTrack {
track: FullMeshMediaTrack;
mid: string;
scope: ObservableScope;
source: MediaSource;
}
function createAttempt({
scope,
peer,
signalling,
iceServers,
audioOutputDeviceId$,
onData,
onFailed,
onConnected,
logger,
}: {
scope: ObservableScope;
peer: Peer;
signalling: Signalling;
iceServers: RTCIceServer[];
audioOutputDeviceId$: Behavior<string | undefined>;
onData: (text: string) => void;
onFailed: (reason: string) => void;
onConnected: () => void;
logger: Logger;
}): Attempt {
const connectionId = uuidv4();
let failed = false;
const fail = (reason: string): void => {
if (failed) return;
failed = true;
onFailed(reason);
};
const base = createPeerConnectionBase({
scope,
peer,
connectionId,
signalling,
iceServers,
onControlMessage: (message) => handle(message),
logger,
});
const { pc, enqueue } = base;
const metadata$ = new BehaviorSubject<TrackMetadata>({});
const received$ = new BehaviorSubject<ReceivedTrack[] | null>(null);
const wrappers = new Map<RTCRtpTransceiver, ReceivedTrack>();
pc.addEventListener("datachannel", ({ channel }) => {
if (channel.protocol !== DATA_PROTOCOL) return;
channel.addEventListener("message", ({ data }: MessageEvent) => {
if (typeof data === "string") onData(data);
});
});
/** The tracks of the transceivers that receive, after each negotiation. */
const updateTracks = (): void => {
const metadata = metadata$.value;
const live = new Set<RTCRtpTransceiver>();
for (const transceiver of pc.getTransceivers()) {
const { mid, receiver } = transceiver;
const track = receiver.track;
const entry = mid === null ? undefined : metadata[mid];
if (
transceiver.currentDirection !== "recvonly" ||
mid === null ||
entry === undefined ||
!isMediaPurpose(entry.purpose)
)
continue;
const kind = track.kind === "audio" ? "audio" : "video";
const source = sourceOf(entry.purpose, kind);
live.add(transceiver);
const existing = wrappers.get(transceiver);
if (existing?.source === source) continue;
existing?.scope.end();
const trackScope = new ObservableScope();
wrappers.set(transceiver, {
scope: trackScope,
source,
mid,
track: createFullMeshMediaTrack(trackScope, {
source,
kind,
id: `${connectionId}:${mid}`,
track$: trackScope.behavior(of(track)),
muted$: trackScope.behavior(
metadata$.pipe(
map((m) => m[mid]?.muted ?? false),
distinctUntilChanged(),
),
),
stats: async () => rtpStats(pc, track, "inbound-rtp"),
audioOutputDeviceId$,
observeViewport: true,
}),
});
}
for (const [transceiver, { scope: s }] of wrappers)
if (!live.has(transceiver)) {
s.end();
wrappers.delete(transceiver);
}
received$.next([...wrappers.values()]);
};
// How the videos are shown; the first goes out once the control channel
// is open, as everything is sent at full size until then anyway
received$
.pipe(
switchMap((received) => {
const video = (received ?? []).filter((r) => r.track.kind === "video");
return video.length === 0
? of({})
: combineLatest(video.map((r) => r.track.viewport$)).pipe(
map((viewports) => consumptionOf(video, viewports)),
);
}),
distinctUntilChanged((a, b) => JSON.stringify(a) === JSON.stringify(b)),
debounceTime(CONSUMPTION_DEBOUNCE_MS),
scope.bind(),
)
.subscribe((metadata) =>
base.send({
type: "consumption_metadata",
consumption_metadata: metadata,
}),
);
base.state$
.pipe(
filter((s) => s instanceof Error || s === MediaConnectionState.Connected),
scope.bind(),
)
.subscribe((s) => {
if (s instanceof Error) {
base.close("failed", true);
fail(s.message);
} else onConnected();
});
function handle(message: ConnectionMessage): void {
switch (message.type) {
case "description":
if (message.description.type !== "offer") return;
enqueue(async () => {
if (message.track_metadata) metadata$.next(message.track_metadata);
await pc.setRemoteDescription(message.description);
base.openControlChannel();
await pc.setLocalDescription(await pc.createAnswer());
updateTracks();
base.send({
type: "description",
description: { type: "answer", sdp: pc.localDescription!.sdp },
});
});
break;
case "candidates":
base.addCandidates(message.candidates);
break;
case "track_metadata":
metadata$.next(message.track_metadata);
// A track may only become known with its metadata
enqueue(async () => Promise.resolve(updateTracks()));
break;
case "close":
base.close(message.reason, false);
fail(`closed by the publisher: ${message.reason}`);
break;
default:
logger.debug(`Ignoring ${message.type}`);
}
}
scope.onEnd(() => {
for (const { scope: s } of wrappers.values()) s.end();
wrappers.clear();
});
signalling.send(peer, connectionId, { type: "subscribe" });
return {
connectionId,
base,
tracks$: scope.behavior(
received$.pipe(map((r) => r?.map(({ track }) => track) ?? null)),
),
handle,
end: () => scope.end(),
};
}
/**
* The consumption metadata for the received videos, given their viewports
* in order: a video no element shows isn't in use, and one shown at a known
* size needs no more than that.
*/
function consumptionOf(
video: ReceivedTrack[],
viewports: (Viewport | null)[],
): ConsumptionMetadata {
const metadata: ConsumptionMetadata = {};
video.forEach(({ mid }, i) => {
const viewport = viewports[i];
if (viewport === null) metadata[mid] = { in_use: false };
else if (viewport.width !== undefined && viewport.height !== undefined)
metadata[mid] = {
max_width: viewport.width,
max_height: viewport.height,
};
});
return metadata;
}
function sourceOf(purpose: MediaPurpose, kind: "audio" | "video"): MediaSource {
if (purpose === "m.usermedia")
return kind === "audio" ? "microphone" : "camera";
return kind === "audio" ? "screenShareAudio" : "screenShare";
}
@@ -1,52 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { describe, expect, it } from "vitest";
import { parseConsumptionMetadata, parseTrackMetadata } from "./metadata";
describe("parseTrackMetadata", () => {
it("keeps the well-formed entries", () => {
expect(
parseTrackMetadata({
"0": { purpose: "m.usermedia", muted: true, extra: 1 },
"1": { purpose: "m.screenshare", muted: "yes" },
"2": { muted: true },
"3": [],
}),
).toEqual({
"0": { purpose: "m.usermedia", muted: true },
"1": { purpose: "m.screenshare" },
});
});
it("rejects what isn't metadata at all", () => {
expect(parseTrackMetadata("x")).toBeNull();
expect(parseTrackMetadata([])).toBeNull();
});
it("reads no more entries than any member publishes", () => {
const many = Object.fromEntries(
Array.from({ length: 100 }, (_, i) => [i, { purpose: "m.usermedia" }]),
);
expect(Object.keys(parseTrackMetadata(many)!)).toHaveLength(64);
});
});
describe("parseConsumptionMetadata", () => {
it("bounds the sizes to what makes sense", () => {
expect(
parseConsumptionMetadata({
"0": { in_use: false, max_width: 1e9, max_height: -1 },
"1": { max_width: 320.4, max_height: Number.NaN },
}),
).toEqual({
"0": { in_use: false, max_width: 16_384 },
"1": { max_width: 320 },
});
});
});
@@ -1,95 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
/**
* The track and consumption metadata of the full mesh transport (MSC4575),
* and their validation: both come from the other member and are untrusted.
* Both are keyed by the `mid` of a track's transceiver.
*/
/**
* The data channel every publisher connection has for `sendData`, and its
* protocol: one JSON object `{ topic, text }` per message.
*/
export const DATA_CHANNEL_LABEL = "io.element.matrixrtc.data";
export const DATA_PROTOCOL = "io.element.matrixrtc.topic_text";
export type MediaPurpose = "m.usermedia" | "m.screenshare";
export interface TrackMetadataEntry {
purpose: string;
muted?: boolean;
}
export type TrackMetadata = Record<string, TrackMetadataEntry>;
export interface ConsumptionMetadataEntry {
in_use?: boolean;
max_width?: number;
max_height?: number;
}
export type ConsumptionMetadata = Record<string, ConsumptionMetadataEntry>;
/** Larger than any video we capture; a bound on what a subscriber can ask for. */
const MAX_DIMENSION = 16_384;
/** More tracks than any member publishes; a bound on the work a message causes. */
const MAX_TRACKS = 64;
export function isMediaPurpose(purpose: string): purpose is MediaPurpose {
return purpose === "m.usermedia" || purpose === "m.screenshare";
}
/** The entries of `value` that are well formed, or null if it isn't metadata at all. */
export function parseTrackMetadata(value: unknown): TrackMetadata | null {
const entries = recordEntries(value);
if (entries === null) return null;
const metadata: TrackMetadata = {};
for (const [mid, e] of entries) {
if (typeof e.purpose !== "string") continue;
const entry: TrackMetadataEntry = { purpose: e.purpose };
if (typeof e.muted === "boolean") entry.muted = e.muted;
metadata[mid] = entry;
}
return metadata;
}
/** As `parseTrackMetadata`, with the sizes bounded to what makes sense. */
export function parseConsumptionMetadata(
value: unknown,
): ConsumptionMetadata | null {
const entries = recordEntries(value);
if (entries === null) return null;
const metadata: ConsumptionMetadata = {};
for (const [mid, e] of entries) {
const entry: ConsumptionMetadataEntry = {};
if (typeof e.in_use === "boolean") entry.in_use = e.in_use;
const width = bounded(e.max_width);
if (width !== undefined) entry.max_width = width;
const height = bounded(e.max_height);
if (height !== undefined) entry.max_height = height;
metadata[mid] = entry;
}
return metadata;
}
function recordEntries(
value: unknown,
): [string, Record<string, unknown>][] | null {
if (typeof value !== "object" || value === null || Array.isArray(value))
return null;
return Object.entries(value)
.filter(
(e): e is [string, Record<string, unknown>] =>
typeof e[1] === "object" && e[1] !== null && !Array.isArray(e[1]),
)
.slice(0, MAX_TRACKS);
}
function bounded(value: unknown): number | undefined {
return typeof value === "number" && Number.isFinite(value) && value > 0
? Math.round(Math.min(value, MAX_DIMENSION))
: undefined;
}
@@ -1,199 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { EventEmitter } from "events";
import { ClientEvent, EventType, MatrixEvent } from "matrix-js-sdk";
import { logger } from "matrix-js-sdk/lib/logger";
import { ObservableScope } from "../../reactive/ObservableScope";
import {
createSignalling,
SEQ_GAP_TIMEOUT_MS,
SeqOrderer,
SignallingEventType,
type SignallingClient,
type SignallingMessage,
} from "./signalling";
describe("SeqOrderer", () => {
beforeEach(() => vi.useFakeTimers());
afterEach(() => vi.useRealTimers());
it("delivers in seq order", () => {
const delivered: string[] = [];
const orderer = new SeqOrderer<string>((s) => delivered.push(s));
orderer.push(1, "b");
orderer.push(2, "c");
expect(delivered).toEqual([]);
orderer.push(0, "a");
expect(delivered).toEqual(["a", "b", "c"]);
});
it("skips a gap that isn't filled in time, then drops the late message", () => {
const delivered: string[] = [];
const orderer = new SeqOrderer<string>((s) => delivered.push(s));
orderer.push(0, "a");
orderer.push(2, "c");
vi.advanceTimersByTime(SEQ_GAP_TIMEOUT_MS);
expect(delivered).toEqual(["a", "c"]);
orderer.push(1, "b");
expect(delivered).toEqual(["a", "c"]);
});
});
describe("createSignalling", () => {
const own = { userId: "@me:x", deviceId: "MINE", memberId: "me" };
function setup(): {
client: SignallingClient & EventEmitter;
received: SignallingMessage[];
scope: ObservableScope;
} {
const client = Object.assign(new EventEmitter(), {
encryptAndSendToDevice: vi.fn(async () => Promise.resolve()),
}) as unknown as SignallingClient & EventEmitter;
const scope = new ObservableScope();
const signalling = createSignalling({
scope,
client,
roomId: "!room:x",
slotId: "m.call#ROOM",
own,
logger,
});
const received: SignallingMessage[] = [];
signalling.messages$.subscribe((m) => received.push(m));
return { client, received, scope };
}
function toDevice(
type: string,
content: Record<string, unknown>,
encrypted = true,
): MatrixEvent {
const event = new MatrixEvent({ type, sender: "@peer:x", content });
if (encrypted)
vi.spyOn(event, "getWireType").mockReturnValue(
EventType.RoomMessageEncrypted,
);
return event;
}
const common = {
room_id: "!room:x",
slot_id: "m.call#ROOM",
member_id: "peer",
target_member_id: "me",
connection_id: "c1",
device_id: "PEER",
};
it("passes valid messages on and drops cleartext or misaddressed ones", () => {
const { client, received, scope } = setup();
client.emit(
ClientEvent.ToDeviceEvent,
toDevice(SignallingEventType.subscribe, { ...common, seq: 0 }, false),
);
client.emit(
ClientEvent.ToDeviceEvent,
toDevice(SignallingEventType.subscribe, {
...common,
target_member_id: "someone else",
seq: 0,
}),
);
expect(received).toEqual([]);
client.emit(
ClientEvent.ToDeviceEvent,
toDevice(SignallingEventType.subscribe, { ...common, seq: 0 }),
);
expect(received).toEqual([
{
type: "subscribe",
memberId: "peer",
userId: "@peer:x",
deviceId: "PEER",
connectionId: "c1",
},
]);
scope.end();
});
it("orders by seq across connection ids", () => {
const { client, received, scope } = setup();
client.emit(
ClientEvent.ToDeviceEvent,
toDevice(SignallingEventType.subscribe, {
...common,
connection_id: "c2",
seq: 2,
}),
);
client.emit(
ClientEvent.ToDeviceEvent,
toDevice(SignallingEventType.close, { ...common, seq: 1 }),
);
client.emit(
ClientEvent.ToDeviceEvent,
toDevice(SignallingEventType.subscribe, { ...common, seq: 0 }),
);
expect(received.map((m) => [m.type, m.connectionId])).toEqual([
["subscribe", "c1"],
["close", "c1"],
["subscribe", "c2"],
]);
scope.end();
});
it("keeps the well-formed metadata of a description", () => {
const { client, received, scope } = setup();
client.emit(
ClientEvent.ToDeviceEvent,
toDevice(SignallingEventType.description, {
...common,
seq: 0,
description: { type: "offer", sdp: "offer" },
track_metadata: {
"0": { purpose: "m.usermedia", muted: "yes" },
"1": { muted: true },
"2": "not an object",
},
}),
);
expect(received).toEqual([
expect.objectContaining({
type: "description",
track_metadata: { "0": { purpose: "m.usermedia" } },
}),
]);
scope.end();
});
it("sends each payload as its event type", () => {
const { client, scope } = setup();
const signalling = createSignalling({
scope,
client,
roomId: "!room:x",
slotId: "m.call#ROOM",
own,
logger,
});
signalling.send(
{ userId: "@peer:x", deviceId: "PEER", memberId: "peer" },
"c1",
{ type: "close", reason: "bye" },
);
expect(client.encryptAndSendToDevice).toHaveBeenCalledWith(
SignallingEventType.close,
[{ userId: "@peer:x", deviceId: "PEER" }],
expect.objectContaining({ reason: "bye", connection_id: "c1", seq: 0 }),
);
scope.end();
});
});
@@ -1,291 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
/**
* The to-device signalling of the full mesh transport (MSC4575): four event
* types, the fields every one carries, and the ordering by `seq`. It sets a
* connection up; the control channel carries the rest.
*/
import { type Logger } from "matrix-js-sdk/lib/logger";
import {
ClientEvent,
EventType,
type MatrixClient,
type MatrixEvent,
} from "matrix-js-sdk";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import { fromEvent, Observable, share } from "rxjs";
import { type ObservableScope } from "../../reactive/ObservableScope";
import { type ControlMessage } from "./ControlChannel";
import { parseTrackMetadata } from "./metadata";
export const FULL_MESH_TRANSPORT_TYPE = "org.matrix.msc4575.full_mesh";
const prefix = "org.matrix.msc4575.rtc.full_mesh";
/** The event type of each payload type. */
export const SignallingEventType = {
subscribe: `${prefix}.subscribe`,
description: `${prefix}.description`,
candidates: `${prefix}.candidates`,
close: `${prefix}.close`,
} as const;
export const signallingEventTypes: string[] =
Object.values(SignallingEventType);
/** How long a gap in `seq` is waited for before the missing messages are skipped. */
export const SEQ_GAP_TIMEOUT_MS = 10_000;
interface CommonContent {
room_id: string;
slot_id: string;
member_id: string;
target_member_id: string;
connection_id: string;
seq: number;
/**
* Not in the MSC, which takes the device from the Olm session. The js-sdk
* does not expose that for to-device events, so, like the key transport's
* `claimed_device_id`, the sender states it.
*/
device_id: string;
}
/** What a connection hears from its peer, over to-device messages or the control channel. */
export type ConnectionMessage =
| ControlMessage
| { type: "close"; reason?: string };
export type SignallingPayload =
| { type: "subscribe" }
| Extract<
ConnectionMessage,
{ type: "description" | "candidates" | "close" }
>;
export type SignallingMessage = SignallingPayload & {
/** The member that sent it, as named in `member_id`. */
memberId: string;
userId: string;
deviceId: string;
connectionId: string;
};
/** Another member, as far as signalling needs to know it. */
export interface Peer {
userId: string;
deviceId: string;
memberId: string;
}
export interface Signalling {
send(peer: Peer, connectionId: string, payload: SignallingPayload): void;
/** Every valid message for us, per sender in `seq` order. */
messages$: Observable<SignallingMessage>;
}
export type SignallingClient = Pick<
MatrixClient,
"encryptAndSendToDevice" | "on" | "off"
>;
interface Props {
scope: ObservableScope;
client: SignallingClient;
roomId: string;
slotId: string;
own: CallMembershipIdentityParts;
logger: Logger;
}
export function createSignalling({
scope,
client,
roomId,
slotId,
own,
logger: parentLogger,
}: Props): Signalling {
const logger = parentLogger.getChild("[Signalling]");
const nextSeqTo = new Map<string, number>();
const send = (
peer: Peer,
connectionId: string,
{ type, ...payload }: SignallingPayload,
): void => {
const eventType = SignallingEventType[type];
const seq = nextSeqTo.get(peer.memberId) ?? 0;
nextSeqTo.set(peer.memberId, seq + 1);
const content: CommonContent & Record<string, unknown> = {
...payload,
room_id: roomId,
slot_id: slotId,
member_id: own.memberId,
target_member_id: peer.memberId,
connection_id: connectionId,
seq,
device_id: own.deviceId,
};
client
.encryptAndSendToDevice(
eventType,
[{ userId: peer.userId, deviceId: peer.deviceId }],
content,
)
.catch((e: unknown) => logger.warn(`Failed to send ${eventType}`, e));
};
const messages$ = new Observable<SignallingMessage>((subscriber) => {
const orderers = new Map<string, SeqOrderer<SignallingMessage>>();
const sub = fromEvent<MatrixEvent>(client, ClientEvent.ToDeviceEvent)
.pipe(scope.bind())
.subscribe((event) => {
const parsed = parse(event, roomId, slotId, own.memberId, logger);
if (parsed === null) return;
let orderer = orderers.get(parsed.message.memberId);
if (orderer === undefined) {
orderer = new SeqOrderer((m) => subscriber.next(m), logger);
orderers.set(parsed.message.memberId, orderer);
}
orderer.push(parsed.seq, parsed.message);
});
return () => {
sub.unsubscribe();
for (const orderer of orderers.values()) orderer.dispose();
};
}).pipe(share());
return { send, messages$ };
}
function parse(
event: MatrixEvent,
roomId: string,
slotId: string,
ownMemberId: string,
logger: Logger,
): { seq: number; message: SignallingMessage } | null {
const type = event.getType();
if (!signallingEventTypes.includes(type)) return null;
// Via the widget API, to-device events are reconstructed as encrypted
if (event.getWireType() !== EventType.RoomMessageEncrypted) {
logger.warn(`Dropping ${type} sent in clear from ${event.getSender()}`);
return null;
}
const c = event.getContent<
Partial<CommonContent> & Record<string, unknown>
>();
const userId = event.getSender();
if (
userId === undefined ||
c.room_id !== roomId ||
c.slot_id !== slotId ||
c.target_member_id !== ownMemberId ||
typeof c.member_id !== "string" ||
typeof c.connection_id !== "string" ||
typeof c.device_id !== "string" ||
typeof c.seq !== "number"
)
return null;
const common = {
memberId: c.member_id,
userId,
deviceId: c.device_id,
connectionId: c.connection_id,
};
let payload: SignallingPayload;
switch (type) {
case SignallingEventType.subscribe:
payload = { type: "subscribe" };
break;
case SignallingEventType.description: {
const d = c.description as { type?: unknown; sdp?: unknown } | undefined;
if (
(d?.type !== "offer" && d?.type !== "answer") ||
typeof d.sdp !== "string"
)
return null;
payload = {
type: "description",
description: { type: d.type, sdp: d.sdp },
track_metadata: parseTrackMetadata(c.track_metadata) ?? undefined,
};
break;
}
case SignallingEventType.candidates:
if (!Array.isArray(c.candidates)) return null;
payload = {
type: "candidates",
candidates: c.candidates as RTCIceCandidateInit[],
};
break;
case SignallingEventType.close:
payload = {
type: "close",
reason: typeof c.reason === "string" ? c.reason : undefined,
};
break;
default:
return null;
}
return { seq: c.seq, message: { ...payload, ...common } };
}
/**
* Delivers the messages of one sender in `seq` order. A gap is waited for
* `SEQ_GAP_TIMEOUT_MS`, then skipped. A message below the next expected `seq`
* came after its gap was skipped, and is dropped.
*/
export class SeqOrderer<T> {
private next = 0;
private readonly buffer = new Map<number, T>();
private gapTimer: ReturnType<typeof setTimeout> | undefined;
public constructor(
private readonly deliver: (item: T) => void,
private readonly logger?: Logger,
private readonly gapTimeoutMs = SEQ_GAP_TIMEOUT_MS,
) {}
public push(seq: number, item: T): void {
if (seq < this.next) {
this.logger?.warn(`Dropping seq ${seq}, already past it`);
return;
}
this.buffer.set(seq, item);
this.flush();
}
public dispose(): void {
clearTimeout(this.gapTimer);
this.buffer.clear();
}
private flush(): void {
let item = this.buffer.get(this.next);
while (item !== undefined) {
this.buffer.delete(this.next);
this.next++;
this.deliver(item);
item = this.buffer.get(this.next);
}
clearTimeout(this.gapTimer);
this.gapTimer = undefined;
if (this.buffer.size > 0)
this.gapTimer = setTimeout(() => this.skipGap(), this.gapTimeoutMs);
}
private skipGap(): void {
const lowest = Math.min(...this.buffer.keys());
this.logger?.warn(`Skipping seq ${this.next} to ${lowest - 1}`);
this.next = lowest;
this.flush();
}
}
-380
View File
@@ -1,380 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
/**
* Fakes for the parts of WebRTC that jsdom lacks, and for the signalling, so
* that the full mesh connections can run in unit tests.
*/
import { vi } from "vitest";
import { BehaviorSubject, Subject } from "rxjs";
import { CONTROL_CHANNEL_LABEL } from "./ControlChannel";
import { type LocalPublication } from "./LocalMedia";
import {
type ConnectionMessage,
type Peer,
type Signalling,
type SignallingMessage,
type SignallingPayload,
} from "./signalling";
let nextId = 0;
export class FakeMediaStream {
public readonly id: string;
public constructor(public readonly tracks: MediaStreamTrack[] = []) {
this.id = `stream-${nextId++}`;
}
public getTracks(): MediaStreamTrack[] {
return this.tracks;
}
}
export function fakeTrack(
kind: "audio" | "video",
settings: MediaTrackSettings = {},
): MediaStreamTrack {
return {
id: `track-${nextId++}`,
kind,
enabled: true,
stop: vi.fn(),
getSettings: () => settings,
} as unknown as MediaStreamTrack;
}
/** A data channel whose opening and incoming messages the test drives. */
export class FakeDataChannel extends EventTarget {
public readyState: RTCDataChannelState = "connecting";
public readonly sent: string[] = [];
public readonly protocol: string;
public constructor(
public readonly label: string,
public readonly init: RTCDataChannelInit = {},
) {
super();
this.protocol = init.protocol ?? "";
}
public send(data: string): void {
if (this.readyState !== "open") throw new Error("The channel isn't open");
this.sent.push(data);
}
/** What was sent, parsed. */
public sentMessages(): unknown[] {
return this.sent.map((data) => JSON.parse(data) as unknown);
}
public open(): void {
this.readyState = "open";
this.dispatchEvent(new Event("open"));
}
public close(): void {
if (this.readyState === "closed") return;
this.readyState = "closed";
this.dispatchEvent(new Event("close"));
}
/** A message from the peer; an object is sent as JSON. */
public receive(data: unknown): void {
this.dispatchEvent(
Object.assign(new Event("message"), {
data: typeof data === "string" ? data : JSON.stringify(data),
}),
);
}
}
interface FakeTransceiver {
kind: string;
/** Null until a local description gives it one, as in a browser. */
mid: string | null;
direction: RTCRtpTransceiverDirection;
currentDirection: RTCRtpTransceiverDirection | null;
streams: MediaStream[];
stopped: boolean;
stop(): void;
sender: {
track: MediaStreamTrack | null;
replaceTrack: ReturnType<typeof vi.fn>;
parameters: RTCRtpSendParameters;
getParameters(): RTCRtpSendParameters;
setParameters: ReturnType<typeof vi.fn>;
};
receiver: { track: MediaStreamTrack };
}
/** Enough of `RTCPeerConnection` for one side of a connection, driven by the test. */
export class FakePeerConnection extends EventTarget {
public static instances: FakePeerConnection[] = [];
public signalingState: RTCSignalingState = "stable";
public connectionState: RTCPeerConnectionState = "new";
public iceConnectionState: RTCIceConnectionState = "new";
public localDescription: RTCSessionDescriptionInit | null = null;
public remoteDescription: RTCSessionDescriptionInit | null = null;
public readonly transceivers: FakeTransceiver[] = [];
public readonly addedCandidates: RTCIceCandidateInit[] = [];
public readonly dataChannels: FakeDataChannel[] = [];
public offers = 0;
public closed = false;
public constructor(public readonly config: RTCConfiguration) {
super();
FakePeerConnection.instances.push(this);
}
public addTransceiver(
kind: string,
init: RTCRtpTransceiverInit,
): FakeTransceiver {
const transceiver: FakeTransceiver = {
kind,
mid: null,
direction: init.direction ?? "sendrecv",
currentDirection: null,
streams: init.streams ?? [],
stopped: false,
stop(): void {
this.stopped = true;
},
sender: {
track: null,
replaceTrack: vi.fn(async (track: MediaStreamTrack | null) => {
transceiver.sender.track = track;
return Promise.resolve();
}),
parameters: {
encodings: [{}],
} as unknown as RTCRtpSendParameters,
getParameters: (): RTCRtpSendParameters =>
structuredClone(transceiver.sender.parameters),
setParameters: vi.fn(async (parameters: RTCRtpSendParameters) => {
transceiver.sender.parameters = parameters;
return Promise.resolve();
}),
},
receiver: { track: fakeTrack(kind === "audio" ? "audio" : "video") },
};
this.transceivers.push(transceiver);
return transceiver;
}
public createDataChannel(
label: string,
init?: RTCDataChannelInit,
): FakeDataChannel {
const channel = new FakeDataChannel(label, init);
this.dataChannels.push(channel);
return channel;
}
/** Our side of the control channel. */
public control(): FakeDataChannel {
const channel = this.dataChannels.find(
(c) => c.label === CONTROL_CHANNEL_LABEL,
);
if (channel === undefined) throw new Error("No control channel");
return channel;
}
/** A data channel the remote side created, as `ondatachannel` brings it. */
public announceDataChannel(label: string, protocol = ""): FakeDataChannel {
const channel = new FakeDataChannel(label, { protocol });
this.dispatchEvent(Object.assign(new Event("datachannel"), { channel }));
return channel;
}
public getTransceivers(): FakeTransceiver[] {
return this.transceivers;
}
public getSenders(): FakeTransceiver["sender"][] {
return this.transceivers.map((t) => t.sender);
}
public async createOffer(): Promise<RTCSessionDescriptionInit> {
return Promise.resolve({ type: "offer", sdp: `offer-${++this.offers}` });
}
public async createAnswer(): Promise<RTCSessionDescriptionInit> {
return Promise.resolve({ type: "answer", sdp: "answer" });
}
public async setLocalDescription(
description: RTCSessionDescriptionInit,
): Promise<void> {
this.localDescription = description;
this.transceivers.forEach((t, i) => (t.mid ??= String(i)));
this.signalingState =
description.type === "offer" ? "have-local-offer" : "stable";
return Promise.resolve();
}
public async setRemoteDescription(
description: RTCSessionDescriptionInit,
): Promise<void> {
this.remoteDescription = description;
this.signalingState =
description.type === "offer" ? "have-remote-offer" : "stable";
return Promise.resolve();
}
public async addIceCandidate(candidate: RTCIceCandidateInit): Promise<void> {
if (candidate.candidate === "bad")
return Promise.reject(new Error("bad candidate"));
this.addedCandidates.push(candidate);
return Promise.resolve();
}
public async getStats(): Promise<RTCStatsReport> {
return Promise.resolve(new Map() as unknown as RTCStatsReport);
}
public close(): void {
this.closed = true;
this.signalingState = "closed";
this.connectionState = "closed";
for (const channel of this.dataChannels) channel.close();
}
/** Gathers a local candidate; null ends the gathering. */
public gather(candidate: RTCIceCandidateInit | null): void {
this.dispatchEvent(
Object.assign(new Event("icecandidate"), {
candidate: candidate && { toJSON: () => candidate },
}),
);
}
public setConnectionState(state: RTCPeerConnectionState): void {
this.connectionState = state;
this.dispatchEvent(new Event("connectionstatechange"));
}
/** A track the remote side sends, as the next negotiation would bring it. */
public receive(kind: "audio" | "video", mid: string): FakeTransceiver {
const transceiver = this.addTransceiver(kind, { direction: "recvonly" });
transceiver.currentDirection = "recvonly";
transceiver.mid = mid;
return transceiver;
}
}
class FakeAudioContext {
public destination = {};
public createMediaStreamSource(): object {
return { connect: vi.fn(), disconnect: vi.fn() };
}
public createAnalyser(): object {
return {
fftSize: 512,
connect: vi.fn(),
disconnect: vi.fn(),
getFloatTimeDomainData: (samples: Float32Array): void => {
samples.fill(0);
},
};
}
public createGain(): object {
return { gain: { value: 1 }, connect: vi.fn(), disconnect: vi.fn() };
}
}
/** Replaces the WebRTC and Web Audio globals jsdom doesn't have. Undo with `vi.unstubAllGlobals()`. */
export function installWebRtcFakes(): void {
FakePeerConnection.instances = [];
vi.stubGlobal("RTCPeerConnection", FakePeerConnection);
vi.stubGlobal("MediaStream", FakeMediaStream);
vi.stubGlobal("AudioContext", FakeAudioContext);
}
/** The last peer connection created. */
export function lastPeerConnection(): FakePeerConnection {
const pc = FakePeerConnection.instances.at(-1);
if (pc === undefined) throw new Error("No peer connection was created");
return pc;
}
/** Lets every queued signalling operation run. */
export async function settle(): Promise<void> {
for (let i = 0; i < 20; i++) await Promise.resolve();
}
export const peer: Peer = {
userId: "@peer:example.org",
deviceId: "PEERDEVICE",
memberId: "peerMember",
};
export interface FakeSignalling extends Signalling {
send: ReturnType<typeof vi.fn<Signalling["send"]>>;
messages$: Subject<SignallingMessage>;
/** The payloads sent so far, optionally only those of one type. */
sent(type?: string): SignallingPayload[];
/** The connection ids sent with the payloads, in the same order. */
sentConnectionIds(type?: string): string[];
}
export function fakeSignalling(): FakeSignalling {
const send = vi.fn<Signalling["send"]>();
const calls = (type?: string): Parameters<Signalling["send"]>[] =>
send.mock.calls.filter(([, , p]) => type === undefined || p.type === type);
return {
send,
messages$: new Subject(),
sent: (type) => calls(type).map(([, , p]) => p),
sentConnectionIds: (type) => calls(type).map(([, id]) => id),
};
}
/** A to-device message of a connection, as signalling delivers it. */
export type PeerMessage = ConnectionMessage & Omit<SignallingMessage, "type">;
/** A to-device message from `peer` on a connection. */
export function fromPeer<P extends SignallingPayload>(
connectionId: string,
payload: P,
): P & Omit<SignallingMessage, "type"> {
return { ...payload, ...peer, connectionId };
}
/** The local media a publisher connection reads. */
export function fakeLocalMedia(): {
publications$: BehaviorSubject<LocalPublication[]>;
publishing$: BehaviorSubject<boolean>;
} {
return {
publications$: new BehaviorSubject<LocalPublication[]>([]),
publishing$: new BehaviorSubject(true),
};
}
export function fakePublication(
kind: "audio" | "video",
{
source = kind === "audio" ? "microphone" : "camera",
stream = new FakeMediaStream() as unknown as MediaStream,
}: Partial<Pick<LocalPublication, "source" | "stream">> = {},
): LocalPublication & {
track$: BehaviorSubject<MediaStreamTrack | null>;
muted$: BehaviorSubject<boolean>;
} {
const muted$ = new BehaviorSubject(false);
return {
source,
kind,
stream,
track$: new BehaviorSubject<MediaStreamTrack | null>(fakeTrack(kind)),
muted$,
mediaTrack: { muted$ } as unknown as LocalPublication["mediaTrack"],
};
}
+4 -11
View File
@@ -101,11 +101,8 @@ export class ConnectionMap {
interface Props {
scope: ObservableScope;
connectionFactory: ConnectionFactory;
/**
* The transport we publish on; the one connection with the publisher role.
* Null for a backend that only subscribes, while another backend publishes.
*/
localTransport: TransportLocator | null;
/** The transport we publish on; the one connection with the publisher role. */
localTransport: TransportLocator;
/** Every transport some member is on, deduplicated against the local one. */
remoteTransports$: Behavior<Epoch<TransportLocator[]>>;
logger: Logger;
@@ -131,11 +128,7 @@ export function createConnectionMap$({
scope.behavior(
remoteTransports$.pipe(
mapEpoch((remoteTransports) =>
removeDuplicateTransports(
localTransport
? [localTransport, ...remoteTransports]
: remoteTransports,
),
removeDuplicateTransports([localTransport, ...remoteTransports]),
),
tap((transports) =>
logger.debug("localAndRemoteTransports$ = ", transports),
@@ -143,7 +136,7 @@ export function createConnectionMap$({
),
);
const localTransportKey = localTransport && keyFor(localTransport);
const localTransportKey = keyFor(localTransport);
const connections$ = scope.behavior(
localAndRemoteTransports$.pipe(
@@ -258,23 +258,6 @@ describe("createLivekitBackend", () => {
},
);
it("prepares a remote SFU's transport on its homeserver, where delegation is not possible", async () => {
// Our own homeserver would delegate, so only the SFU being remote can
// explain delegation being skipped
fetchMock.post(homeserverEndpoint, 401);
const remoteTransport: UnstableLivekitTransport = {
type: "livekit",
url: "wss://sfu.sample.com",
server_name: "sample.com",
};
const capabilities = await backend().prepareLocalTransport(remoteTransport);
expect(capabilities).toEqual({ canDelegateDelayedLeave: false });
expect(connections[0].connection.role).toBe("publisher");
expect(connections[0].connection.serverName).toBe("sample.com");
});
it("waits for the publisher connection to hold its token before the join", async () => {
fetchMock.post(homeserverEndpoint, 404);
fetchMock.post(`${exampleServiceUrl}/delegate_delayed_leave`, 404);
@@ -505,27 +488,6 @@ describe("createLivekitBackend", () => {
expect(remote?.connection.transport).toEqual(carlsTransport);
expect(remote?.connection.serverName).toBe("sample.com");
});
it("connects to a remote publisher's transport on the SFU's homeserver", async () => {
const b = await prepared();
// Dave has no SFU of his own and publishes on sample.com's
const remoteTransport: UnstableLivekitTransport = {
type: "livekit",
url: "wss://sfu.sample.com",
server_name: "sample.com",
};
b.mediaFor(testScope(), {
...bob(remoteTransport, ["dave-hashed"]),
userId: "@dave:example.org",
});
await flushPromises();
const remote = connections.find(
(c) => c.connection.role === "subscriber",
);
expect(remote?.connection.transport).toEqual(remoteTransport);
expect(remote?.connection.serverName).toBe("sample.com");
});
});
});
+24 -47
View File
@@ -34,7 +34,7 @@ import {
trackEpoch,
} from "../../reactive/ObservableScope";
import { mapScoped } from "../../utils/mapScoped";
import { serverNameOf, serverNameOfTransport } from "../../utils/serverName";
import { serverNameOf } from "../../utils/serverName";
import {
type BackendConnection,
type MediaBackend,
@@ -115,14 +115,9 @@ export function createLivekitBackend(
);
// A new map, in a scope of its own, whenever the local transport changes:
// the publisher role sits on one connection, and which one is fixed per map.
// Without a local transport the map only subscribes, for when another
// backend publishes; boxed so that `mapScoped` builds a map for null too
const boxedLocalTransport$ = scope.behavior(
localTransport$.pipe(map((localTransport) => ({ localTransport }))),
);
// the publisher role sits on one connection, and which one is fixed per map
const connectionMap$ = scope.behavior<Epoch<ConnectionMap>>(
mapScoped(scope, boxedLocalTransport$, (mapScope, { localTransport }) =>
mapScoped(scope, localTransport$, (mapScope, localTransport) =>
createConnectionMap$({
scope: mapScope,
connectionFactory,
@@ -144,48 +139,33 @@ export function createLivekitBackend(
logger,
});
// Probed once, when first needed: the answer cannot change within a
// session, and a backend that only subscribes never needs it
let homeserverProbe: Promise<boolean> | undefined;
const homeserverSupportsDelegation = async (): Promise<boolean> =>
(homeserverProbe ??= supportsDelegation(
`${client.baseUrl}/_matrix/client/unstable/io.element.msc4195/rtc/livekit/delegate_delayed_leave`,
"homeserver",
logger,
));
const ownServerName = serverNameOf(ownMembershipIdentity.userId);
// Probed once: the answer cannot change within a session
const homeserverSupportsDelegation = supportsDelegation(
`${client.baseUrl}/_matrix/client/unstable/io.element.msc4195/rtc/livekit/delegate_delayed_leave`,
"homeserver",
logger,
);
const prepareLocalTransport = async (
transport: Transport,
): Promise<TransportCapabilities> => {
if (!isUnstableLivekitTransport(transport))
throw new MatrixRTCError(`Not a LiveKit transport: ${transport.type}`);
const serverName = serverNameOfTransport(
const locator: TransportLocator = {
transport,
ownMembershipIdentity.userId,
);
const locator: TransportLocator = { transport, serverName };
let canDelegate: Promise<boolean>;
if (serverName !== ownServerName) {
// Our homeserver only takes over leaves for its own SFUs
logger.info(
`Publishing on a remote SFU of ${serverName}, delegation is not possible`,
);
canDelegate = Promise.resolve(false);
} else {
// Delegation goes through whichever service authenticates us as a
// publisher, so only that service's support counts
const flow = chooseAuthFlow(transport, "publisher");
canDelegate =
flow.kind === "openID"
? supportsDelegation(
`${flow.serviceUrl}/delegate_delayed_leave`,
`transport ${flow.serviceUrl}`,
logger,
)
: homeserverSupportsDelegation();
}
serverName: serverNameOf(ownMembershipIdentity.userId),
};
// Delegation goes through whichever service authenticates us as a
// publisher, so only that service's support counts
const flow = chooseAuthFlow(transport, "publisher");
const canDelegate =
flow.kind === "openID"
? supportsDelegation(
`${flow.serviceUrl}/delegate_delayed_leave`,
`transport ${flow.serviceUrl}`,
logger,
)
: homeserverSupportsDelegation;
localTransport$.next(locator);
// The membership is only sent once the publisher connection holds its
// token, so that the LiveKit room exists before anyone subscribes to it:
@@ -224,10 +204,7 @@ export function createLivekitBackend(
map((transport): TransportLocator | undefined => {
const raw = transport?.raw;
return isUnstableLivekitTransport(raw)
? {
transport: raw,
serverName: serverNameOfTransport(raw, member.userId),
}
? { transport: raw, serverName: serverNameOf(member.userId) }
: undefined;
}),
distinctUntilChanged((a, b) => (a && keyFor(a)) === (b && keyFor(b))),
@@ -1,172 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { MatrixError } from "matrix-js-sdk";
import { logger } from "matrix-js-sdk/lib/logger";
import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
import { beforeEach, describe, expect, it, type Mocked, vi } from "vitest";
import { mockRtcMembership, ownMemberMock } from "../../utils/test";
import { type ClientGetTokenParts } from "./auth/csApi";
import { findRemotePublishLivekitTransport } from "./RemotePublishTransport";
/** A JWT whose payload states whether the issuing server lets us publish. */
function jwtWithPublishGrant(canPublish: boolean): string {
return [
{}, // header
{ sub: "identity", video: { room: "room", canPublish } }, // payload
{}, // signature
]
.map((d) => global.btoa(JSON.stringify(d)))
.join(".");
}
const bobSfu = { type: "livekit", url: "wss://sfu.bob.com" };
const carolSfu = { type: "livekit", url: "wss://sfu.carol.com" };
const bobsTransport = { ...bobSfu, server_name: "bob.com" };
const carolsTransport = { ...carolSfu, server_name: "carol.com" };
// Bob is the oldest member, Carol joined after him
const bob = mockRtcMembership("@bob:bob.com", "BOB", {
fociPreferred: [bobSfu],
membership: { created_ts: 100 },
});
const carol = mockRtcMembership("@carol:carol.com", "CAROL", {
fociPreferred: [carolSfu],
membership: { created_ts: 200 },
});
// Bob's other device, and Dave who publishes on Bob's SFU remotely
const bobsLaptop = mockRtcMembership("@bob:bob.com", "LAPTOP", {
fociPreferred: [bobSfu],
membership: { created_ts: 150 },
});
const dave = mockRtcMembership("@dave:dave.com", "DAVE", {
fociPreferred: [bobsTransport],
membership: { created_ts: 160 },
});
const legacyErin = mockRtcMembership("@erin:erin.com", "ERIN", {
fociPreferred: [
{ type: "livekit", livekit_service_url: "https://jwt.erin.com" },
],
membership: { created_ts: 50 },
});
const ourselves = mockRtcMembership(
ownMemberMock.userId,
ownMemberMock.deviceId,
{
fociPreferred: [{ type: "livekit", url: "wss://sfu.example.org" }],
membership: { created_ts: 10 },
},
);
const csApiMissing = new MatrixError(
{ errcode: "M_UNRECOGNIZED", error: "" },
404,
);
describe("findRemotePublishLivekitTransport", () => {
let client: Mocked<ClientGetTokenParts>;
const find = async (
memberships: CallMembership[],
): ReturnType<typeof findRemotePublishLivekitTransport> =>
findRemotePublishLivekitTransport({
client,
memberships,
ownMembershipIdentity: ownMemberMock,
roomId: "!room:example.org",
slotId: "m.call#ROOM",
logger,
});
beforeEach(() => {
client = {
_unstable_getLivekitToken: vi
.fn()
.mockResolvedValue({ jwt: jwtWithPublishGrant(true) }),
};
});
it("publishes on the oldest member's SFU with its server_name", async () => {
await expect(find([carol, bob])).resolves.toEqual(bobsTransport);
expect(client._unstable_getLivekitToken).toHaveBeenCalledTimes(1);
expect(client._unstable_getLivekitToken).toHaveBeenCalledWith({
url: "wss://sfu.bob.com",
server_name: "bob.com",
room_id: "!room:example.org",
slot_id: "m.call#ROOM",
member_id: ownMemberMock.memberId,
});
});
it("asks about an SFU once however many members are on it", async () => {
client._unstable_getLivekitToken.mockResolvedValue({
jwt: jwtWithPublishGrant(false),
});
await expect(find([dave, bobsLaptop, bob])).resolves.toBeNull();
expect(client._unstable_getLivekitToken).toHaveBeenCalledTimes(1);
});
it("skips our own membership", async () => {
await expect(find([ourselves, bob])).resolves.toEqual(bobsTransport);
expect(client._unstable_getLivekitToken).toHaveBeenCalledTimes(1);
});
it("ignores members publishing through a legacy JWT service", async () => {
await expect(find([legacyErin, bob])).resolves.toEqual(bobsTransport);
expect(client._unstable_getLivekitToken).toHaveBeenCalledTimes(1);
});
it("moves on when a homeserver does not let us publish", async () => {
client._unstable_getLivekitToken.mockImplementation(
async ({ server_name }) =>
Promise.resolve({
jwt: jwtWithPublishGrant(server_name === "carol.com"),
}),
);
await expect(find([bob, carol])).resolves.toEqual(carolsTransport);
expect(client._unstable_getLivekitToken).toHaveBeenCalledTimes(2);
});
it("moves on when a homeserver refuses to issue a token", async () => {
client._unstable_getLivekitToken.mockImplementation(
async ({ server_name }) =>
server_name === "bob.com"
? Promise.reject(
new MatrixError(
{ errcode: "M_FORBIDDEN", error: "Not joined" },
403,
),
)
: Promise.resolve({ jwt: jwtWithPublishGrant(true) }),
);
await expect(find([bob, carol])).resolves.toEqual(carolsTransport);
});
it("finds nothing once all candidates are exhausted", async () => {
client._unstable_getLivekitToken.mockResolvedValue({
jwt: jwtWithPublishGrant(false),
});
await expect(find([bob, carol])).resolves.toBeNull();
expect(client._unstable_getLivekitToken).toHaveBeenCalledTimes(2);
});
it("gives up immediately if our homeserver lacks the CS API", async () => {
client._unstable_getLivekitToken.mockRejectedValue(csApiMissing);
await expect(find([bob, carol])).resolves.toBeNull();
expect(client._unstable_getLivekitToken).toHaveBeenCalledTimes(1);
});
it("finds nothing where there are no other members", async () => {
await expect(find([ourselves])).resolves.toBeNull();
expect(client._unstable_getLivekitToken).not.toHaveBeenCalled();
});
});
@@ -1,105 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type Logger } from "matrix-js-sdk/lib/logger";
import {
type CallMembership,
isUnstableLivekitTransport,
type Transport,
} from "matrix-js-sdk/lib/matrixrtc";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import { isCsApiUnavailable } from "./auth";
import * as csApi from "./auth/csApi";
import { decodeSFUJWTPayload } from "./auth/types";
export interface RemotePublishTransportParams {
client: csApi.ClientGetTokenParts;
/** The members whose SFUs are the candidates. */
memberships: CallMembership[];
ownMembershipIdentity: CallMembershipIdentityParts;
roomId: string;
slotId: string;
logger: Logger;
}
/**
* The SFU of another member that its homeserver lets us publish on, as the
* transport to advertise, or null where none does. Candidates are tried
* oldest membership first, each SFU once, and each is checked by asking for a
* publisher token through our homeserver (MSC4195): a homeserver may restrict
* remote publishers, which shows as a token without the publish grant.
*/
export async function findRemotePublishLivekitTransport({
client,
memberships,
ownMembershipIdentity: own,
roomId,
slotId,
logger,
}: RemotePublishTransportParams): Promise<Transport | null> {
logger.info(
"No local transport available, looking for another member's SFU to publish on",
);
const tried = new Set<string>();
const byAge = [...memberships].sort((a, b) => a.createdTs() - b.createdTs());
for (const membership of byAge) {
if (
membership.userId === own.userId &&
membership.deviceId === own.deviceId
)
continue;
for (const transport of membership.transports) {
// Publishing remotely goes through the client-server API, which a JWT
// service cannot stand in for, so only a `url` qualifies
if (!isUnstableLivekitTransport(transport) || !("url" in transport))
continue;
const { url } = transport;
const serverName = membership.getTransportServerName(transport);
const key = JSON.stringify([url, serverName]);
if (tried.has(key)) continue;
tried.add(key);
try {
const { jwt } = await csApi.getSFUConfig({
client,
membership: own,
url,
serverName,
roomId,
slotId,
});
try {
// The SFU treats an absent canPublish grant as publishing allowed.
if (decodeSFUJWTPayload(jwt).video.canPublish !== false) {
logger.info(`Publishing on remote SFU ${url} of ${serverName}`);
return { type: "livekit", url, server_name: serverName };
}
logger.info(
`${serverName} does not let us publish on ${url}, skipping`,
);
continue;
} catch (e) {
logger.info(
`${serverName}: failed to decode SFUJWT payload: ${e}, skipping`,
);
continue;
}
} catch (e) {
if (isCsApiUnavailable(e)) {
// Without the endpoint no further candidate can work
logger.warn(
`Cannot use the MSC4195 get_token endpoint (${e}), so no remote SFU can be used`,
);
return null;
}
logger.warn(`Failed to get a token for ${url} of ${serverName}:`, e);
continue;
}
}
}
return null;
}
+1 -1
View File
@@ -48,7 +48,7 @@ function isEndpointUnsupported(e: unknown): boolean {
* (MSC4533), which the widget API reports as a plain error. A connection
* error is neither: the endpoint may well exist.
*/
export function isCsApiUnavailable(e: unknown): boolean {
function isCsApiUnavailable(e: unknown): boolean {
if (isEndpointUnsupported(e)) return true;
return !(e instanceof MatrixError) && !(e instanceof ConnectionError);
}
+3 -10
View File
@@ -34,13 +34,11 @@ export interface SFUConfig {
}
/** The parts of the JWT payload the backend reads. */
export interface SFUJWTPayload {
interface SFUJWTPayload {
/** The subject: the participant identity the token carries. */
sub: string;
video: {
room: string;
/** Whether the token lets us publish; a homeserver may deny this to remote publishers. */
canPublish?: boolean;
};
}
@@ -49,7 +47,8 @@ export function extractFullConfigFromToken(sfuConfig: {
url: string;
jwt: string;
}): SFUConfig {
const payload = decodeSFUJWTPayload(sfuConfig.jwt);
const [, payloadStr] = sfuConfig.jwt.split(".");
const payload = JSON.parse(global.atob(payloadStr)) as SFUJWTPayload;
return {
jwt: sfuConfig.jwt,
url: sfuConfig.url,
@@ -57,9 +56,3 @@ export function extractFullConfigFromToken(sfuConfig: {
livekitIdentity: payload.sub,
};
}
/** Decodes the payload of an SFU JWT, without verifying its signature. */
export function decodeSFUJWTPayload(jwt: string): SFUJWTPayload {
const [, payloadStr] = jwt.split(".");
return JSON.parse(global.atob(payloadStr)) as SFUJWTPayload;
}
-16
View File
@@ -5,23 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type Transport } from "matrix-js-sdk/lib/matrixrtc";
/** The homeserver a Matrix user id belongs to: everything after the first colon. */
export function serverNameOf(userId: string): string {
return userId.replace(/^.*?:/, "");
}
/**
* The homeserver that operates a transport: the one the transport names, else
* the one of the member advertising it. A member publishing on another
* homeserver's SFU (MSC4195) names that homeserver in `server_name`.
*/
export function serverNameOfTransport(
transport: Transport,
userId: string,
): string {
return typeof transport.server_name === "string"
? transport.server_name
: serverNameOf(userId);
}
+1 -2
View File
@@ -7,7 +7,6 @@
padding-top: env(safe-area-inset-top);
opacity: 1;
transition: opacity 0.15s;
text-align: center;
}
/* Pseudo-element for the gradient background */
@@ -53,7 +52,7 @@
". . ."
"primaryButton title secondaryButton"
". . .";
place-items: center stretch;
place-items: center;
column-gap: var(--cpd-space-2x);
}
+6 -1
View File
@@ -6,11 +6,12 @@ Please see LICENSE in the repository root for full details.
*/
import { describe, expect, it, vi } from "vitest";
import { BehaviorSubject } from "rxjs";
import { testScope, mockMuteStates, mockMediaDevices } from "../utils/test";
import { constant } from "../state/Behavior";
import type { CallViewModel } from "../state/CallViewModel/CallViewModel";
import type { Layout } from "../state/layout-types";
import type { Alignment, Layout } from "../state/layout-types";
import type { SpotlightTileViewModel } from "../state/TileViewModel";
import type { DeviceLabel } from "../state/MediaDevices";
import { createCallFooterViewModel } from "./CallFooterViewModel";
@@ -64,6 +65,10 @@ function buildMinimalCallViewModel(layout: Layout): CallViewModel {
const gridLayout: Layout = {
type: "grid",
grid: [],
spotlightAlignment$: new BehaviorSubject<Alignment>({
inline: "end",
block: "end",
}),
setVisibleTiles: (_: number) => {},
};
+150 -100
View File
@@ -5,25 +5,28 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { render } from "@testing-library/react";
import { expect, test, vi } from "vitest";
import { act, render, screen } from "@testing-library/react";
import { BehaviorSubject } from "rxjs";
import { describe, expect, it, vi } from "vitest";
import { type CallLayout } from "./CallLayout";
import { arrangeTiles, type Bounds } from "./CallLayout";
import { makeGridLayout } from "./GridLayout";
import { makeSpotlightLandscapeLayout } from "./SpotlightLandscapeLayout";
import { makeSpotlightPortraitLayout } from "./SpotlightPortraitLayout";
import { type LayoutProps } from "./Grid";
import { type DragCallback, type LayoutProps } from "./Grid";
import { constant } from "../state/Behavior";
import { type Alignment } from "../state/layout-types";
import {
type Alignment,
type GridLayout,
type SpotlightLandscapeLayout,
type SpotlightPortraitLayout,
} from "../state/layout-types";
import { type UserMediaViewModel } from "../state/media/UserMediaViewModel";
import {
GridTileViewModel,
SpotlightTileViewModel,
type TileViewModel,
} from "../state/TileViewModel";
import { makeOneOnOneMobileLayout } from "./OneOnOneMobileLayout";
import { makeSpotlightExpandedLayout } from "./SpotlightExpandedLayout";
import { makeOneOnOneDesktopLayout } from "./OneOnOneDesktopLayout";
// Grid supplies these hooks to the layouts it renders; on their own the
// layouts have no grid to report to
@@ -33,105 +36,152 @@ vi.mock("./Grid", async (importOriginal) => ({
useVisibleTiles: (): void => {},
}));
const minBounds$ = constant({ width: 800, height: 800 });
describe("grid layout", () => {
it("sizes the tiles from the minimum bounds", () => {
const minBounds$ = new BehaviorSubject<Bounds>({ width: 800, height: 600 });
const { scrolling: Scrolling } = makeGridLayout({ minBounds$ });
render(<Scrolling model={gridModel()} Slot={Slot} />);
const layer = screen.getByTestId("slot-0").parentElement!;
expect(layer.style.width).toBe("800px");
expect(layer.style.getPropertyValue("--width")).toBe(
`${Math.floor(arrangeTiles(800, 600, 3).tileWidth)}px`,
);
const spotlight = {
vm: new SpotlightTileViewModel(
constant([]),
constant(false),
constant("solid"),
),
alignment$: constant<Alignment>({ inline: "end", block: "end" }),
onDrag: () => {},
};
const grid = Array.from(
{ length: 3 },
() => new GridTileViewModel(constant({} as UserMediaViewModel)),
act(() => minBounds$.next({ width: 400, height: 300 }));
expect(layer.style.width).toBe("400px");
expect(layer.style.getPropertyValue("--width")).toBe(
`${Math.floor(arrangeTiles(400, 300, 3).tileWidth)}px`,
);
});
it("moves the spotlight only when a drag reaches another corner", () => {
const model = gridModel();
const { fixed: Fixed } = makeGridLayout({ minBounds$: bounds });
render(<Fixed model={model} Slot={Slot} />);
const alignments: Alignment[] = [];
model.spotlightAlignment$.subscribe((a) => alignments.push(a));
const drag = drags.get("spotlight")!;
// Still in the bottom right corner
act(() => drag({ x: 0, y: 0, xRatio: 0.9, yRatio: 0.9 }));
// Over to the bottom left
act(() => drag({ x: 0, y: 0, xRatio: 0.1, yRatio: 0.9 }));
// And a little further, still bottom left
act(() => drag({ x: 0, y: 0, xRatio: 0.2, yRatio: 0.8 }));
// Up to the top left
act(() => drag({ x: 0, y: 0, xRatio: 0.2, yRatio: 0.1 }));
expect(alignments).toEqual([
{ inline: "end", block: "end" },
{ inline: "start", block: "end" },
{ inline: "start", block: "start" },
]);
const slot = screen.getByTestId("slot-spotlight");
expect(slot.getAttribute("data-inline-alignment")).toBe("start");
expect(slot.getAttribute("data-block-alignment")).toBe("start");
});
});
describe("spotlight landscape layout", () => {
it("puts the spotlight in the fixed layer and the grid in the scrolling one", () => {
const { fixed: Fixed, scrolling: Scrolling } = makeSpotlightLandscapeLayout(
{ minBounds$: bounds },
);
const model = landscapeModel();
const { unmount } = render(<Fixed model={model} Slot={Slot} />);
expect(screen.getByTestId("slot-spotlight")).toBeInTheDocument();
unmount();
render(<Scrolling model={model} Slot={Slot} />);
expect(slotIds()).toEqual(model.grid.map((tile) => tile.id));
});
});
describe("spotlight portrait layout", () => {
it("sizes the grid tiles from the minimum width", () => {
const minBounds$ = new BehaviorSubject<Bounds>({ width: 300, height: 700 });
const { scrolling: Scrolling } = makeSpotlightPortraitLayout({
minBounds$,
});
const model = portraitModel();
render(<Scrolling model={model} Slot={Slot} />);
expect(slotIds()).toEqual(model.grid.map((tile) => tile.id));
const layer = screen.getByTestId(`slot-${model.grid[0].id}`).parentElement!
.parentElement!;
expect(layer.style.getPropertyValue("--grid-tile-width")).toBe(
`${Math.floor(arrangeTiles(300, 300, 3).tileWidth)}px`,
);
act(() => minBounds$.next({ width: 500, height: 700 }));
expect(layer.style.getPropertyValue("--grid-tile-width")).toBe(
`${Math.floor(arrangeTiles(500, 500, 3).tileWidth)}px`,
);
});
});
const bounds = constant<Bounds>({ width: 800, height: 600 });
// The layouts only pass tile view models through to their slots
const media = {} as UserMediaViewModel;
const spotlight = new SpotlightTileViewModel(
constant([]),
constant(false),
constant("solid"),
);
const pip = {
alignment$: constant<Alignment>({ inline: "end", block: "end" }),
onDrag: () => {},
vm: { id: "pip" } as GridTileViewModel,
size$: constant("sm" as const),
};
const tiles = (): GridTileViewModel[] =>
Array.from({ length: 3 }, () => new GridTileViewModel(constant(media)));
function gridModel(): GridLayout {
return {
type: "grid",
spotlight,
grid: tiles(),
spotlightAlignment$: new BehaviorSubject<Alignment>({
inline: "end",
block: "end",
}),
setVisibleTiles: () => {},
};
}
function landscapeModel(): SpotlightLandscapeLayout {
return {
type: "spotlight-landscape",
spotlight,
grid: tiles(),
setVisibleTiles: () => {},
};
}
function portraitModel(): SpotlightPortraitLayout {
return {
type: "spotlight-portrait",
spotlight,
grid: tiles(),
setVisibleTiles: () => {},
};
}
// Slots record their drag callback so a test can drag them, and show the id
// of the tile they hold
const drags = new Map<string, DragCallback | undefined>();
const Slot: LayoutProps<unknown, TileViewModel, HTMLDivElement>["Slot"] = ({
id,
model,
onDrag,
children,
...props
}) => (
<div data-testid={`slot-${id}`} {...props}>
{children}
</div>
);
function snapshotLayout<Model>(layout: CallLayout<Model>, model: Model): void {
const {
foreground,
scrolling: Scrolling,
fixed: Fixed,
} = layout({ minBounds$ });
const scrollingLayer = <Scrolling model={model} Slot={Slot} />;
const fixedLayer = <Fixed model={model} Slot={Slot} />;
const { container } = render(
foreground === "fixed" ? (
<>
{scrollingLayer}
{fixedLayer}
</>
) : (
<>
{fixedLayer}
{scrollingLayer}
</>
),
}) => {
drags.set(id, onDrag);
return (
<div data-testid={`slot-${id}`} {...props}>
{model instanceof GridTileViewModel ? model.id : "spotlight"}
</div>
);
expect(container).toMatchSnapshot();
};
function slotIds(): string[] {
return screen
.getAllByTestId(/^slot-/)
.map((slot) => slot.textContent ?? "")
.filter((id) => id !== "spotlight");
}
test("grid layout", () =>
snapshotLayout(makeGridLayout, {
type: "grid",
spotlight,
grid,
setVisibleTiles: () => {},
}));
test("spotlight landscape layout", () =>
snapshotLayout(makeSpotlightLandscapeLayout, {
type: "spotlight-landscape",
spotlight,
grid,
setVisibleTiles: () => {},
}));
test("spotlight portrait layout", () =>
snapshotLayout(makeSpotlightPortraitLayout, {
type: "spotlight-portrait",
spotlight,
grid,
setVisibleTiles: () => {},
}));
test("spotlight expanded layout", () =>
snapshotLayout(makeSpotlightExpandedLayout, {
type: "spotlight-expanded",
spotlight: spotlight.vm,
pip,
}));
test("one-on-one desktop layout", () =>
snapshotLayout(makeOneOnOneDesktopLayout, {
type: "one-on-one-desktop",
spotlight: { id: "spotlight" } as GridTileViewModel,
pip,
}));
test("one-on-one mobile layout", () =>
snapshotLayout(makeOneOnOneMobileLayout, {
type: "one-on-one-mobile",
spotlight: spotlight.vm,
pip,
}));
+29 -12
View File
@@ -40,7 +40,6 @@ import { useMergedRefs } from "../useMergedRefs";
import { TileWrapper } from "./TileWrapper";
import { usePrefersReducedMotion } from "../usePrefersReducedMotion";
import { useInitial } from "../useInitial";
import { type DragCallback } from "../state/layout-types";
const MAX_ANIMATED_TILES = 50; // Capped for performance reasons
@@ -84,10 +83,7 @@ interface DragState {
cursorY: number;
}
export interface SlotProps<Model> extends Omit<
ComponentProps<"div">,
"onDrag"
> {
interface SlotProps<Model> extends Omit<ComponentProps<"div">, "onDrag"> {
id: string;
model: Model;
onDrag?: DragCallback;
@@ -185,6 +181,27 @@ export interface TileProps<Model, R extends HTMLElement> {
model: Model;
}
interface Drag {
/**
* The X coordinate of the dragged tile in grid space.
*/
x: number;
/**
* The Y coordinate of the dragged tile in grid space.
*/
y: number;
/**
* The X coordinate of the dragged tile, as a scalar of the grid width.
*/
xRatio: number;
/**
* The Y coordinate of the dragged tile, as a scalar of the grid height.
*/
yRatio: number;
}
export type DragCallback = (drag: Drag) => void;
interface LayoutMemoProps<
LayoutModel,
TileModel,
@@ -447,13 +464,13 @@ export function Grid<
)
.catch(logger.error);
callback({
x: tileX,
y: tileY,
xRatio: tileX / (gridBounds.width - tile.width),
yRatio: tileY / (gridBounds.height - tile.height),
endOfGesture,
});
if (endOfGesture)
callback({
x: tileX,
y: tileY,
xRatio: tileX / (gridBounds.width - tile.width),
yRatio: tileY / (gridBounds.height - tile.height),
});
};
// Callback for useDrag. We could call useDrag here, but the default
+25 -30
View File
@@ -8,21 +8,15 @@ Please see LICENSE in the repository root for full details.
import {
type CSSProperties,
type ReactNode,
useCallback,
useMemo,
type FC,
type ComponentType,
} from "react";
import { type GridLayout as GridLayoutModel } from "../state/layout-types.ts";
import styles from "./GridLayout.module.css";
import { type CallLayout, arrangeTiles } from "./CallLayout";
import { type SlotProps, useUpdateLayout, useVisibleTiles } from "./Grid";
import { type DragCallback, useUpdateLayout, useVisibleTiles } from "./Grid";
import { useBehavior } from "../useBehavior";
import {
type SpotlightTileViewModel,
type TileViewModel,
} from "../state/TileViewModel.ts";
import { type FloatingTile } from "../state/FloatingTile.ts";
interface GridCSSProperties extends CSSProperties {
"--gap": string;
@@ -30,27 +24,6 @@ interface GridCSSProperties extends CSSProperties {
"--height": string;
}
interface SpotlightSlotProps {
model: FloatingTile & { vm: SpotlightTileViewModel };
Slot: ComponentType<SlotProps<TileViewModel>>;
}
const SpotlightSlot: FC<SpotlightSlotProps> = ({ model, Slot }) => {
useUpdateLayout();
const alignment = useBehavior(model.alignment$);
return (
<Slot
className={styles.slot}
id="spotlight"
model={model.vm}
onDrag={model.onDrag}
data-block-alignment={alignment.block}
data-inline-alignment={alignment.inline}
/>
);
};
/**
* An implementation of the "grid" layout, in which all participants are shown
* together in a scrolling grid.
@@ -64,10 +37,32 @@ export const makeGridLayout: CallLayout<GridLayoutModel> = ({
// lives
fixed: function GridLayoutFixed({ ref, model, Slot }): ReactNode {
useUpdateLayout();
const alignment = useBehavior(model.spotlightAlignment$);
const onDragSpotlight: DragCallback = useCallback(
({ xRatio, yRatio }) => {
const block = yRatio < 0.5 ? "start" : "end";
const inline = xRatio < 0.5 ? "start" : "end";
// A drag reports the same alignment on every move; only a change is
// worth a re-render
const current = model.spotlightAlignment$.value;
if (current.block !== block || current.inline !== inline)
model.spotlightAlignment$.next({ block, inline });
},
[model.spotlightAlignment$],
);
return (
<div ref={ref} className={styles.fixed}>
{model.spotlight && (
<SpotlightSlot model={model.spotlight} Slot={Slot} />
<Slot
className={styles.slot}
id="spotlight"
model={model.spotlight}
onDrag={onDragSpotlight}
data-block-alignment={alignment.block}
data-inline-alignment={alignment.inline}
/>
)}
</div>
);
+15 -6
View File
@@ -6,13 +6,13 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type ReactNode, useMemo } from "react";
import { type ReactNode, useCallback, useMemo } from "react";
import classNames from "classnames";
import { type OneOnOneDesktopLayout as OneOnOneDesktopLayoutModel } from "../state/layout-types.ts";
import { type CallLayout, arrangeTiles } from "./CallLayout";
import styles from "./OneOnOneDesktopLayout.module.css";
import { useUpdateLayout } from "./Grid";
import { type DragCallback, useUpdateLayout } from "./Grid";
import { useBehavior } from "../useBehavior";
/**
@@ -37,12 +37,21 @@ export const makeOneOnOneDesktopLayout: CallLayout<
}): ReactNode {
useUpdateLayout();
const { width, height } = useBehavior(minBounds$);
const pipAlignment = useBehavior(model.pip.alignment$);
const pipAlignment = useBehavior(model.pipAlignment$);
const { tileWidth, tileHeight } = useMemo(
() => arrangeTiles(width, height, 1),
[width, height],
);
const onDragLocalTile: DragCallback = useCallback(
({ xRatio, yRatio }) =>
model.pipAlignment$.next({
block: yRatio < 0.5 ? "start" : "end",
inline: xRatio < 0.5 ? "start" : "end",
}),
[model.pipAlignment$],
);
return (
<div ref={ref} className={styles.layer}>
<Slot
@@ -53,9 +62,9 @@ export const makeOneOnOneDesktopLayout: CallLayout<
>
<Slot
className={classNames(styles.slot, styles.local)}
id={model.pip.vm.id}
model={model.pip.vm}
onDrag={model.pip.onDrag}
id={model.pip.id}
model={model.pip}
onDrag={onDragLocalTile}
data-block-alignment={pipAlignment.block}
data-inline-alignment={pipAlignment.inline}
/>
+24 -32
View File
@@ -6,43 +6,14 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type ComponentType, type FC, type ReactNode } from "react";
import { type ReactNode, useCallback } from "react";
import classNames from "classnames";
import { type OneOnOneMobileLayout as OneOnOneMobileLayoutModel } from "../state/layout-types.ts";
import { type CallLayout } from "./CallLayout";
import styles from "./OneOnOneMobileLayout.module.css";
import { type SlotProps, useUpdateLayout } from "./Grid";
import { type DragCallback, useUpdateLayout } from "./Grid";
import { useBehavior } from "../useBehavior";
import { type FloatingTile } from "../state/FloatingTile.ts";
import {
type GridTileViewModel,
type TileViewModel,
} from "../state/TileViewModel.ts";
import { type Behavior } from "../state/Behavior.ts";
interface PipSlotProps {
model: FloatingTile & { vm: GridTileViewModel; size$: Behavior<"sm" | "lg"> };
Slot: ComponentType<SlotProps<TileViewModel>>;
}
const PipSlot: FC<PipSlotProps> = ({ model, Slot }) => {
useUpdateLayout();
const size = useBehavior(model.size$);
const alignment = useBehavior(model.alignment$);
return (
<Slot
className={classNames(styles.pip)}
id={model.vm.id}
model={model.vm}
onDrag={model.onDrag}
data-size={size}
data-block-alignment={alignment.block}
data-inline-alignment={alignment.inline}
/>
);
};
/**
* An implementation of the "one-on-one" layout for mobile platforms, in which
@@ -73,9 +44,30 @@ export const makeOneOnOneMobileLayout: CallLayout<
Slot,
}): ReactNode {
useUpdateLayout();
const pipSize = useBehavior(model.pipSize$);
const pipAlignment = useBehavior(model.pipAlignment$);
const onDragLocalTile: DragCallback = useCallback(
({ xRatio, yRatio }) =>
model.pipAlignment$.next({
block: yRatio < 0.5 ? "start" : "end",
inline: xRatio < 0.5 ? "start" : "end",
}),
[model.pipAlignment$],
);
return (
<div ref={ref} className={styles.layer}>
{model.pip && <PipSlot model={model.pip} Slot={Slot} />}
{model.pip && (
<Slot
className={classNames(styles.pip)}
id={model.pip.id}
model={model.pip}
onDrag={onDragLocalTile}
data-size={pipSize}
data-block-alignment={pipAlignment.block}
data-inline-alignment={pipAlignment.inline}
/>
)}
</div>
);
},
+24 -29
View File
@@ -5,39 +5,13 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type ReactNode, type ComponentType, type FC } from "react";
import { type ReactNode, useCallback } from "react";
import { type SpotlightExpandedLayout as SpotlightExpandedLayoutModel } from "../state/layout-types.ts";
import { type CallLayout } from "./CallLayout";
import { type SlotProps, useUpdateLayout } from "./Grid";
import { type DragCallback, useUpdateLayout } from "./Grid";
import styles from "./SpotlightExpandedLayout.module.css";
import { useBehavior } from "../useBehavior";
import { type FloatingTile } from "../state/FloatingTile.ts";
import {
type GridTileViewModel,
type TileViewModel,
} from "../state/TileViewModel.ts";
interface PipSlotProps {
model: FloatingTile & { vm: GridTileViewModel };
Slot: ComponentType<SlotProps<TileViewModel>>;
}
const PipSlot: FC<PipSlotProps> = ({ model, Slot }) => {
useUpdateLayout();
const alignment = useBehavior(model.alignment$);
return (
<Slot
className={styles.pip}
id={model.vm.id}
model={model.vm}
onDrag={model.onDrag}
data-block-alignment={alignment.block}
data-inline-alignment={alignment.inline}
/>
);
};
/**
* An implementation of the "expanded spotlight" layout, in which the spotlight
@@ -54,6 +28,7 @@ export const makeSpotlightExpandedLayout: CallLayout<
Slot,
}): ReactNode {
useUpdateLayout();
return (
<div ref={ref} className={styles.layer}>
<Slot
@@ -71,9 +46,29 @@ export const makeSpotlightExpandedLayout: CallLayout<
Slot,
}): ReactNode {
useUpdateLayout();
const pipAlignment = useBehavior(model.pipAlignment$);
const onDragPip: DragCallback = useCallback(
({ xRatio, yRatio }) =>
model.pipAlignment$.next({
block: yRatio < 0.5 ? "start" : "end",
inline: xRatio < 0.5 ? "start" : "end",
}),
[model.pipAlignment$],
);
return (
<div ref={ref} className={styles.layer}>
{model.pip && <PipSlot model={model.pip} Slot={Slot} />}
{model.pip && (
<Slot
className={styles.pip}
id={model.pip.id}
model={model.pip}
onDrag={onDragPip}
data-block-alignment={pipAlignment.block}
data-inline-alignment={pipAlignment.inline}
/>
)}
</div>
);
},
+2 -2
View File
@@ -40,7 +40,7 @@ export const makeSpotlightLandscapeLayout: CallLayout<
<Slot
className={styles.slot}
id="spotlight"
model={model.spotlight.vm}
model={model.spotlight}
/>
</div>
<div className={styles.grid} />
@@ -59,7 +59,7 @@ export const makeSpotlightLandscapeLayout: CallLayout<
return (
<div ref={ref} className={styles.layer}>
<SpotlightSlot media$={model.spotlight.vm.media$} />
<SpotlightSlot media$={model.spotlight.media$} />
<div className={styles.grid}>
{model.grid.map((m) => (
<Slot key={m.id} className={styles.slot} id={m.id} model={m} />
+2 -2
View File
@@ -43,7 +43,7 @@ export const makeSpotlightPortraitLayout: CallLayout<
<Slot
className={styles.slot}
id="spotlight"
model={model.spotlight.vm}
model={model.spotlight}
/>
</div>
</div>
@@ -65,7 +65,7 @@ export const makeSpotlightPortraitLayout: CallLayout<
width,
model.grid.length,
);
const withIndicators = useBehavior(model.spotlight.vm.media$).length > 1;
const withIndicators = useBehavior(model.spotlight.media$).length > 1;
return (
<div
@@ -1,186 +0,0 @@
// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
exports[`grid layout 1`] = `
<div>
<div
class="_scrolling_e8555d"
style="width: 800px; --gap: 20px; --width: 370px; --height: 277px;"
>
<div
class="_slot_e8555d"
data-testid="slot-0"
/>
<div
class="_slot_e8555d"
data-testid="slot-1"
/>
<div
class="_slot_e8555d"
data-testid="slot-2"
/>
</div>
<div
class="_fixed_e8555d"
>
<div
class="_slot_e8555d"
data-block-alignment="end"
data-inline-alignment="end"
data-testid="slot-spotlight"
/>
</div>
</div>
`;
exports[`one-on-one desktop layout 1`] = `
<div>
<div
class="_layer_185815"
>
<div
class="_container_185815"
data-testid="slot-spotlight"
style="width: 760px; height: 570px;"
>
<div
class="_slot_185815 _local_185815"
data-block-alignment="end"
data-inline-alignment="end"
data-testid="slot-pip"
/>
</div>
</div>
<div />
</div>
`;
exports[`one-on-one mobile layout 1`] = `
<div>
<div
class="_layer_c81c30"
>
<div
class="_spotlight_c81c30"
data-testid="slot-spotlight"
/>
</div>
<div
class="_layer_c81c30"
>
<div
class="_pip_c81c30"
data-block-alignment="end"
data-inline-alignment="end"
data-size="sm"
data-testid="slot-pip"
/>
</div>
</div>
`;
exports[`spotlight expanded layout 1`] = `
<div>
<div
class="_layer_e70191"
>
<div
class="_spotlight_e70191"
data-testid="slot-spotlight"
/>
</div>
<div
class="_layer_e70191"
>
<div
class="_pip_e70191"
data-block-alignment="end"
data-inline-alignment="end"
data-testid="slot-pip"
/>
</div>
</div>
`;
exports[`spotlight landscape layout 1`] = `
<div>
<div
class="_layer_29809c"
>
<div
class="_spotlight_29809c"
>
<div
class="_slot_29809c"
data-testid="slot-spotlight"
/>
</div>
<div
class="_grid_29809c"
/>
</div>
<div
class="_layer_29809c"
>
<div
class="_spotlight_29809c"
/>
<div
class="_grid_29809c"
>
<div
class="_slot_29809c"
data-testid="slot-0"
/>
<div
class="_slot_29809c"
data-testid="slot-1"
/>
<div
class="_slot_29809c"
data-testid="slot-2"
/>
</div>
</div>
</div>
`;
exports[`spotlight portrait layout 1`] = `
<div>
<div
class="_layer_065bbc"
style="--grid-gap: 20px; --grid-tile-width: 370px; --grid-tile-height: 277px;"
>
<div
class="_spotlight_065bbc"
/>
<div
class="_grid_065bbc"
>
<div
class="_slot_065bbc"
data-testid="slot-0"
/>
<div
class="_slot_065bbc"
data-testid="slot-1"
/>
<div
class="_slot_065bbc"
data-testid="slot-2"
/>
</div>
</div>
<div
class="_layer_065bbc"
>
<div
class="_spotlight_065bbc"
>
<div
class="_slot_065bbc"
data-testid="slot-spotlight"
/>
</div>
</div>
</div>
`;
+1 -5
View File
@@ -70,7 +70,6 @@ import { ReactionsOverlay } from "./ReactionsOverlay";
import { CallEventAudioRenderer } from "./CallEventAudioRenderer";
import {
createRTCSlot,
FULL_MESH_TRANSPORT_TYPE,
type RTCParticipationOptions,
type RTCSlotOptions,
constant,
@@ -85,7 +84,6 @@ import {
cameraFramerate,
cameraResolution,
customTransport,
fullMeshPublishing,
echoCancellationSetting,
matrixRTCMode as matrixRTCModeSetting,
noiseSuppressionSetting,
@@ -825,9 +823,7 @@ export function rtcParticipationOptions(
delegatedDelayedLeave: session.delegated_delayed_leave,
},
mediaQuality: config.media_quality,
transport: fullMeshPublishing.value$.value
? { type: FULL_MESH_TRANSPORT_TYPE }
: (customTransport.value$.value ?? undefined),
transport: customTransport.value$.value ?? undefined,
fallbackTransport: config.livekit && {
type: "livekit",
livekit_service_url: config.livekit.livekit_service_url,
@@ -1228,7 +1228,7 @@ exports[`should have a close button when the host can dismiss us 1`] = `
Call is not supported
</h1>
<p>
The server is not configured to work with Element Call and no other call participant provides a usable transport. Please contact your server admin (Domain: example.com, Error Code: MISSING_MATRIX_RTC_TRANSPORT).
The server is not configured to work with Element Call. Please contact your server admin (Domain: example.com, Error Code: MISSING_MATRIX_RTC_TRANSPORT).
</p>
<button
class="_button_1nw83_8"
@@ -1381,7 +1381,7 @@ exports[`should render the error page with link back to home 1`] = `
Call is not supported
</h1>
<p>
The server is not configured to work with Element Call and no other call participant provides a usable transport. Please contact your server admin (Domain: example.com, Error Code: MISSING_MATRIX_RTC_TRANSPORT).
The server is not configured to work with Element Call. Please contact your server admin (Domain: example.com, Error Code: MISSING_MATRIX_RTC_TRANSPORT).
</p>
<button
class="_button_1nw83_8"
@@ -1534,7 +1534,7 @@ exports[`should report correct error for 'Call is not supported' 1`] = `
Call is not supported
</h1>
<p>
The server is not configured to work with Element Call and no other call participant provides a usable transport. Please contact your server admin (Domain: example.com, Error Code: MISSING_MATRIX_RTC_TRANSPORT).
The server is not configured to work with Element Call. Please contact your server admin (Domain: example.com, Error Code: MISSING_MATRIX_RTC_TRANSPORT).
</p>
<button
class="_button_1nw83_8"
@@ -30,6 +30,9 @@ import {
} from "@element-hq/matrixrtc-sdk";
import type * as MatrixRTCSdk from "@element-hq/matrixrtc-sdk";
import { exampleTransport, mockConfig } from "../utils/test";
import { GroupCallErrorBoundary } from "../room/GroupCallErrorBoundary";
import { LeaveToHomeProvider } from "../LeaveToHomeContext";
import { ElementCallError, ErrorCode } from "../utils/errors";
// Mock url params hook to avoid environment-dependent snapshot churn.
vi.mock("../UrlParams", () => ({
@@ -493,4 +496,38 @@ describe("DeveloperSettingsTab", () => {
);
});
});
describe("throw test error", () => {
it("throws an ElementCallError that the error boundary shows", async () => {
const user = userEvent.setup();
const client = createMockMatrixClient();
const onError = vi.fn();
render(
<LeaveToHomeProvider value={vi.fn()}>
<GroupCallErrorBoundary
onError={onError}
recoveryActionHandler={vi.fn()}
>
<TooltipProvider>
<DeveloperSettingsTab
client={client}
env={{} as unknown as ImportMetaEnv}
vm={outOfCallDeveloperSettingsTabViewModel}
/>
</TooltipProvider>
</GroupCallErrorBoundary>
</LeaveToHomeProvider>,
);
await user.click(
screen.getByRole("button", { name: "Throw test error" }),
);
expect(await screen.findByText("Something went wrong")).toBeVisible();
expect(onError).toHaveBeenCalledOnce();
const error = onError.mock.calls[0][0] as unknown;
expect(error).toBeInstanceOf(ElementCallError);
expect((error as ElementCallError).code).toBe(ErrorCode.UNKNOWN_ERROR);
});
});
});
+25 -29
View File
@@ -22,6 +22,7 @@ import {
import { logger } from "matrix-js-sdk/lib/logger";
import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import {
Button,
EditInPlace,
ErrorMessage,
Root as Form,
@@ -45,7 +46,6 @@ import {
alwaysShowIphoneEarpiece as alwaysShowIphoneEarpieceSetting,
matrixRTCMode as matrixRTCModeSetting,
customTransport as customTransportSetting,
fullMeshPublishing as fullMeshPublishingSetting,
advancedScreenShare as advancedScreenShareSetting,
screenShareResolution as screenShareResolutionSetting,
screenShareFramerate as screenShareFramerateSetting,
@@ -69,11 +69,11 @@ import { Slider } from "../Slider";
import { useUrlParams } from "../UrlParams";
import {
authenticateWithTransport,
FULL_MESH_TRANSPORT_TYPE,
type TransportMetadata,
} from "@element-hq/matrixrtc-sdk";
import { useBehavior } from "../useBehavior";
import { type ViewModel } from "../state/ViewModel.ts";
import { type ElementCallError, UnknownCallError } from "../utils/errors.ts";
/**
* The state of MatrixRTC's media key rotation.
@@ -177,9 +177,6 @@ export const DeveloperSettingsTab: FC<Props> = ({
const [customTransportUpdateError, setCustomTransportUpdateError] = useState<
string | null
>(null);
const [fullMeshPublishing, setFullMeshPublishing] = useSetting(
fullMeshPublishingSetting,
);
const [customTransport, setCustomTransport] = useSetting(
customTransportSetting,
);
@@ -194,6 +191,22 @@ export const DeveloperSettingsTab: FC<Props> = ({
const urlParams = useUrlParams();
// An error thrown from a click handler never reaches an error boundary, so
// the button only records it and the next render throws it instead.
const [errorToThrow, setErrorToThrow] = useState<ElementCallError | null>(
null,
);
const throwTestError = useCallback(
() =>
setErrorToThrow(
new UnknownCallError(
new Error("Test error thrown from the developer settings"),
),
),
[],
);
if (errorToThrow !== null) throw errorToThrow;
const MediaQualitySettings: React.FC<{
id: string;
header: string;
@@ -422,17 +435,6 @@ export const DeveloperSettingsTab: FC<Props> = ({
)}
/>
</FieldRow>
<FieldRow>
<InputField
id="fullMeshPublishing"
type="checkbox"
checked={fullMeshPublishing}
label={t("developer_mode.full_mesh_publishing")}
onChange={(event: ChangeEvent<HTMLInputElement>): void =>
setFullMeshPublishing(event.target.checked)
}
/>
</FieldRow>
<FieldRow>
<InputField
id="debugTileLayout"
@@ -533,7 +535,7 @@ export const DeveloperSettingsTab: FC<Props> = ({
client,
membership: { userId, deviceId, memberId: "" },
transport,
serverName: transport.server_name ?? serverName,
serverName,
roomId,
slotId: "org.example.fake-slot#room",
role: "publisher",
@@ -665,6 +667,10 @@ export const DeveloperSettingsTab: FC<Props> = ({
<Separator />
<AudioProcessingSettings />
<Separator />
<Button kind="secondary" destructive onClick={throwTestError}>
{t("developer_mode.throw_error_button")}
</Button>
<Separator />
<p>{t("developer_mode.environment_variables")}</p>
<pre className={styles.pre}>{JSON.stringify(env, null, 2)}</pre>
<p>{t("developer_mode.url_params")}</p>
@@ -681,21 +687,11 @@ function transportText(transport: UnstableLivekitTransport | null): string {
const TransportBox: FC<TransportInfo> = ({ transport, local }) => {
const { t } = useTranslation();
const resolved = useBehavior(transport.resolved$);
const fullMesh = transport.type === FULL_MESH_TRANSPORT_TYPE;
return (
<div className={styles.livekit_room_box}>
<h4>
{fullMesh
? t("developer_mode.full_mesh_transport", {
device: transport.raw.device_id,
})
: t("developer_mode.livekit_sfu", { url: transport.id })}
</h4>
<h4>{t("developer_mode.livekit_sfu", { url: transport.id })}</h4>
<p>
{fullMesh
? t("developer_mode.full_mesh_peer_connections")
: t("developer_mode.livekit_server_info")}
({local ? "local" : "remote"})
{t("developer_mode.livekit_server_info")}({local ? "local" : "remote"})
</p>
<pre className={styles.pre}>
{resolved === undefined
@@ -51,43 +51,6 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
>
<input
aria-describedby="_r_2_"
id="fullMeshPublishing"
type="checkbox"
/>
<label
for="fullMeshPublishing"
>
<div
class="_checkbox_1bd8c0"
>
<svg
fill="none"
height="24"
stroke="#000"
stroke-linecap="round"
stroke-linejoin="round"
stroke-width="2"
viewBox="0 0 24 24"
width="24"
xmlns="http://www.w3.org/2000/svg"
>
<path
d="m20 6-11 11-5-5"
/>
</svg>
</div>
Publish on full mesh (MSC4575) instead of LiveKit
</label>
</div>
</div>
<div
class="_fieldRow_1bd8c0"
>
<div
class="_field_1bd8c0 _checkboxField_1bd8c0"
>
<input
aria-describedby="_r_3_"
id="debugTileLayout"
type="checkbox"
/>
@@ -124,7 +87,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
class="_field_1bd8c0 _checkboxField_1bd8c0"
>
<input
aria-describedby="_r_4_"
aria-describedby="_r_3_"
id="showConnectionStats"
type="checkbox"
/>
@@ -161,7 +124,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
class="_field_1bd8c0 _checkboxField_1bd8c0"
>
<input
aria-describedby="_r_5_"
aria-describedby="_r_4_"
id="muteAllAudio"
type="checkbox"
/>
@@ -199,7 +162,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
class="_field_1bd8c0 _checkboxField_1bd8c0"
>
<input
aria-describedby="_r_6_"
aria-describedby="_r_5_"
id="alwaysShowIphoneEarpiece"
type="checkbox"
/>
@@ -236,7 +199,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
class="_field_1bd8c0 _checkboxField_1bd8c0"
>
<input
aria-describedby="_r_7_"
aria-describedby="_r_6_"
id="enableLivekitExtendedLogs"
type="checkbox"
/>
@@ -274,7 +237,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
>
<label
class="_label_1o4d9_60"
for="radix-_r_8_"
for="radix-_r_7_"
>
Custom transport JSON
</label>
@@ -282,9 +245,9 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
class="_controls_17lij_8"
>
<input
aria-describedby="radix-_r_9_"
aria-describedby="radix-_r_8_"
class="_control_d83jn_10"
id="radix-_r_8_"
id="radix-_r_7_"
name="input"
title=""
value=""
@@ -292,7 +255,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
</div>
<span
class="_message_1o4d9_86 _help-message_1o4d9_92"
id="radix-_r_9_"
id="radix-_r_8_"
>
Currently, no overwrite is set. Url from config is used.
</span>
@@ -322,10 +285,10 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
class="_container_1ug7n_10"
>
<input
aria-describedby="radix-_r_b_ radix-_r_d_"
aria-describedby="radix-_r_a_ radix-_r_c_"
checked=""
class="_input_1ug7n_18"
id="radix-_r_a_"
id="radix-_r_9_"
name="_r_0_"
title=""
type="radio"
@@ -341,13 +304,13 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
>
<label
class="_label_1o4d9_60"
for="radix-_r_a_"
for="radix-_r_9_"
>
Compatibility: state events & multi SFU
</label>
<span
class="_message_1o4d9_86 _help-message_1o4d9_92"
id="radix-_r_b_"
id="radix-_r_a_"
>
Compatible with homeservers that do not support sticky events (but all other EC clients are v0.17.0 or later)
</span>
@@ -363,9 +326,9 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
class="_container_1ug7n_10"
>
<input
aria-describedby="radix-_r_b_ radix-_r_d_"
aria-describedby="radix-_r_a_ radix-_r_c_"
class="_input_1ug7n_18"
id="radix-_r_c_"
id="radix-_r_b_"
name="_r_0_"
title=""
type="radio"
@@ -381,13 +344,13 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
>
<label
class="_label_1o4d9_60"
for="radix-_r_c_"
for="radix-_r_b_"
>
Matrix 2.0: sticky events & multi SFU
</label>
<span
class="_message_1o4d9_86 _help-message_1o4d9_92"
id="radix-_r_d_"
id="radix-_r_c_"
>
Compatible only with homservers supporting sticky events and all EC clients v0.17.0 or later
</span>
@@ -452,7 +415,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
class="_field_1bd8c0 _checkboxField_1bd8c0"
>
<input
aria-describedby="_r_j_"
aria-describedby="_r_i_"
id="cameraToggle"
type="checkbox"
/>
@@ -482,7 +445,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
</label>
<p
class="_description_1bd8c0"
id="_r_j_"
id="_r_i_"
>
Configure resolution, framerate, bitrate, and codec for camera video. Changes apply on next call join.
</p>
@@ -504,7 +467,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
class="_field_1bd8c0 _checkboxField_1bd8c0"
>
<input
aria-describedby="_r_k_"
aria-describedby="_r_j_"
id="screenShareToggle"
type="checkbox"
/>
@@ -534,7 +497,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
</label>
<p
class="_description_1bd8c0"
id="_r_k_"
id="_r_j_"
>
Configure resolution, framerate, bitrate, and codec for screen sharing
</p>
@@ -559,7 +522,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
class="_field_1bd8c0 _checkboxField_1bd8c0"
>
<input
aria-describedby="_r_l_"
aria-describedby="_r_k_"
checked=""
id="echoCancellation"
type="checkbox"
@@ -597,7 +560,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
class="_field_1bd8c0 _checkboxField_1bd8c0"
>
<input
aria-describedby="_r_m_"
aria-describedby="_r_l_"
checked=""
id="noiseSuppression"
type="checkbox"
@@ -635,7 +598,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
class="_field_1bd8c0 _checkboxField_1bd8c0"
>
<input
aria-describedby="_r_n_"
aria-describedby="_r_m_"
checked=""
id="autoGainControl"
type="checkbox"
@@ -672,6 +635,21 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
data-orientation="horizontal"
role="separator"
/>
<button
class="_button_1nw83_8 _destructive_1nw83_110"
data-kind="secondary"
data-size="lg"
role="button"
tabindex="0"
>
Throw test error
</button>
<div
class="_separator_13qwf_8"
data-kind="primary"
data-orientation="horizontal"
role="separator"
/>
<p>
Environment variables
</p>
-9
View File
@@ -159,15 +159,6 @@ export const customTransport = new Setting<UnstableLivekitTransport | null>(
null,
);
/**
* Publish on the full mesh transport (MSC4575) instead of LiveKit. Members on
* either transport are received whatever this says.
*/
export const fullMeshPublishing = new Setting<boolean>(
"full-mesh-publishing",
false,
);
export type VideoCodec = "vp8" | "vp9" | "h264" | "av1";
/**
+10 -10
View File
@@ -155,7 +155,7 @@ function summarizeLayout$(l$: Observable<Layout>): Observable<LayoutSummary> {
case "grid":
return combineLatest(
[
l.spotlight?.vm.media$ ?? constant(undefined),
l.spotlight?.media$ ?? constant(undefined),
...l.grid.map((vm) => vm.media$),
],
(spotlight, ...grid) => ({
@@ -167,7 +167,7 @@ function summarizeLayout$(l$: Observable<Layout>): Observable<LayoutSummary> {
case "spotlight-landscape":
case "spotlight-portrait":
return combineLatest(
[l.spotlight.vm.media$, ...l.grid.map((vm) => vm.media$)],
[l.spotlight.media$, ...l.grid.map((vm) => vm.media$)],
(spotlight, ...grid) => ({
type: l.type,
spotlight: spotlight.map((vm) => vm.id),
@@ -176,7 +176,7 @@ function summarizeLayout$(l$: Observable<Layout>): Observable<LayoutSummary> {
);
case "spotlight-expanded":
return combineLatest(
[l.spotlight.media$, l.pip?.vm.media$ ?? constant(undefined)],
[l.spotlight.media$, l.pip?.media$ ?? constant(undefined)],
(spotlight, pip) => ({
type: l.type,
spotlight: spotlight.map((vm) => vm.id),
@@ -185,7 +185,7 @@ function summarizeLayout$(l$: Observable<Layout>): Observable<LayoutSummary> {
);
case "one-on-one-desktop":
return combineLatest(
[l.spotlight.media$, l.pip.vm.media$],
[l.spotlight.media$, l.pip.media$],
(spotlight, pip) => ({
type: l.type,
spotlight: spotlight.id,
@@ -196,8 +196,8 @@ function summarizeLayout$(l$: Observable<Layout>): Observable<LayoutSummary> {
return combineLatest(
[
l.spotlight.media$,
l.pip?.vm.media$ ?? constant(undefined),
l.pip?.size$ ?? constant(undefined),
l.pip?.media$ ?? constant(undefined),
l.pipSize$,
],
(spotlight, pip, pipSize) => ({
type: l.type,
@@ -506,7 +506,7 @@ describe.each(modes)("CallViewModel (%s mode)", (mode) => {
type: "one-on-one-mobile",
spotlight: [`${aliceId}:0`],
pip: undefined,
pipSize: undefined,
pipSize: "lg",
},
b: {
type: "one-on-one-mobile",
@@ -544,7 +544,7 @@ describe.each(modes)("CallViewModel (%s mode)", (mode) => {
type: "one-on-one-mobile",
spotlight: [`${aliceId}:0`],
pip: undefined,
pipSize: undefined,
pipSize: "lg",
},
});
// It wouldn't be clear whether Alice or Bob is the remote video tile,
@@ -592,7 +592,7 @@ describe.each(modes)("CallViewModel (%s mode)", (mode) => {
type: "one-on-one-mobile",
spotlight: [`${aliceId}:0`],
pip: undefined,
pipSize: undefined,
pipSize: "sm",
},
b: {
type: "spotlight-landscape",
@@ -791,7 +791,7 @@ describe.each(modes)("CallViewModel (%s mode)", (mode) => {
type: "one-on-one-mobile",
spotlight: [`${aliceId}:0`],
pip: undefined,
pipSize: undefined,
pipSize: "lg",
},
b: {
// In a larger window, expect the one-on-one desktop layout
+32 -32
View File
@@ -78,13 +78,14 @@ import {
type ReactionInfo,
type ReactionOption,
} from "../../reactions";
import { shallowEquals as arrayShallowEquals } from "../../utils/array";
import { shallowEquals as shallowArrayEquals } from "../../utils/array";
import { shallowEquals as shallowObjectEquals } from "../../utils/object";
import { type MediaDevice, type MediaDevices } from "../MediaDevices";
import { type Handler, type MuteStates } from "../MuteStates";
import { HeaderStyle, type UrlParams } from "../../UrlParams";
import { type HostBridge, nullHostBridge } from "../../HostBridge";
import {
layoutShallowEquals,
type Alignment,
type Layout,
type LayoutMedia,
type WindowMode,
@@ -125,7 +126,6 @@ import {
type RingingMediaViewModel,
} from "../media/RingingMediaViewModel.ts";
import { type GridTileViewModel } from "../TileViewModel.ts";
import { createFloatingTile } from "../FloatingTile.ts";
/** How a call captures and encodes the tracks it publishes. */
export interface CaptureSettings {
@@ -830,7 +830,7 @@ export function createCallViewModel$(
bins.sort(([, bin1], [, bin2]) => bin1 - bin2).map(([m]) => m),
);
}),
distinctUntilChanged(arrayShallowEquals),
distinctUntilChanged(shallowArrayEquals),
),
);
@@ -862,18 +862,13 @@ export function createCallViewModel$(
[ringingMedia$, screenShares$, spotlightSpeaker$, localUserMediaForPip$],
(ringing, screenShares, speaker, localPip) =>
computeSpotlight({ ringing, screenShares, speaker, localPip }),
).pipe(
distinctUntilChanged(
(a, b) =>
arrayShallowEquals(a.spotlight, b.spotlight) && a.pip === b.pip,
),
),
).pipe(distinctUntilChanged(shallowObjectEquals)),
);
const spotlight$ = scope.behavior<MediaViewModel[]>(
spotlightAndPip$.pipe(
map(({ spotlight }) => spotlight),
distinctUntilChanged<MediaViewModel[]>(arrayShallowEquals),
distinctUntilChanged<MediaViewModel[]>(shallowArrayEquals),
),
);
@@ -1034,7 +1029,7 @@ export function createCallViewModel$(
desktop: platform === "desktop",
}),
),
distinctUntilChanged(layoutShallowEquals),
distinctUntilChanged(shallowObjectEquals<LayoutMedia>),
),
);
@@ -1226,28 +1221,32 @@ export function createCallViewModel$(
);
/**
* State of the floating spotlight tile, if present.
* The alignment of the floating spotlight tile, if present.
*/
const floatingSpotlightTile = createFloatingTile(scope, {
const spotlightAlignment$ = new BehaviorSubject<Alignment>({
inline: "end",
block: "end",
});
/**
* State of the small picture-in-picture tile, if present, when in landscape.
* The size of the small picture-in-picture tile, if present, when in portrait.
*/
const landscapePipTile = createFloatingTile(scope, {
const portraitPipSize$ = scope.behavior(
showFooter$.pipe(map((showFooter) => (showFooter ? "lg" : "sm"))),
);
/**
* The alignment of the small picture-in-picture tile, if present, when in portrait.
*/
const portraitPipAlignment$ = new BehaviorSubject<Alignment>({
inline: "end",
block: "end",
});
/**
* The alignment of the small picture-in-picture tile, if present, when in landscape.
*/
const landscapePipAlignment$ = new BehaviorSubject<Alignment>({
inline: "end",
block: "start",
});
/**
* State of the small picture-in-picture tile, if present, when in portrait.
*/
const portraitPipTile = {
...createFloatingTile(scope, { inline: "end", block: "end" }),
size$: scope.behavior(
showFooter$.pipe(map((showFooter) => (showFooter ? "lg" : "sm"))),
),
};
// There is a cyclical dependency here: the layout algorithms want to know
// which tiles are on screen, but to know which tiles are on screen we have to
@@ -1277,7 +1276,7 @@ export function createCallViewModel$(
case "spotlight-portrait":
[layout, newTiles] = gridLikeLayout(
media,
floatingSpotlightTile,
spotlightAlignment$,
visibleTiles,
setVisibleTiles,
prevTiles,
@@ -1287,25 +1286,26 @@ export function createCallViewModel$(
case "spotlight-expanded":
[layout, newTiles] = spotlightExpandedLayout(
media,
landscapePipTile,
landscapePipAlignment$,
prevTiles,
);
break;
case "one-on-one-desktop":
[layout, newTiles] = oneOnOneDesktopLayout(
media,
landscapePipTile,
landscapePipAlignment$,
prevTiles,
);
pip = layout.pip.vm;
pip = layout.pip;
break;
case "one-on-one-mobile":
[layout, newTiles] = oneOnOneMobileLayout(
media,
portraitPipTile,
portraitPipSize$,
portraitPipAlignment$,
prevTiles,
);
pip = layout.pip?.vm;
pip = layout.pip;
break;
case "pip":
[layout, newTiles] = pipLayout(media, prevTiles);
@@ -1330,7 +1330,7 @@ export function createCallViewModel$(
layoutInternals$.pipe(
map(({ layout }) => layout),
// Drop redundant layout updates before they would hit React.
distinctUntilChanged(layoutShallowEquals),
distinctUntilChanged(shallowObjectEquals<Layout>),
),
);
-38
View File
@@ -1,38 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { test } from "vitest";
import { withTestScheduler } from "../utils/test";
import { createFloatingTile } from "./FloatingTile";
test("FloatingTile changes alignment on drag", () =>
withTestScheduler(({ scope, expectObservable, schedule }) => {
// Tile starts in the top left
const { alignment$, onDrag } = createFloatingTile(scope, {
inline: "start",
block: "start",
});
// Drag to all four corners, and then repeatedly into the top left.
schedule("-↗↘↙↖↖", {
"↗": () =>
onDrag({ x: 100, y: 0, xRatio: 1, yRatio: 0, endOfGesture: false }),
"↘": () =>
onDrag({ x: 100, y: 100, xRatio: 1, yRatio: 1, endOfGesture: false }),
"↙": () =>
onDrag({ x: 0, y: 100, xRatio: 0, yRatio: 1, endOfGesture: false }),
"↖": () =>
onDrag({ x: 0, y: 0, xRatio: 0, yRatio: 0, endOfGesture: false }),
});
// Alignment should change accordingly. Notably, the alignment should *not*
// update when repeatedly dragged into the same corner, for performance.
expectObservable(alignment$).toBe("↖↗↘↙↖-", {
"↗": { inline: "end", block: "start" },
"↘": { inline: "end", block: "end" },
"↙": { inline: "start", block: "end" },
"↖": { inline: "start", block: "start" },
});
}));
-43
View File
@@ -1,43 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { distinctUntilChanged, map, startWith, Subject } from "rxjs";
import { type Behavior } from "./Behavior";
import { type DragCallback, type Alignment, type Drag } from "./layout-types";
import { type ObservableScope } from "./ObservableScope";
/**
* Represents a floating tile which may be dragged to different corners of the
* call interface.
*/
export interface FloatingTile {
alignment$: Behavior<Alignment>;
onDrag: DragCallback;
}
export function createFloatingTile(
scope: ObservableScope,
defaultAlignment: Alignment,
): FloatingTile {
const dragEvents$ = new Subject<Drag>();
return {
alignment$: scope.behavior(
dragEvents$.pipe(
map<Drag, Alignment>(({ xRatio, yRatio }) => ({
inline: xRatio < 0.5 ? "start" : "end",
block: yRatio < 0.5 ? "start" : "end",
})),
startWith(defaultAlignment),
distinctUntilChanged(
(a1, a2) => a1.inline === a2.inline && a1.block === a2.block,
),
),
),
onDrag: (drag) => dragEvents$.next(drag),
};
}
+10 -7
View File
@@ -5,9 +5,14 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type Layout, type LayoutMedia } from "./layout-types.ts";
import { type BehaviorSubject } from "rxjs";
import {
type Alignment,
type Layout,
type LayoutMedia,
} from "./layout-types.ts";
import { type TileStore } from "./TileStore";
import { type FloatingTile } from "./FloatingTile.ts";
export type GridLikeLayoutType =
| "grid"
@@ -20,7 +25,7 @@ export type GridLikeLayoutType =
*/
export function gridLikeLayout(
media: LayoutMedia & { type: GridLikeLayoutType },
floatingSpotlightTile: FloatingTile,
spotlightAlignment$: BehaviorSubject<Alignment>,
visibleTiles: number,
setVisibleTiles: (value: number) => void,
prevTiles: TileStore,
@@ -37,11 +42,9 @@ export function gridLikeLayout(
return [
{
type: media.type,
spotlight: tiles.spotlightTile && {
vm: tiles.spotlightTile,
...floatingSpotlightTile,
},
spotlight: tiles.spotlightTile,
grid: tiles.gridTiles,
spotlightAlignment$,
setVisibleTiles,
} as Layout & { type: GridLikeLayoutType },
tiles,
+6 -3
View File
@@ -6,19 +6,21 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type BehaviorSubject } from "rxjs";
import {
type Alignment,
type OneOnOneDesktopLayout,
type OneOnOneDesktopLayoutMedia,
} from "./layout-types";
import { type TileStore } from "./TileStore";
import { type FloatingTile } from "./FloatingTile";
/**
* Produces a one-on-one desktop layout with the given media.
*/
export function oneOnOneDesktopLayout(
media: OneOnOneDesktopLayoutMedia,
pipTile: FloatingTile,
pipAlignment$: BehaviorSubject<Alignment>,
prevTiles: TileStore,
): [OneOnOneDesktopLayout, TileStore] {
const update = prevTiles.from(2);
@@ -30,7 +32,8 @@ export function oneOnOneDesktopLayout(
{
type: media.type,
spotlight: tiles.gridTilesByMedia.get(media.spotlight)!,
pip: { vm: tiles.gridTilesByMedia.get(media.pip)!, ...pipTile },
pip: tiles.gridTilesByMedia.get(media.pip)!,
pipAlignment$,
},
tiles,
];
+8 -4
View File
@@ -6,20 +6,23 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type BehaviorSubject } from "rxjs";
import {
type Alignment,
type OneOnOneMobileLayout,
type OneOnOneMobileLayoutMedia,
} from "./layout-types";
import { type TileStore } from "./TileStore";
import { type Behavior } from "./Behavior";
import { type FloatingTile } from "./FloatingTile";
/**
* Produces a one-on-one mobile layout with the given media.
*/
export function oneOnOneMobileLayout(
media: OneOnOneMobileLayoutMedia,
pipTile: FloatingTile & { size$: Behavior<"sm" | "lg"> },
pipSize$: Behavior<"sm" | "lg">,
pipAlignment$: BehaviorSubject<Alignment>,
prevTiles: TileStore,
): [OneOnOneMobileLayout, TileStore] {
const update = prevTiles.from(media.pip === undefined ? 0 : 1);
@@ -27,12 +30,13 @@ export function oneOnOneMobileLayout(
if (media.pip !== undefined) update.registerGridTile(media.pip);
const tiles = update.build();
const pipVm = media.pip && tiles.gridTilesByMedia.get(media.pip);
return [
{
type: media.type,
spotlight: tiles.spotlightTile!,
pip: pipVm && { vm: pipVm, ...pipTile },
pip: media.pip && tiles.gridTilesByMedia.get(media.pip),
pipSize$,
pipAlignment$,
},
tiles,
];
+6 -4
View File
@@ -5,19 +5,21 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type BehaviorSubject } from "rxjs";
import {
type Alignment,
type SpotlightExpandedLayout,
type SpotlightExpandedLayoutMedia,
} from "./layout-types";
import { type TileStore } from "./TileStore";
import { type FloatingTile } from "./FloatingTile";
/**
* Produces an expanded spotlight layout with the given media.
*/
export function spotlightExpandedLayout(
media: SpotlightExpandedLayoutMedia,
pipTile: FloatingTile,
pipAlignment$: BehaviorSubject<Alignment>,
prevTiles: TileStore,
): [SpotlightExpandedLayout, TileStore] {
const update = prevTiles.from(1);
@@ -25,12 +27,12 @@ export function spotlightExpandedLayout(
if (media.pip !== undefined) update.registerPipTile(media.pip);
const tiles = update.build();
const pipVm = tiles.gridTiles.at(0);
return [
{
type: media.type,
spotlight: tiles.spotlightTile!,
pip: pipVm && { vm: pipVm, ...pipTile },
pip: tiles.gridTiles.at(0),
pipAlignment$,
},
tiles,
];
-82
View File
@@ -1,82 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { test, expect } from "vitest";
import { layoutShallowEquals, type Layout } from "./layout-types";
import {
type SpotlightTileViewModel,
type GridTileViewModel,
} from "./TileViewModel";
import { type FloatingTile } from "./FloatingTile";
const spotlightTile = {} as unknown as SpotlightTileViewModel;
const gridTile = {} as unknown as GridTileViewModel;
const pipTile = {} as unknown as FloatingTile & { vm: GridTileViewModel };
const spotlightExpanded: Layout = {
type: "spotlight-expanded",
spotlight: spotlightTile,
};
const spotlightPortrait: Layout = {
type: "spotlight-portrait",
spotlight: { vm: spotlightTile },
grid: [gridTile],
setVisibleTiles: () => {},
};
test("layoutShallowEquals considers a layout to be equal to its shallow clone", () =>
expect(layoutShallowEquals(spotlightExpanded, { ...spotlightExpanded })).toBe(
true,
));
test("layoutShallowEquals detects a missing key", () => {
expect(
layoutShallowEquals(spotlightExpanded, {
...spotlightExpanded,
pip: pipTile,
}),
).toBe(false);
expect(
layoutShallowEquals(
{ ...spotlightExpanded, pip: pipTile },
spotlightExpanded,
),
).toBe(false);
});
test("layoutShallowEquals considers tile wrappers with equal contents to be equal", () =>
expect(
layoutShallowEquals(spotlightPortrait, {
...spotlightPortrait,
spotlight: { vm: spotlightTile },
}),
).toBe(true));
test("layoutShallowEquals detects tile wrappers with different contents", () =>
expect(
layoutShallowEquals(spotlightPortrait, {
...spotlightPortrait,
spotlight: { vm: {} as unknown as SpotlightTileViewModel },
}),
).toBe(false));
test("layoutShallowEquals considers grid arrays with equal contents to be equal", () =>
expect(
layoutShallowEquals(spotlightPortrait, {
...spotlightPortrait,
grid: [...spotlightPortrait.grid],
}),
).toBe(true));
test("layoutShallowEquals detects grid arrays with different contents", () =>
expect(
layoutShallowEquals(spotlightPortrait, {
...spotlightPortrait,
grid: [...spotlightPortrait.grid, gridTile],
}),
).toBe(false));
+13 -65
View File
@@ -5,6 +5,8 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type BehaviorSubject } from "rxjs";
import { type LocalUserMediaViewModel } from "./media/LocalUserMediaViewModel.ts";
import { type MediaViewModel } from "./media/MediaViewModel.ts";
import { type RingingMediaViewModel } from "./media/RingingMediaViewModel.ts";
@@ -14,9 +16,6 @@ import {
type SpotlightTileViewModel,
} from "./TileViewModel.ts";
import { type Behavior } from "./Behavior.ts";
import { type FloatingTile } from "./FloatingTile.ts";
import { shallowEquals } from "../utils/object.ts";
import { shallowEquals as arrayShallowEquals } from "../utils/array.ts";
/**
* The general shape of the space the call is drawn in. Called a window because
@@ -87,48 +86,24 @@ export interface Alignment {
block: "start" | "end";
}
export interface Drag {
/**
* The X offset of the dragged tile relative to its container.
*/
x: number;
/**
* The Y offset of the dragged tile relative to its container.
*/
y: number;
/**
* The X coordinate of the dragged tile, as a scalar of the container's width.
*/
xRatio: number;
/**
* The Y coordinate of the dragged tile, as a scalar of the container's height.
*/
yRatio: number;
/**
* Whether this event ends the drag gesture.
*/
endOfGesture: boolean;
}
export type DragCallback = (drag: Drag) => void;
export interface GridLayout {
type: "grid";
spotlight?: FloatingTile & { vm: SpotlightTileViewModel };
spotlight?: SpotlightTileViewModel;
grid: GridTileViewModel[];
spotlightAlignment$: BehaviorSubject<Alignment>;
setVisibleTiles: (value: number) => void;
}
export interface SpotlightLandscapeLayout {
type: "spotlight-landscape";
spotlight: { vm: SpotlightTileViewModel };
spotlight: SpotlightTileViewModel;
grid: GridTileViewModel[];
setVisibleTiles: (value: number) => void;
}
export interface SpotlightPortraitLayout {
type: "spotlight-portrait";
spotlight: { vm: SpotlightTileViewModel };
spotlight: SpotlightTileViewModel;
grid: GridTileViewModel[];
setVisibleTiles: (value: number) => void;
}
@@ -136,19 +111,23 @@ export interface SpotlightPortraitLayout {
export interface SpotlightExpandedLayout {
type: "spotlight-expanded";
spotlight: SpotlightTileViewModel;
pip?: FloatingTile & { vm: GridTileViewModel };
pip?: GridTileViewModel;
pipAlignment$: BehaviorSubject<Alignment>;
}
export interface OneOnOneDesktopLayout {
type: "one-on-one-desktop";
spotlight: GridTileViewModel;
pip: FloatingTile & { vm: GridTileViewModel };
pip: GridTileViewModel;
pipAlignment$: BehaviorSubject<Alignment>;
}
export interface OneOnOneMobileLayout {
type: "one-on-one-mobile";
spotlight: SpotlightTileViewModel;
pip?: FloatingTile & { vm: GridTileViewModel; size$: Behavior<"sm" | "lg"> };
pip?: GridTileViewModel;
pipSize$: Behavior<"sm" | "lg">;
pipAlignment$: BehaviorSubject<Alignment>;
}
export interface PipLayout {
@@ -168,34 +147,3 @@ export type Layout =
| OneOnOneDesktopLayout
| OneOnOneMobileLayout
| PipLayout;
/**
* Tests whether the top-level properties, tile wrapper properties, and array
* elements of layout `a` are equal to those of layout `b`. Useful for deduping
* redundant layout updates.
*/
export function layoutShallowEquals<T extends Layout | LayoutMedia>(
a: T,
b: T,
): boolean {
return shallowEquals<T>(a, b, (aValue, bValue) => {
// Special case for tile wrapper objects so we can detect when the
// spotlight or PiP tile is essentially the same
if (
typeof aValue === "object" &&
aValue !== null &&
"vm" in aValue &&
typeof bValue === "object" &&
bValue !== null &&
"vm" in bValue
)
return shallowEquals(aValue, bValue);
// Special case for arrays so we can detect when the grid tile arrays are
// essentially the same
if (Array.isArray(aValue) && Array.isArray(bValue))
return arrayShallowEquals(aValue, bValue);
return aValue === bValue;
});
}
+4 -19
View File
@@ -7,7 +7,6 @@ Please see LICENSE in the repository root for full details.
import { test, expect } from "vitest";
import { shallowEquals } from "./object";
import { shallowEquals as arrayShallowEquals } from "./array";
interface Thing {
type: string;
@@ -30,26 +29,12 @@ test("shallowEquals detects a missing key", () => {
test("shallowEquals detects a differing value", () =>
expect(shallowEquals(withArray, { ...withArray, value: 2 })).toBe(false));
const comparator = (aValue: unknown, bValue: unknown): boolean => {
if (Array.isArray(aValue) && Array.isArray(bValue))
return arrayShallowEquals(aValue, bValue);
return aValue === bValue;
};
test("shallowEquals with comparator considers arrays with equal contents to be equal", () =>
test("shallowEquals considers arrays with equal contents to be equal", () =>
expect(
shallowEquals(
withArray,
{ ...withArray, items: [...withArray.items] },
comparator,
),
shallowEquals(withArray, { ...withArray, items: [...withArray.items] }),
).toBe(true));
test("shallowEquals with comparator detects arrays with different contents", () =>
test("shallowEquals detects arrays with different contents", () =>
expect(
shallowEquals(
withArray,
{ ...withArray, items: [...withArray.items, {}] },
comparator,
),
shallowEquals(withArray, { ...withArray, items: [...withArray.items, {}] }),
).toBe(false));
+8 -6
View File
@@ -5,17 +5,17 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { shallowEquals as arrayShallowEquals } from "./array.ts";
/**
* Determine whether two objects are equal by shallow comparison of their
* top-level properties, using the given `comparator`.
* top-level properties. Array-valued properties are themselves compared
* shallowly, so two objects holding distinct arrays with the same elements are
* considered equal.
*/
export function shallowEquals<T extends object>(
a: Readonly<T>,
b: Readonly<T>,
comparator: (aValue: unknown, bValue: unknown) => boolean = (
aValue,
bValue,
) => aValue === bValue,
): boolean {
// If a and b have the same number of keys and every key in a is also in b,
// then they have the same keys.
@@ -29,7 +29,9 @@ export function shallowEquals<T extends object>(
// Now check that they have the same values.
const aValue = (a as any)[key];
const bValue = (b as any)[key];
if (!comparator(aValue, bValue)) return false;
if (Array.isArray(aValue) && Array.isArray(bValue)) {
if (!arrayShallowEquals(aValue, bValue)) return false;
} else if (aValue !== bValue) return false;
}
return true;
+1 -3
View File
@@ -109,10 +109,8 @@ describe("widget", () => {
receiveState,
sendToDevice: sendRecvToDevice,
receiveToDevice: sendRecvToDevice,
turnServers: true,
turnServers: false,
rtcTransports: true,
rtcLivekitGetToken: true,
rtcLivekitDelegateDelayedLeave: true,
sendDelayedEvents: true,
updateDelayedEvents: true,
sendSticky: true,
+2 -13
View File
@@ -21,12 +21,8 @@ import type { IWidgetApiRequest } from "matrix-widget-api";
import { LazyEventEmitter } from "./LazyEventEmitter";
import { getUrlParams } from "./UrlParams";
import { Config } from "./config/Config";
import {
fullMeshPublishing,
seedSettingsFromConfig,
} from "./settings/settings";
import { seedSettingsFromConfig } from "./settings/settings";
import { ElementCallReactionEventType } from "./reactions";
import { fullMeshSignallingEventTypes } from "@element-hq/matrixrtc-sdk";
// Subset of the actions in element-web
export enum ElementWidgetActions {
@@ -166,10 +162,6 @@ export const initializeWidget = (
EventType.CallSDPStreamMetadataChangedPrefix,
EventType.CallReplaces,
EventType.CallEncryptionKeysPrefix,
// Full mesh signalling (MSC4575), only where it is turned on: Element
// Web doesn't auto-approve these for Element Call yet, and would ask
// the user on every call
...(fullMeshPublishing.getValue() ? fullMeshSignallingEventTypes : []),
];
const client = createRoomWidgetClient(
@@ -181,11 +173,8 @@ export const initializeWidget = (
receiveState,
sendToDevice: sendRecvToDevice,
receiveToDevice: sendRecvToDevice,
// Full mesh connects through the homeserver's TURN server
turnServers: true,
turnServers: false,
rtcTransports: true,
rtcLivekitGetToken: true,
rtcLivekitDelegateDelayedLeave: true,
sendDelayedEvents: true,
updateDelayedEvents: true,
sendSticky: true,