From 682edd25a9ea3370e7c8d36361a17af2a70c3df1 Mon Sep 17 00:00:00 2001 From: "Timo K." Date: Thu, 5 Nov 2026 15:51:16 +0100 Subject: [PATCH] Publish on full mesh from Element Call A developer setting switches the published transport to full mesh; members on either transport are received whatever it says. The widget allows the full mesh signalling events while the setting is on, and asks for TURN servers. The developer panel shows a full mesh transport with its peer connections. A widget e2e spec covers two members calling over full mesh. Co-Authored-By: Claude Fable 5.1 --- locales/en/app.json | 3 + playwright/widget/full-mesh.test.ts | 167 ++++++++++++++++++ playwright/widget/test-helpers.ts | 8 +- src/room/InCallView.tsx | 6 +- src/settings/DeveloperSettingsTab.tsx | 30 +++- .../DeveloperSettingsTab.test.tsx.snap | 83 ++++++--- src/settings/settings.ts | 9 + src/widget.test.ts | 2 +- src/widget.ts | 13 +- 9 files changed, 291 insertions(+), 30 deletions(-) create mode 100644 playwright/widget/full-mesh.test.ts diff --git a/locales/en/app.json b/locales/en/app.json index aa753edc9..ec6c92de2 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -78,6 +78,9 @@ "device_id": "Device ID: {{id}}", "duplicate_tiles_label": "Number of additional tile copies per participant", "environment_variables": "Environment variables", + "full_mesh_peer_connections": "Full mesh peer connections", + "full_mesh_publishing": "Publish on full mesh (MSC4575) instead of LiveKit", + "full_mesh_transport": "Full mesh: {{device}}", "hostname": "Hostname: {{hostname}}", "livekit_server_info": "LiveKit Server Info", "livekit_sfu": "LiveKit SFU: {{url}}", diff --git a/playwright/widget/full-mesh.test.ts b/playwright/widget/full-mesh.test.ts new file mode 100644 index 000000000..a361533b8 --- /dev/null +++ b/playwright/widget/full-mesh.test.ts @@ -0,0 +1,167 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { expect, type Page, test } from "@playwright/test"; + +import { widgetTest } from "../fixtures/widget-user"; +import { HOST1, TestHelpers } from "./test-helpers"; + +/** + * Records the LiveKit signalling websockets a page opens: the SFU is reached + * with an access token in the url, which full mesh never needs. + */ +function recordSfuSockets(page: Page): string[] { + const urls: string[] = []; + page.on("websocket", (ws) => { + if (ws.url().includes("access_token")) urls.push(ws.url()); + }); + return urls; +} + +/** + * Starts recording the types of the to-device messages the widget on a page + * asks its host to send, and returns a function reading them. On the wire + * they are encrypted, so the type is only visible in the widget API. + */ +async function recordWidgetToDevice( + page: Page, +): Promise<() => Promise> { + await page.evaluate(() => { + const types: string[] = []; + Object.assign(window, { widgetToDeviceTypes: types }); + window.addEventListener("message", ({ data }) => { + const { + action, + response, + data: request, + } = (data ?? {}) as { + action?: string; + response?: unknown; + data?: { type?: string }; + }; + if (action === "send_to_device" && response === undefined) + types.push(request?.type ?? ""); + }); + }); + return async () => + page.evaluate( + () => + (window as unknown as { widgetToDeviceTypes: string[] }) + .widgetToDeviceTypes, + ); +} + +const cases: { name: string; fullMesh: [boolean, boolean] }[] = [ + // Both publish on full mesh: the media flows without any SFU + { name: "both on full mesh", fullMesh: [true, true] }, + // One on each: Florian receives Valere over LiveKit without publishing there + { name: "full mesh and LiveKit mixed", fullMesh: [true, false] }, +]; + +cases.forEach(({ name, fullMesh: [florianMesh, valereMesh] }) => { + widgetTest(`Full mesh call: ${name}`, async ({ addUser, browserName }) => { + test.skip( + browserName === "firefox", + "The is test is not working on firefox CI environment. No mic/audio device inputs so cam/mic are disabled", + ); + test.slow(); + + // Distinct per case: the cases run in parallel and register at once + const suffix = `${Number(florianMesh)}${Number(valereMesh)}`; + const [florian, valere] = await Promise.all([ + addUser(`florian${suffix}`, HOST1), + addUser(`valere${suffix}`, HOST1), + ]); + // Element Web doesn't auto-approve the full mesh signalling for Element + // Call yet, so it asks for it whenever a widget with full mesh opens + for (const { page } of [florian, valere]) + await page.addLocatorHandler( + page.getByRole("heading", { name: "Approve widget permissions" }), + async () => page.getByRole("button", { name: "Approve" }).click(), + ); + const sockets = { + florian: recordSfuSockets(florian.page), + valere: recordSfuSockets(valere.page), + }; + + const roomName = "Full Mesh Room"; + await TestHelpers.createRoom(roomName, florian.page, [valere.mxId]); + await TestHelpers.acceptRoomInvite(roomName, valere.page); + + await TestHelpers.openWidgetSetEmbeddedElementCallRtcModeCloseWidget( + florian.page, + "2_0", + { fullMesh: florianMesh }, + ); + await TestHelpers.openWidgetSetEmbeddedElementCallRtcModeCloseWidget( + valere.page, + "2_0", + { fullMesh: valereMesh }, + ); + + await TestHelpers.startCallInCurrentRoom(florian.page, false); + await TestHelpers.joinCallFromLobby(florian.page); + await TestHelpers.joinCallInCurrentRoom(valere.page); + + // A widget only asks for the full mesh signalling with full mesh turned + // on, so a LiveKit publisher in Element Web can't receive full mesh yet: + // the other's media arrives if it is on LiveKit or we have full mesh + const receives = { + florian: !valereMesh || florianMesh, + valere: !florianMesh || valereMesh, + }; + // In parallel: assertions on a page also run its permission handler + await Promise.all( + (["florian", "valere"] as const).map(async (name) => { + const frame = { florian, valere }[name].page + .locator('iframe[title="Element Call"]') + .contentFrame(); + await expect(frame.getByTestId("videoTile")).toHaveCount(2, { + timeout: 15000, + }); + const waiting = frame.getByText("Waiting for media..."); + if (receives[name]) { + // Over whichever transport carries it + await expect(waiting).not.toBeVisible({ timeout: 20000 }); + await TestHelpers.expectVisibleVideoCount(frame, 2); + } else { + await expect(waiting).toBeVisible({ timeout: 20000 }); + await TestHelpers.expectVisibleVideoCount(frame, 1); + } + }), + ); + + if (florianMesh && valereMesh) { + // Nobody published on LiveKit, so nobody connected to an SFU + expect(sockets.florian).toEqual([]); + expect(sockets.valere).toEqual([]); + } else { + // Florian receives Valere's media from the SFU without publishing there + expect(sockets.florian.length).toBeGreaterThan(0); + } + + // Florian sees Valere's camera go off: over the track metadata for full + // mesh, without renegotiating + const toDevice = await recordWidgetToDevice(valere.page); + const valereFrame = valere.page + .locator('iframe[title="Element Call"]') + .contentFrame(); + await valereFrame.getByRole("switch", { name: "Stop video" }).click(); + const florianFrame = florian.page + .locator('iframe[title="Element Call"]') + .contentFrame(); + await TestHelpers.expectVisibleVideoCount(florianFrame, 1); + // On the connection's control channel, not through the homeserver. + // Candidates may still trickle there until the channel is up + if (florianMesh && valereMesh) + expect( + (await toDevice()).filter( + (type) => type.includes("full_mesh") && !type.endsWith(".candidates"), + ), + ).toEqual([]); + }); +}); diff --git a/playwright/widget/test-helpers.ts b/playwright/widget/test-helpers.ts index 4e255f869..d62fb3791 100644 --- a/playwright/widget/test-helpers.ts +++ b/playwright/widget/test-helpers.ts @@ -306,6 +306,7 @@ export class TestHelpers { public static async openWidgetSetEmbeddedElementCallRtcModeCloseWidget( page: Page, mode: RtcMode, + { fullMesh = false }: { fullMesh?: boolean } = {}, ): Promise { await page.getByRole("button", { name: "Video call" }).click({ timeout: 5000, @@ -314,7 +315,7 @@ export class TestHelpers { timeout: 10000, }); - await TestHelpers.setEmbeddedElementCallRtcMode(page, mode); + await TestHelpers.setEmbeddedElementCallRtcMode(page, mode, { fullMesh }); await page.getByRole("button", { name: "Close lobby" }).click(); } @@ -327,6 +328,7 @@ export class TestHelpers { public static async setEmbeddedElementCallRtcMode( page: Page, mode: RtcMode, + { fullMesh = false }: { fullMesh?: boolean } = {}, ): Promise { const iframe = page.locator('iframe[title="Element Call"]').contentFrame(); @@ -344,6 +346,10 @@ export class TestHelpers { // compat await iframe.getByText("Compatibility: state events").click(); } + // Publish on the full mesh transport (MSC4575) instead of LiveKit + await iframe + .getByText("Publish on full mesh", { exact: false }) + .setChecked(fullMesh); await iframe.getByTestId("modal_close").click(); } diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 3459d5565..84a7ec5c8 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -71,6 +71,7 @@ import { ReactionsOverlay } from "./ReactionsOverlay"; import { CallEventAudioRenderer } from "./CallEventAudioRenderer"; import { createRTCSlot, + FULL_MESH_TRANSPORT_TYPE, type RTCParticipationOptions, type RTCSlotOptions, constant, @@ -85,6 +86,7 @@ import { cameraFramerate, cameraResolution, customTransport, + fullMeshPublishing, echoCancellationSetting, matrixRTCMode as matrixRTCModeSetting, noiseSuppressionSetting, @@ -837,7 +839,9 @@ export function rtcParticipationOptions( delegatedDelayedLeave: session.delegated_delayed_leave, }, mediaQuality: config.media_quality, - transport: customTransport.value$.value ?? undefined, + transport: fullMeshPublishing.value$.value + ? { type: FULL_MESH_TRANSPORT_TYPE } + : (customTransport.value$.value ?? undefined), fallbackTransport: config.livekit && { type: "livekit", livekit_service_url: config.livekit.livekit_service_url, diff --git a/src/settings/DeveloperSettingsTab.tsx b/src/settings/DeveloperSettingsTab.tsx index e303a4d0b..5c2e3cf8d 100644 --- a/src/settings/DeveloperSettingsTab.tsx +++ b/src/settings/DeveloperSettingsTab.tsx @@ -45,6 +45,7 @@ import { alwaysShowIphoneEarpiece as alwaysShowIphoneEarpieceSetting, matrixRTCMode as matrixRTCModeSetting, customTransport as customTransportSetting, + fullMeshPublishing as fullMeshPublishingSetting, advancedScreenShare as advancedScreenShareSetting, screenShareResolution as screenShareResolutionSetting, screenShareFramerate as screenShareFramerateSetting, @@ -68,6 +69,7 @@ import { Slider } from "../Slider"; import { useUrlParams } from "../UrlParams"; import { authenticateWithTransport, + FULL_MESH_TRANSPORT_TYPE, type TransportMetadata, } from "@element-hq/matrixrtc-sdk"; import { useBehavior } from "../useBehavior"; @@ -175,6 +177,9 @@ export const DeveloperSettingsTab: FC = ({ const [customTransportUpdateError, setCustomTransportUpdateError] = useState< string | null >(null); + const [fullMeshPublishing, setFullMeshPublishing] = useSetting( + fullMeshPublishingSetting, + ); const [customTransport, setCustomTransport] = useSetting( customTransportSetting, ); @@ -417,6 +422,17 @@ export const DeveloperSettingsTab: FC = ({ )} /> + + ): void => + setFullMeshPublishing(event.target.checked) + } + /> + = ({ transport, local }) => { const { t } = useTranslation(); const resolved = useBehavior(transport.resolved$); + const fullMesh = transport.type === FULL_MESH_TRANSPORT_TYPE; return (
-

{t("developer_mode.livekit_sfu", { url: transport.id })}

+

+ {fullMesh + ? t("developer_mode.full_mesh_transport", { + device: transport.raw.device_id, + }) + : t("developer_mode.livekit_sfu", { url: transport.id })} +

- {t("developer_mode.livekit_server_info")}({local ? "local" : "remote"}) + {fullMesh + ? t("developer_mode.full_mesh_peer_connections") + : t("developer_mode.livekit_server_info")} + ({local ? "local" : "remote"}) 

         {resolved === undefined
diff --git a/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap b/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap
index a2e8671c1..e34df26e2 100644
--- a/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap
+++ b/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap
@@ -51,6 +51,43 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
     >
       
+      
+
+ +
+
+ @@ -87,7 +124,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = ` class="_field_1bd8c0 _checkboxField_1bd8c0" > @@ -124,7 +161,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = ` class="_field_1bd8c0 _checkboxField_1bd8c0" > @@ -162,7 +199,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = ` class="_field_1bd8c0 _checkboxField_1bd8c0" > @@ -199,7 +236,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = ` class="_field_1bd8c0 _checkboxField_1bd8c0" > @@ -237,7 +274,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = ` > @@ -245,9 +282,9 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = ` class="_controls_17lij_8" > renders and matches snapshot 1`] = `
Currently, no overwrite is set. Url from config is used. @@ -285,10 +322,10 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = ` class="_container_1ug7n_10" > renders and matches snapshot 1`] = ` > Compatible with homeservers that do not support sticky events (but all other EC clients are v0.17.0 or later) @@ -326,9 +363,9 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = ` class="_container_1ug7n_10" > renders and matches snapshot 1`] = ` > Compatible only with homservers supporting sticky events and all EC clients v0.17.0 or later @@ -415,7 +452,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = ` class="_field_1bd8c0 _checkboxField_1bd8c0" > @@ -445,7 +482,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `

Configure resolution, framerate, bitrate, and codec for camera video. Changes apply on next call join.

@@ -467,7 +504,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = ` class="_field_1bd8c0 _checkboxField_1bd8c0" > @@ -497,7 +534,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `

Configure resolution, framerate, bitrate, and codec for screen sharing

@@ -522,7 +559,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = ` class="_field_1bd8c0 _checkboxField_1bd8c0" > renders and matches snapshot 1`] = ` class="_field_1bd8c0 _checkboxField_1bd8c0" > renders and matches snapshot 1`] = ` class="_field_1bd8c0 _checkboxField_1bd8c0" > ( null, ); +/** + * Publish on the full mesh transport (MSC4575) instead of LiveKit. Members on + * either transport are received whatever this says. + */ +export const fullMeshPublishing = new Setting( + "full-mesh-publishing", + false, +); + export type VideoCodec = "vp8" | "vp9" | "h264" | "av1"; /** diff --git a/src/widget.test.ts b/src/widget.test.ts index 908159158..50a58ea24 100644 --- a/src/widget.test.ts +++ b/src/widget.test.ts @@ -109,7 +109,7 @@ describe("widget", () => { receiveState, sendToDevice: sendRecvToDevice, receiveToDevice: sendRecvToDevice, - turnServers: false, + turnServers: true, rtcTransports: true, rtcLivekitGetToken: true, rtcLivekitDelegateDelayedLeave: true, diff --git a/src/widget.ts b/src/widget.ts index f1847004b..764e13d9a 100644 --- a/src/widget.ts +++ b/src/widget.ts @@ -21,8 +21,12 @@ import type { IWidgetApiRequest } from "matrix-widget-api"; import { LazyEventEmitter } from "./LazyEventEmitter"; import { getUrlParams } from "./UrlParams"; import { Config } from "./config/Config"; -import { seedSettingsFromConfig } from "./settings/settings"; +import { + fullMeshPublishing, + seedSettingsFromConfig, +} from "./settings/settings"; import { ElementCallReactionEventType } from "./reactions"; +import { fullMeshSignallingEventTypes } from "@element-hq/matrixrtc-sdk"; // Subset of the actions in element-web export enum ElementWidgetActions { @@ -162,6 +166,10 @@ export const initializeWidget = ( EventType.CallSDPStreamMetadataChangedPrefix, EventType.CallReplaces, EventType.CallEncryptionKeysPrefix, + // Full mesh signalling (MSC4575), only where it is turned on: Element + // Web doesn't auto-approve these for Element Call yet, and would ask + // the user on every call + ...(fullMeshPublishing.getValue() ? fullMeshSignallingEventTypes : []), ]; const client = createRoomWidgetClient( @@ -173,7 +181,8 @@ export const initializeWidget = ( receiveState, sendToDevice: sendRecvToDevice, receiveToDevice: sendRecvToDevice, - turnServers: false, + // Full mesh connects through the homeserver's TURN server + turnServers: true, rtcTransports: true, rtcLivekitGetToken: true, rtcLivekitDelegateDelayedLeave: true,