diff --git a/locales/en/app.json b/locales/en/app.json index aa753edc9..ec6c92de2 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -78,6 +78,9 @@ "device_id": "Device ID: {{id}}", "duplicate_tiles_label": "Number of additional tile copies per participant", "environment_variables": "Environment variables", + "full_mesh_peer_connections": "Full mesh peer connections", + "full_mesh_publishing": "Publish on full mesh (MSC4575) instead of LiveKit", + "full_mesh_transport": "Full mesh: {{device}}", "hostname": "Hostname: {{hostname}}", "livekit_server_info": "LiveKit Server Info", "livekit_sfu": "LiveKit SFU: {{url}}", diff --git a/playwright/widget/full-mesh.test.ts b/playwright/widget/full-mesh.test.ts new file mode 100644 index 000000000..a361533b8 --- /dev/null +++ b/playwright/widget/full-mesh.test.ts @@ -0,0 +1,167 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { expect, type Page, test } from "@playwright/test"; + +import { widgetTest } from "../fixtures/widget-user"; +import { HOST1, TestHelpers } from "./test-helpers"; + +/** + * Records the LiveKit signalling websockets a page opens: the SFU is reached + * with an access token in the url, which full mesh never needs. + */ +function recordSfuSockets(page: Page): string[] { + const urls: string[] = []; + page.on("websocket", (ws) => { + if (ws.url().includes("access_token")) urls.push(ws.url()); + }); + return urls; +} + +/** + * Starts recording the types of the to-device messages the widget on a page + * asks its host to send, and returns a function reading them. On the wire + * they are encrypted, so the type is only visible in the widget API. + */ +async function recordWidgetToDevice( + page: Page, +): Promise<() => Promise> { + await page.evaluate(() => { + const types: string[] = []; + Object.assign(window, { widgetToDeviceTypes: types }); + window.addEventListener("message", ({ data }) => { + const { + action, + response, + data: request, + } = (data ?? {}) as { + action?: string; + response?: unknown; + data?: { type?: string }; + }; + if (action === "send_to_device" && response === undefined) + types.push(request?.type ?? ""); + }); + }); + return async () => + page.evaluate( + () => + (window as unknown as { widgetToDeviceTypes: string[] }) + .widgetToDeviceTypes, + ); +} + +const cases: { name: string; fullMesh: [boolean, boolean] }[] = [ + // Both publish on full mesh: the media flows without any SFU + { name: "both on full mesh", fullMesh: [true, true] }, + // One on each: Florian receives Valere over LiveKit without publishing there + { name: "full mesh and LiveKit mixed", fullMesh: [true, false] }, +]; + +cases.forEach(({ name, fullMesh: [florianMesh, valereMesh] }) => { + widgetTest(`Full mesh call: ${name}`, async ({ addUser, browserName }) => { + test.skip( + browserName === "firefox", + "The is test is not working on firefox CI environment. No mic/audio device inputs so cam/mic are disabled", + ); + test.slow(); + + // Distinct per case: the cases run in parallel and register at once + const suffix = `${Number(florianMesh)}${Number(valereMesh)}`; + const [florian, valere] = await Promise.all([ + addUser(`florian${suffix}`, HOST1), + addUser(`valere${suffix}`, HOST1), + ]); + // Element Web doesn't auto-approve the full mesh signalling for Element + // Call yet, so it asks for it whenever a widget with full mesh opens + for (const { page } of [florian, valere]) + await page.addLocatorHandler( + page.getByRole("heading", { name: "Approve widget permissions" }), + async () => page.getByRole("button", { name: "Approve" }).click(), + ); + const sockets = { + florian: recordSfuSockets(florian.page), + valere: recordSfuSockets(valere.page), + }; + + const roomName = "Full Mesh Room"; + await TestHelpers.createRoom(roomName, florian.page, [valere.mxId]); + await TestHelpers.acceptRoomInvite(roomName, valere.page); + + await TestHelpers.openWidgetSetEmbeddedElementCallRtcModeCloseWidget( + florian.page, + "2_0", + { fullMesh: florianMesh }, + ); + await TestHelpers.openWidgetSetEmbeddedElementCallRtcModeCloseWidget( + valere.page, + "2_0", + { fullMesh: valereMesh }, + ); + + await TestHelpers.startCallInCurrentRoom(florian.page, false); + await TestHelpers.joinCallFromLobby(florian.page); + await TestHelpers.joinCallInCurrentRoom(valere.page); + + // A widget only asks for the full mesh signalling with full mesh turned + // on, so a LiveKit publisher in Element Web can't receive full mesh yet: + // the other's media arrives if it is on LiveKit or we have full mesh + const receives = { + florian: !valereMesh || florianMesh, + valere: !florianMesh || valereMesh, + }; + // In parallel: assertions on a page also run its permission handler + await Promise.all( + (["florian", "valere"] as const).map(async (name) => { + const frame = { florian, valere }[name].page + .locator('iframe[title="Element Call"]') + .contentFrame(); + await expect(frame.getByTestId("videoTile")).toHaveCount(2, { + timeout: 15000, + }); + const waiting = frame.getByText("Waiting for media..."); + if (receives[name]) { + // Over whichever transport carries it + await expect(waiting).not.toBeVisible({ timeout: 20000 }); + await TestHelpers.expectVisibleVideoCount(frame, 2); + } else { + await expect(waiting).toBeVisible({ timeout: 20000 }); + await TestHelpers.expectVisibleVideoCount(frame, 1); + } + }), + ); + + if (florianMesh && valereMesh) { + // Nobody published on LiveKit, so nobody connected to an SFU + expect(sockets.florian).toEqual([]); + expect(sockets.valere).toEqual([]); + } else { + // Florian receives Valere's media from the SFU without publishing there + expect(sockets.florian.length).toBeGreaterThan(0); + } + + // Florian sees Valere's camera go off: over the track metadata for full + // mesh, without renegotiating + const toDevice = await recordWidgetToDevice(valere.page); + const valereFrame = valere.page + .locator('iframe[title="Element Call"]') + .contentFrame(); + await valereFrame.getByRole("switch", { name: "Stop video" }).click(); + const florianFrame = florian.page + .locator('iframe[title="Element Call"]') + .contentFrame(); + await TestHelpers.expectVisibleVideoCount(florianFrame, 1); + // On the connection's control channel, not through the homeserver. + // Candidates may still trickle there until the channel is up + if (florianMesh && valereMesh) + expect( + (await toDevice()).filter( + (type) => type.includes("full_mesh") && !type.endsWith(".candidates"), + ), + ).toEqual([]); + }); +}); diff --git a/playwright/widget/test-helpers.ts b/playwright/widget/test-helpers.ts index 4e255f869..d62fb3791 100644 --- a/playwright/widget/test-helpers.ts +++ b/playwright/widget/test-helpers.ts @@ -306,6 +306,7 @@ export class TestHelpers { public static async openWidgetSetEmbeddedElementCallRtcModeCloseWidget( page: Page, mode: RtcMode, + { fullMesh = false }: { fullMesh?: boolean } = {}, ): Promise { await page.getByRole("button", { name: "Video call" }).click({ timeout: 5000, @@ -314,7 +315,7 @@ export class TestHelpers { timeout: 10000, }); - await TestHelpers.setEmbeddedElementCallRtcMode(page, mode); + await TestHelpers.setEmbeddedElementCallRtcMode(page, mode, { fullMesh }); await page.getByRole("button", { name: "Close lobby" }).click(); } @@ -327,6 +328,7 @@ export class TestHelpers { public static async setEmbeddedElementCallRtcMode( page: Page, mode: RtcMode, + { fullMesh = false }: { fullMesh?: boolean } = {}, ): Promise { const iframe = page.locator('iframe[title="Element Call"]').contentFrame(); @@ -344,6 +346,10 @@ export class TestHelpers { // compat await iframe.getByText("Compatibility: state events").click(); } + // Publish on the full mesh transport (MSC4575) instead of LiveKit + await iframe + .getByText("Publish on full mesh", { exact: false }) + .setChecked(fullMesh); await iframe.getByTestId("modal_close").click(); } diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 3459d5565..84a7ec5c8 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -71,6 +71,7 @@ import { ReactionsOverlay } from "./ReactionsOverlay"; import { CallEventAudioRenderer } from "./CallEventAudioRenderer"; import { createRTCSlot, + FULL_MESH_TRANSPORT_TYPE, type RTCParticipationOptions, type RTCSlotOptions, constant, @@ -85,6 +86,7 @@ import { cameraFramerate, cameraResolution, customTransport, + fullMeshPublishing, echoCancellationSetting, matrixRTCMode as matrixRTCModeSetting, noiseSuppressionSetting, @@ -837,7 +839,9 @@ export function rtcParticipationOptions( delegatedDelayedLeave: session.delegated_delayed_leave, }, mediaQuality: config.media_quality, - transport: customTransport.value$.value ?? undefined, + transport: fullMeshPublishing.value$.value + ? { type: FULL_MESH_TRANSPORT_TYPE } + : (customTransport.value$.value ?? undefined), fallbackTransport: config.livekit && { type: "livekit", livekit_service_url: config.livekit.livekit_service_url, diff --git a/src/settings/DeveloperSettingsTab.tsx b/src/settings/DeveloperSettingsTab.tsx index e303a4d0b..5c2e3cf8d 100644 --- a/src/settings/DeveloperSettingsTab.tsx +++ b/src/settings/DeveloperSettingsTab.tsx @@ -45,6 +45,7 @@ import { alwaysShowIphoneEarpiece as alwaysShowIphoneEarpieceSetting, matrixRTCMode as matrixRTCModeSetting, customTransport as customTransportSetting, + fullMeshPublishing as fullMeshPublishingSetting, advancedScreenShare as advancedScreenShareSetting, screenShareResolution as screenShareResolutionSetting, screenShareFramerate as screenShareFramerateSetting, @@ -68,6 +69,7 @@ import { Slider } from "../Slider"; import { useUrlParams } from "../UrlParams"; import { authenticateWithTransport, + FULL_MESH_TRANSPORT_TYPE, type TransportMetadata, } from "@element-hq/matrixrtc-sdk"; import { useBehavior } from "../useBehavior"; @@ -175,6 +177,9 @@ export const DeveloperSettingsTab: FC = ({ const [customTransportUpdateError, setCustomTransportUpdateError] = useState< string | null >(null); + const [fullMeshPublishing, setFullMeshPublishing] = useSetting( + fullMeshPublishingSetting, + ); const [customTransport, setCustomTransport] = useSetting( customTransportSetting, ); @@ -417,6 +422,17 @@ export const DeveloperSettingsTab: FC = ({ )} /> + + ): void => + setFullMeshPublishing(event.target.checked) + } + /> + = ({ transport, local }) => { const { t } = useTranslation(); const resolved = useBehavior(transport.resolved$); + const fullMesh = transport.type === FULL_MESH_TRANSPORT_TYPE; return (
-

{t("developer_mode.livekit_sfu", { url: transport.id })}

+

+ {fullMesh + ? t("developer_mode.full_mesh_transport", { + device: transport.raw.device_id, + }) + : t("developer_mode.livekit_sfu", { url: transport.id })} +

- {t("developer_mode.livekit_server_info")}({local ? "local" : "remote"}) + {fullMesh + ? t("developer_mode.full_mesh_peer_connections") + : t("developer_mode.livekit_server_info")} + ({local ? "local" : "remote"}) 

         {resolved === undefined
diff --git a/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap b/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap
index a2e8671c1..e34df26e2 100644
--- a/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap
+++ b/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap
@@ -51,6 +51,43 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
     >
       
+      
+
+ +
+
+ @@ -87,7 +124,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = ` class="_field_1bd8c0 _checkboxField_1bd8c0" > @@ -124,7 +161,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = ` class="_field_1bd8c0 _checkboxField_1bd8c0" > @@ -162,7 +199,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = ` class="_field_1bd8c0 _checkboxField_1bd8c0" > @@ -199,7 +236,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = ` class="_field_1bd8c0 _checkboxField_1bd8c0" > @@ -237,7 +274,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = ` > @@ -245,9 +282,9 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = ` class="_controls_17lij_8" > renders and matches snapshot 1`] = `
Currently, no overwrite is set. Url from config is used. @@ -285,10 +322,10 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = ` class="_container_1ug7n_10" > renders and matches snapshot 1`] = ` > Compatible with homeservers that do not support sticky events (but all other EC clients are v0.17.0 or later) @@ -326,9 +363,9 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = ` class="_container_1ug7n_10" > renders and matches snapshot 1`] = ` > Compatible only with homservers supporting sticky events and all EC clients v0.17.0 or later @@ -415,7 +452,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = ` class="_field_1bd8c0 _checkboxField_1bd8c0" > @@ -445,7 +482,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `

Configure resolution, framerate, bitrate, and codec for camera video. Changes apply on next call join.

@@ -467,7 +504,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = ` class="_field_1bd8c0 _checkboxField_1bd8c0" > @@ -497,7 +534,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `

Configure resolution, framerate, bitrate, and codec for screen sharing

@@ -522,7 +559,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = ` class="_field_1bd8c0 _checkboxField_1bd8c0" > renders and matches snapshot 1`] = ` class="_field_1bd8c0 _checkboxField_1bd8c0" > renders and matches snapshot 1`] = ` class="_field_1bd8c0 _checkboxField_1bd8c0" > ( null, ); +/** + * Publish on the full mesh transport (MSC4575) instead of LiveKit. Members on + * either transport are received whatever this says. + */ +export const fullMeshPublishing = new Setting( + "full-mesh-publishing", + false, +); + export type VideoCodec = "vp8" | "vp9" | "h264" | "av1"; /** diff --git a/src/widget.test.ts b/src/widget.test.ts index 908159158..50a58ea24 100644 --- a/src/widget.test.ts +++ b/src/widget.test.ts @@ -109,7 +109,7 @@ describe("widget", () => { receiveState, sendToDevice: sendRecvToDevice, receiveToDevice: sendRecvToDevice, - turnServers: false, + turnServers: true, rtcTransports: true, rtcLivekitGetToken: true, rtcLivekitDelegateDelayedLeave: true, diff --git a/src/widget.ts b/src/widget.ts index f1847004b..764e13d9a 100644 --- a/src/widget.ts +++ b/src/widget.ts @@ -21,8 +21,12 @@ import type { IWidgetApiRequest } from "matrix-widget-api"; import { LazyEventEmitter } from "./LazyEventEmitter"; import { getUrlParams } from "./UrlParams"; import { Config } from "./config/Config"; -import { seedSettingsFromConfig } from "./settings/settings"; +import { + fullMeshPublishing, + seedSettingsFromConfig, +} from "./settings/settings"; import { ElementCallReactionEventType } from "./reactions"; +import { fullMeshSignallingEventTypes } from "@element-hq/matrixrtc-sdk"; // Subset of the actions in element-web export enum ElementWidgetActions { @@ -162,6 +166,10 @@ export const initializeWidget = ( EventType.CallSDPStreamMetadataChangedPrefix, EventType.CallReplaces, EventType.CallEncryptionKeysPrefix, + // Full mesh signalling (MSC4575), only where it is turned on: Element + // Web doesn't auto-approve these for Element Call yet, and would ask + // the user on every call + ...(fullMeshPublishing.getValue() ? fullMeshSignallingEventTypes : []), ]; const client = createRoomWidgetClient( @@ -173,7 +181,8 @@ export const initializeWidget = ( receiveState, sendToDevice: sendRecvToDevice, receiveToDevice: sendRecvToDevice, - turnServers: false, + // Full mesh connects through the homeserver's TURN server + turnServers: true, rtcTransports: true, rtcLivekitGetToken: true, rtcLivekitDelegateDelayedLeave: true,