From e133289a7fef85cdfab94c4a12ef4a4279c1c479 Mon Sep 17 00:00:00 2001 From: Timo K Date: Mon, 4 Aug 2025 18:50:57 +0200 Subject: [PATCH 01/15] Make convert remaining js config files to ts Co-authored-by: hughns Signed-off-by: Timo K --- i18next-parser.config.ts | 12 +++++++++- ...edded.config.js => vite-embedded.config.ts | 10 +++++++- vite.config.js => vite.config.ts | 23 +++++++++++++------ vitest.config.js => vitest.config.ts | 9 +++++++- 4 files changed, 44 insertions(+), 10 deletions(-) rename vite-embedded.config.js => vite-embedded.config.ts (84%) rename vite.config.js => vite.config.ts (90%) rename vitest.config.js => vitest.config.ts (81%) diff --git a/i18next-parser.config.ts b/i18next-parser.config.ts index 3acf2b5e..e07021ae 100644 --- a/i18next-parser.config.ts +++ b/i18next-parser.config.ts @@ -1,4 +1,12 @@ -export default { +/* +Copyright 2024 New Vector Ltd. +SPDX-License-Identifier: AGPL-3.0-only +Please see LICENSE in the repository root for full details. +*/ + +import type { UserConfig } from "i18next-parser"; + +const config: UserConfig = { keySeparator: ".", namespaceSeparator: false, contextSeparator: "|", @@ -26,3 +34,5 @@ export default { input: ["src/**/*.{ts,tsx}"], sort: true, }; + +export default config; diff --git a/vite-embedded.config.js b/vite-embedded.config.ts similarity index 84% rename from vite-embedded.config.js rename to vite-embedded.config.ts index 8f5bcba8..27a42fbb 100644 --- a/vite-embedded.config.js +++ b/vite-embedded.config.ts @@ -1,7 +1,15 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + import { defineConfig, mergeConfig } from "vite"; -import fullConfig from "./vite.config"; import generateFile from "vite-plugin-generate-file"; +import fullConfig from "./vite.config"; + const base = "./"; // Config for embedded deployments (possibly hosted under a non-root path) diff --git a/vite.config.js b/vite.config.ts similarity index 90% rename from vite.config.js rename to vite.config.ts index 5fe3a99b..115565bb 100644 --- a/vite.config.js +++ b/vite.config.ts @@ -5,7 +5,12 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { defineConfig, loadEnv, searchForWorkspaceRoot } from "vite"; +import { + loadEnv, + searchForWorkspaceRoot, + type ConfigEnv, + type UserConfig, +} from "vite"; import svgrPlugin from "vite-plugin-svgr"; import { createHtmlPlugin } from "vite-plugin-html"; import { codecovVitePlugin } from "@codecov/vite-plugin"; @@ -13,12 +18,16 @@ import { sentryVitePlugin } from "@sentry/vite-plugin"; import react from "@vitejs/plugin-react"; import { realpathSync } from "fs"; import * as fs from "node:fs"; - +import { logger } from "matrix-js-sdk/lib/logger"; // https://vitejs.dev/config/ -export default defineConfig(({ mode, packageType }) => { +// Modified type helper from defineConfig to allow for packageType (see defineConfig from vite) +export default ({ + mode, + packageType, +}: ConfigEnv & { packageType?: "full" | "embedded" }): UserConfig => { const env = loadEnv(mode, process.cwd()); // Environment variables with the VITE_ prefix are accessible at runtime. - // So, we set this to allow for build/package specific behaviour. + // So, we set this to allow for build/package specific behavior. // In future we might be able to do what is needed via code splitting at // build time. process.env.VITE_PACKAGE = packageType ?? "full"; @@ -77,7 +86,7 @@ export default defineConfig(({ mode, packageType }) => { allow.push(realpathSync(path)); } catch {} } - console.log("Allowed vite paths:", allow); + logger.log("Allowed vite paths:", allow); return { server: { @@ -93,7 +102,7 @@ export default defineConfig(({ mode, packageType }) => { sourcemap: true, rollupOptions: { output: { - assetFileNames: ({ originalFileNames }) => { + assetFileNames: ({ originalFileNames }): string => { if (originalFileNames) { for (const name of originalFileNames) { // Custom asset name for locales to include the locale code in the filename @@ -143,4 +152,4 @@ export default defineConfig(({ mode, packageType }) => { exclude: ["@matrix-org/matrix-sdk-crypto-wasm"], }, }; -}); +}; diff --git a/vitest.config.js b/vitest.config.ts similarity index 81% rename from vitest.config.js rename to vitest.config.ts index a6c3107f..ff8947b0 100644 --- a/vitest.config.js +++ b/vitest.config.ts @@ -1,5 +1,12 @@ +/* +Copyright 2024 New Vector Ltd. +SPDX-License-Identifier: AGPL-3.0-only +Please see LICENSE in the repository root for full details. +*/ + import { defineConfig, mergeConfig } from "vitest/config"; -import viteConfig from "./vite.config.js"; + +import viteConfig from "./vite.config"; export default defineConfig((configEnv) => mergeConfig( From 02f9d0822fdc497107df77ee6e333a8847ff6e35 Mon Sep 17 00:00:00 2001 From: Timo K Date: Tue, 5 Aug 2025 09:17:50 +0200 Subject: [PATCH 02/15] update knip.ts Signed-off-by: Timo K --- knip.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/knip.ts b/knip.ts index 2381356c..6b378e29 100644 --- a/knip.ts +++ b/knip.ts @@ -1,8 +1,15 @@ -import { KnipConfig } from "knip"; +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type KnipConfig } from "knip"; export default { vite: { - config: ["vite.config.js", "vite-embedded.config.js"], + config: ["vite.config.ts", "vite-embedded.config.ts"], }, entry: ["src/main.tsx", "i18next-parser.config.ts"], ignoreBinaries: [ From 1a3e88c19ab568d97b2c3d98c7e968744c372adb Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 5 Aug 2025 15:13:04 +0200 Subject: [PATCH 03/15] Ensure that the the jwt service is called before starting a call --- playwright/restricted-sfu.spec.ts | 75 ++++++++++++++++ playwright/sfu-reconnect-bug.spec.ts | 2 +- src/rtcSessionHelpers.ts | 125 +++++++++++++++++++++------ 3 files changed, 175 insertions(+), 27 deletions(-) create mode 100644 playwright/restricted-sfu.spec.ts diff --git a/playwright/restricted-sfu.spec.ts b/playwright/restricted-sfu.spec.ts new file mode 100644 index 00000000..a9e07d38 --- /dev/null +++ b/playwright/restricted-sfu.spec.ts @@ -0,0 +1,75 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { expect, test } from "@playwright/test"; +import { sleep } from "matrix-js-sdk/lib/utils.js"; + +test("Should request JWT token before starting the call", async ({ page }) => { + await page.goto("/"); + + let sfGetTimestamp = 0; + let sendStateEventTimestamp = 0; + await page.route( + "**/matrix-rtc.m.localhost/livekit/jwt/sfu/get", + async (route) => { + await sleep(2000); // Simulate very slow request + await route.continue(); + sfGetTimestamp = Date.now(); + }, + ); + + await page.route( + "**/state/org.matrix.msc3401.call.member/**", + async (route) => { + await route.continue(); + sendStateEventTimestamp = Date.now(); + }, + ); + + await page.getByTestId("home_callName").click(); + await page.getByTestId("home_callName").fill("HelloCall"); + await page.getByTestId("home_displayName").click(); + await page.getByTestId("home_displayName").fill("John Doe"); + await page.getByTestId("home_go").click(); + + // Join the call + await page.getByTestId("lobby_joinCall").click(); + await page.waitForTimeout(4000); + // Ensure that the call is connected + await page + .locator("div") + .filter({ hasText: /^HelloCall$/ }) + .click(); + + expect(sfGetTimestamp).toBeGreaterThan(0); + expect(sendStateEventTimestamp).toBeGreaterThan(0); + expect(sfGetTimestamp).toBeLessThan(sendStateEventTimestamp); +}); + +test("Error when pre-warming the focus are caught by the ErrorBoundary", async ({ + page, +}) => { + await page.goto("/"); + + await page.route("**/openid/request_token", async (route) => { + await route.fulfill({ + status: 418, // Simulate an error not retryable + }); + }); + + await page.getByTestId("home_callName").click(); + await page.getByTestId("home_callName").fill("HelloCall"); + await page.getByTestId("home_displayName").click(); + await page.getByTestId("home_displayName").fill("John Doe"); + await page.getByTestId("home_go").click(); + + // Join the call + await page.getByTestId("lobby_joinCall").click(); + + // Should fail + await expect(page.getByText("Something went wrong")).toBeVisible(); +}); diff --git a/playwright/sfu-reconnect-bug.spec.ts b/playwright/sfu-reconnect-bug.spec.ts index c756570a..6138eb78 100644 --- a/playwright/sfu-reconnect-bug.spec.ts +++ b/playwright/sfu-reconnect-bug.spec.ts @@ -100,5 +100,5 @@ test("When creator left, avoid reconnect to the same SFU", async ({ // https://github.com/element-hq/element-call/issues/3344 // The app used to request a new jwt token then to reconnect to the SFU expect(wsConnectionCount).toBe(1); - expect(sfuGetCallCount).toBe(1); + expect(sfuGetCallCount).toBe(2 /* the first one is for the warmup */); }); diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index be5eedf1..3e2ff9cd 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -5,14 +5,14 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; -import { logger } from "matrix-js-sdk/lib/logger"; import { isLivekitFocus, isLivekitFocusConfig, type LivekitFocus, type LivekitFocusActive, + type MatrixRTCSession, } from "matrix-js-sdk/lib/matrixrtc"; +import { logger } from "matrix-js-sdk/lib/logger"; import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery"; import { PosthogAnalytics } from "./analytics/PosthogAnalytics"; @@ -20,6 +20,7 @@ import { Config } from "./config/Config"; import { ElementWidgetActions, widget, type WidgetHelpers } from "./widget"; import { MatrixRTCFocusMissingError } from "./utils/errors"; import { getUrlParams } from "./UrlParams"; +import { getSFUConfigWithOpenID } from "./livekit/openIDSFU.ts"; const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci"; @@ -51,32 +52,13 @@ async function makePreferredLivekitFoci( if (domain) { // we use AutoDiscovery instead of relying on the MatrixClient having already // been fully configured and started - const wellKnownFoci = (await AutoDiscovery.getRawClientConfig(domain))?.[ - FOCI_WK_KEY - ]; - if (Array.isArray(wellKnownFoci)) { - preferredFoci.push( - ...wellKnownFoci - .filter((f) => !!f) - .filter(isLivekitFocusConfig) - .map((wellKnownFocus) => { - logger.log( - "Adding livekit focus from well known: ", - wellKnownFocus, - ); - return { ...wellKnownFocus, livekit_alias: livekitAlias }; - }), - ); - } + const wellKnownFoci = await getFocusListFromWellKnown(domain, livekitAlias); + logger.log("Adding livekit focus from well known: ", wellKnownFoci); + preferredFoci.push(...wellKnownFoci); } - const urlFromConf = Config.get().livekit?.livekit_service_url; - if (urlFromConf) { - const focusFormConf: LivekitFocus = { - type: "livekit", - livekit_service_url: urlFromConf, - livekit_alias: livekitAlias, - }; + const focusFormConf = getFocusListFromConfig(livekitAlias); + if (focusFormConf) { logger.log("Adding livekit focus from config: ", focusFormConf); preferredFoci.push(focusFormConf); } @@ -94,6 +76,93 @@ async function makePreferredLivekitFoci( // if (focusOtherMembers) preferredFoci.push(focusOtherMembers); } +async function getFocusListFromWellKnown( + domain: string, + alias: string, +): Promise { + if (domain) { + // we use AutoDiscovery instead of relying on the MatrixClient having already + // been fully configured and started + const wellKnownFoci = (await AutoDiscovery.getRawClientConfig(domain))?.[ + FOCI_WK_KEY + ]; + if (Array.isArray(wellKnownFoci)) { + return wellKnownFoci + .filter((f) => !!f) + .filter(isLivekitFocusConfig) + .map((wellKnownFocus) => { + return { ...wellKnownFocus, livekit_alias: alias }; + }); + } + } + return []; +} + +function getFocusListFromConfig(livekitAlias: string): LivekitFocus | null { + const urlFromConf = Config.get().livekit?.livekit_service_url; + if (urlFromConf) { + return { + type: "livekit", + livekit_service_url: urlFromConf, + livekit_alias: livekitAlias, + }; + } + return null; +} + +export async function getMyPreferredLivekitFoci( + domain: string | null, + livekitAlias: string, +): Promise { + if (domain) { + // we use AutoDiscovery instead of relying on the MatrixClient having already + // been fully configured and started + const wellKnownFociList = await getFocusListFromWellKnown( + domain, + livekitAlias, + ); + if (wellKnownFociList.length > 0) { + return wellKnownFociList[0]; + } + } + + const urlFromConf = Config.get().livekit?.livekit_service_url; + if (urlFromConf) { + return { + type: "livekit", + livekit_service_url: urlFromConf, + livekit_alias: livekitAlias, + }; + } + throw new MatrixRTCFocusMissingError(domain ?? ""); +} + +// Stop-gap solution for pre-warming the SFU. +// This is needed to ensure that the livekit room is created before we try to join the rtc session. +// This is because the livekit room creation is done by the auth service and this can be restricted to +// only specific users, so we need to ensure that the room is created before we try to join it. +async function preWarmSFU(rtcSession: MatrixRTCSession, livekitAlias: string) { + const client = rtcSession.room.client; + // We need to make sure that the livekit room is created before sending the membership event + // because other joiners might not be able to join the call if the room does not exist yet. + const fociToWarmup = await getMyPreferredLivekitFoci( + client.getDomain(), + livekitAlias + ); + + // Request a token in advance to warm up the livekit room. + // Let it throw if it fails, errors will be handled by the ErrorBoundary, if it fails now + // it will fail later when we try to join the room. + await getSFUConfigWithOpenID(client, fociToWarmup); + // For now we don't do anything with the token returned by `getSFUConfigWithOpenID`, it is just to ensure that we + // call the `sfu/get` endpoint so that the auth service create the room in advance if it can. + // Note: This is not actually checking that the room was created! If the roon creation is + // not done by the auth service, the call will fail later when we try to join the room; that case + // is a miss-configuration of the auth service, you should be able to create room in your selected SFU. + // A solution could be to call the internal `/validate` endpoint to check that the room exists, but this needs + // to access livekit internal APIs, so we don't do it for now. +} + export async function enterRTCSession( rtcSession: MatrixRTCSession, encryptMedia: boolean, @@ -112,6 +181,10 @@ export async function enterRTCSession( const { features, matrix_rtc_session: matrixRtcSessionConfig } = Config.get(); const useDeviceSessionMemberEvents = features?.feature_use_device_session_member_events; + + // Pre-warm the SFU to ensure that the room is created before anyone tries to join it. + await preWarmSFU(rtcSession, livekitAlias); + rtcSession.joinRoomSession( await makePreferredLivekitFoci(rtcSession, livekitAlias), makeActiveFocus(), From dba8a434958d236bf4b6f83ce253c560a2ec4dee Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 5 Aug 2025 15:30:15 +0200 Subject: [PATCH 04/15] fixup test mock --- src/rtcSessionHelpers.test.ts | 12 ++++++++++++ src/rtcSessionHelpers.ts | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/rtcSessionHelpers.test.ts b/src/rtcSessionHelpers.test.ts index ecfd44f7..2ef9e3f1 100644 --- a/src/rtcSessionHelpers.test.ts +++ b/src/rtcSessionHelpers.test.ts @@ -70,6 +70,12 @@ test("It joins the correct Session", async () => { roomId: "roomId", client: { getDomain: vi.fn().mockReturnValue("example.org"), + getOpenIdToken: vi.fn().mockResolvedValue({ + access_token: "ACCCESS_TOKEN", + token_type: "Bearer", + matrix_server_name: "localhost", + expires_in: 10000, + }), }, }, memberships: [], @@ -195,6 +201,12 @@ test("It should not fail with configuration error if homeserver config has livek roomId: "roomId", client: { getDomain: vi.fn().mockReturnValue("example.org"), + getOpenIdToken: vi.fn().mockResolvedValue({ + access_token: "ACCCESS_TOKEN", + token_type: "Bearer", + matrix_server_name: "localhost", + expires_in: 10000, + }), }, }, memberships: [], diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index 3e2ff9cd..e1d97db9 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -147,7 +147,7 @@ async function preWarmSFU(rtcSession: MatrixRTCSession, livekitAlias: string) { // because other joiners might not be able to join the call if the room does not exist yet. const fociToWarmup = await getMyPreferredLivekitFoci( client.getDomain(), - livekitAlias + livekitAlias, ); // Request a token in advance to warm up the livekit room. From ef7c9a166a5d49fc6ab131485972d2ebcd651b04 Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 5 Aug 2025 15:58:46 +0200 Subject: [PATCH 05/15] fix eslint --- src/rtcSessionHelpers.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index e1d97db9..75c1ff60 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -141,7 +141,10 @@ export async function getMyPreferredLivekitFoci( // This is needed to ensure that the livekit room is created before we try to join the rtc session. // This is because the livekit room creation is done by the auth service and this can be restricted to // only specific users, so we need to ensure that the room is created before we try to join it. -async function preWarmSFU(rtcSession: MatrixRTCSession, livekitAlias: string) { +async function preWarmSFU( + rtcSession: MatrixRTCSession, + livekitAlias: string, +): Promise { const client = rtcSession.room.client; // We need to make sure that the livekit room is created before sending the membership event // because other joiners might not be able to join the call if the room does not exist yet. From fce7b6d4569165d483d48c2e7896034d099c1ff6 Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 5 Aug 2025 17:44:22 +0200 Subject: [PATCH 06/15] refactor: move warmup to makePreferredLivekitFoci --- src/rtcSessionHelpers.ts | 133 ++++++++++----------------------------- 1 file changed, 34 insertions(+), 99 deletions(-) diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index 75c1ff60..33150b48 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -47,18 +47,47 @@ async function makePreferredLivekitFoci( preferredFoci.push(focusInUse); } + // Stop-gap solution for pre-warming the SFU. + // This is needed to ensure that the livekit room is created before we try to join the rtc session. + // This is because the livekit room creation is done by the auth service and this can be restricted to + // only specific users, so we need to ensure that the room is created before we try to send state events. + let shouldWarmup = true; + // Prioritize the .well-known/matrix/client, if available, over the configured SFU const domain = rtcSession.room.client.getDomain(); if (domain) { // we use AutoDiscovery instead of relying on the MatrixClient having already // been fully configured and started - const wellKnownFoci = await getFocusListFromWellKnown(domain, livekitAlias); - logger.log("Adding livekit focus from well known: ", wellKnownFoci); - preferredFoci.push(...wellKnownFoci); + const wellKnownFoci = (await AutoDiscovery.getRawClientConfig(domain))?.[ + FOCI_WK_KEY + ]; + if (Array.isArray(wellKnownFoci)) { + const validWellKnownFoci = wellKnownFoci + .filter((f) => !!f) + .filter(isLivekitFocusConfig) + .map((wellKnownFocus) => { + logger.log("Adding livekit focus from well known: ", wellKnownFocus); + return { ...wellKnownFocus, livekit_alias: livekitAlias }; + }); + if (validWellKnownFoci.length > 0) { + const toWarmup = validWellKnownFoci[0]; + await getSFUConfigWithOpenID(rtcSession.room.client, toWarmup); + shouldWarmup = false; + } + preferredFoci.push(...validWellKnownFoci); + } } - const focusFormConf = getFocusListFromConfig(livekitAlias); - if (focusFormConf) { + const urlFromConf = Config.get().livekit?.livekit_service_url; + if (urlFromConf) { + const focusFormConf: LivekitFocus = { + type: "livekit", + livekit_service_url: urlFromConf, + livekit_alias: livekitAlias, + }; + if (shouldWarmup) { + await getSFUConfigWithOpenID(rtcSession.room.client, focusFormConf); + } logger.log("Adding livekit focus from config: ", focusFormConf); preferredFoci.push(focusFormConf); } @@ -76,96 +105,6 @@ async function makePreferredLivekitFoci( // if (focusOtherMembers) preferredFoci.push(focusOtherMembers); } -async function getFocusListFromWellKnown( - domain: string, - alias: string, -): Promise { - if (domain) { - // we use AutoDiscovery instead of relying on the MatrixClient having already - // been fully configured and started - const wellKnownFoci = (await AutoDiscovery.getRawClientConfig(domain))?.[ - FOCI_WK_KEY - ]; - if (Array.isArray(wellKnownFoci)) { - return wellKnownFoci - .filter((f) => !!f) - .filter(isLivekitFocusConfig) - .map((wellKnownFocus) => { - return { ...wellKnownFocus, livekit_alias: alias }; - }); - } - } - return []; -} - -function getFocusListFromConfig(livekitAlias: string): LivekitFocus | null { - const urlFromConf = Config.get().livekit?.livekit_service_url; - if (urlFromConf) { - return { - type: "livekit", - livekit_service_url: urlFromConf, - livekit_alias: livekitAlias, - }; - } - return null; -} - -export async function getMyPreferredLivekitFoci( - domain: string | null, - livekitAlias: string, -): Promise { - if (domain) { - // we use AutoDiscovery instead of relying on the MatrixClient having already - // been fully configured and started - const wellKnownFociList = await getFocusListFromWellKnown( - domain, - livekitAlias, - ); - if (wellKnownFociList.length > 0) { - return wellKnownFociList[0]; - } - } - - const urlFromConf = Config.get().livekit?.livekit_service_url; - if (urlFromConf) { - return { - type: "livekit", - livekit_service_url: urlFromConf, - livekit_alias: livekitAlias, - }; - } - throw new MatrixRTCFocusMissingError(domain ?? ""); -} - -// Stop-gap solution for pre-warming the SFU. -// This is needed to ensure that the livekit room is created before we try to join the rtc session. -// This is because the livekit room creation is done by the auth service and this can be restricted to -// only specific users, so we need to ensure that the room is created before we try to join it. -async function preWarmSFU( - rtcSession: MatrixRTCSession, - livekitAlias: string, -): Promise { - const client = rtcSession.room.client; - // We need to make sure that the livekit room is created before sending the membership event - // because other joiners might not be able to join the call if the room does not exist yet. - const fociToWarmup = await getMyPreferredLivekitFoci( - client.getDomain(), - livekitAlias, - ); - - // Request a token in advance to warm up the livekit room. - // Let it throw if it fails, errors will be handled by the ErrorBoundary, if it fails now - // it will fail later when we try to join the room. - await getSFUConfigWithOpenID(client, fociToWarmup); - // For now we don't do anything with the token returned by `getSFUConfigWithOpenID`, it is just to ensure that we - // call the `sfu/get` endpoint so that the auth service create the room in advance if it can. - // Note: This is not actually checking that the room was created! If the roon creation is - // not done by the auth service, the call will fail later when we try to join the room; that case - // is a miss-configuration of the auth service, you should be able to create room in your selected SFU. - // A solution could be to call the internal `/validate` endpoint to check that the room exists, but this needs - // to access livekit internal APIs, so we don't do it for now. -} - export async function enterRTCSession( rtcSession: MatrixRTCSession, encryptMedia: boolean, @@ -184,10 +123,6 @@ export async function enterRTCSession( const { features, matrix_rtc_session: matrixRtcSessionConfig } = Config.get(); const useDeviceSessionMemberEvents = features?.feature_use_device_session_member_events; - - // Pre-warm the SFU to ensure that the room is created before anyone tries to join it. - await preWarmSFU(rtcSession, livekitAlias); - rtcSession.joinRoomSession( await makePreferredLivekitFoci(rtcSession, livekitAlias), makeActiveFocus(), From 6dcc44b631abf3346db12daa81dcb1871febf630 Mon Sep 17 00:00:00 2001 From: Valere Date: Wed, 6 Aug 2025 09:26:07 +0200 Subject: [PATCH 07/15] comment --- src/rtcSessionHelpers.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index 33150b48..ba0b39f4 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -71,6 +71,7 @@ async function makePreferredLivekitFoci( }); if (validWellKnownFoci.length > 0) { const toWarmup = validWellKnownFoci[0]; + // this will call the jwt/sfu/get endpoint to pre create the livekit room. await getSFUConfigWithOpenID(rtcSession.room.client, toWarmup); shouldWarmup = false; } @@ -86,6 +87,7 @@ async function makePreferredLivekitFoci( livekit_alias: livekitAlias, }; if (shouldWarmup) { + // this will call the jwt/sfu/get endpoint to pre create the livekit room. await getSFUConfigWithOpenID(rtcSession.room.client, focusFormConf); } logger.log("Adding livekit focus from config: ", focusFormConf); From 8509efb48bcee81075d97c41aa47130faee35038 Mon Sep 17 00:00:00 2001 From: Valere Date: Wed, 6 Aug 2025 13:59:53 +0200 Subject: [PATCH 08/15] review --- src/rtcSessionHelpers.ts | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index ba0b39f4..8fe45821 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -47,11 +47,8 @@ async function makePreferredLivekitFoci( preferredFoci.push(focusInUse); } - // Stop-gap solution for pre-warming the SFU. - // This is needed to ensure that the livekit room is created before we try to join the rtc session. - // This is because the livekit room creation is done by the auth service and this can be restricted to - // only specific users, so we need to ensure that the room is created before we try to send state events. - let shouldWarmup = true; + // Warm up the first focus we owned, to ensure livekit room is created before any state event sent. + let toWarmUp: LivekitFocus | undefined; // Prioritize the .well-known/matrix/client, if available, over the configured SFU const domain = rtcSession.room.client.getDomain(); @@ -70,10 +67,7 @@ async function makePreferredLivekitFoci( return { ...wellKnownFocus, livekit_alias: livekitAlias }; }); if (validWellKnownFoci.length > 0) { - const toWarmup = validWellKnownFoci[0]; - // this will call the jwt/sfu/get endpoint to pre create the livekit room. - await getSFUConfigWithOpenID(rtcSession.room.client, toWarmup); - shouldWarmup = false; + toWarmUp = validWellKnownFoci[0]; } preferredFoci.push(...validWellKnownFoci); } @@ -86,14 +80,17 @@ async function makePreferredLivekitFoci( livekit_service_url: urlFromConf, livekit_alias: livekitAlias, }; - if (shouldWarmup) { - // this will call the jwt/sfu/get endpoint to pre create the livekit room. - await getSFUConfigWithOpenID(rtcSession.room.client, focusFormConf); + if (!toWarmUp) { + toWarmUp = focusFormConf; } logger.log("Adding livekit focus from config: ", focusFormConf); preferredFoci.push(focusFormConf); } + if (toWarmUp) { + // this will call the jwt/sfu/get endpoint to pre create the livekit room. + await getSFUConfigWithOpenID(rtcSession.room.client, toWarmUp); + } if (preferredFoci.length === 0) throw new MatrixRTCFocusMissingError(domain ?? ""); return Promise.resolve(preferredFoci); From e4d14b4892be72327bfadf32d0f77a62e7cecd6c Mon Sep 17 00:00:00 2001 From: Valere Fedronic Date: Wed, 6 Aug 2025 14:22:10 +0200 Subject: [PATCH 09/15] Update src/rtcSessionHelpers.ts Co-authored-by: Timo <16718859+toger5@users.noreply.github.com> --- src/rtcSessionHelpers.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index 8fe45821..73f58cea 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -80,9 +80,7 @@ async function makePreferredLivekitFoci( livekit_service_url: urlFromConf, livekit_alias: livekitAlias, }; - if (!toWarmUp) { - toWarmUp = focusFormConf; - } + toWarmUp = toWarmUp ?? focusFormConf; logger.log("Adding livekit focus from config: ", focusFormConf); preferredFoci.push(focusFormConf); } From c0aab96968ac9e8d7b208e454297e0094dfbcf70 Mon Sep 17 00:00:00 2001 From: Timo <16718859+toger5@users.noreply.github.com> Date: Thu, 7 Aug 2025 15:16:41 +0200 Subject: [PATCH 10/15] Add intents for DM (#3445) * Add intents for DM Signed-off-by: Timo K * review Signed-off-by: Timo K --------- Signed-off-by: Timo K --- docs/url-params.md | 2 +- src/UrlParams.ts | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/docs/url-params.md b/docs/url-params.md index e76c976e..3fac185a 100644 --- a/docs/url-params.md +++ b/docs/url-params.md @@ -48,6 +48,7 @@ These parameters are relevant to both [widget](./embedded-standalone.md) and [st | Name | Values | Required for widget | Required for SPA | Description | | ------------------------------------------------------- | ---------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | ---------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `intent` | `start_call`, `join_existing`, `start_call_dm`, `join_existing_dm. | No, defaults to `start_call` | No, defaults to `start_call` | The intent is a special url parameter that defines the defaults for all the other parameters. In most cases it should be enough to only set the intent to setup element-call. | | `allowIceFallback` | `true` or `false` | No, defaults to `false` | No, defaults to `false` | Allows use of fallback STUN servers for ICE if the user's homeserver doesn’t provide any. | | `analyticsID` (deprecated: use `posthogUserId` instead) | Posthog analytics ID | No | No | Available only with user's consent for sharing telemetry in Element Web. | | `appPrompt` | `true` or `false` | No, defaults to `true` | No, defaults to `true` | Prompts the user to launch the native mobile app upon entering a room, applicable only on Android and iOS, and must be enabled in config. | @@ -59,7 +60,6 @@ These parameters are relevant to both [widget](./embedded-standalone.md) and [st | `header` | `none`, `standard` or `app_bar` | No, defaults to `standard` | No, defaults to `standard` | The style of headers to show. `standard` is the default arrangement, `none` hides the header entirely, and `app_bar` produces a header with a back button like you might see in mobile apps. The callback for the back button is `window.controls.onBackButtonPressed`. | | `hideScreensharing` | `true` or `false` | No, defaults to `false` | No, defaults to `false` | Hides the screen-sharing button. | | `homeserver` | | Not applicable | No | Homeserver for registering a new (guest) user, configures non-default guest user server when creating a spa link. | -| `intent` | `start_call` or `join_existing` | No, defaults to `start_call` | No, defaults to `start_call` | The intent of the user with respect to the call. e.g. if they clicked a Start Call button, this would be `start_call`. If it was a Join Call button, it would be `join_existing`. | | `lang` | [BCP 47](https://www.rfc-editor.org/info/bcp47) code | No | No | The language the app should use. | | `password` | | No | No | E2EE password when using a shared secret. (For individual sender keys in embedded mode this is not required.) | | `perParticipantE2EE` | `true` or `false` | No, defaults to `false` | No, defaults to `false` | Enables per participant encryption with Keys exchanged over encrypted matrix room messages. | diff --git a/src/UrlParams.ts b/src/UrlParams.ts index 65e3d901..94fd3c14 100644 --- a/src/UrlParams.ts +++ b/src/UrlParams.ts @@ -23,9 +23,10 @@ interface RoomIdentifier { } export enum UserIntent { - // TODO: add DM vs room call StartNewCall = "start_call", JoinExistingCall = "join_existing", + StartNewCallDM = "start_call_dm", + JoinExistingCallDM = "join_existing_dm", Unknown = "unknown", } @@ -347,6 +348,20 @@ export const getUrlParams = ( skipLobby: false, }; break; + case UserIntent.StartNewCallDM: + intentPreset = { + ...inAppDefault, + skipLobby: true, + // autoLeaveWhenOthersLeft: true, // TODO: add this once available + }; + break; + case UserIntent.JoinExistingCallDM: + intentPreset = { + ...inAppDefault, + skipLobby: true, + // autoLeaveWhenOthersLeft: true, // TODO: add this once available + }; + break; // Non widget usecase defaults default: intentPreset = { From 6667fc54c0a758531476f697022a0001654e1f17 Mon Sep 17 00:00:00 2001 From: Timo <16718859+toger5@users.noreply.github.com> Date: Fri, 8 Aug 2025 17:09:25 +0200 Subject: [PATCH 11/15] Add a fullscreen button that uses the element request Fullscreen browser api (#3447) * Add a fullscreen button that uses the element request Fullscreen browser api Signed-off-by: Timo K * use body instead of root node Signed-off-by: Timo K --------- Signed-off-by: Timo K --- src/icons/FullScreenMaximise.svg | 6 ++++ src/icons/FullScreenMinimise.svg | 6 ++++ src/tile/SpotlightTile.module.css | 22 ++++++++++----- src/tile/SpotlightTile.tsx | 47 ++++++++++++++++++++++++++----- 4 files changed, 67 insertions(+), 14 deletions(-) create mode 100644 src/icons/FullScreenMaximise.svg create mode 100644 src/icons/FullScreenMinimise.svg diff --git a/src/icons/FullScreenMaximise.svg b/src/icons/FullScreenMaximise.svg new file mode 100644 index 00000000..1814f16e --- /dev/null +++ b/src/icons/FullScreenMaximise.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/src/icons/FullScreenMinimise.svg b/src/icons/FullScreenMinimise.svg new file mode 100644 index 00000000..204259e2 --- /dev/null +++ b/src/icons/FullScreenMinimise.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/src/tile/SpotlightTile.module.css b/src/tile/SpotlightTile.module.css index 78831571..622496d2 100644 --- a/src/tile/SpotlightTile.module.css +++ b/src/tile/SpotlightTile.module.css @@ -88,40 +88,48 @@ Please see LICENSE in the repository root for full details. padding: var(--cpd-space-2x); border: none; border-radius: var(--cpd-radius-pill-effect); - background: var(--cpd-color-alpha-gray-1400); + background: rgba(from var(--cpd-color-gray-100) r g b / 0.6); box-shadow: var(--small-drop-shadow); transition: opacity 0.15s, background-color 0.1s; - position: absolute; z-index: 1; --inset: 6px; inset-block-end: var(--inset); inset-inline-end: var(--inset); } +.bottomRightButtons { + display: flex; + gap: var(--cpd-space-2x); + position: absolute; + inset-block-end: var(--cpd-space-1x); + inset-inline-end: var(--cpd-space-1x); + z-index: 1; +} + .expand > svg { display: block; - color: var(--cpd-color-icon-on-solid-primary); + color: var(--cpd-color-icon-primary); } @media (hover) { .expand:hover { - background: var(--cpd-color-bg-action-primary-hovered); + background: var(--cpd-color-gray-400); } } .expand:active { - background: var(--cpd-color-bg-action-primary-pressed); + background: var(--cpd-color-gray-100); } @media (hover) { - .tile:hover > button { + .tile:hover > div > button { opacity: 1; } } -.tile:has(:focus-visible) > button { +.tile:has(:focus-visible) > div > button { opacity: 1; } diff --git a/src/tile/SpotlightTile.tsx b/src/tile/SpotlightTile.tsx index 8495e88b..8bc45a81 100644 --- a/src/tile/SpotlightTile.tsx +++ b/src/tile/SpotlightTile.tsx @@ -29,6 +29,8 @@ import classNames from "classnames"; import { type TrackReferenceOrPlaceholder } from "@livekit/components-core"; import { type RoomMember } from "matrix-js-sdk"; +import FullScreenMaximiseIcon from "../icons/FullScreenMaximise.svg?react"; +import FullScreenMinimiseIcon from "../icons/FullScreenMinimise.svg?react"; import { MediaView } from "./MediaView"; import styles from "./SpotlightTile.module.css"; import { @@ -210,6 +212,26 @@ export const SpotlightTile: FC = ({ const canGoBack = visibleIndex > 0; const canGoToNext = visibleIndex !== -1 && visibleIndex < media.length - 1; + const isFullscreen = useCallback((): boolean => { + const rootElement = document.body; + if (rootElement && document.fullscreenElement) return true; + return false; + }, []); + + const FullScreenIcon = isFullscreen() + ? FullScreenMinimiseIcon + : FullScreenMaximiseIcon; + + const onToggleFullscreen = useCallback(() => { + const rootElement = document.body; + if (!rootElement) return; + if (isFullscreen()) { + void document?.exitFullscreen(); + } else { + void rootElement.requestFullscreen(); + } + }, [isFullscreen]); + // To keep track of which item is visible, we need an intersection observer // hooked up to the root element and the items. Because the items will run // their effects before their parent does, we need to do this dance with an @@ -292,17 +314,28 @@ export const SpotlightTile: FC = ({ /> ))} - {onToggleExpanded && ( +
- )} + + {onToggleExpanded && ( + + )} +
+ {canGoToNext && (