Merge branch 'livekit' into fkwp/feature_deepfilternet

This commit is contained in:
fkwp
2026-04-09 18:32:54 +02:00
36 changed files with 2623 additions and 1180 deletions

View File

@@ -15,6 +15,7 @@ module.exports = {
"plugin:matrix-org/typescript",
"prettier",
"plugin:rxjs/recommended",
"plugin:storybook/recommended",
],
parserOptions: {
ecmaVersion: "latest",

View File

@@ -5,7 +5,7 @@ on:
# This is safe because we do not use actions/checkout or execute untrusted code.
# Using pull_request_target is necessary to allow status writes for PRs from forks.
pull_request_target:
types: [labeled, unlabeled, opened]
types: [labeled, unlabeled, opened, synchronize]
permissions:
pull-requests: read

View File

@@ -97,7 +97,7 @@ jobs:
run: find ${FILENAME_PREFIX} -type f -print0 | sort -z | xargs -0 sha256sum | tee ${FILENAME_PREFIX}.sha256
- name: Upload
if: ${{ needs.versioning.outputs.DRY_RUN == 'false' }}
uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2
uses: softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2
with:
files: |
${{ env.FILENAME_PREFIX }}.tar.gz
@@ -297,7 +297,7 @@ jobs:
NEEDS_PUBLISH_IOS_OUTPUTS_ARTIFACT_VERSION: ${{ needs.publish_ios.outputs.ARTIFACT_VERSION }}
- name: Add release notes
if: ${{ needs.versioning.outputs.DRY_RUN == 'false' }}
uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2
uses: softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2
with:
append_body: true
body: |

View File

@@ -42,7 +42,7 @@ jobs:
- name: Create Checksum
run: find ${FILENAME_PREFIX} -type f -print0 | sort -z | xargs -0 sha256sum | tee ${FILENAME_PREFIX}.sha256
- name: Upload
uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2
uses: softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2
with:
files: |
${{ env.FILENAME_PREFIX }}.tar.gz
@@ -71,7 +71,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Add release note
uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2
uses: softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2
with:
append_body: true
body: |

View File

@@ -24,7 +24,7 @@ jobs:
- name: Vitest
run: "yarn run test:coverage"
- name: Upload to codecov
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5
uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:

3
.gitignore vendored
View File

@@ -31,3 +31,6 @@ yarn-error.log
/playwright-report/
/blob-report/
/playwright/.cache/
*storybook.log
storybook-static

8
.storybook/main.ts Normal file
View File

@@ -0,0 +1,8 @@
import type { StorybookConfig } from "@storybook/react-vite";
const config: StorybookConfig = {
stories: ["../src/**/*.stories.@(js|jsx|mjs|ts|tsx)"],
addons: ["@storybook/addon-docs"],
framework: "@storybook/react-vite",
};
export default config;

24
.storybook/manager.ts Normal file
View File

@@ -0,0 +1,24 @@
import { create } from "storybook/theming";
import { addons } from "storybook/manager-api";
addons.setConfig({
theme: create({
base: "light",
colorPrimary: "#1b1d22",
colorSecondary: "#0467dd",
// Typography
fontBase: '"Inter", sans-serif',
fontCode: '"Inconsolata", monospace',
// Text colors
textColor: "#1b1d22",
appBg: "#ffffff",
barBg: "#ffffff",
brandTitle: "Element Call",
brandUrl: "https://element.io/",
brandImage: "/src/icons/Logo.svg",
brandTarget: "_self",
}),
});

49
.storybook/preview.tsx Normal file
View File

@@ -0,0 +1,49 @@
import type { Preview } from "@storybook/react-vite";
import { TooltipProvider } from "@vector-im/compound-web";
import i18n from "i18next";
import { logger } from "matrix-js-sdk/lib/logger";
import EN from "../locales/en/app.json";
import { initReactI18next } from "react-i18next";
import "../src/index.css";
// Bare-minimum i18n config
i18n
.use(initReactI18next)
.init({
lng: "en",
fallbackLng: "en",
supportedLngs: ["en"],
// We embed the translations, so that it never needs to fetch
resources: {
en: {
translation: EN,
},
},
interpolation: {
escapeValue: false, // React has built-in XSS protections
},
})
.catch((e) => logger.warn("Failed to init i18n for stories", e));
const preview: Preview = {
parameters: {
layout: "centered",
controls: {
matchers: {
color: /(background|color)$/i,
date: /Date$/i,
},
},
},
tags: ["autodocs"],
decorators: [
(Story) => (
<TooltipProvider>
<Story />
</TooltipProvider>
),
],
};
export default preview;

View File

@@ -34,7 +34,9 @@
"test:playwright": "playwright test",
"test:playwright:open": "yarn test:playwright --ui",
"links:enable": "mv .links.disabled.yaml .links.yaml & touch .links.yaml",
"links:disable": "mv .links.yaml .links.disabled.yaml"
"links:disable": "mv .links.yaml .links.disabled.yaml",
"storybook": "storybook dev -p 6006",
"build-storybook": "storybook build"
},
"devDependencies": {
"@babel/core": "^7.16.5",
@@ -58,6 +60,8 @@
"@react-spring/web": "^10.0.0",
"@sentry/react": "^8.0.0",
"@sentry/vite-plugin": "^3.0.0",
"@storybook/addon-docs": "^10.3.3",
"@storybook/react-vite": "^10.3.3",
"@stylistic/eslint-plugin": "^3.0.0",
"@testing-library/dom": "^10.1.0",
"@testing-library/jest-dom": "^6.6.3",
@@ -78,7 +82,7 @@
"@typescript-eslint/eslint-plugin": "^8.31.0",
"@typescript-eslint/parser": "^8.31.0",
"@use-gesture/react": "^10.2.11",
"@vector-im/compound-design-tokens": "^6.0.0",
"@vector-im/compound-design-tokens": "^9.0.0",
"@vector-im/compound-web": "^8.0.0",
"@vitejs/plugin-react": "^4.0.1",
"@vitest/coverage-v8": "^4.0.18",
@@ -88,7 +92,7 @@
"eslint": "^8.14.0",
"eslint-config-google": "^0.14.0",
"eslint-config-prettier": "^10.0.0",
"eslint-plugin-deprecate": "^0.8.2",
"eslint-plugin-deprecate": "^0.9.0",
"eslint-plugin-import": "^2.26.0",
"eslint-plugin-jsdoc": "^61.5.0",
"eslint-plugin-jsx-a11y": "^6.5.1",
@@ -96,6 +100,7 @@
"eslint-plugin-react": "^7.29.4",
"eslint-plugin-react-hooks": "^5.0.0",
"eslint-plugin-rxjs": "^5.0.3",
"eslint-plugin-storybook": "^10.3.3",
"eslint-plugin-unicorn": "^56.0.0",
"fetch-mock": "11.1.5",
"global-jsdom": "^26.0.0",
@@ -120,17 +125,18 @@
"qrcode": "^1.5.4",
"react": "19",
"react-dom": "19",
"react-i18next": "^16.0.0 <16.6.0",
"react-i18next": "^16.0.0 <16.7.0",
"react-router-dom": "^7.0.0",
"react-use-measure": "^2.1.1",
"rxjs": "^7.8.1",
"sass": "^1.42.1",
"storybook": "^10.3.3",
"typescript": "^5.8.3",
"typescript-eslint-language-service": "^5.0.5",
"unique-names-generator": "^4.6.0",
"uuid": "^13.0.0",
"vaul": "^1.0.0",
"vite": "^7.3.0",
"vite": "^8.0.0",
"vite-plugin-generate-file": "^0.3.0",
"vite-plugin-html": "^3.2.2",
"vite-plugin-node-stdlib-browser": "^0.2.1",

View File

@@ -8,6 +8,7 @@ Please see LICENSE in the repository root for full details.
import { expect, test } from "@playwright/test";
import { widgetTest } from "../fixtures/widget-user.ts";
import { TestHelpers } from "./test-helpers.ts";
// Skip test, including Fixtures
widgetTest.skip(
@@ -20,19 +21,7 @@ widgetTest("Start a new call as widget", async ({ asWidget, browserName }) => {
const { brooks, whistler } = asWidget;
await expect(
brooks.page.getByRole("button", { name: "Video call" }),
).toBeVisible();
await brooks.page.getByRole("button", { name: "Video call" }).click();
await expect(
brooks.page.getByRole("menuitem", { name: "Legacy Call" }),
).toBeVisible();
await expect(
brooks.page.getByRole("menuitem", { name: "Element Call" }),
).toBeVisible();
await brooks.page.getByRole("menuitem", { name: "Element Call" }).click();
await TestHelpers.startCallInCurrentRoom(brooks.page, false);
await expect(
brooks.page
@@ -56,11 +45,7 @@ widgetTest("Start a new call as widget", async ({ asWidget, browserName }) => {
).toBeVisible();
// Join from the other side
await expect(whistler.page.getByText("Video call started")).toBeVisible();
await expect(
whistler.page.getByRole("button", { name: "Join" }),
).toBeVisible();
await whistler.page.getByRole("button", { name: "Join" }).click();
await TestHelpers.joinCallInCurrentRoom(whistler.page);
// Currently disabled due to recent Element Web is bypassing Lobby
// await expect(

View File

@@ -235,9 +235,11 @@ export class TestHelpers {
): Promise<void> {
await page.getByRole("button", { name: "Video call" }).click();
await page.getByRole("menuitem", { name: "Element Call" }).click();
await TestHelpers.setEmbeddedElementCallRtcMode(page, mode);
await page.getByRole("button", { name: "Close lobby" }).click();
}
/**
* Goes to the settings to set the RTC mode.
* then closes the settings modal.

12
src/@types/mdx.d.ts vendored Normal file
View File

@@ -0,0 +1,12 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { JSX as ReactJSX } from "react";
declare module "mdx/types.js" {
export import JSX = ReactJSX;
}

View File

@@ -12,7 +12,7 @@ import { type FC, type PropsWithChildren } from "react";
import { type WidgetApi } from "matrix-widget-api";
import { ClientContextProvider } from "./ClientContext";
import { Avatar } from "./Avatar";
import { Avatar, getAvatarFromWidgetAPI } from "./Avatar";
import { mockMatrixRoomMember, mockRtcMembership } from "./utils/test";
import { widget } from "./widget";
@@ -178,3 +178,36 @@ test("should attempt to use widget API if running as a widget", async () => {
expect(widget!.api.downloadFile).toBeCalledWith(expectedMXCUrl);
});
test("Supports download files as base64", async () => {
const expectedMXCUrl = "mxc://example.org/alice-avatar";
const expectedBase64 =
"iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAADIElEQVR4nAAQA+/8ApxhEfFNuwna" +
"+DO1pFMx5YDg6gb8p1WFkbFSox9H6r5c8jp1gxlHXrDfA/oQFi4A0gTXH9YBNgwRm12xO68QP6lv" +
"ZLKH9qW1VM6kz6zA3T1Ui8J+Xbnh2BZ7oXDe/2gajzoA6j1JGotpz99xO+T2NR634Nhx3zhuera/" +
"UdrpMLdEpwWXLnSqZRasGsrl93FjdTwRBMaqsx6vJksnPOmV9ttbXFIOb0XDGPbVythSC2n7P/bS" +
"Zv0U0QqbBLk/5Wu1werYzAHiz11Bj8bEylQ92Pxvo+PwF6/KbGnIHTvGZkFzDkMnqz3g7Pw3NOSP" +
"oV+qfyJuSI0AeZmrPejFQ8kzBSDWO8D7lr4+6ePRBRmZtKCf+fNjSCOyb5jqwhBnD2cycbJtQQbR" +
"A4qdPG2ONfTPeQgi96+zT7grBI0JwvgFBceJdLJd4BX1VQIyY+j7OYueNWqEpf8iYgMj78I95eRt" +
"nfPLwlxhVns84iL4Yvw8jDrB9vQi8ktpsdJOMiDwKrBGD3q56COD2oIA96CCBgiro4tkvkumZSAc" +
"ZKXRLsziUFGytWJLaPjwnzXv2hicPy6k9AXsF3QkysOZAkB3m9XPpixhq9b0OKqV/zZx3L79o6wZ" +
"Dr40J7sj7f+ARd545CP01r5omHt94tbnjgA46HsM2OhP+qQ882LN+Bhscq2WSHGSHT4J9MQcsWZP" +
"2+N2LdPy61MN4/1++BJHmDcDLQBUEwLvjZp1fRfzxV7yirwIiOA7Vr8z+1yvS/pSkfUzkjswybOd" +
"M5i0I8Q69MTXAKxqtR0/tyGkfCmHfupGASp/SAT9J8f3aQV+gDbpva592v4w8Cv5EMm7CzZPwThF" +
"kgTChNPts7F03ccxpblfIz0EiAON1DKk71rX07BvDlLHY1ItPuqZ7hjy19jrAgl+QqEE1btHVA5R" +
"uAnRXpEWc6rjARlJY5G1wbMk12rrqpr8rhR3YpFgLgOx4BtQ0D/hGe7KANSGBMQojmObId0asCmd" +
"XzmnQI9P8QnwsO9vtqZlgIoU4g+f2/G8Q3/nVMX7dujniwEAAP//KmiQs7P8MeIAAAAASUVORK5C" +
"YII=";
const mockWidgetAPI = {
downloadFile: vi.fn().mockImplementation(async (contentUri) => {
if (contentUri !== expectedMXCUrl) {
return Promise.reject(new Error("Unexpected content URI"));
}
return { file: expectedBase64 };
}),
} as unknown as WidgetApi;
const blob = await getAvatarFromWidgetAPI(mockWidgetAPI, expectedMXCUrl);
expect(blob).toBeInstanceOf(Blob);
});

View File

@@ -173,7 +173,8 @@ async function getAvatarFromServer(
return blob;
}
async function getAvatarFromWidgetAPI(
// export for testing
export async function getAvatarFromWidgetAPI(
api: WidgetApi,
src: string,
): Promise<Blob> {
@@ -181,9 +182,14 @@ async function getAvatarFromWidgetAPI(
const file = response.file;
// element-web sends a Blob, and the MSC4039 is considering changing the spec to strictly Blob, so only handling that
if (!(file instanceof Blob)) {
throw new Error("Downloaded file is not a Blob");
if (file instanceof Blob) {
return file;
} else if (typeof file === "string") {
// it is a base64 string
const bytes = Uint8Array.from(atob(file), (c) => c.charCodeAt(0));
return new Blob([bytes]);
}
return file;
throw new Error(
"Downloaded file format is not supported: " + typeof file + "",
);
}

View File

@@ -25,7 +25,7 @@ export abstract class TranslatedError extends Error {
messageKey: ParseKeys<DefaultNamespace, TOptions>,
translationFn: TFunction<DefaultNamespace>,
) {
super(translationFn(messageKey, { lng: "en" } as TOptions));
super(translationFn(messageKey, { lng: "en" }));
this.translatedMessage = translationFn(messageKey);
}
}

View File

@@ -7,6 +7,7 @@ Please see LICENSE in the repository root for full details.
.tile.draggable {
cursor: grab;
box-shadow: var(--big-drop-shadow);
}
.tile.draggable:active {

View File

@@ -1,3 +1,3 @@
<svg width="28" height="26" viewBox="0 0 28 26" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M14 21.0267L22.24 26.0001L20.0533 16.6267L27.3333 10.3201L17.7466 9.50675L14 0.666748L10.2533 9.50675L0.666626 10.3201L7.94663 16.6267L5.75996 26.0001L14 21.0267Z" fill="white"/>
</svg>
<path d="M14 21.0267L22.24 26.0001L20.0533 16.6267L27.3333 10.3201L17.7466 9.50675L14 0.666748L10.2533 9.50675L0.666626 10.3201L7.94663 16.6267L5.75996 26.0001L14 21.0267Z" fill="currentColor"/>
</svg>

Before

Width:  |  Height:  |  Size: 290 B

After

Width:  |  Height:  |  Size: 298 B

View File

@@ -1,4 +1,4 @@
<svg width="28" height="26" viewBox="0 0 28 26" fill="none" xmlns="http://www.w3.org/2000/svg">
<path id="Vector" d="M14 7.50675L15.2933 10.5601L15.92 12.0401L17.52 12.1734L20.8133 12.4534L18.3066 14.6267L17.0933 15.6801L17.4533 17.2534L18.2 20.4667L15.3733 18.7601L14 17.9067L12.6266 18.7334L9.79996 20.4401L10.5466 17.2267L10.9066 15.6534L9.69329 14.6001L7.18663 12.4267L10.48 12.1467L12.08 12.0134L12.7066 10.5334L14 7.50675M14 0.666748L10.2533 9.50675L0.666626 10.3201L7.94663 16.6267L5.75996 26.0001L14 21.0267L22.24 26.0001L20.0533 16.6267L27.3333 10.3201L17.7466 9.50675L14 0.666748Z" fill="white"/>
<path id="Vector" d="M14 7.50675L15.2933 10.5601L15.92 12.0401L17.52 12.1734L20.8133 12.4534L18.3066 14.6267L17.0933 15.6801L17.4533 17.2534L18.2 20.4667L15.3733 18.7601L14 17.9067L12.6266 18.7334L9.79996 20.4401L10.5466 17.2267L10.9066 15.6534L9.69329 14.6001L7.18663 12.4267L10.48 12.1467L12.08 12.0134L12.7066 10.5334L14 7.50675M14 0.666748L10.2533 9.50675L0.666626 10.3201L7.94663 16.6267L5.75996 26.0001L14 21.0267L22.24 26.0001L20.0533 16.6267L27.3333 10.3201L17.7466 9.50675L14 0.666748Z" fill="currentColor"/>
</svg>

Before

Width:  |  Height:  |  Size: 620 B

After

Width:  |  Height:  |  Size: 628 B

View File

@@ -43,6 +43,7 @@ layer(compound);
max(var(--cpd-space-4x), calc((100vw - 900px) / 3))
);
--small-drop-shadow: 0px 1.2px 2.4px 0px rgba(0, 0, 0, 0.15);
--big-drop-shadow: 0px 0px 24px 0px #1b1d221a;
--subtle-drop-shadow: 0px 1px 2px 0px rgba(16, 24, 40, 0.05);
--background-gradient: url("graphics/backgroundGradient.svg");

View File

@@ -0,0 +1,25 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { fn } from "storybook/test";
import type { Meta, StoryObj } from "@storybook/react-vite";
import { StarRatingInput } from "./StarRatingInput";
const meta = {
component: StarRatingInput,
} satisfies Meta<typeof StarRatingInput>;
export default meta;
type Story = StoryObj<typeof meta>;
export const Default: Story = {
args: {
starCount: 5,
onChange: fn(),
},
};

View File

@@ -19,12 +19,14 @@ import fetchMock from "fetch-mock";
import { getSFUConfigWithOpenID, type OpenIDClientParts } from "./openIDSFU";
import { testJWTToken } from "../utils/test-fixtures";
import { ownMemberMock } from "../utils/test";
import { FailToGetOpenIdToken } from "../utils/errors";
const sfuUrl = "https://sfu.example.org";
describe("getSFUConfigWithOpenID", () => {
let matrixClient: MockedObject<OpenIDClientParts>;
beforeEach(() => {
fetchMock.catch(404);
matrixClient = {
getOpenIdToken: vitest.fn(),
getDeviceId: vitest.fn(),
@@ -71,9 +73,10 @@ describe("getSFUConfigWithOpenID", () => {
"https://sfu.example.org",
"!example_room_id",
);
} catch (ex) {
expect((ex as Error).message).toEqual(
"SFU Config fetch failed with status code 500",
} catch (ex: unknown) {
expect(ex).toBeInstanceOf(FailToGetOpenIdToken);
expect((ex as FailToGetOpenIdToken).cause).toEqual(
new Error("SFU Config fetch failed with status code 500"),
);
void (await fetchMock.flush());
return;
@@ -106,8 +109,9 @@ describe("getSFUConfigWithOpenID", () => {
},
);
} catch (ex) {
expect((ex as Error).message).toEqual(
"SFU Config fetch failed with status code 500",
expect(ex).toBeInstanceOf(FailToGetOpenIdToken);
expect((ex as FailToGetOpenIdToken).cause).toEqual(
new Error("SFU Config fetch failed with status code 500"),
);
void (await fetchMock.flush());
}
@@ -160,8 +164,9 @@ describe("getSFUConfigWithOpenID", () => {
},
);
} catch (ex) {
expect((ex as Error).message).toEqual(
"SFU Config fetch failed with status code 500",
expect(ex).toBeInstanceOf(FailToGetOpenIdToken);
expect((ex as FailToGetOpenIdToken).cause).toEqual(
new Error("SFU Config fetch failed with status code 500"),
);
void (await fetchMock.flush());
}

View File

@@ -5,11 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
retryNetworkOperation,
type IOpenIDToken,
type MatrixClient,
} from "matrix-js-sdk";
import { type IOpenIDToken, type MatrixClient } from "matrix-js-sdk";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import { type Logger } from "matrix-js-sdk/lib/logger";
@@ -70,6 +66,7 @@ export type OpenIDClientParts = Pick<
MatrixClient,
"getOpenIdToken" | "getDeviceId"
>;
/**
* Gets a bearer token from the homeserver and then use it to authenticate
* to the matrix RTC backend in order to get acces to the SFU.
@@ -113,9 +110,6 @@ export async function getSFUConfigWithOpenID(
);
}
logger?.debug("Got openID token", openIdToken);
logger?.info(`Trying to get JWT for focus ${serviceUrl}...`);
let sfuConfig: { url: string; jwt: string } | undefined;
const tryBothJwtEndpoints = opts?.forceJwtEndpoint === undefined; // This is for SFUs where we do not publish.
@@ -127,7 +121,10 @@ export async function getSFUConfigWithOpenID(
// if we can use both or if we are forced to use the new one.
if (tryBothJwtEndpoints || forceMatrix2Jwt) {
try {
sfuConfig = await getLiveKitJWTWithDelayDelegation(
logger?.info(
`Trying to get JWT with delegation for focus ${serviceUrl}...`,
);
const sfuConfig = await getLiveKitJWTWithDelayDelegation(
membership,
serviceUrl,
roomId,
@@ -135,32 +132,24 @@ export async function getSFUConfigWithOpenID(
opts?.delayEndpointBaseUrl,
opts?.delayId,
);
logger?.info(`Got JWT from call's active focus URL.`);
return extractFullConfigFromToken(sfuConfig);
} catch (e) {
if (e instanceof NotSupportedError) {
logger?.warn(
`Failed fetching jwt with matrix 2.0 endpoint (retry with legacy) Not supported`,
e,
);
sfuConfig = undefined;
} else {
logger?.warn(
`Failed fetching jwt with matrix 2.0 endpoint other issues ->`,
`(not going to try with legacy endpoint: forceOldJwtEndpoint is set to false, we did not get a not supported error from the sfu)`,
e,
);
// Make this throw a hard error in case we force the matrix2.0 endpoint.
if (forceMatrix2Jwt)
throw new NoMatrix2AuthorizationService(e as Error);
// NEVER get bejond this point if we forceMatrix2 and it failed!
logger?.debug(`Failed fetching jwt with matrix 2.0 endpoint:`, e);
// Make this throw a hard error in case we force the matrix2.0 endpoint.
if (forceMatrix2Jwt) {
throw new NoMatrix2AuthorizationService(e as Error);
}
}
}
// DEPRECATED
// here we either have a sfuConfig or we alredy exited because of `if (forceMatrix2) throw ...`
// here we either have a sfuConfig or we already exited because of `if (forceMatrix2) throw ...`
// The only case we can get into this condition is, if `forceMatrix2` is `false`
if (sfuConfig === undefined) {
try {
logger?.info(
`Trying to get JWT with legacy endpoint for focus ${serviceUrl}...`,
);
sfuConfig = await getLiveKitJWT(
membership.deviceId,
serviceUrl,
@@ -168,15 +157,19 @@ export async function getSFUConfigWithOpenID(
openIdToken,
);
logger?.info(`Got JWT from call's active focus URL.`);
return extractFullConfigFromToken(sfuConfig);
} catch (ex) {
throw new FailToGetOpenIdToken(
ex instanceof Error ? ex : new Error(`Unknown error ${ex}`),
);
}
}
if (!sfuConfig) {
throw new Error("No `sfuConfig` after trying with old and new endpoints");
}
// Pull the details from the JWT
function extractFullConfigFromToken(sfuConfig: {
url: string;
jwt: string;
}): SFUConfig {
const [, payloadStr] = sfuConfig.jwt.split(".");
// TODO: Prefer Uint8Array.fromBase64 when widely available
const payload = JSON.parse(global.atob(payloadStr)) as SFUJWTPayload;
return {
jwt: sfuConfig.jwt,
@@ -188,16 +181,15 @@ export async function getSFUConfigWithOpenID(
livekitIdentity: payload.sub,
};
}
const RETRIES = 4;
async function getLiveKitJWT(
deviceId: string,
livekitServiceURL: string,
matrixRoomId: string,
openIDToken: IOpenIDToken,
): Promise<{ url: string; jwt: string }> {
let res: Response | undefined;
await retryNetworkOperation(RETRIES, async () => {
res = await fetch(livekitServiceURL + "/sfu/get", {
const res = await doNetworkOperationWithRetry(async () => {
return await fetch(livekitServiceURL + "/sfu/get", {
method: "POST",
headers: {
"Content-Type": "application/json",
@@ -210,11 +202,7 @@ async function getLiveKitJWT(
}),
});
});
if (!res) {
throw new Error(
`Network error while connecting to jwt service after ${RETRIES} retries`,
);
}
if (!res.ok) {
throw new Error("SFU Config fetch failed with status code " + res.status);
}
@@ -261,10 +249,8 @@ export async function getLiveKitJWTWithDelayDelegation(
};
}
let res: Response | undefined;
await retryNetworkOperation(RETRIES, async () => {
res = await fetch(livekitServiceURL + "/get_token", {
const res = await doNetworkOperationWithRetry(async () => {
return await fetch(livekitServiceURL + "/get_token", {
method: "POST",
headers: {
"Content-Type": "application/json",
@@ -273,11 +259,6 @@ export async function getLiveKitJWTWithDelayDelegation(
});
});
if (!res) {
throw new Error(
`Network error while connecting to jwt service after ${RETRIES} retries`,
);
}
if (!res.ok) {
const msg = "SFU Config fetch failed with status code " + res.status;
if (res.status === 404) {

View File

@@ -0,0 +1,25 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { fn } from "storybook/test";
import type { Meta, StoryObj } from "@storybook/react-vite";
import { LayoutToggle } from "./LayoutToggle";
const meta = {
component: LayoutToggle,
} satisfies Meta<typeof LayoutToggle>;
export default meta;
type Story = StoryObj<typeof meta>;
export const Default: Story = {
args: {
layout: "grid",
setLayout: fn(),
},
};

View File

@@ -18,11 +18,11 @@ import styles from "./LayoutToggle.module.css";
export type Layout = "spotlight" | "grid";
interface Props {
type Props = {
layout: Layout;
setLayout: (layout: Layout) => void;
className?: string;
}
};
export const LayoutToggle: FC<Props> = ({ layout, setLayout, className }) => {
const { t } = useTranslation();

View File

@@ -750,6 +750,53 @@ describe.each([
});
});
test("PiP tile in expanded spotlight layout avoids redundantly showing local user", () => {
withTestScheduler(({ behavior, schedule, expectObservable }) => {
// Switch to spotlight immediately
const modeInputMarbles = " s";
// And expand the spotlight immediately
const expandInputMarbles = " a";
// First no one else is in the call, then Alice joins
const participantInputMarbles = "ab";
// First local user should be in the spotlight, then they appear in PiP
// only once Alice has joined
const expectedLayoutMarbles = " ab";
withCallViewModel(
{
rtcMembers$: behavior(participantInputMarbles, {
a: [localRtcMember],
b: [localRtcMember, aliceRtcMember],
}),
},
(vm) => {
schedule(modeInputMarbles, {
s: () => vm.setGridMode("spotlight"),
});
schedule(expandInputMarbles, {
a: () => vm.toggleSpotlightExpanded$.value!(),
});
expectObservable(summarizeLayout$(vm.layout$)).toBe(
expectedLayoutMarbles,
{
a: {
type: "spotlight-expanded",
spotlight: [`${localId}:0`],
pip: undefined,
},
b: {
type: "spotlight-expanded",
spotlight: [`${aliceId}:0`],
pip: `${localId}:0`,
},
},
);
},
);
});
});
test("spotlight remembers whether it's expanded", () => {
withTestScheduler(({ schedule, expectObservable }) => {
// Start in spotlight mode, then switch to grid and back to spotlight a

View File

@@ -951,7 +951,7 @@ export function createCallViewModel$(
const spotlightAndPip$ = scope.behavior<{
spotlight: MediaViewModel[];
pip$: Behavior<UserMediaViewModel | undefined>;
pip$: Observable<UserMediaViewModel | undefined>;
}>(
ringingMedia$.pipe(
switchMap((ringingMedia) => {
@@ -966,7 +966,10 @@ export function createCallViewModel$(
return spotlightSpeaker$.pipe(
map((speaker) => ({
spotlight: speaker ? [speaker] : [],
pip$: localUserMediaForPip$,
// Hide PiP if redundant (i.e. if local user is already in spotlight)
pip$: localUserMediaForPip$.pipe(
map((m) => (m === speaker ? undefined : m)),
),
})),
);
}),

View File

@@ -131,6 +131,9 @@ export function withCallViewModel(mode: MatrixRTCMode) {
public getSyncState(): SyncState {
return syncState;
}
public getAccessToken(): string | null {
return "a-token";
}
})() as Partial<MatrixClient> as MatrixClient,
getMembers: () => roomMembers,
getMembersWithMembership: () => roomMembers,

View File

@@ -19,7 +19,7 @@ import {
type CallMembership,
type LivekitTransportConfig,
} from "matrix-js-sdk/lib/matrixrtc";
import { BehaviorSubject, lastValueFrom } from "rxjs";
import { BehaviorSubject, filter, lastValueFrom } from "rxjs";
import fetchMock from "fetch-mock";
import {
@@ -27,8 +27,13 @@ import {
flushPromises,
ownMemberMock,
mockRtcMembership,
testScope,
} from "../../../utils/test";
import { createLocalTransport$, JwtEndpointVersion } from "./LocalTransport";
import {
createLocalTransport$,
JwtEndpointVersion,
type LocalTransportWithSFUConfig,
} from "./LocalTransport";
import { constant } from "../../Behavior";
import { Epoch, ObservableScope, trackEpoch } from "../../ObservableScope";
import {
@@ -47,19 +52,18 @@ describe("LocalTransport", () => {
livekitIdentity: "@lk_user:ABCDEF",
};
let scope: ObservableScope;
beforeEach(() => (scope = new ObservableScope()));
afterEach(() => scope.end());
beforeEach(() => vi.clearAllMocks());
it("throws if config is missing", async () => {
const { advertised$, active$ } = createLocalTransport$({
scope,
scope: testScope(),
roomId: "!room:example.org",
useOldestMember: false,
memberships$: constant(new Epoch<CallMembership[]>([])),
client: {
// eslint-disable-next-line @typescript-eslint/naming-convention
_unstable_getRTCTransports: async () => Promise.resolve([]),
getAccessToken: vi.fn().mockReturnValue("access_token"),
getDomain: () => "",
baseUrl: "example.org",
// These won't be called in this error path but satisfy the type
@@ -102,6 +106,7 @@ describe("LocalTransport", () => {
baseUrl: "https://lk.example.org",
// Use empty domain to skip .well-known and use config directly
getDomain: () => "",
getAccessToken: vi.fn().mockReturnValue("access_token"),
// eslint-disable-next-line @typescript-eslint/naming-convention
_unstable_getRTCTransports: async () => Promise.resolve([]),
getOpenIdToken: vi.fn(),
@@ -138,7 +143,7 @@ describe("LocalTransport", () => {
);
const { advertised$, active$ } = createLocalTransport$({
scope,
scope: testScope(),
roomId: "!room:example.org",
useOldestMember: false,
memberships$: constant(new Epoch<CallMembership[]>([])),
@@ -149,6 +154,7 @@ describe("LocalTransport", () => {
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
baseUrl: "https://lk.example.org",
getAccessToken: vi.fn().mockReturnValue("access_token"),
},
ownMembershipIdentity: ownMemberMock,
forceJwtEndpoint: JwtEndpointVersion.Legacy,
@@ -208,6 +214,7 @@ describe("LocalTransport", () => {
// Initially, Alice is the only member
const memberships$ = new BehaviorSubject([aliceMembership]);
const scope = testScope();
const { advertised$, active$ } = createLocalTransport$({
scope,
roomId: "!example_room_id",
@@ -217,6 +224,7 @@ describe("LocalTransport", () => {
getDomain: () => "",
// eslint-disable-next-line @typescript-eslint/naming-convention
_unstable_getRTCTransports: async () => Promise.resolve([]),
getAccessToken: vi.fn().mockReturnValue("access_token"),
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
baseUrl: "https://lk.example.org",
@@ -263,6 +271,7 @@ describe("LocalTransport", () => {
// Initially, there are no members
const memberships$ = new BehaviorSubject<CallMembership[]>([]);
const scope = testScope();
const { advertised$, active$ } = createLocalTransport$({
scope,
roomId: "!example_room_id",
@@ -273,6 +282,7 @@ describe("LocalTransport", () => {
// eslint-disable-next-line @typescript-eslint/naming-convention
_unstable_getRTCTransports: async () =>
Promise.resolve([aliceTransport]),
getAccessToken: vi.fn().mockReturnValue("access_token"),
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
baseUrl: "https://lk.example.org",
@@ -312,7 +322,7 @@ describe("LocalTransport", () => {
customLivekitUrl.setValue(customLivekitUrl.defaultValue);
localTransportOpts = {
ownMembershipIdentity: ownMemberMock,
scope,
scope: testScope(),
roomId: "!example_room_id",
useOldestMember: false,
forceJwtEndpoint: JwtEndpointVersion.Legacy,
@@ -323,6 +333,7 @@ describe("LocalTransport", () => {
getDomain: vi.fn().mockReturnValue(""),
// eslint-disable-next-line @typescript-eslint/naming-convention
_unstable_getRTCTransports: vi.fn().mockResolvedValue([]),
getAccessToken: vi.fn().mockReturnValue("access_token"),
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
},
@@ -410,6 +421,42 @@ describe("LocalTransport", () => {
});
});
it("Should not call _unstable_getRTCTransports in widget mode but use well-known", async () => {
mockConfig({
livekit: { livekit_service_url: "https://do-not-use.lk.example.org" },
});
localTransportOpts.client.getDomain.mockReturnValue("example.org");
fetchMock.getOnce("https://example.org/.well-known/matrix/client", {
"org.matrix.msc4143.rtc_foci": [
{
type: "livekit",
livekit_service_url: "https://use-me.jwt.call.example.org",
},
],
});
localTransportOpts.client.getAccessToken.mockReturnValue(null);
const { advertised$, active$ } =
createLocalTransport$(localTransportOpts);
openIdResolver.resolve?.(openIdResponse);
expect(advertised$.value).toBe(null);
expect(active$.value).toBe(null);
await flushPromises();
expect(
localTransportOpts.client._unstable_getRTCTransports,
).not.toHaveBeenCalled();
const expectedTransport = {
type: "livekit",
livekit_service_url: "https://use-me.jwt.call.example.org",
};
expect(advertised$.value).toStrictEqual(expectedTransport);
});
it("fails fast if the openID request fails for backend config", async () => {
localTransportOpts.client._unstable_getRTCTransports.mockResolvedValue([
{ type: "livekit", livekit_service_url: "https://lk.example.org" },
@@ -469,7 +516,7 @@ describe("LocalTransport", () => {
it("throws if no options are available", async () => {
const { advertised$, active$ } = createLocalTransport$({
scope,
scope: testScope(),
ownMembershipIdentity: ownMemberMock,
roomId: "!example_room_id",
useOldestMember: false,
@@ -481,6 +528,7 @@ describe("LocalTransport", () => {
baseUrl: "https://example.org",
// eslint-disable-next-line @typescript-eslint/naming-convention
_unstable_getRTCTransports: async () => Promise.resolve([]),
getAccessToken: vi.fn().mockReturnValue("access_token"),
// These won't be called in this error path but satisfy the type
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
@@ -496,4 +544,86 @@ describe("LocalTransport", () => {
);
});
});
it("should not update advertised/active transport on delayID changes, but delay Id delegation should be called", async () => {
// For simplicity, we'll just use the config livekit
customLivekitUrl.setValue("https://lk.example.org");
const authCallSpy = vi
.spyOn(openIDSFU, "getSFUConfigWithOpenID")
.mockResolvedValue(openIdResponse);
const delayId$ = new BehaviorSubject<string | null>(null);
const { advertised$, active$ } = createLocalTransport$({
scope: testScope(),
ownMembershipIdentity: ownMemberMock,
roomId: "!example_room_id",
// We want multi-sdu
useOldestMember: false,
forceJwtEndpoint: JwtEndpointVersion.Legacy,
delayId$: delayId$,
memberships$: constant(new Epoch<CallMembership[]>([])),
client: {
getDomain: () => "",
baseUrl: "https://example.org",
// eslint-disable-next-line @typescript-eslint/naming-convention
_unstable_getRTCTransports: async () => Promise.resolve([]),
getAccessToken: vi.fn().mockReturnValue("access_token"),
// These won't be called in this error path but satisfy the type
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
},
});
const advertisedValues: LivekitTransportConfig[] = [];
const activeValues: LocalTransportWithSFUConfig[] = [];
advertised$
.pipe(filter((v) => v !== null))
.subscribe((t) => advertisedValues.push(t));
active$
.pipe(filter((v) => v !== null))
.subscribe((t) => activeValues.push(t));
await flushPromises();
// we have now an active and an advertised
expect(advertisedValues.length).toEqual(1);
expect(activeValues.length).toEqual(1);
expect(advertisedValues[0]!.livekit_service_url).toEqual(
"https://lk.example.org",
);
expect(activeValues[0]!.transport.livekit_service_url).toEqual(
"https://lk.example.org",
);
expect(authCallSpy).toHaveBeenCalledTimes(2);
// Now emits 3 new delays id
delayId$.next("delay_id_1");
await flushPromises();
delayId$.next("delay_id_2");
await flushPromises();
delayId$.next("delay_id_3");
await flushPromises();
// No new emissions should've happened, it is the same transport.
expect(advertisedValues.length).toEqual(1);
expect(activeValues.length).toEqual(1);
// Still we should have updated the delayID to auth
expect(authCallSpy).toHaveBeenCalledTimes(
4 * 2 /* 2 calls for each delayId ?? why */,
);
expect(authCallSpy).toHaveBeenLastCalledWith(
expect.anything(),
expect.anything(),
expect.anything(),
expect.anything(),
expect.objectContaining({
delayId: "delay_id_3",
}),
expect.anything(),
);
});
});

View File

@@ -8,16 +8,18 @@ Please see LICENSE in the repository root for full details.
import {
type CallMembership,
isLivekitTransportConfig,
type Transport,
type LivekitTransportConfig,
} from "matrix-js-sdk/lib/matrixrtc";
import { MatrixError, type MatrixClient } from "matrix-js-sdk";
import { type MatrixClient } from "matrix-js-sdk";
import {
catchError,
combineLatest,
distinctUntilChanged,
first,
from,
map,
merge,
type Observable,
of,
startWith,
switchMap,
@@ -42,6 +44,7 @@ import {
} from "../../../livekit/openIDSFU.ts";
import { areLivekitTransportsEqual } from "../remoteMembers/MatrixLivekitMembers.ts";
import { customLivekitUrl } from "../../../settings/settings.ts";
import { RtcTransportAutoDiscovery } from "./RtcTransportAutoDiscovery.ts";
const logger = rootLogger.getChild("[LocalTransport]");
@@ -56,7 +59,7 @@ interface Props {
memberships$: Behavior<Epoch<CallMembership[]>>;
client: Pick<
MatrixClient,
"getDomain" | "baseUrl" | "_unstable_getRTCTransports"
"getDomain" | "baseUrl" | "_unstable_getRTCTransports" | "getAccessToken"
> &
OpenIDClientParts;
// Used by the jwt service to create the livekit room and compute the livekit alias.
@@ -137,11 +140,116 @@ export const createLocalTransport$ = ({
forceJwtEndpoint,
delayId$,
}: Props): LocalTransport => {
/**
* The LiveKit transport in use by the oldest RTC membership. `null` when the
* oldest member has no such transport.
*/
const oldestMemberTransport$ = scope.behavior<LivekitTransportConfig | null>(
// The LiveKit transport in use by the oldest RTC membership. `null` when the
// oldest member has no such transport.
const oldestMemberTransport$ = observerOldestMembership$(scope, memberships$);
const transportDiscovery = new RtcTransportAutoDiscovery({
client: client,
resolvedConfig: Config.get(),
wellKnownFetcher: AutoDiscovery.getRawClientConfig.bind(AutoDiscovery),
logger: logger,
});
// Get the preferred transport from the current deployment.
const discoveredTransport$ = from(
transportDiscovery.discoverPreferredTransport(),
);
const preferredConfig$ = customLivekitUrl.value$
.pipe(
switchMap((customUrl) => {
if (customUrl) {
return of({
type: "livekit",
livekit_service_url: customUrl,
} as LivekitTransportConfig);
} else {
return discoveredTransport$;
}
}),
)
.pipe(
map((config) => {
if (!config) {
// Bubbled up from the preferredConfig$ observable.
throw new MatrixRTCTransportMissingError(client.getDomain() ?? "");
}
return config;
}),
distinctUntilChanged(areLivekitTransportsEqual),
);
const preferredTransport$ = combineLatest([preferredConfig$, delayId$]).pipe(
switchMap(async ([transport, delayId]) => {
try {
return await doOpenIdAndJWTFromUrl(
transport,
forceJwtEndpoint,
ownMembershipIdentity,
roomId,
client,
delayId ?? undefined,
);
} catch (e) {
logger.error(
`Failed to authenticate to transport ${transport.livekit_service_url}`,
e,
);
throw mapAuthErrorToUserFriendlyError(e);
}
}),
);
if (useOldestMember) {
return observeLocalTransportForOldestMembership(
scope,
oldestMemberTransport$,
preferredTransport$,
client,
ownMembershipIdentity,
roomId,
);
}
// --- Multi-SFU mode ---
// Always publish on and advertise the preferred transport.
return {
advertised$: scope.behavior(
preferredTransport$.pipe(
map((t) => t.transport),
distinctUntilChanged(areLivekitTransportsEqual),
),
null,
),
active$: scope.behavior(
preferredTransport$.pipe(
// XXX: WORK AROUND due to a reconnection glitch.
// To remove when we have a proper way to refresh the delegation event ID without refreshing
// the whole credentials.
// We deliberately hide any changes to the SFU config because we
// do not want the app to reconnect whenever the JWT
// token changes due to us delegating a new delayed event. The
// initial SFU config for the transport is all the app needs.
distinctUntilChanged((prev, next) =>
areLivekitTransportsEqual(prev.transport, next.transport),
),
),
null,
),
};
};
/**
* Observes the oldest member in the room and returns the transport that it uses if it is a livekit transport.
* @param scope - The observable scope.
* @param memberships$ - The observable of the call's memberships.'
*/
function observerOldestMembership$(
scope: ObservableScope,
memberships$: Behavior<Epoch<CallMembership[]>>,
): Behavior<LivekitTransportConfig | null> {
return scope.behavior<LivekitTransportConfig | null>(
memberships$.pipe(
map((memberships) => {
const oldestMember = memberships.value[0];
@@ -170,292 +278,141 @@ export const createLocalTransport$ = ({
distinctUntilChanged(areLivekitTransportsEqual),
),
);
/**
* The transport that we would personally prefer to publish on (if not for the
* transport preferences of others, perhaps). `null` until fetched and
* validated.
*
* @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken
*/
const preferredTransport$ =
scope.behavior<LocalTransportWithSFUConfig | null>(
// preferredTransport$ (used for multi sfu) needs to know if we are using the old or new
// jwt endpoint (`get_token` vs `sfu/get`) based on that the jwt endpoint will compute the rtcBackendIdentity
// differently. (sha(`${userId}|${deviceId}|${memberId}`) vs `${userId}|${deviceId}|${memberId}`)
// When using sticky events (we need to use the new endpoint).
customLivekitUrl.value$.pipe(
switchMap((customUrl) =>
startWith<LocalTransportWithSFUConfig | null>(null)(
// Fetch the SFU config, and repeat this asynchronously for every
// change in delay ID.
delayId$.pipe(
switchMap(async (delayId) => {
logger.info(
"Creating preferred transport based on: ",
"customUrl: ",
customUrl,
"delayId: ",
delayId,
"forceJwtEndpoint: ",
forceJwtEndpoint,
);
return makeTransport(
client,
ownMembershipIdentity,
roomId,
customUrl,
forceJwtEndpoint,
delayId ?? undefined,
);
}),
// We deliberately hide any changes to the SFU config because we
// do not actually want the app to reconnect whenever the JWT
// token changes due to us delegating a new delayed event. The
// initial SFU config for the transport is all the app needs.
distinctUntilChanged((prev, next) =>
areLivekitTransportsEqual(prev.transport, next.transport),
),
),
),
),
),
);
if (useOldestMember) {
// --- Oldest member mode ---
return {
// Never update the transport that we advertise in our membership. Just
// take the first valid oldest member or preferred transport that we learn
// about, and stick with that. This avoids unnecessary SFU hops and room
// state changes.
advertised$: scope.behavior(
merge(
oldestMemberTransport$,
preferredTransport$.pipe(map((t) => t?.transport ?? null)),
).pipe(
first((t) => t !== null),
tap((t) =>
logger.info(`Advertise transport: ${t.livekit_service_url}`),
),
),
null,
),
// Publish on the transport used by the oldest member.
active$: scope.behavior(
oldestMemberTransport$.pipe(
switchMap((transport) => {
// Oldest member not available (or invalid SFU config).
if (transport === null) return of(null);
// Oldest member available: fetch the SFU config.
const fetchOldestMemberTransport =
async (): Promise<LocalTransportWithSFUConfig> => ({
transport,
sfuConfig: await getSFUConfigWithOpenID(
client,
ownMembershipIdentity,
transport.livekit_service_url,
roomId,
{ forceJwtEndpoint: JwtEndpointVersion.Legacy },
logger,
),
});
return from(fetchOldestMemberTransport()).pipe(startWith(null));
}),
tap((t) =>
logger.info(
`Publish on transport: ${t?.transport.livekit_service_url}`,
),
),
),
),
};
}
// --- Multi-SFU mode ---
// Always publish on and advertise the preferred transport.
return {
advertised$: scope.behavior(
preferredTransport$.pipe(
map((t) => t?.transport ?? null),
distinctUntilChanged(areLivekitTransportsEqual),
),
),
active$: preferredTransport$,
};
};
const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci";
}
/**
* Determine the correct Transport for the current session, including
* validating auth against the service to ensure it's correct.
* Prefers in order:
* Utility to ensure the user can authenticate with the SFU.
* We will call `getSFUConfigWithOpenID` once per transport here as it's our
* only mechanism of validation. This means we will also ask the
* homeserver for a OpenID token a few times. Since OpenID tokens are single
* use we don't want to risk any issues by re-using a token.
*
* 1. The `urlFromDevSettings` value. If this cannot be validated, the function will throw.
* 2. The transports returned via the homeserver.
* 3. The transports returned via .well-known.
* 4. The transport configured in Element Call's config.
* @param transport The transport to authenticate with.
* @param forceJwtEndpoint Whether to force the JWT endpoint to be used.
* @param membership The identity of the local member.
* @param roomId The room ID to use for the JWT.
* @param client The client to use for the OpenID token.
* @param delayId The delayId to use for the JWT.
*
* @param client The authenticated Matrix client for the current user
* @param membership The membership identity of the user.
* @param roomId The ID of the room to be connected to.
* @param urlFromDevSettings Override URL provided by the user's local config.
* @param forceJwtEndpoint Whether to force a specific JWT endpoint
* - `Legacy` / `Matrix_2_0`
* - `get_token` / `sfu/get`
* - not hashing / hashing the backendIdentity
* @param delayId the delay id passed to the jwt service.
*
* @returns A fully validated transport config.
* @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken
* @throws FailToGetOpenIdToken, NoMatrix2AuthorizationService
*/
async function makeTransport(
client: Pick<
MatrixClient,
"getDomain" | "baseUrl" | "_unstable_getRTCTransports"
> &
OpenIDClientParts,
async function doOpenIdAndJWTFromUrl(
transport: LivekitTransportConfig,
forceJwtEndpoint: JwtEndpointVersion,
membership: CallMembershipIdentityParts,
roomId: string,
urlFromDevSettings: string | null,
forceJwtEndpoint: JwtEndpointVersion,
client: Pick<
MatrixClient,
"getDomain" | "baseUrl" | "_unstable_getRTCTransports" | "getAccessToken"
> &
OpenIDClientParts,
delayId?: string,
): Promise<LocalTransportWithSFUConfig> {
logger.trace("Searching for a preferred transport");
async function doOpenIdAndJWTFromUrl(
url: string,
): Promise<LocalTransportWithSFUConfig> {
const sfuConfig = await getSFUConfigWithOpenID(
client,
membership,
url,
roomId,
{
forceJwtEndpoint: forceJwtEndpoint,
delayEndpointBaseUrl: client.baseUrl,
delayId,
},
logger,
);
return {
transport: {
type: "livekit",
livekit_service_url: url,
},
sfuConfig,
};
}
// We will call `getSFUConfigWithOpenID` once per transport here as it's our
// only mechanism of valiation. This means we will also ask the
// homeserver for a OpenID token a few times. Since OpenID tokens are single
// use we don't want to risk any issues by re-using a token.
//
// If the OpenID request were to fail then it's acceptable for us to fail
// this function early, as we assume the homeserver has got some problems.
// DEVTOOL: Highest priority: Load from devtool setting
if (urlFromDevSettings !== null) {
// Validate that the SFU is up. Otherwise, we want to fail on this
// as we don't permit other SFUs.
// This will call the jwt/sfu/get endpoint to pre create the livekit room.
logger.info("Using LiveKit transport from dev tools: ", urlFromDevSettings);
return await doOpenIdAndJWTFromUrl(urlFromDevSettings);
}
async function getFirstUsableTransport(
transports: Transport[],
): Promise<LocalTransportWithSFUConfig | null> {
for (const potentialTransport of transports) {
if (isLivekitTransportConfig(potentialTransport)) {
try {
// This will call the jwt/sfu/get endpoint to pre create the livekit room.
return await doOpenIdAndJWTFromUrl(
potentialTransport.livekit_service_url,
);
} catch (ex) {
// Explictly throw these
if (ex instanceof FailToGetOpenIdToken) {
throw ex;
}
if (ex instanceof NoMatrix2AuthorizationService) {
throw ex;
}
logger.debug(
`Could not use SFU service "${potentialTransport.livekit_service_url}" as SFU`,
ex,
);
}
}
}
return null;
}
// MSC4143: Attempt to fetch transports from backend.
if ("_unstable_getRTCTransports" in client) {
try {
const transportList = await client._unstable_getRTCTransports();
const selectedTransport = await getFirstUsableTransport(transportList);
if (selectedTransport) {
logger.info(
"Using backend-configured (client.getRTCTransports) SFU",
selectedTransport,
);
return selectedTransport;
}
} catch (ex) {
if (ex instanceof MatrixError && ex.httpStatus === 404) {
// Expected, this is an unstable endpoint and it's not required.
// There will be expected 404 errors in the console. When we check if synapse supports the endpoint.
logger.debug(
"Matrix homeserver does not provide any RTC transports via `/rtc/transports` (will retry with well-known.)",
);
} else if (ex instanceof FailToGetOpenIdToken) {
throw ex;
} else {
// We got an error that wasn't just missing support for the feature, so log it loudly.
logger.error(
"Unexpected error fetching RTC transports from backend",
ex,
);
}
}
}
// Legacy MSC4143 (to be removed) WELL_KNOWN: Prioritize the .well-known/matrix/client, if available.
const domain = client.getDomain();
if (domain) {
// we use AutoDiscovery instead of relying on the MatrixClient having already
// been fully configured and started
const wellKnownFoci = (await AutoDiscovery.getRawClientConfig(domain))?.[
FOCI_WK_KEY
];
const selectedTransport = Array.isArray(wellKnownFoci)
? await getFirstUsableTransport(wellKnownFoci)
: null;
if (selectedTransport) {
logger.info("Using .well-known SFU", selectedTransport);
return selectedTransport;
}
}
// CONFIG: Least prioritized; Load from config file
const urlFromConf = Config.get().livekit?.livekit_service_url;
if (urlFromConf) {
try {
// This will call the jwt/sfu/get endpoint to pre create the livekit room.
logger.info("Using config SFU", urlFromConf);
return await doOpenIdAndJWTFromUrl(urlFromConf);
} catch (ex) {
if (ex instanceof FailToGetOpenIdToken) {
throw ex;
}
logger.error("Failed to validate config SFU", ex);
}
}
// If we do not have returned a transport by now we throw an error
throw new MatrixRTCTransportMissingError(domain ?? "");
const sfuConfig = await getSFUConfigWithOpenID(
client,
membership,
transport.livekit_service_url,
roomId,
{
forceJwtEndpoint: forceJwtEndpoint,
delayEndpointBaseUrl: client.baseUrl,
delayId,
},
logger,
);
return {
transport,
sfuConfig,
};
}
function observeLocalTransportForOldestMembership(
scope: ObservableScope,
oldestMemberTransport$: Behavior<LivekitTransportConfig | null>,
preferredTransport$: Observable<LocalTransportWithSFUConfig>,
client: Pick<
MatrixClient,
"getDomain" | "baseUrl" | "_unstable_getRTCTransports" | "getAccessToken"
> &
OpenIDClientParts,
ownMembershipIdentity: CallMembershipIdentityParts,
roomId: string,
): LocalTransport {
// Ensure we can authenticate with the SFU.
const authenticatedOldestMemberTransport$ = oldestMemberTransport$.pipe(
switchMap((transport) => {
// Oldest member not available -we are first- (or invalid SFU config).
if (transport === null) return of(null);
// Whenever there is transport change we want to revert
// to no transport while we do the authentication.
// So do a from(promise) here to be able to startWith(null)
return from(
doOpenIdAndJWTFromUrl(
transport,
JwtEndpointVersion.Legacy,
ownMembershipIdentity,
roomId,
client,
undefined,
),
).pipe(
catchError((e: unknown) => {
logger.error(
`Failed to authenticate to transport ${transport.livekit_service_url}`,
e,
);
throw mapAuthErrorToUserFriendlyError(e);
}),
startWith(null),
);
}),
);
// --- Oldest member mode ---
return {
// Never update the transport that we advertise in our membership. Just
// take the first valid oldest member or preferred transport that we learn
// about, and stick with that. This avoids unnecessary SFU hops and room
// state changes.
advertised$: scope.behavior(
merge(
authenticatedOldestMemberTransport$.pipe(
map((t) => t?.transport ?? null),
),
preferredTransport$.pipe(map((t) => t.transport)),
).pipe(
first((t) => t !== null),
tap((t) =>
logger.info(`Advertise transport: ${t.livekit_service_url}`),
),
),
null,
),
// Publish on the transport used by the oldest member.
active$: scope.behavior(
authenticatedOldestMemberTransport$.pipe(
tap((t) =>
logger.info(
`Publish on transport: ${t?.transport.livekit_service_url}`,
),
),
),
null,
),
};
}
function mapAuthErrorToUserFriendlyError(e: unknown): Error {
if (
e instanceof FailToGetOpenIdToken ||
e instanceof NoMatrix2AuthorizationService
) {
// rethrow as is
return e;
}
// Catch others and rethrow as FailToGetOpenIdToken that has user friendly message.
return new FailToGetOpenIdToken(
e instanceof Error ? e : new Error(String(e)),
);
}

View File

@@ -0,0 +1,233 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
beforeEach,
describe,
expect,
it,
type MockedObject,
vi,
} from "vitest";
import { type IClientWellKnown, MatrixError } from "matrix-js-sdk";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import {
type LivekitTransportConfig,
type Transport,
} from "matrix-js-sdk/lib/matrixrtc";
import type { ResolvedConfigOptions } from "../../../config/ConfigOptions.ts";
import {
RtcTransportAutoDiscovery,
type RtcTransportAutoDiscoveryProps,
} from "./RtcTransportAutoDiscovery.ts";
type DiscoveryClient = RtcTransportAutoDiscoveryProps["client"];
const backendTransport: LivekitTransportConfig = {
type: "livekit",
livekit_service_url: "https://backend.example.org",
};
const wellKnownTransport: LivekitTransportConfig = {
type: "livekit",
livekit_service_url: "https://well-known.example.org",
};
function makeClient(): MockedObject<DiscoveryClient> {
return {
getDomain: vi.fn().mockReturnValue("example.org"),
baseUrl: "https://matrix.example.org",
_unstable_getRTCTransports: vi.fn().mockResolvedValue([]),
getAccessToken: vi.fn().mockReturnValue("access_token"),
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
} as unknown as MockedObject<DiscoveryClient>;
}
function makeResolvedConfig(livekitServiceUrl?: string): ResolvedConfigOptions {
return {
livekit: livekitServiceUrl
? {
livekit_service_url: livekitServiceUrl,
}
: undefined,
} as ResolvedConfigOptions;
}
function makeWellKnown(rtcFoci?: Transport[]): IClientWellKnown {
return {
"org.matrix.msc4143.rtc_foci": rtcFoci,
} as unknown as IClientWellKnown;
}
describe("RtcTransportAutoDiscovery", () => {
beforeEach(() => {
vi.clearAllMocks();
});
const VALID_TEST_CASES: Array<{ transports: Transport[] }> = [
{ transports: [backendTransport] },
// will pick the first livekit transport in the list, even if there are other non-livekit transports
{ transports: [{ type: "not_livekit" }, backendTransport] },
];
it.each(VALID_TEST_CASES)(
"prefers backend transport over well-known and app config $transports",
async ({ transports }) => {
// it("prefers backend transport over well-known and app config", async () => {
const client = makeClient();
client._unstable_getRTCTransports.mockResolvedValue(transports);
const wellKnownFetcher = vi
.fn<(domain: string) => Promise<IClientWellKnown>>()
.mockResolvedValue(makeWellKnown([wellKnownTransport]));
const discovery = new RtcTransportAutoDiscovery({
client,
resolvedConfig: makeResolvedConfig("https://config.example.org"),
wellKnownFetcher,
logger: rootLogger,
});
await expect(
discovery.discoverPreferredTransport(),
).resolves.toStrictEqual(backendTransport);
expect(client._unstable_getRTCTransports).toHaveBeenCalledTimes(1);
expect(wellKnownFetcher).not.toHaveBeenCalled();
},
);
it("Retries limit_exceeded backend transport over well-known", async () => {
const client = makeClient();
client._unstable_getRTCTransports
.mockRejectedValueOnce(
new MatrixError(
{
errcode: "M_LIMIT_EXCEEDED",
error: "Too many requests",
retry_after_ms: 100,
},
429,
),
)
.mockResolvedValue([backendTransport]);
const wellKnownFetcher = vi
.fn<(domain: string) => Promise<IClientWellKnown>>()
.mockResolvedValue(makeWellKnown([wellKnownTransport]));
const discovery = new RtcTransportAutoDiscovery({
client,
resolvedConfig: makeResolvedConfig("https://config.example.org"),
wellKnownFetcher,
logger: rootLogger,
});
await expect(discovery.discoverPreferredTransport()).resolves.toStrictEqual(
backendTransport,
);
expect(client._unstable_getRTCTransports).toHaveBeenCalledTimes(2);
expect(wellKnownFetcher).not.toHaveBeenCalled();
});
const INVALID_TEST_CASES: Array<{ transports: Transport[] }> = [
{ transports: [] },
{ transports: [{ type: "not_livekit" }] },
];
it.each(INVALID_TEST_CASES)(
"falls back to well-known when backend has no (valid) livekit transports $transports",
async ({ transports }) => {
const client = makeClient();
client._unstable_getRTCTransports.mockResolvedValue(transports);
const wellKnownFetcher = vi
.fn<(domain: string) => Promise<IClientWellKnown>>()
.mockResolvedValue(makeWellKnown([wellKnownTransport]));
const discovery = new RtcTransportAutoDiscovery({
client,
resolvedConfig: makeResolvedConfig("https://config.example.org"),
wellKnownFetcher,
logger: rootLogger,
});
await expect(
discovery.discoverPreferredTransport(),
).resolves.toStrictEqual(wellKnownTransport);
expect(wellKnownFetcher).toHaveBeenCalledWith("example.org");
},
);
it("skips backend discovery in widget mode and uses well-known", async () => {
const client = makeClient();
// widget mode is detected by the absence of an access token
client.getAccessToken.mockReturnValue(null);
const wellKnownFetcher = vi
.fn<(domain: string) => Promise<IClientWellKnown>>()
.mockResolvedValue(makeWellKnown([wellKnownTransport]));
const discovery = new RtcTransportAutoDiscovery({
client,
resolvedConfig: makeResolvedConfig("https://config.example.org"),
wellKnownFetcher,
logger: rootLogger,
});
await expect(discovery.discoverPreferredTransport()).resolves.toStrictEqual(
wellKnownTransport,
);
expect(client._unstable_getRTCTransports).not.toHaveBeenCalled();
expect(wellKnownFetcher).toHaveBeenCalledWith("example.org");
});
it("falls back to app config when backend fails and well-known has no rtc_foci", async () => {
const client = makeClient();
client._unstable_getRTCTransports.mockRejectedValue(
new MatrixError({ errcode: "M_UNKNOWN" }, 404),
);
const wellKnownFetcher = vi
.fn<(domain: string) => Promise<IClientWellKnown>>()
.mockResolvedValue({} as IClientWellKnown);
const discovery = new RtcTransportAutoDiscovery({
client,
resolvedConfig: makeResolvedConfig("https://config.example.org"),
wellKnownFetcher,
logger: rootLogger,
});
await expect(discovery.discoverPreferredTransport()).resolves.toStrictEqual(
{
type: "livekit",
livekit_service_url: "https://config.example.org",
},
);
});
it("returns null when backend, well-known and config are all unavailable", async () => {
const client = makeClient();
client._unstable_getRTCTransports.mockResolvedValue([]);
const wellKnownFetcher = vi
.fn<(domain: string) => Promise<IClientWellKnown>>()
.mockResolvedValue({} as IClientWellKnown);
const discovery = new RtcTransportAutoDiscovery({
client,
resolvedConfig: makeResolvedConfig(undefined),
wellKnownFetcher,
logger: rootLogger,
});
await expect(discovery.discoverPreferredTransport()).resolves.toBeNull();
});
});

View File

@@ -0,0 +1,172 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-IdFentifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
isLivekitTransportConfig,
type LivekitTransportConfig,
} from "matrix-js-sdk/lib/matrixrtc";
import { type IClientWellKnown, type MatrixClient } from "matrix-js-sdk";
import { type Logger } from "matrix-js-sdk/lib/logger";
import type { ResolvedConfigOptions } from "../../../config/ConfigOptions.ts";
import { doNetworkOperationWithRetry } from "../../../utils/matrix.ts";
type TransportDiscoveryClient = Pick<
MatrixClient,
"getDomain" | "_unstable_getRTCTransports" | "getAccessToken"
>;
export interface RtcTransportAutoDiscoveryProps {
client: TransportDiscoveryClient;
resolvedConfig: ResolvedConfigOptions;
wellKnownFetcher: (domain: string) => Promise<IClientWellKnown>;
logger: Logger;
}
export class RtcTransportAutoDiscovery {
private readonly client: TransportDiscoveryClient;
private readonly resolvedConfig: ResolvedConfigOptions;
private readonly wellKnownFetcher: (
domain: string,
) => Promise<IClientWellKnown>;
private readonly logger: Logger;
public constructor({
client,
resolvedConfig,
wellKnownFetcher,
logger,
}: RtcTransportAutoDiscoveryProps) {
this.client = client;
this.resolvedConfig = resolvedConfig;
this.wellKnownFetcher = wellKnownFetcher;
this.logger = logger.getChild("[RtcTransportAutoDiscovery]");
}
public async discoverPreferredTransport(): Promise<LivekitTransportConfig | null> {
// 1) backend transports
const backendTransport = await this.tryBackendTransports();
if (backendTransport) {
this.logger.info(
`Found backend transport: ${backendTransport.livekit_service_url}`,
);
return backendTransport;
}
this.logger.info("No backend transport found, falling back to well-known");
// 2) .well-known transports
const wellKnownTransport = await this.tryWellKnownTransports();
if (wellKnownTransport) {
this.logger.info(
`Found .well-known transport: ${wellKnownTransport.livekit_service_url}`,
);
return wellKnownTransport;
}
this.logger.info(
"No .well-known transport found, falling back to app config",
);
// 3) app config URL
const configTransport = this.tryConfigTransport();
if (configTransport) {
this.logger.info(
`Found app config transport: ${configTransport.livekit_service_url}`,
);
return configTransport;
}
return null;
}
/**
* Fetches the first rtc_foci from the backend.
* This will not throw errors, but instead just log them and return null if the expected config is not found or malformed.
* @private
*/
private async tryBackendTransports(): Promise<LivekitTransportConfig | null> {
const client = this.client;
// MSC4143: Attempt to fetch transports from backend.
// TODO: Workaround for an issue in the js-sdk RoomWidgetClient that
// is not yet implementing _unstable_getRTCTransports properly (via widget API new action).
// For now we just skip this call if we are in a widget.
// In widget mode the client is a `RoomWidgetClient` which has no access token (it is using the widget API).
// Could be removed once the js-sdk is fixed (https://github.com/matrix-org/matrix-js-sdk/issues/5245)
const isSPA = !!client.getAccessToken();
if (isSPA && "_unstable_getRTCTransports" in client) {
this.logger.info("First try to use getRTCTransports end point ...");
try {
const transportList = await doNetworkOperationWithRetry(async () =>
client._unstable_getRTCTransports(),
);
const first = transportList.filter(isLivekitTransportConfig)[0];
if (first) {
return first;
} else {
this.logger.info(
`No livekit transport found in getRTCTransports end point`,
transportList,
);
}
} catch (ex) {
this.logger.info(`Failed to use getRTCTransports end point: ${ex}`);
}
} else {
this.logger.debug(`getRTCTransports end point not available`);
}
return null;
}
/**
* Fetches the first rtc_foci from the .well-known/matrix/client.
* This will not throw errors, but instead just log them and return null if the expected config is not found or malformed.
* @private
*/
private async tryWellKnownTransports(): Promise<LivekitTransportConfig | null> {
// Legacy MSC4143 (to be removed) WELL_KNOWN: Prioritize the .well-known/matrix/client, if available.
const client = this.client;
const domain = client.getDomain();
if (domain) {
// we use AutoDiscovery instead of relying on the MatrixClient having already
// been fully configured and started
const wellKnownFoci = await this.wellKnownFetcher(domain);
const fociConfig = wellKnownFoci["org.matrix.msc4143.rtc_foci"];
if (fociConfig) {
if (!Array.isArray(fociConfig)) {
this.logger.warn(
`org.matrix.msc4143.rtc_foci is not an array in .well-known`,
);
} else {
return fociConfig[0];
}
} else {
this.logger.info(
`No .well-known "org.matrix.msc4143.rtc_foci" found for ${domain}`,
wellKnownFoci,
);
}
} else {
// Should never happen, but just in case
this.logger.warn(`No domain configured for client`);
}
return null;
}
private tryConfigTransport(): LivekitTransportConfig | null {
const url = this.resolvedConfig.livekit?.livekit_service_url;
if (url) {
return {
type: "livekit",
livekit_service_url: url,
};
}
return null;
}
}

View File

@@ -8,6 +8,7 @@ Please see LICENSE in the repository root for full details.
import {
afterEach,
beforeEach,
describe,
expect,
it,
@@ -151,6 +152,19 @@ afterEach(() => {
});
describe("Start connection states", () => {
beforeEach(() => {
fetchMock.post(
`https://matrix-rtc.example.org/livekit/jwt/get_token`,
() => {
return {
// Return a non-retryable error, if not, the retry logic will
// wait and fail the test with a timeout.
status: 404,
};
},
);
});
it("start in initialized state", () => {
setupTest();

View File

@@ -66,17 +66,19 @@ export default ({
);
}
plugins.push(
createHtmlPlugin({
entry: "src/main.tsx",
inject: {
data: {
brand: env.VITE_PRODUCT_NAME || "Element Call",
packageType: process.env.VITE_PACKAGE,
if (!process.env.STORYBOOK) {
plugins.push(
createHtmlPlugin({
entry: "src/main.tsx",
inject: {
data: {
brand: env.VITE_PRODUCT_NAME || "Element Call",
packageType: process.env.VITE_PACKAGE,
},
},
},
}),
);
}),
);
}
// The crypto WASM module is imported dynamically. Since it's common
// for developers to use a linked copy of matrix-js-sdk or Rust
@@ -135,10 +137,6 @@ export default ({
// Default naming fallback
return "assets/[name]-[hash][extname]";
},
manualChunks: {
// we should be able to remove this one https://github.com/matrix-org/matrix-rust-sdk-crypto-wasm/pull/167 lands
"matrix-sdk-crypto-wasm": ["@matrix-org/matrix-sdk-crypto-wasm"],
},
},
},
},

View File

@@ -19,7 +19,7 @@ export default defineConfig((configEnv) =>
reporter: ["html", "json"],
include: ["src/"],
exclude: [
"src/**/*.{d,test}.{ts,tsx}",
"src/**/*.{d,test,stories}.{ts,tsx}",
"src/utils/test.ts",
"src/utils/test-viewmodel.ts",
"src/utils/test-fixtures.ts",

2251
yarn.lock

File diff suppressed because it is too large Load Diff