Compare commits

...
Author SHA1 Message Date
Timo K. 376dff02eb fix Config.get before ready 2026-11-13 13:55:27 +01:00
Timo K.andClaude Fable 5.1 07e1fa56d5 Let a deployment serve MediaPipe from its own URL
Background effects run on MediaPipe's 11 MB runtime, which the
component ships as a file beside its module. A deployment that would
rather have it on a CDN, or served with cache headers of its own, can
now point `background_effects.mediapipe_wasm_url` at a directory
holding the `wasm` files of `@mediapipe/tasks-vision` at the version
Element Call depends on; the transformer then resolves the runtime
through MediaPipe's fileset resolver instead of the bundled copy.
Unset, nothing changes.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-11-13 13:55:19 +01:00
renovate[bot] e9c1e25542 Update ghcr.io/element-hq/element-web:develop Docker digest to 2901117 (#4453)
Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
2026-11-13 09:06:50 +00:00
Timo f88d8d5830 Merge pull request #4456 from element-hq/toger5/getRTCTrasnport_and_getDomain
Use non deprecated non unstable names for getRTCTransports.
2026-11-12 16:12:02 +01:00
Timo K. 6c66f9a3e7 use non deprecated non unstable names for getRTCTransports.
Also remove getDomain where possible
2026-11-12 16:01:11 +01:00
13 changed files with 111 additions and 49 deletions
+4 -1
View File
@@ -271,7 +271,10 @@ API is in `component/index.tsx`, `component/api.ts` and `component/host.ts`.
showing its own post-call and error screens.
- Sounds, images and the MediaPipe runtime are separate files the host's
bundler copies (webpack 5, Vite, Rollup); MediaPipe is fetched only when a
background effect is first used.
background effect is first used. To serve MediaPipe yourself (from a CDN,
say), point `background_effects.mediapipe_wasm_url` in the config at a
directory holding the `wasm` files of `@mediapipe/tasks-vision`, at the
version Element Call depends on.
##### Develop
+2 -2
View File
@@ -19,10 +19,10 @@ services:
- ./backend/playwright_homeserver-othersite.yaml:/data/cfg/homeserver.yaml:Z
element-web:
# Pin to a SHA so that upstream cannot break our tests. Renovate handles regular updates.
image: ghcr.io/element-hq/element-web:develop@sha256:72791f0b2541208f8f7edd9c6b5a7eaf88accd8ed1f0c4db1bc62bfe18eb90fe
image: ghcr.io/element-hq/element-web:develop@sha256:fd1c291a228374aff127ceb6eea3c20291b233623bbbd96dcae53eca679fd9d7
element-web-1:
# Pin to a SHA so that upstream cannot break our tests. Renovate handles regular updates.
image: ghcr.io/element-hq/element-web:develop@sha256:72791f0b2541208f8f7edd9c6b5a7eaf88accd8ed1f0c4db1bc62bfe18eb90fe
image: ghcr.io/element-hq/element-web:develop@sha256:fd1c291a228374aff127ceb6eea3c20291b233623bbbd96dcae53eca679fd9d7
volumes:
# The other site opens the embedded package under test in place of the
# copy its image bundles (the first site opens the dev server's app.
+1 -1
View File
@@ -107,7 +107,7 @@
"livekit-client": "^2.18.1",
"lodash-es": "^4.17.21",
"loglevel": "^1.9.1",
"matrix-js-sdk": "github:matrix-org/matrix-js-sdk#dc8610e796c92553c658300a4882dde961a0678a",
"matrix-js-sdk": "github:matrix-org/matrix-js-sdk#10e4faabbb49baca655492d523b58faa9eea04b9",
"matrix-widget-api": "^1.18.0",
"node-stdlib-browser": "^1.3.1",
"normalize.css": "^8.0.1",
+10 -10
View File
@@ -191,8 +191,8 @@ importers:
specifier: ^1.9.1
version: 1.9.2
matrix-js-sdk:
specifier: github:matrix-org/matrix-js-sdk#dc8610e796c92553c658300a4882dde961a0678a
version: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/dc8610e796c92553c658300a4882dde961a0678a
specifier: github:matrix-org/matrix-js-sdk#10e4faabbb49baca655492d523b58faa9eea04b9
version: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/10e4faabbb49baca655492d523b58faa9eea04b9
matrix-widget-api:
specifier: ^1.18.0
version: 1.20.0
@@ -1195,8 +1195,8 @@ packages:
'@types/dom-mediacapture-transform': ^0.1.9
livekit-client: ^1.12.0 || ^2.1.0
'@matrix-org/matrix-sdk-crypto-wasm@18.9.0':
resolution: {integrity: sha512-NtLzNDTEfNASBjNCTGZHuk7N0jrQhS6Zx2CM857rCHZYYh/rpbKGQNrJ7azktm4S1P5Jy3WU3nwJTGohGw26qQ==}
'@matrix-org/matrix-sdk-crypto-wasm@18.9.2':
resolution: {integrity: sha512-GHeuSeb5Fj6Wg2+NE/WoLJJs9U3nOzttO+2sj7lOaS5gNBpGIksc48N/Q/kK/Z+Lxh7YMhopB7M8b6HyASXnGQ==}
engines: {node: '>= 18'}
'@mdx-js/react@3.1.1':
@@ -4734,9 +4734,9 @@ packages:
matrix-events-sdk@0.0.1:
resolution: {integrity: sha512-1QEOsXO+bhyCroIe2/A5OwaxHvBm7EsSQ46DEDn8RBIfQwN5HWBpFvyWWR4QY0KHPPnnJdI99wgRiAl7Ad5qaA==}
matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/dc8610e796c92553c658300a4882dde961a0678a:
resolution: {gitHosted: true, integrity: sha512-Ldt/VkCzBtre6m1cn0FqgrzRV0GrgWIoARgQUQO58ef2Edu2ERFCINPzp/psFS5RwoN3RIy9Rm4oPVc1ATkEWA==, tarball: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/dc8610e796c92553c658300a4882dde961a0678a}
version: 45.0.0
matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/10e4faabbb49baca655492d523b58faa9eea04b9:
resolution: {gitHosted: true, integrity: sha512-7cbPN96wGt93B73jZieW2HBtTnasgKJ7wzTgiV0SjwkNyb50EK3vDa17LdSJpyOwEjFP5edyPRprk865HQJqww==, tarball: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/10e4faabbb49baca655492d523b58faa9eea04b9}
version: 45.1.0
engines: {node: '>=22.0.0'}
matrix-widget-api@1.20.0:
@@ -7178,7 +7178,7 @@ snapshots:
'@types/dom-mediacapture-transform': 0.1.11
livekit-client: 2.23.0(@types/dom-mediacapture-record@1.0.22)
'@matrix-org/matrix-sdk-crypto-wasm@18.9.0': {}
'@matrix-org/matrix-sdk-crypto-wasm@18.9.2': {}
'@mdx-js/react@3.1.1(@types/react@19.2.18)(react@19.3.1)':
dependencies:
@@ -10413,10 +10413,10 @@ snapshots:
matrix-events-sdk@0.0.1: {}
matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/dc8610e796c92553c658300a4882dde961a0678a:
matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/10e4faabbb49baca655492d523b58faa9eea04b9:
dependencies:
'@babel/runtime': 8.0.8
'@matrix-org/matrix-sdk-crypto-wasm': 18.9.0
'@matrix-org/matrix-sdk-crypto-wasm': 18.9.2
another-json: 0.2.0
bs58: 6.0.0
content-type: 3.1.1
+1 -1
View File
@@ -140,7 +140,7 @@ function setup(
getDeviceId: () => deviceId,
getDomain: () => "example.org",
getSyncState: () => SyncState.Syncing,
_unstable_getRTCTransports: vi.fn().mockResolvedValue(homeserverTransports),
getRTCTransports: vi.fn().mockResolvedValue(homeserverTransports),
_unstable_getLivekitToken: vi.fn().mockResolvedValue({ jwt: publisherJWT }),
} as unknown as MatrixClient;
const members$ = new BehaviorSubject<RTCMember[]>([]);
+2 -2
View File
@@ -31,7 +31,7 @@ function client(
): Parameters<typeof discoverLocalTransport>[0] {
return {
getDomain: () => "example.org",
_unstable_getRTCTransports: vi.fn().mockResolvedValue(advertised),
getRTCTransports: vi.fn().mockResolvedValue(advertised),
};
}
@@ -54,7 +54,7 @@ describe("discoverLocalTransport", () => {
logger,
),
).resolves.toBe(hostTransport);
expect(c._unstable_getRTCTransports).not.toHaveBeenCalled();
expect(c.getRTCTransports).not.toHaveBeenCalled();
});
it("prefers the homeserver's transport over the host's fallback", async () => {
+1 -1
View File
@@ -22,7 +22,7 @@ import { TransportAutoDiscovery } from "./TransportAutoDiscovery";
* @throws {MatrixRTCTransportMissingError} where none of these exists.
*/
export async function discoverLocalTransport(
client: Pick<MatrixClient, "getDomain" | "_unstable_getRTCTransports">,
client: Pick<MatrixClient, "getDomain" | "getRTCTransports">,
transportTypes: readonly string[],
{
transport: override,
@@ -36,11 +36,7 @@ const configTransport: Transport = {
function makeClient(): MockedObject<DiscoveryClient> {
return {
getDomain: vi.fn().mockReturnValue("example.org"),
baseUrl: "https://matrix.example.org",
_unstable_getRTCTransports: vi.fn().mockResolvedValue([]),
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
getRTCTransports: vi.fn().mockResolvedValue([]),
} as unknown as MockedObject<DiscoveryClient>;
}
@@ -57,7 +53,7 @@ describe("TransportAutoDiscovery", () => {
"prefers backend transport other app config $transports",
async ({ transports }) => {
const client = makeClient();
client._unstable_getRTCTransports.mockResolvedValue(transports);
client.getRTCTransports.mockResolvedValue(transports);
const discovery = new TransportAutoDiscovery({
client,
@@ -71,7 +67,7 @@ describe("TransportAutoDiscovery", () => {
expect(discoveredTransport).toStrictEqual(backendTransport);
expect(discoveredTransport).not.toStrictEqual(configTransport);
expect(client._unstable_getRTCTransports).toHaveBeenCalledTimes(1);
expect(client.getRTCTransports).toHaveBeenCalledTimes(1);
},
);
@@ -81,10 +77,7 @@ describe("TransportAutoDiscovery", () => {
url: "wss://sfu.example.org",
};
const client = makeClient();
client._unstable_getRTCTransports.mockResolvedValue([
backendTransport,
mLivekit,
]);
client.getRTCTransports.mockResolvedValue([backendTransport, mLivekit]);
const discovery = new TransportAutoDiscovery({
client,
@@ -100,7 +93,7 @@ describe("TransportAutoDiscovery", () => {
it("Retries limit_exceeded backend transport", async () => {
const client = makeClient();
client._unstable_getRTCTransports
client.getRTCTransports
.mockRejectedValueOnce(
new MatrixError(
{
@@ -124,7 +117,7 @@ describe("TransportAutoDiscovery", () => {
backendTransport,
);
expect(client._unstable_getRTCTransports).toHaveBeenCalledTimes(2);
expect(client.getRTCTransports).toHaveBeenCalledTimes(2);
});
const INVALID_TEST_CASES: Array<{ transports: Transport[] }> = [
@@ -135,7 +128,7 @@ describe("TransportAutoDiscovery", () => {
"falls back to config when backend has no (valid) livekit transports $transports",
async ({ transports }) => {
const client = makeClient();
client._unstable_getRTCTransports.mockResolvedValue(transports);
client.getRTCTransports.mockResolvedValue(transports);
const discovery = new TransportAutoDiscovery({
client,
@@ -152,7 +145,7 @@ describe("TransportAutoDiscovery", () => {
it("returns null when backend and config are all unavailable", async () => {
const client = makeClient();
client._unstable_getRTCTransports.mockResolvedValue([]);
client.getRTCTransports.mockResolvedValue([]);
const discovery = new TransportAutoDiscovery({
client,
+2 -5
View File
@@ -10,10 +10,7 @@ import { type Logger } from "matrix-js-sdk/lib/logger";
import { doNetworkOperationWithRetry } from "../utils/network";
type TransportDiscoveryClient = Pick<
MatrixClient,
"getDomain" | "_unstable_getRTCTransports"
>;
type TransportDiscoveryClient = Pick<MatrixClient, "getRTCTransports">;
export interface TransportAutoDiscoveryProps {
client: TransportDiscoveryClient;
@@ -74,7 +71,7 @@ export class TransportAutoDiscovery {
this.logger.info("First try to use getRTCTransports end point ...");
try {
const transportList = await doNetworkOperationWithRetry(async () =>
client._unstable_getRTCTransports(),
client.getRTCTransports(),
);
for (const type of this.transportTypes) {
const first = transportList.find(
+11
View File
@@ -181,6 +181,17 @@ export interface ConfigOptions {
enable_video?: boolean;
};
background_effects?: {
/**
* Where to fetch the MediaPipe runtime behind background effects from
* instead of the copy Element Call ships: the URL of a directory holding
* `vision_wasm_internal.js` and `vision_wasm_internal.wasm`, as the `wasm`
* directory of the `@mediapipe/tasks-vision` package ships them, at the
* version Element Call depends on.
*/
mediapipe_wasm_url?: string;
};
/**
* Grace period in milliseconds to wait before reporting the sync loop as disconnected.
* This allows brief sync interruptions without triggering a reconnection message.
+31 -1
View File
@@ -11,7 +11,7 @@ import {
VideoTransformer,
type VideoTransformerInitOptions,
} from "@livekit/track-processors";
import { ImageSegmenter } from "@mediapipe/tasks-vision";
import { FilesetResolver, ImageSegmenter } from "@mediapipe/tasks-vision";
import {
BackgroundEffectTransformer,
@@ -53,11 +53,13 @@ describe("BackgroundEffectTransformer", () => {
const create = vi
.spyOn(ImageSegmenter, "createFromOptions")
.mockResolvedValue({} as ImageSegmenter);
const resolve = vi.spyOn(FilesetResolver, "forVisionTasks");
await new BackgroundEffectTransformer({ backgroundDisabled: true }).init(
{} as VideoTransformerInitOptions,
);
expect(resolve).not.toHaveBeenCalled();
const [fileset, options] = create.mock.calls[0];
expect(fileset.wasmLoaderPath).toMatch(/vision_wasm_internal\.js/);
expect(fileset.wasmBinaryPath).toMatch(/vision_wasm_internal\.wasm/);
@@ -66,6 +68,34 @@ describe("BackgroundEffectTransformer", () => {
);
});
it("runs MediaPipe from where the deployment serves it, if it does", async () => {
vi.spyOn(VideoTransformer.prototype, "init").mockResolvedValue();
const create = vi
.spyOn(ImageSegmenter, "createFromOptions")
.mockResolvedValue({} as ImageSegmenter);
const served = {
wasmLoaderPath:
"https://cdn.example.org/mediapipe/vision_wasm_internal.js",
wasmBinaryPath:
"https://cdn.example.org/mediapipe/vision_wasm_internal.wasm",
};
const resolve = vi
.spyOn(FilesetResolver, "forVisionTasks")
.mockResolvedValue(served);
const wasmBaseUrl = vi.fn(() => "https://cdn.example.org/mediapipe");
const transformer = new BackgroundEffectTransformer(
{ backgroundDisabled: true },
{ wasmBaseUrl },
);
expect(wasmBaseUrl).not.toHaveBeenCalled();
await transformer.init({} as VideoTransformerInitOptions);
expect(resolve).toHaveBeenCalledWith("https://cdn.example.org/mediapipe");
expect(create.mock.calls[0][0]).toBe(served);
});
it("reports the first frame carrying an effect, once", async () => {
// The library reports every frame it draws.
vi.spyOn(BackgroundTransformer.prototype, "transform").mockImplementation(
+28 -6
View File
@@ -14,7 +14,7 @@ import {
type SegmenterOptions,
} from "@livekit/track-processors";
import { logger } from "matrix-js-sdk/lib/logger";
import { ImageSegmenter } from "@mediapipe/tasks-vision";
import { FilesetResolver, ImageSegmenter } from "@mediapipe/tasks-vision";
// Only the SIMD build of MediaPipe's runtime: every browser Element Call
// supports has WebAssembly SIMD. As files rather than data URLs, so that a
@@ -31,7 +31,17 @@ interface WasmFileset {
wasmBinaryPath: string;
}
const wasmFileset: WasmFileset = { wasmLoaderPath, wasmBinaryPath };
const bundledFileset: WasmFileset = { wasmLoaderPath, wasmBinaryPath };
export interface BackgroundEffectTransformerSources {
/**
* Where to fetch MediaPipe's runtime from instead of the bundled copy: a
* directory holding its `vision_wasm_internal.js` and `.wasm` files.
* Consulted only when the runtime is about to load, so a transformer can be
* built before the deployment's config has arrived.
*/
wasmBaseUrl?: () => string | undefined;
}
/**
* Track processor that applies effects such as blurring to a user's background.
@@ -48,7 +58,10 @@ export class BackgroundEffectTransformer extends BackgroundTransformer {
public onFirstFrame: (() => void) | undefined;
private drewAFrame = false;
public constructor(opts: BackgroundOptions) {
public constructor(
opts: BackgroundOptions,
private readonly sources: BackgroundEffectTransformerSources = {},
) {
super(opts);
// Called by the library for a frame it drew, never for one it passed on
// untouched.
@@ -104,6 +117,7 @@ export class BackgroundEffectTransformer extends BackgroundTransformer {
});
this.imageSegmenter = await createSegmenter(
this.sources,
this.canvas,
this.options.segmenterOptions,
);
@@ -125,9 +139,11 @@ export class BackgroundEffectTransformer extends BackgroundTransformer {
* Whether a segmenter can be built here, asked once before any camera depends
* on one: a pipeline that fails to build takes the camera down with it.
*/
export async function canSegment(): Promise<boolean> {
export async function canSegment(
sources: BackgroundEffectTransformerSources = {},
): Promise<boolean> {
try {
(await createSegmenter(new OffscreenCanvas(1, 1))).close();
(await createSegmenter(sources, new OffscreenCanvas(1, 1))).close();
return true;
} catch (e) {
logger.warn("Background effects cannot run here", e);
@@ -137,11 +153,17 @@ export async function canSegment(): Promise<boolean> {
/** On the GPU, or on the CPU where the GPU refuses, as a blocklisted one does. */
async function createSegmenter(
sources: BackgroundEffectTransformerSources,
canvas: HTMLCanvasElement | OffscreenCanvas | undefined,
segmenterOptions?: SegmenterOptions,
): Promise<ImageSegmenter> {
const wasmBaseUrl = sources.wasmBaseUrl?.();
const fileset =
wasmBaseUrl === undefined
? bundledFileset
: await FilesetResolver.forVisionTasks(wasmBaseUrl);
const build = async (delegate: "GPU" | "CPU"): Promise<ImageSegmenter> =>
ImageSegmenter.createFromOptions(wasmFileset, {
ImageSegmenter.createFromOptions(fileset, {
baseOptions: { modelAssetPath, delegate, ...segmenterOptions },
canvas,
runningMode: "VIDEO",
+10 -4
View File
@@ -12,6 +12,7 @@ import {
import { createContext, type FC, type JSX, use } from "react";
import { backgroundEffect as backgroundEffectSetting } from "../settings/settings";
import { Config } from "../config/Config";
import {
BackgroundEffectTransformer,
canSegment,
@@ -51,15 +52,20 @@ export function useBackgroundEffectsState$(): Behavior<BackgroundEffectsState> {
export function createAppBackgroundEffects(
scope: ObservableScope,
): BackgroundEffects {
const transformer = new BackgroundEffectTransformer({
backgroundDisabled: true,
});
const sources = {
wasmBaseUrl: (): string | undefined =>
Config.get().background_effects?.mediapipe_wasm_url,
};
const transformer = new BackgroundEffectTransformer(
{ backgroundDisabled: true },
sources,
);
return createBackgroundEffects(scope, {
supported: supportsBackgroundProcessors(),
effect$: backgroundEffectSetting.value$,
setEffect: backgroundEffectSetting.setValue,
added$: addedBackgrounds.added$,
canSegment,
canSegment: async () => canSegment(sources),
pipeline: new BackgroundProcessorWrapper(transformer, "background-effect"),
transformer,
});