From 797a86815bca87150b2893fb853624c2a8037748 Mon Sep 17 00:00:00 2001 From: "Timo K." Date: Fri, 6 Nov 2026 14:34:14 +0100 Subject: [PATCH 1/2] Ship the component's assets as files beside the module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The component module was 18 MB, of which 15 MB was MediaPipe's runtime as a base64 data URL, with the segmentation model, every sound and the background pictures inlined the same way: a library build inlines every asset unless its import says `no-inline`. Every host downloaded all of it before the first call, whether or not a background effect was ever switched on, and a host with a Content Security Policy without `script-src data:` could not run effects at all, since MediaPipe injects its loader as a script tag. The imports now say `no-inline`, and the build renders each reference as `new URL("./assets/…", import.meta.url)`, which webpack 5, Vite and Rollup follow as an asset of the host's own; the `./` is spelled out because a bare `assets/…` is read as a package name. The module is 2.3 MB. The runtime is fetched when an effect is first switched on and cached on its own. MediaPipe's files come through the package's `exports` rather than a path into node_modules. MediaPipe's JavaScript (160 kB) stays in the module: LiveKit's track processors import it statically, so a dynamic import moves nothing. Co-Authored-By: Claude Fable 5.1 --- README.md | 7 +++ src/reactions/index.ts | 48 +++++++++---------- src/room/CallEventAudioRenderer.tsx | 24 +++++----- src/room/RingingAudioRenderer.tsx | 4 +- .../BackgroundEffectTransformer.test.ts | 18 +++++++ src/tracks/BackgroundEffectTransformer.ts | 29 ++++------- src/tracks/backgroundEffectCatalog.ts | 4 +- vite-component.config.ts | 25 ++++++++++ 8 files changed, 98 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index 17eae35d9..ebc26f936 100644 --- a/README.md +++ b/README.md @@ -268,6 +268,13 @@ widget's `theme` URL parameter: `light`, `dark`, `light-high-contrast` or `dark-high-contrast`. Both can change while a call is running without disturbing it. +The component's sounds, pictures and the MediaPipe runtime behind background +effects are files beside the module rather than data URLs inside it, each +referenced relative to the module (`new URL("./assets/…", import.meta.url)`), +so a host's bundler copies them as assets of its own — webpack 5, Vite and +Rollup all do — and the browser fetches the MediaPipe runtime, the largest of +them by far, only when an effect is first switched on. + A host must call and await `initializeElementCall(config)` once before rendering the component: it loads the `Intl` polyfills, applies the deployment-wide `config.json`-style configuration and sets up translations. diff --git a/src/reactions/index.ts b/src/reactions/index.ts index 26ca7f142..ffc14d43e 100644 --- a/src/reactions/index.ts +++ b/src/reactions/index.ts @@ -7,30 +7,30 @@ Please see LICENSE in the repository root for full details. import { type RelationType } from "matrix-js-sdk"; -import catSoundOgg from "../sound/reactions/cat.ogg?url"; -import catSoundMp3 from "../sound/reactions/cat.mp3?url"; -import clapSoundOgg from "../sound/reactions/clap.ogg?url"; -import clapSoundMp3 from "../sound/reactions/clap.mp3?url"; -import cricketsSoundOgg from "../sound/reactions/crickets.ogg?url"; -import cricketsSoundMp3 from "../sound/reactions/crickets.mp3?url"; -import dogSoundOgg from "../sound/reactions/dog.ogg?url"; -import dogSoundMp3 from "../sound/reactions/dog.mp3?url"; -import genericSoundOgg from "../sound/reactions/generic.ogg?url"; -import genericSoundMp3 from "../sound/reactions/generic.mp3?url"; -import lightbulbSoundOgg from "../sound/reactions/lightbulb.ogg?url"; -import lightbulbSoundMp3 from "../sound/reactions/lightbulb.mp3?url"; -import partySoundOgg from "../sound/reactions/party.ogg?url"; -import partySoundMp3 from "../sound/reactions/party.mp3?url"; -import deerSoundOgg from "../sound/reactions/deer.ogg?url"; -import deerSoundMp3 from "../sound/reactions/deer.mp3?url"; -import rockSoundOgg from "../sound/reactions/rock.ogg?url"; -import rockSoundMp3 from "../sound/reactions/rock.mp3?url"; -import waveSoundOgg from "../sound/reactions/wave.ogg?url"; -import waveSoundMp3 from "../sound/reactions/wave.mp3?url"; -import baduntssSoundOgg from "../sound/reactions/baduntss.ogg?url"; -import baduntssSoundMp3 from "../sound/reactions/baduntss.mp3?url"; -import bellSoundOgg from "../sound/reactions/bell.ogg?url"; -import bellSoundMp3 from "../sound/reactions/bell.mp3?url"; +import catSoundOgg from "../sound/reactions/cat.ogg?url&no-inline"; +import catSoundMp3 from "../sound/reactions/cat.mp3?url&no-inline"; +import clapSoundOgg from "../sound/reactions/clap.ogg?url&no-inline"; +import clapSoundMp3 from "../sound/reactions/clap.mp3?url&no-inline"; +import cricketsSoundOgg from "../sound/reactions/crickets.ogg?url&no-inline"; +import cricketsSoundMp3 from "../sound/reactions/crickets.mp3?url&no-inline"; +import dogSoundOgg from "../sound/reactions/dog.ogg?url&no-inline"; +import dogSoundMp3 from "../sound/reactions/dog.mp3?url&no-inline"; +import genericSoundOgg from "../sound/reactions/generic.ogg?url&no-inline"; +import genericSoundMp3 from "../sound/reactions/generic.mp3?url&no-inline"; +import lightbulbSoundOgg from "../sound/reactions/lightbulb.ogg?url&no-inline"; +import lightbulbSoundMp3 from "../sound/reactions/lightbulb.mp3?url&no-inline"; +import partySoundOgg from "../sound/reactions/party.ogg?url&no-inline"; +import partySoundMp3 from "../sound/reactions/party.mp3?url&no-inline"; +import deerSoundOgg from "../sound/reactions/deer.ogg?url&no-inline"; +import deerSoundMp3 from "../sound/reactions/deer.mp3?url&no-inline"; +import rockSoundOgg from "../sound/reactions/rock.ogg?url&no-inline"; +import rockSoundMp3 from "../sound/reactions/rock.mp3?url&no-inline"; +import waveSoundOgg from "../sound/reactions/wave.ogg?url&no-inline"; +import waveSoundMp3 from "../sound/reactions/wave.mp3?url&no-inline"; +import baduntssSoundOgg from "../sound/reactions/baduntss.ogg?url&no-inline"; +import baduntssSoundMp3 from "../sound/reactions/baduntss.mp3?url&no-inline"; +import bellSoundOgg from "../sound/reactions/bell.ogg?url&no-inline"; +import bellSoundMp3 from "../sound/reactions/bell.mp3?url&no-inline"; export const ElementCallReactionEventType = "io.element.call.reaction"; diff --git a/src/room/CallEventAudioRenderer.tsx b/src/room/CallEventAudioRenderer.tsx index d33f3b843..fa14bcd77 100644 --- a/src/room/CallEventAudioRenderer.tsx +++ b/src/room/CallEventAudioRenderer.tsx @@ -8,18 +8,18 @@ Please see LICENSE in the repository root for full details. import { type ReactNode, useEffect } from "react"; import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; -import joinCallSoundMp3 from "../sound/join_call.mp3"; -import joinCallSoundOgg from "../sound/join_call.ogg"; -import leftCallSoundMp3 from "../sound/left_call.mp3"; -import leftCallSoundOgg from "../sound/left_call.ogg"; -import handSoundOgg from "../sound/raise_hand.ogg"; -import handSoundMp3 from "../sound/raise_hand.mp3"; -import screenShareStartedOgg from "../sound/screen_share_started.ogg"; -import screenShareStartedMp3 from "../sound/screen_share_started.mp3"; -import declineMp3 from "../sound/call_declined.mp3?url"; -import declineOgg from "../sound/call_declined.ogg?url"; -import timeoutMp3 from "../sound/call_timeout.mp3?url"; -import timeoutOgg from "../sound/call_timeout.ogg?url"; +import joinCallSoundMp3 from "../sound/join_call.mp3?url&no-inline"; +import joinCallSoundOgg from "../sound/join_call.ogg?url&no-inline"; +import leftCallSoundMp3 from "../sound/left_call.mp3?url&no-inline"; +import leftCallSoundOgg from "../sound/left_call.ogg?url&no-inline"; +import handSoundOgg from "../sound/raise_hand.ogg?url&no-inline"; +import handSoundMp3 from "../sound/raise_hand.mp3?url&no-inline"; +import screenShareStartedOgg from "../sound/screen_share_started.ogg?url&no-inline"; +import screenShareStartedMp3 from "../sound/screen_share_started.mp3?url&no-inline"; +import declineMp3 from "../sound/call_declined.mp3?url&no-inline"; +import declineOgg from "../sound/call_declined.ogg?url&no-inline"; +import timeoutMp3 from "../sound/call_timeout.mp3?url&no-inline"; +import timeoutOgg from "../sound/call_timeout.ogg?url&no-inline"; import { useAudioContext } from "../useAudioContext"; import { prefetchSounds } from "../soundUtils"; import { useLatest } from "../useLatest"; diff --git a/src/room/RingingAudioRenderer.tsx b/src/room/RingingAudioRenderer.tsx index c0fe45d5b..f0ebd8386 100644 --- a/src/room/RingingAudioRenderer.tsx +++ b/src/room/RingingAudioRenderer.tsx @@ -12,8 +12,8 @@ import { type RingingMediaViewModel } from "../state/media/RingingMediaViewModel import { useBehavior } from "../useBehavior"; import { useInitial } from "../useInitial"; import { prefetchSounds } from "../soundUtils"; -import ringtoneMp3 from "../sound/ringtone.mp3?url"; -import ringtoneOgg from "../sound/ringtone.ogg?url"; +import ringtoneMp3 from "../sound/ringtone.mp3?url&no-inline"; +import ringtoneOgg from "../sound/ringtone.ogg?url&no-inline"; import { type UseAudioContext, useAudioContext } from "../useAudioContext"; import { useLatest } from "../useLatest"; diff --git a/src/tracks/BackgroundEffectTransformer.test.ts b/src/tracks/BackgroundEffectTransformer.test.ts index 400a50416..4e7ed8947 100644 --- a/src/tracks/BackgroundEffectTransformer.test.ts +++ b/src/tracks/BackgroundEffectTransformer.test.ts @@ -41,6 +41,24 @@ describe("BackgroundEffectTransformer", () => { expect(load).toHaveBeenCalledWith("/background.jpg"); }); + it("runs MediaPipe from its own bundle", async () => { + vi.spyOn(VideoTransformer.prototype, "init").mockResolvedValue(); + const create = vi + .spyOn(ImageSegmenter, "createFromOptions") + .mockResolvedValue({} as ImageSegmenter); + + await new BackgroundEffectTransformer({ backgroundDisabled: true }).init( + {} as VideoTransformerInitOptions, + ); + + const [fileset, options] = create.mock.calls[0]; + expect(fileset.wasmLoaderPath).toMatch(/vision_wasm_internal\.js/); + expect(fileset.wasmBinaryPath).toMatch(/vision_wasm_internal\.wasm/); + expect(options.baseOptions?.modelAssetPath).toMatch( + /selfie_segmenter\.tflite/, + ); + }); + it("reports the first frame carrying an effect, once", async () => { vi.spyOn(BackgroundTransformer.prototype, "transform").mockImplementation( async function (this: BackgroundTransformer) { diff --git a/src/tracks/BackgroundEffectTransformer.ts b/src/tracks/BackgroundEffectTransformer.ts index e3c0a9183..8d7221a6d 100644 --- a/src/tracks/BackgroundEffectTransformer.ts +++ b/src/tracks/BackgroundEffectTransformer.ts @@ -15,7 +15,13 @@ import { import { logger } from "matrix-js-sdk/lib/logger"; import { ImageSegmenter } from "@mediapipe/tasks-vision"; -import modelAssetPath from "../mediapipe/imageSegmenter/selfie_segmenter.tflite?url"; +// Only the SIMD build of MediaPipe's runtime: every browser Element Call +// supports has WebAssembly SIMD. As files rather than data URLs, so that a +// host's bundler serves them as assets of its own, and a browser fetches them +// only when an effect is first switched on. +import wasmLoaderPath from "@mediapipe/tasks-vision/vision_wasm_internal.js?url&no-inline"; +import wasmBinaryPath from "@mediapipe/tasks-vision/vision_wasm_internal.wasm?url&no-inline"; +import modelAssetPath from "../mediapipe/imageSegmenter/selfie_segmenter.tflite?url&no-inline"; interface WasmFileset { /** The path to the Wasm loader script. */ @@ -24,26 +30,7 @@ interface WasmFileset { wasmBinaryPath: string; } -// The MediaPipe package, by default, ships some alternative versions of the -// WASM files which avoid SIMD for compatibility with older browsers. But SIMD -// in WASM is actually fine by our support policy, so we include just the SIMD -// versions. -// It's really not ideal that we have to reference these internal files from -// MediaPipe and depend on node_modules having this specific structure. It's -// easy to see this breaking if our dependencies changed and MediaPipe were -// no longer hoisted, or if we switched to another dependency loader such as -// yarn PnP. -// https://github.com/google-ai-edge/mediapipe/issues/5961 -const wasmFileset: WasmFileset = { - wasmLoaderPath: new URL( - "../../node_modules/@mediapipe/tasks-vision/wasm/vision_wasm_internal.js", - import.meta.url, - ).href, - wasmBinaryPath: new URL( - "../../node_modules/@mediapipe/tasks-vision/wasm/vision_wasm_internal.wasm", - import.meta.url, - ).href, -}; +const wasmFileset: WasmFileset = { wasmLoaderPath, wasmBinaryPath }; /** * Track processor that applies effects such as blurring to a user's background. diff --git a/src/tracks/backgroundEffectCatalog.ts b/src/tracks/backgroundEffectCatalog.ts index 71c9e2d5d..9dd083ce1 100644 --- a/src/tracks/backgroundEffectCatalog.ts +++ b/src/tracks/backgroundEffectCatalog.ts @@ -5,8 +5,8 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import arc from "../graphics/background-arc.jpg?url"; -import glow from "../graphics/background-glow.jpg?url"; +import arc from "../graphics/background-arc.jpg?url&no-inline"; +import glow from "../graphics/background-glow.jpg?url&no-inline"; export interface ShippedBackground { /** Stored in the user's settings when chosen, so it must not change. */ diff --git a/vite-component.config.ts b/vite-component.config.ts index 63183c781..fc0c9ac4a 100644 --- a/vite-component.config.ts +++ b/vite-component.config.ts @@ -34,6 +34,22 @@ export default defineConfig(({ mode }) => { // whatever is in `public` — including the developer's own config.json, which // is not in the repository — into the output we would publish. publicDir: false, + // A library does not know where it will be served from, so every file it + // emits beside itself — the sounds, the pictures, MediaPipe's runtime — is + // referenced relative to the module's own URL. (A library build inlines + // each asset as a data URL unless its import says `no-inline`, so the + // imports do.) + base: "./", + experimental: { + // Spelled `./assets/…`, not `assets/…` as the relative base would have it: + // a host's bundler reads a bare name as a package to look up. + renderBuiltUrl: (filename, { hostType }) => + hostType === "js" + ? { + runtime: `new URL(${JSON.stringify(`./${filename}`)}, import.meta.url).href`, + } + : { relative: true }, + }, // A host's document is not ours to style: everything in the stylesheet is // confined to the element Element Call is mounted in css: { postcss: { plugins: [scopeStylesToRoot()] } }, @@ -64,6 +80,15 @@ export default defineConfig(({ mode }) => { cssFileName: "element-call", }, rollupOptions: { + output: { + // Hashed, as the app's are, so that a file a host serves as it is + // stays correct across versions of the package. Not the stylesheet, + // which the package exports by name (see `cssFileName`). + assetFileNames: ({ names }) => + names.some((name) => name.endsWith(".css")) + ? "[name][extname]" + : "assets/[name]-[hash][extname]", + }, // The host already has these, and a second copy of any of them does not // merely bloat the bundle: React would hold two sets of hooks, the // Matrix client would run two sync loops, and a second Compound would From a5374791e407b0482128fec9f56f8697c411266f Mon Sep 17 00:00:00 2001 From: "Timo K." Date: Sat, 7 Nov 2026 16:06:05 +0100 Subject: [PATCH 2/2] Resolve asset references from the chunk that holds them The runtime path was `./` plus the asset's path from the package root, which is right for every chunk the build emits today, all of which sit at that root, and wrong for one emitted under a directory. Vite names the referencing chunk, so the path is now computed relative to it. The sound loader kept a branch for sounds inlined as data URLs, with a comment saying the component build produces them. No build does any more, and no host would inline a sound of this size, so the branch and its tests go. Co-Authored-By: Claude Fable 5.1 --- src/soundUtils.test.ts | 33 +++++++-------------------------- src/soundUtils.ts | 28 +--------------------------- vite-component.config.ts | 20 ++++++++++++-------- 3 files changed, 20 insertions(+), 61 deletions(-) diff --git a/src/soundUtils.test.ts b/src/soundUtils.test.ts index 3d9810fe0..c6b652f9d 100644 --- a/src/soundUtils.test.ts +++ b/src/soundUtils.test.ts @@ -34,23 +34,6 @@ describe("prefetchSounds", () => { expect(bytesOf(sounds.beep)).toEqual([1, 2, 3]); }); - it("decodes base64 data: URLs without fetching them", async () => { - // "OggS" — the bytes 0x4f 0x67 0x67 0x53 - const sounds = await prefetchSounds({ - beep: { ogg: "data:audio/ogg;base64,T2dnUw==" }, - }); - expect(fetchSpy).not.toHaveBeenCalled(); - expect(bytesOf(sounds.beep)).toEqual([0x4f, 0x67, 0x67, 0x53]); - }); - - it("leaves out a sound with a percent-encoded data: URL", async () => { - const sounds = await prefetchSounds({ - beep: { ogg: "data:audio/ogg,Ogg%53" }, - }); - expect(fetchSpy).not.toHaveBeenCalled(); - expect(sounds).toEqual({}); - }); - it("leaves out a sound whose response is not okay", async () => { fetchSpy.mockResolvedValue(new Response(null, { status: 404 })); const sounds = await prefetchSounds({ @@ -60,18 +43,16 @@ describe("prefetchSounds", () => { }); it("leaves out a sound that fails to load, keeping the others", async () => { - fetchSpy.mockRejectedValue(new TypeError("Failed to fetch")); + // Fetched in the order given + fetchSpy + .mockRejectedValueOnce(new TypeError("Failed to fetch")) + .mockResolvedValueOnce( + new Response(new Uint8Array([1, 2, 3]), { status: 200 }), + ); const sounds = await prefetchSounds({ beep: { ogg: "https://example.org/beep.ogg" }, - boop: { ogg: "data:audio/ogg;base64,T2dnUw==" }, + boop: { ogg: "https://example.org/boop.ogg" }, }); expect(Object.keys(sounds)).toEqual(["boop"]); }); - - it("leaves out a sound with a malformed data: URL", async () => { - const sounds = await prefetchSounds({ - beep: { ogg: "data:audio/ogg;base64" }, - }); - expect(sounds).toEqual({}); - }); }); diff --git a/src/soundUtils.ts b/src/soundUtils.ts index 74d2ea33c..7ace7876e 100644 --- a/src/soundUtils.ts +++ b/src/soundUtils.ts @@ -32,38 +32,12 @@ function getPreferredAudioFormat(): "ogg" | "mp3" { const preferredFormat = getPreferredAudioFormat(); -/** - * Decode a `data:` URL into the bytes it carries. - * - * The component build inlines the sound files as `data:` URLs. Those could be - * `fetch`ed like any other URL, but that fetch is subject to the host page's - * Content Security Policy, and `connect-src` lists rarely allow `data:` (a `*` - * source does not cover it). Reading the bytes out of the URL directly needs no - * network access, so no policy can stop it. - * @param url The `data:` URL to decode. - * @returns The decoded bytes. - */ -function decodeDataUrl(url: string): ArrayBuffer { - const separator = url.indexOf(","); - if (!url.startsWith("data:") || separator === -1) - throw new Error("Not a valid data: URL"); - const header = url.slice("data:".length, separator); - // Vite only percent-encodes SVGs; every other inlined asset is base64. - if (!header.split(";").includes("base64")) - throw new Error("Only base64 data: URLs are supported"); - // `decoded` is a binary string: one char per byte - const decoded = atob(url.slice(separator + 1)); - const bytes = Uint8Array.from(decoded, (c) => c.charCodeAt(0)); - return bytes.buffer; -} - /** * Load the bytes of a single sound file. - * @param url Where the sound is: an ordinary URL to fetch, or a `data:` URL. + * @param url Where the sound is. * @returns The bytes, or null if the sound could not be loaded. */ async function loadSound(url: string): Promise { - if (url.startsWith("data:")) return decodeDataUrl(url); const response = await fetch(url); if (isFailure(response)) return null; return await response.arrayBuffer(); diff --git a/vite-component.config.ts b/vite-component.config.ts index fc0c9ac4a..859d2f5ff 100644 --- a/vite-component.config.ts +++ b/vite-component.config.ts @@ -6,6 +6,7 @@ Please see LICENSE in the repository root for full details. */ import { defineConfig } from "vite"; +import { posix } from "node:path"; import { vitePluginsConfig } from "./vite.config.ts"; import { scopeStylesToRoot } from "./component/build/scopeStylesToRoot.ts"; @@ -41,14 +42,17 @@ export default defineConfig(({ mode }) => { // imports do.) base: "./", experimental: { - // Spelled `./assets/…`, not `assets/…` as the relative base would have it: - // a host's bundler reads a bare name as a package to look up. - renderBuiltUrl: (filename, { hostType }) => - hostType === "js" - ? { - runtime: `new URL(${JSON.stringify(`./${filename}`)}, import.meta.url).href`, - } - : { relative: true }, + // Relative to the chunk that references it, wherever that chunk lands, + // and spelled `./assets/…` rather than `assets/…`: a host's bundler reads + // a bare name as a package to look up. + renderBuiltUrl: (filename, { hostId, hostType }) => { + if (hostType !== "js") return { relative: true }; + const relative = posix.relative(posix.dirname(hostId), filename); + const specifier = relative.startsWith(".") ? relative : `./${relative}`; + return { + runtime: `new URL(${JSON.stringify(specifier)}, import.meta.url).href`, + }; + }, }, // A host's document is not ours to style: everything in the stylesheet is // confined to the element Element Call is mounted in