Merge pull request #4428 from element-hq/toger5/component-assets-as-files

Ship the component's assets as files beside the module
This commit is contained in:
Timo
2026-11-10 14:18:30 +01:00
committed by GitHub
10 changed files with 110 additions and 114 deletions
+7
View File
@@ -268,6 +268,13 @@ widget's `theme` URL parameter: `light`, `dark`, `light-high-contrast` or
`dark-high-contrast`. Both can change while a call is running without
disturbing it.
The component's sounds, pictures and the MediaPipe runtime behind background
effects are files beside the module rather than data URLs inside it, each
referenced relative to the module (`new URL("./assets/…", import.meta.url)`),
so a host's bundler copies them as assets of its own — webpack 5, Vite and
Rollup all do — and the browser fetches the MediaPipe runtime, the largest of
them by far, only when an effect is first switched on.
A host must call and await `initializeElementCall(config)` once before
rendering the component: it loads the `Intl` polyfills, applies the
deployment-wide `config.json`-style configuration and sets up translations.
+24 -24
View File
@@ -7,30 +7,30 @@ Please see LICENSE in the repository root for full details.
import { type RelationType } from "matrix-js-sdk";
import catSoundOgg from "../sound/reactions/cat.ogg?url";
import catSoundMp3 from "../sound/reactions/cat.mp3?url";
import clapSoundOgg from "../sound/reactions/clap.ogg?url";
import clapSoundMp3 from "../sound/reactions/clap.mp3?url";
import cricketsSoundOgg from "../sound/reactions/crickets.ogg?url";
import cricketsSoundMp3 from "../sound/reactions/crickets.mp3?url";
import dogSoundOgg from "../sound/reactions/dog.ogg?url";
import dogSoundMp3 from "../sound/reactions/dog.mp3?url";
import genericSoundOgg from "../sound/reactions/generic.ogg?url";
import genericSoundMp3 from "../sound/reactions/generic.mp3?url";
import lightbulbSoundOgg from "../sound/reactions/lightbulb.ogg?url";
import lightbulbSoundMp3 from "../sound/reactions/lightbulb.mp3?url";
import partySoundOgg from "../sound/reactions/party.ogg?url";
import partySoundMp3 from "../sound/reactions/party.mp3?url";
import deerSoundOgg from "../sound/reactions/deer.ogg?url";
import deerSoundMp3 from "../sound/reactions/deer.mp3?url";
import rockSoundOgg from "../sound/reactions/rock.ogg?url";
import rockSoundMp3 from "../sound/reactions/rock.mp3?url";
import waveSoundOgg from "../sound/reactions/wave.ogg?url";
import waveSoundMp3 from "../sound/reactions/wave.mp3?url";
import baduntssSoundOgg from "../sound/reactions/baduntss.ogg?url";
import baduntssSoundMp3 from "../sound/reactions/baduntss.mp3?url";
import bellSoundOgg from "../sound/reactions/bell.ogg?url";
import bellSoundMp3 from "../sound/reactions/bell.mp3?url";
import catSoundOgg from "../sound/reactions/cat.ogg?url&no-inline";
import catSoundMp3 from "../sound/reactions/cat.mp3?url&no-inline";
import clapSoundOgg from "../sound/reactions/clap.ogg?url&no-inline";
import clapSoundMp3 from "../sound/reactions/clap.mp3?url&no-inline";
import cricketsSoundOgg from "../sound/reactions/crickets.ogg?url&no-inline";
import cricketsSoundMp3 from "../sound/reactions/crickets.mp3?url&no-inline";
import dogSoundOgg from "../sound/reactions/dog.ogg?url&no-inline";
import dogSoundMp3 from "../sound/reactions/dog.mp3?url&no-inline";
import genericSoundOgg from "../sound/reactions/generic.ogg?url&no-inline";
import genericSoundMp3 from "../sound/reactions/generic.mp3?url&no-inline";
import lightbulbSoundOgg from "../sound/reactions/lightbulb.ogg?url&no-inline";
import lightbulbSoundMp3 from "../sound/reactions/lightbulb.mp3?url&no-inline";
import partySoundOgg from "../sound/reactions/party.ogg?url&no-inline";
import partySoundMp3 from "../sound/reactions/party.mp3?url&no-inline";
import deerSoundOgg from "../sound/reactions/deer.ogg?url&no-inline";
import deerSoundMp3 from "../sound/reactions/deer.mp3?url&no-inline";
import rockSoundOgg from "../sound/reactions/rock.ogg?url&no-inline";
import rockSoundMp3 from "../sound/reactions/rock.mp3?url&no-inline";
import waveSoundOgg from "../sound/reactions/wave.ogg?url&no-inline";
import waveSoundMp3 from "../sound/reactions/wave.mp3?url&no-inline";
import baduntssSoundOgg from "../sound/reactions/baduntss.ogg?url&no-inline";
import baduntssSoundMp3 from "../sound/reactions/baduntss.mp3?url&no-inline";
import bellSoundOgg from "../sound/reactions/bell.ogg?url&no-inline";
import bellSoundMp3 from "../sound/reactions/bell.mp3?url&no-inline";
export const ElementCallReactionEventType = "io.element.call.reaction";
+12 -12
View File
@@ -8,18 +8,18 @@ Please see LICENSE in the repository root for full details.
import { type ReactNode, useEffect } from "react";
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
import joinCallSoundMp3 from "../sound/join_call.mp3";
import joinCallSoundOgg from "../sound/join_call.ogg";
import leftCallSoundMp3 from "../sound/left_call.mp3";
import leftCallSoundOgg from "../sound/left_call.ogg";
import handSoundOgg from "../sound/raise_hand.ogg";
import handSoundMp3 from "../sound/raise_hand.mp3";
import screenShareStartedOgg from "../sound/screen_share_started.ogg";
import screenShareStartedMp3 from "../sound/screen_share_started.mp3";
import declineMp3 from "../sound/call_declined.mp3?url";
import declineOgg from "../sound/call_declined.ogg?url";
import timeoutMp3 from "../sound/call_timeout.mp3?url";
import timeoutOgg from "../sound/call_timeout.ogg?url";
import joinCallSoundMp3 from "../sound/join_call.mp3?url&no-inline";
import joinCallSoundOgg from "../sound/join_call.ogg?url&no-inline";
import leftCallSoundMp3 from "../sound/left_call.mp3?url&no-inline";
import leftCallSoundOgg from "../sound/left_call.ogg?url&no-inline";
import handSoundOgg from "../sound/raise_hand.ogg?url&no-inline";
import handSoundMp3 from "../sound/raise_hand.mp3?url&no-inline";
import screenShareStartedOgg from "../sound/screen_share_started.ogg?url&no-inline";
import screenShareStartedMp3 from "../sound/screen_share_started.mp3?url&no-inline";
import declineMp3 from "../sound/call_declined.mp3?url&no-inline";
import declineOgg from "../sound/call_declined.ogg?url&no-inline";
import timeoutMp3 from "../sound/call_timeout.mp3?url&no-inline";
import timeoutOgg from "../sound/call_timeout.ogg?url&no-inline";
import { useAudioContext } from "../useAudioContext";
import { prefetchSounds } from "../soundUtils";
import { useLatest } from "../useLatest";
+2 -2
View File
@@ -12,8 +12,8 @@ import { type RingingMediaViewModel } from "../state/media/RingingMediaViewModel
import { useBehavior } from "../useBehavior";
import { useInitial } from "../useInitial";
import { prefetchSounds } from "../soundUtils";
import ringtoneMp3 from "../sound/ringtone.mp3?url";
import ringtoneOgg from "../sound/ringtone.ogg?url";
import ringtoneMp3 from "../sound/ringtone.mp3?url&no-inline";
import ringtoneOgg from "../sound/ringtone.ogg?url&no-inline";
import { type UseAudioContext, useAudioContext } from "../useAudioContext";
import { useLatest } from "../useLatest";
+7 -26
View File
@@ -34,23 +34,6 @@ describe("prefetchSounds", () => {
expect(bytesOf(sounds.beep)).toEqual([1, 2, 3]);
});
it("decodes base64 data: URLs without fetching them", async () => {
// "OggS" — the bytes 0x4f 0x67 0x67 0x53
const sounds = await prefetchSounds({
beep: { ogg: "data:audio/ogg;base64,T2dnUw==" },
});
expect(fetchSpy).not.toHaveBeenCalled();
expect(bytesOf(sounds.beep)).toEqual([0x4f, 0x67, 0x67, 0x53]);
});
it("leaves out a sound with a percent-encoded data: URL", async () => {
const sounds = await prefetchSounds({
beep: { ogg: "data:audio/ogg,Ogg%53" },
});
expect(fetchSpy).not.toHaveBeenCalled();
expect(sounds).toEqual({});
});
it("leaves out a sound whose response is not okay", async () => {
fetchSpy.mockResolvedValue(new Response(null, { status: 404 }));
const sounds = await prefetchSounds({
@@ -60,18 +43,16 @@ describe("prefetchSounds", () => {
});
it("leaves out a sound that fails to load, keeping the others", async () => {
fetchSpy.mockRejectedValue(new TypeError("Failed to fetch"));
// Fetched in the order given
fetchSpy
.mockRejectedValueOnce(new TypeError("Failed to fetch"))
.mockResolvedValueOnce(
new Response(new Uint8Array([1, 2, 3]), { status: 200 }),
);
const sounds = await prefetchSounds({
beep: { ogg: "https://example.org/beep.ogg" },
boop: { ogg: "data:audio/ogg;base64,T2dnUw==" },
boop: { ogg: "https://example.org/boop.ogg" },
});
expect(Object.keys(sounds)).toEqual(["boop"]);
});
it("leaves out a sound with a malformed data: URL", async () => {
const sounds = await prefetchSounds({
beep: { ogg: "data:audio/ogg;base64" },
});
expect(sounds).toEqual({});
});
});
+1 -27
View File
@@ -32,38 +32,12 @@ function getPreferredAudioFormat(): "ogg" | "mp3" {
const preferredFormat = getPreferredAudioFormat();
/**
* Decode a `data:` URL into the bytes it carries.
*
* The component build inlines the sound files as `data:` URLs. Those could be
* `fetch`ed like any other URL, but that fetch is subject to the host page's
* Content Security Policy, and `connect-src` lists rarely allow `data:` (a `*`
* source does not cover it). Reading the bytes out of the URL directly needs no
* network access, so no policy can stop it.
* @param url The `data:` URL to decode.
* @returns The decoded bytes.
*/
function decodeDataUrl(url: string): ArrayBuffer {
const separator = url.indexOf(",");
if (!url.startsWith("data:") || separator === -1)
throw new Error("Not a valid data: URL");
const header = url.slice("data:".length, separator);
// Vite only percent-encodes SVGs; every other inlined asset is base64.
if (!header.split(";").includes("base64"))
throw new Error("Only base64 data: URLs are supported");
// `decoded` is a binary string: one char per byte
const decoded = atob(url.slice(separator + 1));
const bytes = Uint8Array.from(decoded, (c) => c.charCodeAt(0));
return bytes.buffer;
}
/**
* Load the bytes of a single sound file.
* @param url Where the sound is: an ordinary URL to fetch, or a `data:` URL.
* @param url Where the sound is.
* @returns The bytes, or null if the sound could not be loaded.
*/
async function loadSound(url: string): Promise<ArrayBuffer | null> {
if (url.startsWith("data:")) return decodeDataUrl(url);
const response = await fetch(url);
if (isFailure(response)) return null;
return await response.arrayBuffer();
@@ -47,6 +47,24 @@ describe("BackgroundEffectTransformer", () => {
expect(load).toHaveBeenCalledWith("/background.jpg");
});
it("runs MediaPipe from its own bundle", async () => {
vi.spyOn(VideoTransformer.prototype, "init").mockResolvedValue();
const create = vi
.spyOn(ImageSegmenter, "createFromOptions")
.mockResolvedValue({} as ImageSegmenter);
await new BackgroundEffectTransformer({ backgroundDisabled: true }).init(
{} as VideoTransformerInitOptions,
);
const [fileset, options] = create.mock.calls[0];
expect(fileset.wasmLoaderPath).toMatch(/vision_wasm_internal\.js/);
expect(fileset.wasmBinaryPath).toMatch(/vision_wasm_internal\.wasm/);
expect(options.baseOptions?.modelAssetPath).toMatch(
/selfie_segmenter\.tflite/,
);
});
it("reports the first frame carrying an effect, once", async () => {
vi.spyOn(BackgroundTransformer.prototype, "transform").mockImplementation(
async function (this: BackgroundTransformer) {
+8 -21
View File
@@ -16,7 +16,13 @@ import {
import { logger } from "matrix-js-sdk/lib/logger";
import { ImageSegmenter } from "@mediapipe/tasks-vision";
import modelAssetPath from "../mediapipe/imageSegmenter/selfie_segmenter.tflite?url";
// Only the SIMD build of MediaPipe's runtime: every browser Element Call
// supports has WebAssembly SIMD. As files rather than data URLs, so that a
// host's bundler serves them as assets of its own, and a browser fetches them
// only when an effect is first switched on.
import wasmLoaderPath from "@mediapipe/tasks-vision/vision_wasm_internal.js?url&no-inline";
import wasmBinaryPath from "@mediapipe/tasks-vision/vision_wasm_internal.wasm?url&no-inline";
import modelAssetPath from "../mediapipe/imageSegmenter/selfie_segmenter.tflite?url&no-inline";
interface WasmFileset {
/** The path to the Wasm loader script. */
@@ -25,26 +31,7 @@ interface WasmFileset {
wasmBinaryPath: string;
}
// The MediaPipe package, by default, ships some alternative versions of the
// WASM files which avoid SIMD for compatibility with older browsers. But SIMD
// in WASM is actually fine by our support policy, so we include just the SIMD
// versions.
// It's really not ideal that we have to reference these internal files from
// MediaPipe and depend on node_modules having this specific structure. It's
// easy to see this breaking if our dependencies changed and MediaPipe were
// no longer hoisted, or if we switched to another dependency loader such as
// yarn PnP.
// https://github.com/google-ai-edge/mediapipe/issues/5961
const wasmFileset: WasmFileset = {
wasmLoaderPath: new URL(
"../../node_modules/@mediapipe/tasks-vision/wasm/vision_wasm_internal.js",
import.meta.url,
).href,
wasmBinaryPath: new URL(
"../../node_modules/@mediapipe/tasks-vision/wasm/vision_wasm_internal.wasm",
import.meta.url,
).href,
};
const wasmFileset: WasmFileset = { wasmLoaderPath, wasmBinaryPath };
/**
* Track processor that applies effects such as blurring to a user's background.
+2 -2
View File
@@ -5,8 +5,8 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import arc from "../graphics/background-arc.jpg?url";
import glow from "../graphics/background-glow.jpg?url";
import arc from "../graphics/background-arc.jpg?url&no-inline";
import glow from "../graphics/background-glow.jpg?url&no-inline";
export interface ShippedBackground {
/** Stored in the user's settings when chosen, so it must not change. */
+29
View File
@@ -6,6 +6,7 @@ Please see LICENSE in the repository root for full details.
*/
import { defineConfig } from "vite";
import { posix } from "node:path";
import { vitePluginsConfig } from "./vite.config.ts";
import { scopeStylesToRoot } from "./component/build/scopeStylesToRoot.ts";
@@ -34,6 +35,25 @@ export default defineConfig(({ mode }) => {
// whatever is in `public` — including the developer's own config.json, which
// is not in the repository — into the output we would publish.
publicDir: false,
// A library does not know where it will be served from, so every file it
// emits beside itself — the sounds, the pictures, MediaPipe's runtime — is
// referenced relative to the module's own URL. (A library build inlines
// each asset as a data URL unless its import says `no-inline`, so the
// imports do.)
base: "./",
experimental: {
// Relative to the chunk that references it, wherever that chunk lands,
// and spelled `./assets/…` rather than `assets/…`: a host's bundler reads
// a bare name as a package to look up.
renderBuiltUrl: (filename, { hostId, hostType }) => {
if (hostType !== "js") return { relative: true };
const relative = posix.relative(posix.dirname(hostId), filename);
const specifier = relative.startsWith(".") ? relative : `./${relative}`;
return {
runtime: `new URL(${JSON.stringify(specifier)}, import.meta.url).href`,
};
},
},
// A host's document is not ours to style: everything in the stylesheet is
// confined to the element Element Call is mounted in
css: { postcss: { plugins: [scopeStylesToRoot()] } },
@@ -64,6 +84,15 @@ export default defineConfig(({ mode }) => {
cssFileName: "element-call",
},
rollupOptions: {
output: {
// Hashed, as the app's are, so that a file a host serves as it is
// stays correct across versions of the package. Not the stylesheet,
// which the package exports by name (see `cssFileName`).
assetFileNames: ({ names }) =>
names.some((name) => name.endsWith(".css"))
? "[name][extname]"
: "assets/[name]-[hash][extname]",
},
// The host already has these, and a second copy of any of them does not
// merely bloat the bundle: React would hold two sets of hooks, the
// Matrix client would run two sync loops, and a second Compound would