mirror of
https://github.com/vector-im/element-call.git
synced 2026-11-16 01:22:29 +00:00
Merge pull request #4428 from element-hq/toger5/component-assets-as-files
Ship the component's assets as files beside the module
This commit is contained in:
@@ -268,6 +268,13 @@ widget's `theme` URL parameter: `light`, `dark`, `light-high-contrast` or
|
||||
`dark-high-contrast`. Both can change while a call is running without
|
||||
disturbing it.
|
||||
|
||||
The component's sounds, pictures and the MediaPipe runtime behind background
|
||||
effects are files beside the module rather than data URLs inside it, each
|
||||
referenced relative to the module (`new URL("./assets/…", import.meta.url)`),
|
||||
so a host's bundler copies them as assets of its own — webpack 5, Vite and
|
||||
Rollup all do — and the browser fetches the MediaPipe runtime, the largest of
|
||||
them by far, only when an effect is first switched on.
|
||||
|
||||
A host must call and await `initializeElementCall(config)` once before
|
||||
rendering the component: it loads the `Intl` polyfills, applies the
|
||||
deployment-wide `config.json`-style configuration and sets up translations.
|
||||
|
||||
+24
-24
@@ -7,30 +7,30 @@ Please see LICENSE in the repository root for full details.
|
||||
|
||||
import { type RelationType } from "matrix-js-sdk";
|
||||
|
||||
import catSoundOgg from "../sound/reactions/cat.ogg?url";
|
||||
import catSoundMp3 from "../sound/reactions/cat.mp3?url";
|
||||
import clapSoundOgg from "../sound/reactions/clap.ogg?url";
|
||||
import clapSoundMp3 from "../sound/reactions/clap.mp3?url";
|
||||
import cricketsSoundOgg from "../sound/reactions/crickets.ogg?url";
|
||||
import cricketsSoundMp3 from "../sound/reactions/crickets.mp3?url";
|
||||
import dogSoundOgg from "../sound/reactions/dog.ogg?url";
|
||||
import dogSoundMp3 from "../sound/reactions/dog.mp3?url";
|
||||
import genericSoundOgg from "../sound/reactions/generic.ogg?url";
|
||||
import genericSoundMp3 from "../sound/reactions/generic.mp3?url";
|
||||
import lightbulbSoundOgg from "../sound/reactions/lightbulb.ogg?url";
|
||||
import lightbulbSoundMp3 from "../sound/reactions/lightbulb.mp3?url";
|
||||
import partySoundOgg from "../sound/reactions/party.ogg?url";
|
||||
import partySoundMp3 from "../sound/reactions/party.mp3?url";
|
||||
import deerSoundOgg from "../sound/reactions/deer.ogg?url";
|
||||
import deerSoundMp3 from "../sound/reactions/deer.mp3?url";
|
||||
import rockSoundOgg from "../sound/reactions/rock.ogg?url";
|
||||
import rockSoundMp3 from "../sound/reactions/rock.mp3?url";
|
||||
import waveSoundOgg from "../sound/reactions/wave.ogg?url";
|
||||
import waveSoundMp3 from "../sound/reactions/wave.mp3?url";
|
||||
import baduntssSoundOgg from "../sound/reactions/baduntss.ogg?url";
|
||||
import baduntssSoundMp3 from "../sound/reactions/baduntss.mp3?url";
|
||||
import bellSoundOgg from "../sound/reactions/bell.ogg?url";
|
||||
import bellSoundMp3 from "../sound/reactions/bell.mp3?url";
|
||||
import catSoundOgg from "../sound/reactions/cat.ogg?url&no-inline";
|
||||
import catSoundMp3 from "../sound/reactions/cat.mp3?url&no-inline";
|
||||
import clapSoundOgg from "../sound/reactions/clap.ogg?url&no-inline";
|
||||
import clapSoundMp3 from "../sound/reactions/clap.mp3?url&no-inline";
|
||||
import cricketsSoundOgg from "../sound/reactions/crickets.ogg?url&no-inline";
|
||||
import cricketsSoundMp3 from "../sound/reactions/crickets.mp3?url&no-inline";
|
||||
import dogSoundOgg from "../sound/reactions/dog.ogg?url&no-inline";
|
||||
import dogSoundMp3 from "../sound/reactions/dog.mp3?url&no-inline";
|
||||
import genericSoundOgg from "../sound/reactions/generic.ogg?url&no-inline";
|
||||
import genericSoundMp3 from "../sound/reactions/generic.mp3?url&no-inline";
|
||||
import lightbulbSoundOgg from "../sound/reactions/lightbulb.ogg?url&no-inline";
|
||||
import lightbulbSoundMp3 from "../sound/reactions/lightbulb.mp3?url&no-inline";
|
||||
import partySoundOgg from "../sound/reactions/party.ogg?url&no-inline";
|
||||
import partySoundMp3 from "../sound/reactions/party.mp3?url&no-inline";
|
||||
import deerSoundOgg from "../sound/reactions/deer.ogg?url&no-inline";
|
||||
import deerSoundMp3 from "../sound/reactions/deer.mp3?url&no-inline";
|
||||
import rockSoundOgg from "../sound/reactions/rock.ogg?url&no-inline";
|
||||
import rockSoundMp3 from "../sound/reactions/rock.mp3?url&no-inline";
|
||||
import waveSoundOgg from "../sound/reactions/wave.ogg?url&no-inline";
|
||||
import waveSoundMp3 from "../sound/reactions/wave.mp3?url&no-inline";
|
||||
import baduntssSoundOgg from "../sound/reactions/baduntss.ogg?url&no-inline";
|
||||
import baduntssSoundMp3 from "../sound/reactions/baduntss.mp3?url&no-inline";
|
||||
import bellSoundOgg from "../sound/reactions/bell.ogg?url&no-inline";
|
||||
import bellSoundMp3 from "../sound/reactions/bell.mp3?url&no-inline";
|
||||
|
||||
export const ElementCallReactionEventType = "io.element.call.reaction";
|
||||
|
||||
|
||||
@@ -8,18 +8,18 @@ Please see LICENSE in the repository root for full details.
|
||||
import { type ReactNode, useEffect } from "react";
|
||||
|
||||
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import joinCallSoundMp3 from "../sound/join_call.mp3";
|
||||
import joinCallSoundOgg from "../sound/join_call.ogg";
|
||||
import leftCallSoundMp3 from "../sound/left_call.mp3";
|
||||
import leftCallSoundOgg from "../sound/left_call.ogg";
|
||||
import handSoundOgg from "../sound/raise_hand.ogg";
|
||||
import handSoundMp3 from "../sound/raise_hand.mp3";
|
||||
import screenShareStartedOgg from "../sound/screen_share_started.ogg";
|
||||
import screenShareStartedMp3 from "../sound/screen_share_started.mp3";
|
||||
import declineMp3 from "../sound/call_declined.mp3?url";
|
||||
import declineOgg from "../sound/call_declined.ogg?url";
|
||||
import timeoutMp3 from "../sound/call_timeout.mp3?url";
|
||||
import timeoutOgg from "../sound/call_timeout.ogg?url";
|
||||
import joinCallSoundMp3 from "../sound/join_call.mp3?url&no-inline";
|
||||
import joinCallSoundOgg from "../sound/join_call.ogg?url&no-inline";
|
||||
import leftCallSoundMp3 from "../sound/left_call.mp3?url&no-inline";
|
||||
import leftCallSoundOgg from "../sound/left_call.ogg?url&no-inline";
|
||||
import handSoundOgg from "../sound/raise_hand.ogg?url&no-inline";
|
||||
import handSoundMp3 from "../sound/raise_hand.mp3?url&no-inline";
|
||||
import screenShareStartedOgg from "../sound/screen_share_started.ogg?url&no-inline";
|
||||
import screenShareStartedMp3 from "../sound/screen_share_started.mp3?url&no-inline";
|
||||
import declineMp3 from "../sound/call_declined.mp3?url&no-inline";
|
||||
import declineOgg from "../sound/call_declined.ogg?url&no-inline";
|
||||
import timeoutMp3 from "../sound/call_timeout.mp3?url&no-inline";
|
||||
import timeoutOgg from "../sound/call_timeout.ogg?url&no-inline";
|
||||
import { useAudioContext } from "../useAudioContext";
|
||||
import { prefetchSounds } from "../soundUtils";
|
||||
import { useLatest } from "../useLatest";
|
||||
|
||||
@@ -12,8 +12,8 @@ import { type RingingMediaViewModel } from "../state/media/RingingMediaViewModel
|
||||
import { useBehavior } from "../useBehavior";
|
||||
import { useInitial } from "../useInitial";
|
||||
import { prefetchSounds } from "../soundUtils";
|
||||
import ringtoneMp3 from "../sound/ringtone.mp3?url";
|
||||
import ringtoneOgg from "../sound/ringtone.ogg?url";
|
||||
import ringtoneMp3 from "../sound/ringtone.mp3?url&no-inline";
|
||||
import ringtoneOgg from "../sound/ringtone.ogg?url&no-inline";
|
||||
import { type UseAudioContext, useAudioContext } from "../useAudioContext";
|
||||
import { useLatest } from "../useLatest";
|
||||
|
||||
|
||||
+7
-26
@@ -34,23 +34,6 @@ describe("prefetchSounds", () => {
|
||||
expect(bytesOf(sounds.beep)).toEqual([1, 2, 3]);
|
||||
});
|
||||
|
||||
it("decodes base64 data: URLs without fetching them", async () => {
|
||||
// "OggS" — the bytes 0x4f 0x67 0x67 0x53
|
||||
const sounds = await prefetchSounds({
|
||||
beep: { ogg: "data:audio/ogg;base64,T2dnUw==" },
|
||||
});
|
||||
expect(fetchSpy).not.toHaveBeenCalled();
|
||||
expect(bytesOf(sounds.beep)).toEqual([0x4f, 0x67, 0x67, 0x53]);
|
||||
});
|
||||
|
||||
it("leaves out a sound with a percent-encoded data: URL", async () => {
|
||||
const sounds = await prefetchSounds({
|
||||
beep: { ogg: "data:audio/ogg,Ogg%53" },
|
||||
});
|
||||
expect(fetchSpy).not.toHaveBeenCalled();
|
||||
expect(sounds).toEqual({});
|
||||
});
|
||||
|
||||
it("leaves out a sound whose response is not okay", async () => {
|
||||
fetchSpy.mockResolvedValue(new Response(null, { status: 404 }));
|
||||
const sounds = await prefetchSounds({
|
||||
@@ -60,18 +43,16 @@ describe("prefetchSounds", () => {
|
||||
});
|
||||
|
||||
it("leaves out a sound that fails to load, keeping the others", async () => {
|
||||
fetchSpy.mockRejectedValue(new TypeError("Failed to fetch"));
|
||||
// Fetched in the order given
|
||||
fetchSpy
|
||||
.mockRejectedValueOnce(new TypeError("Failed to fetch"))
|
||||
.mockResolvedValueOnce(
|
||||
new Response(new Uint8Array([1, 2, 3]), { status: 200 }),
|
||||
);
|
||||
const sounds = await prefetchSounds({
|
||||
beep: { ogg: "https://example.org/beep.ogg" },
|
||||
boop: { ogg: "data:audio/ogg;base64,T2dnUw==" },
|
||||
boop: { ogg: "https://example.org/boop.ogg" },
|
||||
});
|
||||
expect(Object.keys(sounds)).toEqual(["boop"]);
|
||||
});
|
||||
|
||||
it("leaves out a sound with a malformed data: URL", async () => {
|
||||
const sounds = await prefetchSounds({
|
||||
beep: { ogg: "data:audio/ogg;base64" },
|
||||
});
|
||||
expect(sounds).toEqual({});
|
||||
});
|
||||
});
|
||||
|
||||
+1
-27
@@ -32,38 +32,12 @@ function getPreferredAudioFormat(): "ogg" | "mp3" {
|
||||
|
||||
const preferredFormat = getPreferredAudioFormat();
|
||||
|
||||
/**
|
||||
* Decode a `data:` URL into the bytes it carries.
|
||||
*
|
||||
* The component build inlines the sound files as `data:` URLs. Those could be
|
||||
* `fetch`ed like any other URL, but that fetch is subject to the host page's
|
||||
* Content Security Policy, and `connect-src` lists rarely allow `data:` (a `*`
|
||||
* source does not cover it). Reading the bytes out of the URL directly needs no
|
||||
* network access, so no policy can stop it.
|
||||
* @param url The `data:` URL to decode.
|
||||
* @returns The decoded bytes.
|
||||
*/
|
||||
function decodeDataUrl(url: string): ArrayBuffer {
|
||||
const separator = url.indexOf(",");
|
||||
if (!url.startsWith("data:") || separator === -1)
|
||||
throw new Error("Not a valid data: URL");
|
||||
const header = url.slice("data:".length, separator);
|
||||
// Vite only percent-encodes SVGs; every other inlined asset is base64.
|
||||
if (!header.split(";").includes("base64"))
|
||||
throw new Error("Only base64 data: URLs are supported");
|
||||
// `decoded` is a binary string: one char per byte
|
||||
const decoded = atob(url.slice(separator + 1));
|
||||
const bytes = Uint8Array.from(decoded, (c) => c.charCodeAt(0));
|
||||
return bytes.buffer;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load the bytes of a single sound file.
|
||||
* @param url Where the sound is: an ordinary URL to fetch, or a `data:` URL.
|
||||
* @param url Where the sound is.
|
||||
* @returns The bytes, or null if the sound could not be loaded.
|
||||
*/
|
||||
async function loadSound(url: string): Promise<ArrayBuffer | null> {
|
||||
if (url.startsWith("data:")) return decodeDataUrl(url);
|
||||
const response = await fetch(url);
|
||||
if (isFailure(response)) return null;
|
||||
return await response.arrayBuffer();
|
||||
|
||||
@@ -47,6 +47,24 @@ describe("BackgroundEffectTransformer", () => {
|
||||
expect(load).toHaveBeenCalledWith("/background.jpg");
|
||||
});
|
||||
|
||||
it("runs MediaPipe from its own bundle", async () => {
|
||||
vi.spyOn(VideoTransformer.prototype, "init").mockResolvedValue();
|
||||
const create = vi
|
||||
.spyOn(ImageSegmenter, "createFromOptions")
|
||||
.mockResolvedValue({} as ImageSegmenter);
|
||||
|
||||
await new BackgroundEffectTransformer({ backgroundDisabled: true }).init(
|
||||
{} as VideoTransformerInitOptions,
|
||||
);
|
||||
|
||||
const [fileset, options] = create.mock.calls[0];
|
||||
expect(fileset.wasmLoaderPath).toMatch(/vision_wasm_internal\.js/);
|
||||
expect(fileset.wasmBinaryPath).toMatch(/vision_wasm_internal\.wasm/);
|
||||
expect(options.baseOptions?.modelAssetPath).toMatch(
|
||||
/selfie_segmenter\.tflite/,
|
||||
);
|
||||
});
|
||||
|
||||
it("reports the first frame carrying an effect, once", async () => {
|
||||
vi.spyOn(BackgroundTransformer.prototype, "transform").mockImplementation(
|
||||
async function (this: BackgroundTransformer) {
|
||||
|
||||
@@ -16,7 +16,13 @@ import {
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
import { ImageSegmenter } from "@mediapipe/tasks-vision";
|
||||
|
||||
import modelAssetPath from "../mediapipe/imageSegmenter/selfie_segmenter.tflite?url";
|
||||
// Only the SIMD build of MediaPipe's runtime: every browser Element Call
|
||||
// supports has WebAssembly SIMD. As files rather than data URLs, so that a
|
||||
// host's bundler serves them as assets of its own, and a browser fetches them
|
||||
// only when an effect is first switched on.
|
||||
import wasmLoaderPath from "@mediapipe/tasks-vision/vision_wasm_internal.js?url&no-inline";
|
||||
import wasmBinaryPath from "@mediapipe/tasks-vision/vision_wasm_internal.wasm?url&no-inline";
|
||||
import modelAssetPath from "../mediapipe/imageSegmenter/selfie_segmenter.tflite?url&no-inline";
|
||||
|
||||
interface WasmFileset {
|
||||
/** The path to the Wasm loader script. */
|
||||
@@ -25,26 +31,7 @@ interface WasmFileset {
|
||||
wasmBinaryPath: string;
|
||||
}
|
||||
|
||||
// The MediaPipe package, by default, ships some alternative versions of the
|
||||
// WASM files which avoid SIMD for compatibility with older browsers. But SIMD
|
||||
// in WASM is actually fine by our support policy, so we include just the SIMD
|
||||
// versions.
|
||||
// It's really not ideal that we have to reference these internal files from
|
||||
// MediaPipe and depend on node_modules having this specific structure. It's
|
||||
// easy to see this breaking if our dependencies changed and MediaPipe were
|
||||
// no longer hoisted, or if we switched to another dependency loader such as
|
||||
// yarn PnP.
|
||||
// https://github.com/google-ai-edge/mediapipe/issues/5961
|
||||
const wasmFileset: WasmFileset = {
|
||||
wasmLoaderPath: new URL(
|
||||
"../../node_modules/@mediapipe/tasks-vision/wasm/vision_wasm_internal.js",
|
||||
import.meta.url,
|
||||
).href,
|
||||
wasmBinaryPath: new URL(
|
||||
"../../node_modules/@mediapipe/tasks-vision/wasm/vision_wasm_internal.wasm",
|
||||
import.meta.url,
|
||||
).href,
|
||||
};
|
||||
const wasmFileset: WasmFileset = { wasmLoaderPath, wasmBinaryPath };
|
||||
|
||||
/**
|
||||
* Track processor that applies effects such as blurring to a user's background.
|
||||
|
||||
@@ -5,8 +5,8 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import arc from "../graphics/background-arc.jpg?url";
|
||||
import glow from "../graphics/background-glow.jpg?url";
|
||||
import arc from "../graphics/background-arc.jpg?url&no-inline";
|
||||
import glow from "../graphics/background-glow.jpg?url&no-inline";
|
||||
|
||||
export interface ShippedBackground {
|
||||
/** Stored in the user's settings when chosen, so it must not change. */
|
||||
|
||||
@@ -6,6 +6,7 @@ Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { defineConfig } from "vite";
|
||||
import { posix } from "node:path";
|
||||
|
||||
import { vitePluginsConfig } from "./vite.config.ts";
|
||||
import { scopeStylesToRoot } from "./component/build/scopeStylesToRoot.ts";
|
||||
@@ -34,6 +35,25 @@ export default defineConfig(({ mode }) => {
|
||||
// whatever is in `public` — including the developer's own config.json, which
|
||||
// is not in the repository — into the output we would publish.
|
||||
publicDir: false,
|
||||
// A library does not know where it will be served from, so every file it
|
||||
// emits beside itself — the sounds, the pictures, MediaPipe's runtime — is
|
||||
// referenced relative to the module's own URL. (A library build inlines
|
||||
// each asset as a data URL unless its import says `no-inline`, so the
|
||||
// imports do.)
|
||||
base: "./",
|
||||
experimental: {
|
||||
// Relative to the chunk that references it, wherever that chunk lands,
|
||||
// and spelled `./assets/…` rather than `assets/…`: a host's bundler reads
|
||||
// a bare name as a package to look up.
|
||||
renderBuiltUrl: (filename, { hostId, hostType }) => {
|
||||
if (hostType !== "js") return { relative: true };
|
||||
const relative = posix.relative(posix.dirname(hostId), filename);
|
||||
const specifier = relative.startsWith(".") ? relative : `./${relative}`;
|
||||
return {
|
||||
runtime: `new URL(${JSON.stringify(specifier)}, import.meta.url).href`,
|
||||
};
|
||||
},
|
||||
},
|
||||
// A host's document is not ours to style: everything in the stylesheet is
|
||||
// confined to the element Element Call is mounted in
|
||||
css: { postcss: { plugins: [scopeStylesToRoot()] } },
|
||||
@@ -64,6 +84,15 @@ export default defineConfig(({ mode }) => {
|
||||
cssFileName: "element-call",
|
||||
},
|
||||
rollupOptions: {
|
||||
output: {
|
||||
// Hashed, as the app's are, so that a file a host serves as it is
|
||||
// stays correct across versions of the package. Not the stylesheet,
|
||||
// which the package exports by name (see `cssFileName`).
|
||||
assetFileNames: ({ names }) =>
|
||||
names.some((name) => name.endsWith(".css"))
|
||||
? "[name][extname]"
|
||||
: "assets/[name]-[hash][extname]",
|
||||
},
|
||||
// The host already has these, and a second copy of any of them does not
|
||||
// merely bloat the bundle: React would hold two sets of hooks, the
|
||||
// Matrix client would run two sync loops, and a second Compound would
|
||||
|
||||
Reference in New Issue
Block a user