diff --git a/README.md b/README.md index 17eae35d9..ebc26f936 100644 --- a/README.md +++ b/README.md @@ -268,6 +268,13 @@ widget's `theme` URL parameter: `light`, `dark`, `light-high-contrast` or `dark-high-contrast`. Both can change while a call is running without disturbing it. +The component's sounds, pictures and the MediaPipe runtime behind background +effects are files beside the module rather than data URLs inside it, each +referenced relative to the module (`new URL("./assets/…", import.meta.url)`), +so a host's bundler copies them as assets of its own — webpack 5, Vite and +Rollup all do — and the browser fetches the MediaPipe runtime, the largest of +them by far, only when an effect is first switched on. + A host must call and await `initializeElementCall(config)` once before rendering the component: it loads the `Intl` polyfills, applies the deployment-wide `config.json`-style configuration and sets up translations. diff --git a/src/reactions/index.ts b/src/reactions/index.ts index 26ca7f142..ffc14d43e 100644 --- a/src/reactions/index.ts +++ b/src/reactions/index.ts @@ -7,30 +7,30 @@ Please see LICENSE in the repository root for full details. import { type RelationType } from "matrix-js-sdk"; -import catSoundOgg from "../sound/reactions/cat.ogg?url"; -import catSoundMp3 from "../sound/reactions/cat.mp3?url"; -import clapSoundOgg from "../sound/reactions/clap.ogg?url"; -import clapSoundMp3 from "../sound/reactions/clap.mp3?url"; -import cricketsSoundOgg from "../sound/reactions/crickets.ogg?url"; -import cricketsSoundMp3 from "../sound/reactions/crickets.mp3?url"; -import dogSoundOgg from "../sound/reactions/dog.ogg?url"; -import dogSoundMp3 from "../sound/reactions/dog.mp3?url"; -import genericSoundOgg from "../sound/reactions/generic.ogg?url"; -import genericSoundMp3 from "../sound/reactions/generic.mp3?url"; -import lightbulbSoundOgg from "../sound/reactions/lightbulb.ogg?url"; -import lightbulbSoundMp3 from "../sound/reactions/lightbulb.mp3?url"; -import partySoundOgg from "../sound/reactions/party.ogg?url"; -import partySoundMp3 from "../sound/reactions/party.mp3?url"; -import deerSoundOgg from "../sound/reactions/deer.ogg?url"; -import deerSoundMp3 from "../sound/reactions/deer.mp3?url"; -import rockSoundOgg from "../sound/reactions/rock.ogg?url"; -import rockSoundMp3 from "../sound/reactions/rock.mp3?url"; -import waveSoundOgg from "../sound/reactions/wave.ogg?url"; -import waveSoundMp3 from "../sound/reactions/wave.mp3?url"; -import baduntssSoundOgg from "../sound/reactions/baduntss.ogg?url"; -import baduntssSoundMp3 from "../sound/reactions/baduntss.mp3?url"; -import bellSoundOgg from "../sound/reactions/bell.ogg?url"; -import bellSoundMp3 from "../sound/reactions/bell.mp3?url"; +import catSoundOgg from "../sound/reactions/cat.ogg?url&no-inline"; +import catSoundMp3 from "../sound/reactions/cat.mp3?url&no-inline"; +import clapSoundOgg from "../sound/reactions/clap.ogg?url&no-inline"; +import clapSoundMp3 from "../sound/reactions/clap.mp3?url&no-inline"; +import cricketsSoundOgg from "../sound/reactions/crickets.ogg?url&no-inline"; +import cricketsSoundMp3 from "../sound/reactions/crickets.mp3?url&no-inline"; +import dogSoundOgg from "../sound/reactions/dog.ogg?url&no-inline"; +import dogSoundMp3 from "../sound/reactions/dog.mp3?url&no-inline"; +import genericSoundOgg from "../sound/reactions/generic.ogg?url&no-inline"; +import genericSoundMp3 from "../sound/reactions/generic.mp3?url&no-inline"; +import lightbulbSoundOgg from "../sound/reactions/lightbulb.ogg?url&no-inline"; +import lightbulbSoundMp3 from "../sound/reactions/lightbulb.mp3?url&no-inline"; +import partySoundOgg from "../sound/reactions/party.ogg?url&no-inline"; +import partySoundMp3 from "../sound/reactions/party.mp3?url&no-inline"; +import deerSoundOgg from "../sound/reactions/deer.ogg?url&no-inline"; +import deerSoundMp3 from "../sound/reactions/deer.mp3?url&no-inline"; +import rockSoundOgg from "../sound/reactions/rock.ogg?url&no-inline"; +import rockSoundMp3 from "../sound/reactions/rock.mp3?url&no-inline"; +import waveSoundOgg from "../sound/reactions/wave.ogg?url&no-inline"; +import waveSoundMp3 from "../sound/reactions/wave.mp3?url&no-inline"; +import baduntssSoundOgg from "../sound/reactions/baduntss.ogg?url&no-inline"; +import baduntssSoundMp3 from "../sound/reactions/baduntss.mp3?url&no-inline"; +import bellSoundOgg from "../sound/reactions/bell.ogg?url&no-inline"; +import bellSoundMp3 from "../sound/reactions/bell.mp3?url&no-inline"; export const ElementCallReactionEventType = "io.element.call.reaction"; diff --git a/src/room/CallEventAudioRenderer.tsx b/src/room/CallEventAudioRenderer.tsx index d33f3b843..fa14bcd77 100644 --- a/src/room/CallEventAudioRenderer.tsx +++ b/src/room/CallEventAudioRenderer.tsx @@ -8,18 +8,18 @@ Please see LICENSE in the repository root for full details. import { type ReactNode, useEffect } from "react"; import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; -import joinCallSoundMp3 from "../sound/join_call.mp3"; -import joinCallSoundOgg from "../sound/join_call.ogg"; -import leftCallSoundMp3 from "../sound/left_call.mp3"; -import leftCallSoundOgg from "../sound/left_call.ogg"; -import handSoundOgg from "../sound/raise_hand.ogg"; -import handSoundMp3 from "../sound/raise_hand.mp3"; -import screenShareStartedOgg from "../sound/screen_share_started.ogg"; -import screenShareStartedMp3 from "../sound/screen_share_started.mp3"; -import declineMp3 from "../sound/call_declined.mp3?url"; -import declineOgg from "../sound/call_declined.ogg?url"; -import timeoutMp3 from "../sound/call_timeout.mp3?url"; -import timeoutOgg from "../sound/call_timeout.ogg?url"; +import joinCallSoundMp3 from "../sound/join_call.mp3?url&no-inline"; +import joinCallSoundOgg from "../sound/join_call.ogg?url&no-inline"; +import leftCallSoundMp3 from "../sound/left_call.mp3?url&no-inline"; +import leftCallSoundOgg from "../sound/left_call.ogg?url&no-inline"; +import handSoundOgg from "../sound/raise_hand.ogg?url&no-inline"; +import handSoundMp3 from "../sound/raise_hand.mp3?url&no-inline"; +import screenShareStartedOgg from "../sound/screen_share_started.ogg?url&no-inline"; +import screenShareStartedMp3 from "../sound/screen_share_started.mp3?url&no-inline"; +import declineMp3 from "../sound/call_declined.mp3?url&no-inline"; +import declineOgg from "../sound/call_declined.ogg?url&no-inline"; +import timeoutMp3 from "../sound/call_timeout.mp3?url&no-inline"; +import timeoutOgg from "../sound/call_timeout.ogg?url&no-inline"; import { useAudioContext } from "../useAudioContext"; import { prefetchSounds } from "../soundUtils"; import { useLatest } from "../useLatest"; diff --git a/src/room/RingingAudioRenderer.tsx b/src/room/RingingAudioRenderer.tsx index c0fe45d5b..f0ebd8386 100644 --- a/src/room/RingingAudioRenderer.tsx +++ b/src/room/RingingAudioRenderer.tsx @@ -12,8 +12,8 @@ import { type RingingMediaViewModel } from "../state/media/RingingMediaViewModel import { useBehavior } from "../useBehavior"; import { useInitial } from "../useInitial"; import { prefetchSounds } from "../soundUtils"; -import ringtoneMp3 from "../sound/ringtone.mp3?url"; -import ringtoneOgg from "../sound/ringtone.ogg?url"; +import ringtoneMp3 from "../sound/ringtone.mp3?url&no-inline"; +import ringtoneOgg from "../sound/ringtone.ogg?url&no-inline"; import { type UseAudioContext, useAudioContext } from "../useAudioContext"; import { useLatest } from "../useLatest"; diff --git a/src/soundUtils.test.ts b/src/soundUtils.test.ts index 3d9810fe0..c6b652f9d 100644 --- a/src/soundUtils.test.ts +++ b/src/soundUtils.test.ts @@ -34,23 +34,6 @@ describe("prefetchSounds", () => { expect(bytesOf(sounds.beep)).toEqual([1, 2, 3]); }); - it("decodes base64 data: URLs without fetching them", async () => { - // "OggS" — the bytes 0x4f 0x67 0x67 0x53 - const sounds = await prefetchSounds({ - beep: { ogg: "data:audio/ogg;base64,T2dnUw==" }, - }); - expect(fetchSpy).not.toHaveBeenCalled(); - expect(bytesOf(sounds.beep)).toEqual([0x4f, 0x67, 0x67, 0x53]); - }); - - it("leaves out a sound with a percent-encoded data: URL", async () => { - const sounds = await prefetchSounds({ - beep: { ogg: "data:audio/ogg,Ogg%53" }, - }); - expect(fetchSpy).not.toHaveBeenCalled(); - expect(sounds).toEqual({}); - }); - it("leaves out a sound whose response is not okay", async () => { fetchSpy.mockResolvedValue(new Response(null, { status: 404 })); const sounds = await prefetchSounds({ @@ -60,18 +43,16 @@ describe("prefetchSounds", () => { }); it("leaves out a sound that fails to load, keeping the others", async () => { - fetchSpy.mockRejectedValue(new TypeError("Failed to fetch")); + // Fetched in the order given + fetchSpy + .mockRejectedValueOnce(new TypeError("Failed to fetch")) + .mockResolvedValueOnce( + new Response(new Uint8Array([1, 2, 3]), { status: 200 }), + ); const sounds = await prefetchSounds({ beep: { ogg: "https://example.org/beep.ogg" }, - boop: { ogg: "data:audio/ogg;base64,T2dnUw==" }, + boop: { ogg: "https://example.org/boop.ogg" }, }); expect(Object.keys(sounds)).toEqual(["boop"]); }); - - it("leaves out a sound with a malformed data: URL", async () => { - const sounds = await prefetchSounds({ - beep: { ogg: "data:audio/ogg;base64" }, - }); - expect(sounds).toEqual({}); - }); }); diff --git a/src/soundUtils.ts b/src/soundUtils.ts index 74d2ea33c..7ace7876e 100644 --- a/src/soundUtils.ts +++ b/src/soundUtils.ts @@ -32,38 +32,12 @@ function getPreferredAudioFormat(): "ogg" | "mp3" { const preferredFormat = getPreferredAudioFormat(); -/** - * Decode a `data:` URL into the bytes it carries. - * - * The component build inlines the sound files as `data:` URLs. Those could be - * `fetch`ed like any other URL, but that fetch is subject to the host page's - * Content Security Policy, and `connect-src` lists rarely allow `data:` (a `*` - * source does not cover it). Reading the bytes out of the URL directly needs no - * network access, so no policy can stop it. - * @param url The `data:` URL to decode. - * @returns The decoded bytes. - */ -function decodeDataUrl(url: string): ArrayBuffer { - const separator = url.indexOf(","); - if (!url.startsWith("data:") || separator === -1) - throw new Error("Not a valid data: URL"); - const header = url.slice("data:".length, separator); - // Vite only percent-encodes SVGs; every other inlined asset is base64. - if (!header.split(";").includes("base64")) - throw new Error("Only base64 data: URLs are supported"); - // `decoded` is a binary string: one char per byte - const decoded = atob(url.slice(separator + 1)); - const bytes = Uint8Array.from(decoded, (c) => c.charCodeAt(0)); - return bytes.buffer; -} - /** * Load the bytes of a single sound file. - * @param url Where the sound is: an ordinary URL to fetch, or a `data:` URL. + * @param url Where the sound is. * @returns The bytes, or null if the sound could not be loaded. */ async function loadSound(url: string): Promise { - if (url.startsWith("data:")) return decodeDataUrl(url); const response = await fetch(url); if (isFailure(response)) return null; return await response.arrayBuffer(); diff --git a/src/tracks/BackgroundEffectTransformer.test.ts b/src/tracks/BackgroundEffectTransformer.test.ts index 09de8ef5f..ff4f1a549 100644 --- a/src/tracks/BackgroundEffectTransformer.test.ts +++ b/src/tracks/BackgroundEffectTransformer.test.ts @@ -47,6 +47,24 @@ describe("BackgroundEffectTransformer", () => { expect(load).toHaveBeenCalledWith("/background.jpg"); }); + it("runs MediaPipe from its own bundle", async () => { + vi.spyOn(VideoTransformer.prototype, "init").mockResolvedValue(); + const create = vi + .spyOn(ImageSegmenter, "createFromOptions") + .mockResolvedValue({} as ImageSegmenter); + + await new BackgroundEffectTransformer({ backgroundDisabled: true }).init( + {} as VideoTransformerInitOptions, + ); + + const [fileset, options] = create.mock.calls[0]; + expect(fileset.wasmLoaderPath).toMatch(/vision_wasm_internal\.js/); + expect(fileset.wasmBinaryPath).toMatch(/vision_wasm_internal\.wasm/); + expect(options.baseOptions?.modelAssetPath).toMatch( + /selfie_segmenter\.tflite/, + ); + }); + it("reports the first frame carrying an effect, once", async () => { vi.spyOn(BackgroundTransformer.prototype, "transform").mockImplementation( async function (this: BackgroundTransformer) { diff --git a/src/tracks/BackgroundEffectTransformer.ts b/src/tracks/BackgroundEffectTransformer.ts index 774251994..c9caffaa8 100644 --- a/src/tracks/BackgroundEffectTransformer.ts +++ b/src/tracks/BackgroundEffectTransformer.ts @@ -16,7 +16,13 @@ import { import { logger } from "matrix-js-sdk/lib/logger"; import { ImageSegmenter } from "@mediapipe/tasks-vision"; -import modelAssetPath from "../mediapipe/imageSegmenter/selfie_segmenter.tflite?url"; +// Only the SIMD build of MediaPipe's runtime: every browser Element Call +// supports has WebAssembly SIMD. As files rather than data URLs, so that a +// host's bundler serves them as assets of its own, and a browser fetches them +// only when an effect is first switched on. +import wasmLoaderPath from "@mediapipe/tasks-vision/vision_wasm_internal.js?url&no-inline"; +import wasmBinaryPath from "@mediapipe/tasks-vision/vision_wasm_internal.wasm?url&no-inline"; +import modelAssetPath from "../mediapipe/imageSegmenter/selfie_segmenter.tflite?url&no-inline"; interface WasmFileset { /** The path to the Wasm loader script. */ @@ -25,26 +31,7 @@ interface WasmFileset { wasmBinaryPath: string; } -// The MediaPipe package, by default, ships some alternative versions of the -// WASM files which avoid SIMD for compatibility with older browsers. But SIMD -// in WASM is actually fine by our support policy, so we include just the SIMD -// versions. -// It's really not ideal that we have to reference these internal files from -// MediaPipe and depend on node_modules having this specific structure. It's -// easy to see this breaking if our dependencies changed and MediaPipe were -// no longer hoisted, or if we switched to another dependency loader such as -// yarn PnP. -// https://github.com/google-ai-edge/mediapipe/issues/5961 -const wasmFileset: WasmFileset = { - wasmLoaderPath: new URL( - "../../node_modules/@mediapipe/tasks-vision/wasm/vision_wasm_internal.js", - import.meta.url, - ).href, - wasmBinaryPath: new URL( - "../../node_modules/@mediapipe/tasks-vision/wasm/vision_wasm_internal.wasm", - import.meta.url, - ).href, -}; +const wasmFileset: WasmFileset = { wasmLoaderPath, wasmBinaryPath }; /** * Track processor that applies effects such as blurring to a user's background. diff --git a/src/tracks/backgroundEffectCatalog.ts b/src/tracks/backgroundEffectCatalog.ts index e2039f4af..a9d6d65b5 100644 --- a/src/tracks/backgroundEffectCatalog.ts +++ b/src/tracks/backgroundEffectCatalog.ts @@ -5,8 +5,8 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import arc from "../graphics/background-arc.jpg?url"; -import glow from "../graphics/background-glow.jpg?url"; +import arc from "../graphics/background-arc.jpg?url&no-inline"; +import glow from "../graphics/background-glow.jpg?url&no-inline"; export interface ShippedBackground { /** Stored in the user's settings when chosen, so it must not change. */ diff --git a/vite-component.config.ts b/vite-component.config.ts index 63183c781..859d2f5ff 100644 --- a/vite-component.config.ts +++ b/vite-component.config.ts @@ -6,6 +6,7 @@ Please see LICENSE in the repository root for full details. */ import { defineConfig } from "vite"; +import { posix } from "node:path"; import { vitePluginsConfig } from "./vite.config.ts"; import { scopeStylesToRoot } from "./component/build/scopeStylesToRoot.ts"; @@ -34,6 +35,25 @@ export default defineConfig(({ mode }) => { // whatever is in `public` — including the developer's own config.json, which // is not in the repository — into the output we would publish. publicDir: false, + // A library does not know where it will be served from, so every file it + // emits beside itself — the sounds, the pictures, MediaPipe's runtime — is + // referenced relative to the module's own URL. (A library build inlines + // each asset as a data URL unless its import says `no-inline`, so the + // imports do.) + base: "./", + experimental: { + // Relative to the chunk that references it, wherever that chunk lands, + // and spelled `./assets/…` rather than `assets/…`: a host's bundler reads + // a bare name as a package to look up. + renderBuiltUrl: (filename, { hostId, hostType }) => { + if (hostType !== "js") return { relative: true }; + const relative = posix.relative(posix.dirname(hostId), filename); + const specifier = relative.startsWith(".") ? relative : `./${relative}`; + return { + runtime: `new URL(${JSON.stringify(specifier)}, import.meta.url).href`, + }; + }, + }, // A host's document is not ours to style: everything in the stylesheet is // confined to the element Element Call is mounted in css: { postcss: { plugins: [scopeStylesToRoot()] } }, @@ -64,6 +84,15 @@ export default defineConfig(({ mode }) => { cssFileName: "element-call", }, rollupOptions: { + output: { + // Hashed, as the app's are, so that a file a host serves as it is + // stays correct across versions of the package. Not the stylesheet, + // which the package exports by name (see `cssFileName`). + assetFileNames: ({ names }) => + names.some((name) => name.endsWith(".css")) + ? "[name][extname]" + : "assets/[name]-[hash][extname]", + }, // The host already has these, and a second copy of any of them does not // merely bloat the bundle: React would hold two sets of hooks, the // Matrix client would run two sync loops, and a second Compound would