diff --git a/README.md b/README.md index ebc26f936..022978460 100644 --- a/README.md +++ b/README.md @@ -273,7 +273,11 @@ effects are files beside the module rather than data URLs inside it, each referenced relative to the module (`new URL("./assets/…", import.meta.url)`), so a host's bundler copies them as assets of its own — webpack 5, Vite and Rollup all do — and the browser fetches the MediaPipe runtime, the largest of -them by far, only when an effect is first switched on. +them by far, only when an effect is first switched on. A deployment that would +rather serve MediaPipe itself, from a CDN say, points +`background_effects.mediapipe_wasm_url` in the config at a directory holding the +`wasm` files of `@mediapipe/tasks-vision` at the version Element Call depends +on. A host must call and await `initializeElementCall(config)` once before rendering the component: it loads the `Intl` polyfills, applies the diff --git a/src/config/ConfigOptions.ts b/src/config/ConfigOptions.ts index 65aa04da2..5a22f0b33 100644 --- a/src/config/ConfigOptions.ts +++ b/src/config/ConfigOptions.ts @@ -181,6 +181,17 @@ export interface ConfigOptions { enable_video?: boolean; }; + background_effects?: { + /** + * Where to fetch the MediaPipe runtime behind background effects from + * instead of the copy Element Call ships: the URL of a directory holding + * `vision_wasm_internal.js` and `vision_wasm_internal.wasm`, as the `wasm` + * directory of the `@mediapipe/tasks-vision` package ships them, at the + * version Element Call depends on. + */ + mediapipe_wasm_url?: string; + }; + /** * Grace period in milliseconds to wait before reporting the sync loop as disconnected. * This allows brief sync interruptions without triggering a reconnection message. diff --git a/src/tracks/BackgroundEffectTransformer.test.ts b/src/tracks/BackgroundEffectTransformer.test.ts index 4e7ed8947..32ab17c3a 100644 --- a/src/tracks/BackgroundEffectTransformer.test.ts +++ b/src/tracks/BackgroundEffectTransformer.test.ts @@ -11,7 +11,7 @@ import { VideoTransformer, type VideoTransformerInitOptions, } from "@livekit/track-processors"; -import { ImageSegmenter } from "@mediapipe/tasks-vision"; +import { FilesetResolver, ImageSegmenter } from "@mediapipe/tasks-vision"; import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer"; @@ -46,11 +46,13 @@ describe("BackgroundEffectTransformer", () => { const create = vi .spyOn(ImageSegmenter, "createFromOptions") .mockResolvedValue({} as ImageSegmenter); + const resolve = vi.spyOn(FilesetResolver, "forVisionTasks"); await new BackgroundEffectTransformer({ backgroundDisabled: true }).init( {} as VideoTransformerInitOptions, ); + expect(resolve).not.toHaveBeenCalled(); const [fileset, options] = create.mock.calls[0]; expect(fileset.wasmLoaderPath).toMatch(/vision_wasm_internal\.js/); expect(fileset.wasmBinaryPath).toMatch(/vision_wasm_internal\.wasm/); @@ -59,6 +61,30 @@ describe("BackgroundEffectTransformer", () => { ); }); + it("runs MediaPipe from where the deployment serves it, if it does", async () => { + vi.spyOn(VideoTransformer.prototype, "init").mockResolvedValue(); + const create = vi + .spyOn(ImageSegmenter, "createFromOptions") + .mockResolvedValue({} as ImageSegmenter); + const served = { + wasmLoaderPath: + "https://cdn.example.org/mediapipe/vision_wasm_internal.js", + wasmBinaryPath: + "https://cdn.example.org/mediapipe/vision_wasm_internal.wasm", + }; + const resolve = vi + .spyOn(FilesetResolver, "forVisionTasks") + .mockResolvedValue(served); + + await new BackgroundEffectTransformer( + { backgroundDisabled: true }, + { wasmBaseUrl: "https://cdn.example.org/mediapipe" }, + ).init({} as VideoTransformerInitOptions); + + expect(resolve).toHaveBeenCalledWith("https://cdn.example.org/mediapipe"); + expect(create.mock.calls[0][0]).toBe(served); + }); + it("reports the first frame carrying an effect, once", async () => { vi.spyOn(BackgroundTransformer.prototype, "transform").mockImplementation( async function (this: BackgroundTransformer) { diff --git a/src/tracks/BackgroundEffectTransformer.ts b/src/tracks/BackgroundEffectTransformer.ts index 8d7221a6d..959e1aa55 100644 --- a/src/tracks/BackgroundEffectTransformer.ts +++ b/src/tracks/BackgroundEffectTransformer.ts @@ -13,7 +13,7 @@ import { type BackgroundOptions, } from "@livekit/track-processors"; import { logger } from "matrix-js-sdk/lib/logger"; -import { ImageSegmenter } from "@mediapipe/tasks-vision"; +import { FilesetResolver, ImageSegmenter } from "@mediapipe/tasks-vision"; // Only the SIMD build of MediaPipe's runtime: every browser Element Call // supports has WebAssembly SIMD. As files rather than data URLs, so that a @@ -30,7 +30,15 @@ interface WasmFileset { wasmBinaryPath: string; } -const wasmFileset: WasmFileset = { wasmLoaderPath, wasmBinaryPath }; +const bundledFileset: WasmFileset = { wasmLoaderPath, wasmBinaryPath }; + +export interface BackgroundEffectTransformerSources { + /** + * Where to fetch MediaPipe's runtime from instead of the bundled copy: a + * directory holding its `vision_wasm_internal.js` and `.wasm` files. + */ + wasmBaseUrl?: string; +} /** * Track processor that applies effects such as blurring to a user's background. @@ -43,6 +51,13 @@ export class BackgroundEffectTransformer extends BackgroundTransformer { /** Called once, when the first frame carrying an effect has been drawn. */ public onFirstFrame: (() => void) | undefined; + public constructor( + opts: BackgroundOptions, + private readonly sources: BackgroundEffectTransformerSources = {}, + ) { + super(opts); + } + public override async transform( frame: VideoFrame, controller: TransformStreamDefaultController, @@ -79,7 +94,11 @@ export class BackgroundEffectTransformer extends BackgroundTransformer { inputElement: inputVideo, }); - this.imageSegmenter = await ImageSegmenter.createFromOptions(wasmFileset, { + const fileset = + this.sources.wasmBaseUrl === undefined + ? bundledFileset + : await FilesetResolver.forVisionTasks(this.sources.wasmBaseUrl); + this.imageSegmenter = await ImageSegmenter.createFromOptions(fileset, { baseOptions: { modelAssetPath, delegate: "GPU", diff --git a/src/tracks/BackgroundEffectsContext.tsx b/src/tracks/BackgroundEffectsContext.tsx index d582bc0b3..68168e007 100644 --- a/src/tracks/BackgroundEffectsContext.tsx +++ b/src/tracks/BackgroundEffectsContext.tsx @@ -12,6 +12,7 @@ import { import { createContext, type FC, type JSX, use } from "react"; import { backgroundEffect as backgroundEffectSetting } from "../settings/settings"; +import { Config } from "../config/Config"; import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer"; import { type Behavior } from "../state/Behavior"; import { type ObservableScope } from "../state/ObservableScope"; @@ -47,9 +48,10 @@ export function useBackgroundEffectsState$(): Behavior { export function createAppBackgroundEffects( scope: ObservableScope, ): BackgroundEffects { - const transformer = new BackgroundEffectTransformer({ - backgroundDisabled: true, - }); + const transformer = new BackgroundEffectTransformer( + { backgroundDisabled: true }, + { wasmBaseUrl: Config.get().background_effects?.mediapipe_wasm_url }, + ); return createBackgroundEffects(scope, { supported: supportsBackgroundProcessors(), effect$: backgroundEffectSetting.value$,