Let a deployment serve MediaPipe from its own URL

Background effects run on MediaPipe's 11 MB runtime, which the
component ships as a file beside its module. A deployment that would
rather have it on a CDN, or served with cache headers of its own, can
now point `background_effects.mediapipe_wasm_url` at a directory
holding the `wasm` files of `@mediapipe/tasks-vision` at the version
Element Call depends on; the transformer then resolves the runtime
through MediaPipe's fileset resolver instead of the bundled copy.
Unset, nothing changes.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

README.md

@@ -273,7 +273,11 @@ effects are files beside the module rather than data URLs inside it, each
 referenced relative to the module (`new URL("./assets/…", import.meta.url)`),
 so a host's bundler copies them as assets of its own — webpack 5, Vite and
 Rollup all do — and the browser fetches the MediaPipe runtime, the largest of
-them by far, only when an effect is first switched on.
+them by far, only when an effect is first switched on. A deployment that would
+rather serve MediaPipe itself, from a CDN say, points
+`background_effects.mediapipe_wasm_url` in the config at a directory holding the
+`wasm` files of `@mediapipe/tasks-vision` at the version Element Call depends
+on.
 
 A host must call and await `initializeElementCall(config)` once before
 rendering the component: it loads the `Intl` polyfills, applies the

src/config/ConfigOptions.ts

@@ -181,6 +181,17 @@ export interface ConfigOptions {
     enable_video?: boolean;
   };
 
+  background_effects?: {
+    /**
+     * Where to fetch the MediaPipe runtime behind background effects from
+     * instead of the copy Element Call ships: the URL of a directory holding
+     * `vision_wasm_internal.js` and `vision_wasm_internal.wasm`, as the `wasm`
+     * directory of the `@mediapipe/tasks-vision` package ships them, at the
+     * version Element Call depends on.
+     */
+    mediapipe_wasm_url?: string;
+  };
+
   /**
    * Grace period in milliseconds to wait before reporting the sync loop as disconnected.
    * This allows brief sync interruptions without triggering a reconnection message.

src/tracks/BackgroundEffectTransformer.test.ts

@@ -11,7 +11,7 @@ import {
   VideoTransformer,
   type VideoTransformerInitOptions,
 } from "@livekit/track-processors";
-import { ImageSegmenter } from "@mediapipe/tasks-vision";
+import { FilesetResolver, ImageSegmenter } from "@mediapipe/tasks-vision";
 
 import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer";
 
@@ -46,11 +46,13 @@ describe("BackgroundEffectTransformer", () => {
     const create = vi
       .spyOn(ImageSegmenter, "createFromOptions")
       .mockResolvedValue({} as ImageSegmenter);
+    const resolve = vi.spyOn(FilesetResolver, "forVisionTasks");
 
     await new BackgroundEffectTransformer({ backgroundDisabled: true }).init(
       {} as VideoTransformerInitOptions,
     );
 
+    expect(resolve).not.toHaveBeenCalled();
     const [fileset, options] = create.mock.calls[0];
     expect(fileset.wasmLoaderPath).toMatch(/vision_wasm_internal\.js/);
     expect(fileset.wasmBinaryPath).toMatch(/vision_wasm_internal\.wasm/);
@@ -59,6 +61,30 @@ describe("BackgroundEffectTransformer", () => {
     );
   });
 
+  it("runs MediaPipe from where the deployment serves it, if it does", async () => {
+    vi.spyOn(VideoTransformer.prototype, "init").mockResolvedValue();
+    const create = vi
+      .spyOn(ImageSegmenter, "createFromOptions")
+      .mockResolvedValue({} as ImageSegmenter);
+    const served = {
+      wasmLoaderPath:
+        "https://cdn.example.org/mediapipe/vision_wasm_internal.js",
+      wasmBinaryPath:
+        "https://cdn.example.org/mediapipe/vision_wasm_internal.wasm",
+    };
+    const resolve = vi
+      .spyOn(FilesetResolver, "forVisionTasks")
+      .mockResolvedValue(served);
+
+    await new BackgroundEffectTransformer(
+      { backgroundDisabled: true },
+      { wasmBaseUrl: "https://cdn.example.org/mediapipe" },
+    ).init({} as VideoTransformerInitOptions);
+
+    expect(resolve).toHaveBeenCalledWith("https://cdn.example.org/mediapipe");
+    expect(create.mock.calls[0][0]).toBe(served);
+  });
+
   it("reports the first frame carrying an effect, once", async () => {
     vi.spyOn(BackgroundTransformer.prototype, "transform").mockImplementation(
       async function (this: BackgroundTransformer) {

src/tracks/BackgroundEffectTransformer.ts

@@ -13,7 +13,7 @@ import {
   type BackgroundOptions,
 } from "@livekit/track-processors";
 import { logger } from "matrix-js-sdk/lib/logger";
-import { ImageSegmenter } from "@mediapipe/tasks-vision";
+import { FilesetResolver, ImageSegmenter } from "@mediapipe/tasks-vision";
 
 // Only the SIMD build of MediaPipe's runtime: every browser Element Call
 // supports has WebAssembly SIMD. As files rather than data URLs, so that a
@@ -30,7 +30,15 @@ interface WasmFileset {
   wasmBinaryPath: string;
 }
 
-const wasmFileset: WasmFileset = { wasmLoaderPath, wasmBinaryPath };
+const bundledFileset: WasmFileset = { wasmLoaderPath, wasmBinaryPath };
+
+export interface BackgroundEffectTransformerSources {
+  /**
+   * Where to fetch MediaPipe's runtime from instead of the bundled copy: a
+   * directory holding its `vision_wasm_internal.js` and `.wasm` files.
+   */
+  wasmBaseUrl?: string;
+}
 
 /**
  * Track processor that applies effects such as blurring to a user's background.
@@ -43,6 +51,13 @@ export class BackgroundEffectTransformer extends BackgroundTransformer {
   /** Called once, when the first frame carrying an effect has been drawn. */
   public onFirstFrame: (() => void) | undefined;
 
+  public constructor(
+    opts: BackgroundOptions,
+    private readonly sources: BackgroundEffectTransformerSources = {},
+  ) {
+    super(opts);
+  }
+
   public override async transform(
     frame: VideoFrame,
     controller: TransformStreamDefaultController<VideoFrame>,
@@ -79,7 +94,11 @@ export class BackgroundEffectTransformer extends BackgroundTransformer {
       inputElement: inputVideo,
     });
 
-    this.imageSegmenter = await ImageSegmenter.createFromOptions(wasmFileset, {
+    const fileset =
+      this.sources.wasmBaseUrl === undefined
+        ? bundledFileset
+        : await FilesetResolver.forVisionTasks(this.sources.wasmBaseUrl);
+    this.imageSegmenter = await ImageSegmenter.createFromOptions(fileset, {
       baseOptions: {
         modelAssetPath,
         delegate: "GPU",

src/tracks/BackgroundEffectsContext.tsx

@@ -12,6 +12,7 @@ import {
 import { createContext, type FC, type JSX, use } from "react";
 
 import { backgroundEffect as backgroundEffectSetting } from "../settings/settings";
+import { Config } from "../config/Config";
 import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer";
 import { type Behavior } from "../state/Behavior";
 import { type ObservableScope } from "../state/ObservableScope";
@@ -47,9 +48,10 @@ export function useBackgroundEffectsState$(): Behavior<BackgroundEffectsState> {
 export function createAppBackgroundEffects(
   scope: ObservableScope,
 ): BackgroundEffects {
-  const transformer = new BackgroundEffectTransformer({
-    backgroundDisabled: true,
-  });
+  const transformer = new BackgroundEffectTransformer(
+    { backgroundDisabled: true },
+    { wasmBaseUrl: Config.get().background_effects?.mediapipe_wasm_url },
+  );
   return createBackgroundEffects(scope, {
     supported: supportsBackgroundProcessors(),
     effect$: backgroundEffectSetting.value$,