Attach background effects through one switchable pipeline

Replace the blur-only processor with a single wrapper that switches between no
effect, blur and a background image in place. Switching in place rather than
rebuilding matters: destroy() resets the transformer's first-frame flag, and
that flag makes it emit one unprocessed frame when it next starts processing,
so a rebuild per change would leak a frame of the real background every time.

Attach the pipeline the first time an effect is chosen and keep it attached
afterwards, including at no effect. Measured against a production build, the
segmentation assets are about 11.4MB and take ~143ms to initialise, so always
attaching would charge that to every user on every camera-on call, including
the majority who never choose an effect. Keeping it attached once it is there
costs about 25us per frame, which is within noise of no pipeline at all.

Complete init() while moving it. The old subclass replaced the base class's
init wholesale and dropped its image loading and disabled-mode setup, which
went unnoticed while blur was the only effect and blurRadius was the only
option that mattered.

Keep loading the segmenter from our own bundle. The library's assetPaths option
resolves WASM through MediaPipe's FilesetResolver from a directory, which is
not the same thing, and its defaults reach jsDelivr and googleapis.

Exploration for FEATURES_SPEC/2026-09_Background_Effects.md. Not for merge:
the two shipped backgrounds are stand-in gradients and there is no picker yet,
recorded as S1 and S2 in the spec.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

src/livekit/BackgroundEffectTransformer.ts

@@ -43,13 +43,17 @@ const wasmFileset: WasmFileset = {
 };
 
 /**
- * Track processor that applies effects such as blurring to a user's background.
+ * Track processor that applies background effects — blur, or a replacement
+ * image — to a user's camera.
  *
  * This is just like LiveKit's prebuilt BackgroundTransformer except that it
  * loads the segmentation models from our own bundle rather than as an external
- * resource fetched from the public internet.
+ * resource fetched from the public internet. The library's own `assetPaths`
+ * option cannot do this: it takes a directory and resolves the WASM through
+ * MediaPipe's FilesetResolver, which picks its own variants, whereas we want
+ * only the SIMD ones.
  */
-export class BlurBackgroundTransformer extends BackgroundTransformer {
+export class BackgroundEffectTransformer extends BackgroundTransformer {
   public async init({
     outputCanvas,
     inputElement: inputVideo,
@@ -73,8 +77,16 @@ export class BlurBackgroundTransformer extends BackgroundTransformer {
       outputConfidenceMasks: false,
     });
 
-    if (this.options.blurRadius) {
+    // BackgroundTransformer applies these at the end of its own init. Because
+    // we replace init wholesale rather than extending it, we have to repeat
+    // them, or an effect that was already selected when the pipeline starts is
+    // silently ignored.
+    if (this.options.imagePath) {
+      await this.loadAndSetBackground(this.options.imagePath);
+    }
+    if (typeof this.options.blurRadius === "number") {
       this.gl?.setBlurRadius(this.options.blurRadius);
     }
+    this.gl?.setBackgroundDisabled(this.options.backgroundDisabled ?? false);
   }
 }

src/livekit/TrackProcessorContext.tsx

@@ -6,9 +6,11 @@ Please see LICENSE in the repository root for full details.
 */
 
 import {
-  ProcessorWrapper,
+  BackgroundProcessorWrapper,
+  type ProcessorWrapper,
   supportsBackgroundProcessors as supportsBackgroundProcessorsLivekitSdk,
   type BackgroundOptions,
+  type SwitchBackgroundProcessorOptions,
 } from "@livekit/track-processors";
 import {
   createContext,
@@ -17,6 +19,7 @@ import {
   use,
   useEffect,
   useMemo,
+  useState,
 } from "react";
 import { type LocalVideoTrack } from "livekit-client";
 import { logger } from "matrix-js-sdk/lib/logger";
@@ -24,10 +27,16 @@ import { combineLatest, map, type Observable } from "rxjs";
 import { useObservable } from "observable-hooks";
 
 import {
-  backgroundBlur as backgroundBlurSettings,
+  backgroundEffect as backgroundEffectSetting,
   useSetting,
 } from "../settings/settings";
-import { BlurBackgroundTransformer } from "./BlurBackgroundTransformer";
+import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer";
+import {
+  blurRadius,
+  imagePathFor,
+  parseEffect,
+  type BackgroundEffect,
+} from "./backgroundEffects";
 import { type Behavior } from "../state/Behavior";
 import { type ObservableScope } from "../state/ObservableScope";
 import { platform } from "../Platform";
@@ -129,26 +138,68 @@ function supportsBackgroundProcessors(): boolean {
   return supportsBackgroundProcessorsLivekitSdk() && platform === "desktop";
 }
 
+/** Translates a chosen effect into the pipeline's own vocabulary. */
+function switchOptionsFor(
+  effect: BackgroundEffect,
+): SwitchBackgroundProcessorOptions {
+  switch (effect.kind) {
+    case "blur":
+      return { mode: "background-blur", blurRadius };
+    case "image": {
+      const imagePath = imagePathFor(effect.id);
+      return imagePath
+        ? { mode: "virtual-background", imagePath }
+        : { mode: "disabled" };
+    }
+    default:
+      return { mode: "disabled" };
+  }
+}
+
 export const ProcessorProvider: FC<Props> = ({ children }) => {
-  // The setting the user wants to have
-  const [blurActivated] = useSetting(backgroundBlurSettings);
+  const [effectRaw] = useSetting(backgroundEffectSetting);
   const supported = useMemo(() => supportsBackgroundProcessors(), []);
-  const blur = useMemo(
+
+  // One pipeline for the lifetime of the app, so the pre-join preview and the
+  // call share it and its priming frame is spent before anything is published
+  // (D4).
+  const pipeline = useMemo(
     () =>
-      new ProcessorWrapper(
-        new BlurBackgroundTransformer({ blurRadius: 15 }),
-        "background-blur",
+      new BackgroundProcessorWrapper(
+        new BackgroundEffectTransformer({ backgroundDisabled: true }),
+        "background-effect",
       ),
     [],
   );
 
+  // D5: nothing is attached until an effect is first chosen, so a user who
+  // never chooses one pays neither the segmentation assets nor the time to
+  // initialise them. Once attached it stays attached, including at no effect,
+  // so a later change is a switch rather than a fresh attachment and spends no
+  // further priming frame.
+  const [attached, setAttached] = useState(
+    () => parseEffect(effectRaw).kind !== "none",
+  );
+  useEffect(() => {
+    if (parseEffect(effectRaw).kind !== "none") setAttached(true);
+  }, [effectRaw]);
+
+  // D2: switch the running pipeline in place rather than tearing it down, so
+  // the previous effect stays in force until the new one is live.
+  useEffect(() => {
+    if (!supported || !attached) return;
+    pipeline
+      .switchTo(switchOptionsFor(parseEffect(effectRaw)))
+      .catch((e) => logger.warn("Failed to switch background effect", e));
+  }, [pipeline, supported, attached, effectRaw]);
+
   // This is the actual state exposed through the context
   const processorState = useMemo(
     () => ({
       supported,
-      processor: supported && blurActivated ? blur : undefined,
+      processor: supported && attached ? pipeline : undefined,
     }),
-    [supported, blurActivated, blur],
+    [supported, attached, pipeline],
   );
 
   return <ProcessorContext value={processorState}>{children}</ProcessorContext>;

src/livekit/backgroundEffects.ts

@@ -0,0 +1,58 @@
+/*
+Copyright 2024-2025 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import desktopGradient from "../graphics/desktop-gradient.png?url";
+import mobileGradient from "../graphics/mobile-gradient.png?url";
+
+/** How much to blur, when the chosen effect is blur. */
+export const blurRadius = 15;
+
+export interface ShippedBackground {
+  id: string;
+  imagePath: string;
+}
+
+// EXPLORATION SHORTCUT (S1): the two shipped backgrounds, one indoor and one
+// outdoor, do not exist yet. These stand-ins are existing gradient assets, so
+// the pipeline can be exercised and measured. Never port this.
+export const shippedBackgrounds: ShippedBackground[] = [
+  { id: "indoor", imagePath: desktopGradient },
+  { id: "outdoor", imagePath: mobileGradient },
+];
+
+export type BackgroundEffect =
+  | { kind: "none" }
+  | { kind: "blur" }
+  | { kind: "image"; id: string };
+
+export const noEffect: BackgroundEffect = { kind: "none" };
+
+/** Parses the stored form of a chosen effect, falling back to no effect. */
+export function parseEffect(raw: string): BackgroundEffect {
+  if (raw === "blur") return { kind: "blur" };
+  if (raw.startsWith("image:")) {
+    const id = raw.slice("image:".length);
+    if (shippedBackgrounds.some((b) => b.id === id)) return { kind: "image", id };
+  }
+  return noEffect;
+}
+
+/** The stored form of a chosen effect. */
+export function serializeEffect(effect: BackgroundEffect): string {
+  switch (effect.kind) {
+    case "blur":
+      return "blur";
+    case "image":
+      return `image:${effect.id}`;
+    default:
+      return "none";
+  }
+}
+
+export function imagePathFor(id: string): string | undefined {
+  return shippedBackgrounds.find((b) => b.id === id)?.imagePath;
+}

src/settings/settings.ts

@@ -117,6 +117,15 @@ export const videoInput = new Setting<string | undefined>(
 
 export const backgroundBlur = new Setting<boolean>("background-blur", false);
 
+/**
+ * The background effect applied to the local camera: "none", "blur", or
+ * "image:<id>". See backgroundEffects.ts for the stored form.
+ */
+export const backgroundEffect = new Setting<string>(
+  "background-effect",
+  "none",
+);
+
 export const showHandRaisedTimer = new Setting<boolean>(
   "hand-raised-show-timer",
   false,