Attach background effects through one switchable pipeline

Blur runs through one pipeline, shared by the pre-join preview and the
call: attached the first time an effect is wanted and switched in place
after, never detached. The logic is BackgroundEffects in src/state;
BackgroundEffectsContext only builds it, from the app's view model, so
the standalone app and the component hand the same instance to every
camera. Disabled drops the blur radius, so no effect costs no
segmentation. The SDK gains fromLivekitProcessor beside the adapter,
now toLivekitProcessor.

Ported from spec/background-effects/2-pipeline (#4282): a91dab8f with
its review tidy-ups 445e010c, bbc73d07, 51729bcf, f52d2b79, 813f1e6f
and 584bbaa0. Left out as already solved here: OneStepPipeline (the SDK
runs a processor's steps in turn) and the LiveKit-track sync helpers
(the lobby's camera is an SDK track, synced by followProcessor).

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

component/index.tsx

@@ -72,9 +72,9 @@ import {
   useUrlParams,
 } from "../src/UrlParams";
 import { MediaDevicesContext } from "../src/MediaDevicesContext";
-import { MediaDevices } from "../src/state/MediaDevices";
+import { AppViewModel } from "../src/state/AppViewModel";
 import { ObservableScope } from "../src/state/ObservableScope";
-import { ProcessorProvider } from "../src/tracks/TrackProcessorContext";
+import { BackgroundEffectsProvider } from "../src/tracks/BackgroundEffectsContext";
 import { Config } from "../src/config/Config";
 import { type ConfigOptions } from "../src/config/ConfigOptions";
 import { i18n } from "../src/utils/i18n";
@@ -194,14 +194,12 @@ export const ElementCall: FC<ElementCallProps> = ({
   // observers running for the rest of the page's life. Null until then, which
   // is one render.
   const { controlledAudioDevices, callIntent } = params;
-  const [mediaDevices, setMediaDevices] = useState<MediaDevices | null>(null);
+  const [vm, setVm] = useState<AppViewModel | null>(null);
   useEffect(() => {
     const scope = new ObservableScope();
-    setMediaDevices(
-      new MediaDevices(scope, { controlledAudioDevices, callIntent }),
-    );
+    setVm(new AppViewModel(scope, { controlledAudioDevices, callIntent }));
     return (): void => {
-      setMediaDevices(null);
+      setVm(null);
       scope.end();
     };
   }, [controlledAudioDevices, callIntent]);
@@ -222,8 +220,7 @@ export const ElementCall: FC<ElementCallProps> = ({
   // is told that Element Call has loaded, as the widget tells its client once
   // its own initialisation is over. Once per mount, however often the pieces
   // are later swapped out.
-  const ready =
-    container !== null && rtcSession !== null && mediaDevices !== null;
+  const ready = container !== null && rtcSession !== null && vm !== null;
   const announcedLoaded = useRef(false);
   useEffect(() => {
     if (!ready || announcedLoaded.current) return;
@@ -254,8 +251,10 @@ export const ElementCall: FC<ElementCallProps> = ({
                       <Decoration>
                         <TooltipProvider>
                           <ClientProvider client={client}>
-                            <MediaDevicesContext value={mediaDevices}>
-                              <ProcessorProvider>
+                            <MediaDevicesContext value={vm.mediaDevices}>
+                              <BackgroundEffectsProvider
+                                effects={vm.backgroundEffects}
+                              >
                                 <CallView
                                   client={client}
                                   rtcSession={rtcSession}
@@ -264,7 +263,7 @@ export const ElementCall: FC<ElementCallProps> = ({
                                   preload={params.preload}
                                   skipLobby={params.skipLobby}
                                 />
-                              </ProcessorProvider>
+                              </BackgroundEffectsProvider>
                             </MediaDevicesContext>
                           </ClientProvider>
                         </TooltipProvider>

sdk/index.ts

@@ -63,6 +63,11 @@ export {
 } from "./src/media-backend/livekit/LivekitBackend";
 // Tracks for a lobby, captured ahead of the join and handed to it
 export { createLocalMedia } from "./src/LocalMedia";
+// For a host whose processor is a livekit-client one, such as @livekit/track-processors
+export {
+  fromLivekitProcessor,
+  toLivekitProcessor,
+} from "./src/media-backend/livekit/videoProcessor";
 // For a developer panel to check a transport before using it
 export {
   getSFUConfig as authenticateWithTransport,

sdk/src/media-backend/livekit/LivekitMediaTrack.ts

@@ -50,10 +50,7 @@ import {
   type VideoMediaTrack,
 } from "../../media-api";
 import { MatrixRTCError } from "../../errors";
-import {
-  convertToLivekitProcessor,
-  type LivekitProcessor,
-} from "./videoProcessor";
+import { toLivekitProcessor, type LivekitProcessor } from "./videoProcessor";
 
 export const mediaSources: Record<Track.Source, MediaSource> = {
   [Track.Source.Microphone]: "microphone",
@@ -245,8 +242,7 @@ export function createLocalLivekitMediaTrack(
       if (!track || track.mediaStreamTrack.readyState === "ended") return;
       const current = track.getProcessor() as LivekitProcessor | undefined;
       if (processor === current?.source) return;
-      if (processor)
-        await track.setProcessor(convertToLivekitProcessor(processor));
+      if (processor) await track.setProcessor(toLivekitProcessor(processor));
       else if (current) await track.stopProcessor();
     },
   };

sdk/src/media-backend/livekit/Publisher.ts

@@ -22,7 +22,7 @@ import {
   type PublishRequest,
 } from "../../media-api";
 import { livekitSources, mediaSources } from "./LivekitMediaTrack";
-import { convertToLivekitProcessor } from "./videoProcessor";
+import { toLivekitProcessor } from "./videoProcessor";
 import {
   audioCaptureOptions,
   screenShareCaptureOptions,
@@ -159,10 +159,7 @@ export class Publisher {
     if (muted) await local.mute();
     if (processor && local instanceof LocalVideoTrack)
       await local.setProcessor(
-        convertToLivekitProcessor(
-          processor.processor,
-          processor.processedTrack,
-        ),
+        toLivekitProcessor(processor.processor, processor.processedTrack),
       );
     return this.room.localParticipant.publishTrack(local, {
       source: livekitSources[request.source],

sdk/src/media-backend/livekit/publishOptions.ts

@@ -16,7 +16,7 @@ import {
   type PublishRequest,
   type VideoCaptureSettings,
 } from "../../media-api";
-import { convertToLivekitProcessor } from "./videoProcessor";
+import { toLivekitProcessor } from "./videoProcessor";
 
 export type PublishRequestFor<S extends PublishRequest["source"]> = Extract<
   PublishRequest,
@@ -40,7 +40,7 @@ export function videoCaptureOptions({
   if (!deviceId && !processor && !capture?.resolution) return undefined;
   return {
     ...(deviceId && { deviceId }),
-    ...(processor && { processor: convertToLivekitProcessor(processor) }),
+    ...(processor && { processor: toLivekitProcessor(processor) }),
     ...(capture?.resolution && { resolution: capture.resolution }),
   };
 }

sdk/src/media-backend/livekit/videoProcessor.test.ts

@@ -5,13 +5,17 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import { Track, type VideoProcessorOptions } from "livekit-client";
+import {
+  Track,
+  type TrackProcessor,
+  type VideoProcessorOptions,
+} from "livekit-client";
 import { describe, expect, it, vi } from "vitest";
 
 import { type VideoProcessor } from "../../media-api";
-import { convertToLivekitProcessor } from "./videoProcessor";
+import { fromLivekitProcessor, toLivekitProcessor } from "./videoProcessor";
 
-describe("convertToLivekitProcessor", () => {
+describe("toLivekitProcessor", () => {
   const captured = { id: "captured" } as MediaStreamTrack;
   const processed = { id: "processed" } as MediaStreamTrack;
   const restarted = { id: "restarted" } as MediaStreamTrack;
@@ -33,7 +37,7 @@ describe("convertToLivekitProcessor", () => {
 
   it("holds the processed track where LiveKit reads it", async () => {
     const processor = fakeProcessor();
-    const adapter = convertToLivekitProcessor(processor);
+    const adapter = toLivekitProcessor(processor);
     expect(adapter.name).toBe("blur");
     expect(adapter.processedTrack).toBeUndefined();
 
@@ -59,8 +63,8 @@ describe("convertToLivekitProcessor", () => {
           finishDestroy = resolve;
         }),
     );
-    const previous = convertToLivekitProcessor(processor);
-    const next = convertToLivekitProcessor(processor);
+    const previous = toLivekitProcessor(processor);
+    const next = toLivekitProcessor(processor);
 
     const destroying = previous.destroy();
     const initialising = next.init(options);
@@ -73,3 +77,40 @@ describe("convertToLivekitProcessor", () => {
     expect(next.processedTrack).toBe(processed);
   });
 });
+
+describe("fromLivekitProcessor", () => {
+  const captured = { id: "captured" } as MediaStreamTrack;
+  const processed = { id: "processed" } as MediaStreamTrack;
+
+  function fakeLivekitProcessor(
+    processedTrack: MediaStreamTrack | undefined,
+  ): TrackProcessor<Track.Kind.Video> {
+    return {
+      name: "background-blur",
+      processedTrack,
+      init: vi.fn().mockResolvedValue(undefined),
+      restart: vi.fn().mockResolvedValue(undefined),
+      destroy: vi.fn().mockResolvedValue(undefined),
+    };
+  }
+
+  it("resolves with the track the LiveKit processor produced", async () => {
+    const livekitProcessor = fakeLivekitProcessor(processed);
+    const processor = fromLivekitProcessor(livekitProcessor);
+    expect(processor.name).toBe("background-blur");
+    await expect(processor.init(captured)).resolves.toBe(processed);
+    expect(livekitProcessor.init).toHaveBeenCalledWith({
+      kind: Track.Kind.Video,
+      track: captured,
+    });
+    await expect(processor.restart(captured)).resolves.toBe(processed);
+    await processor.destroy();
+    expect(livekitProcessor.destroy).toHaveBeenCalled();
+  });
+
+  it("rejects where the LiveKit processor produced no track", async () => {
+    await expect(
+      fromLivekitProcessor(fakeLivekitProcessor(undefined)).init(captured),
+    ).rejects.toThrow("produced no track");
+  });
+});

sdk/src/media-backend/livekit/videoProcessor.ts

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import { type Track, type TrackProcessor } from "livekit-client";
+import { Track, type TrackProcessor } from "livekit-client";
 
 import { type VideoProcessor } from "../../media-api";
 import { inTurn } from "../../utils/processorTurns";
@@ -16,14 +16,15 @@ export interface LivekitProcessor extends TrackProcessor<Track.Kind.Video> {
 }
 
 /**
- * A `VideoProcessor` as livekit-client wants it. LiveKit reads the processed
- * track off the processor after `init` and `restart` rather than taking it as
- * their result, so the adapter keeps it there. A processor a lobby already
- * runs on the track is adopted with its output, `running`, so that the join
- * does not initialise it again. Every step takes its turn on the processor,
- * which a stopped track tears down without waiting for the next one's init.
+ * A `VideoProcessor` as a livekit-client processor. LiveKit reads the
+ * processed track off the processor after `init` and `restart` rather than
+ * taking it as their result, so the adapter keeps it there. A processor a
+ * lobby already runs on the track is adopted with its output, `running`, so
+ * that the join does not initialise it again. Every step takes its turn on
+ * the processor, which a stopped track tears down without waiting for the
+ * next one's init.
  */
-export function convertToLivekitProcessor(
+export function toLivekitProcessor(
   processor: VideoProcessor,
   running?: MediaStreamTrack,
 ): LivekitProcessor {
@@ -47,3 +48,30 @@ export function convertToLivekitProcessor(
   };
   return adapter;
 }
+
+/**
+ * A livekit-client processor, such as one from `@livekit/track-processors`,
+ * as a `VideoProcessor`: `init` and `restart` resolve with the track it
+ * leaves on itself.
+ */
+export function fromLivekitProcessor(
+  processor: TrackProcessor<Track.Kind.Video>,
+): VideoProcessor {
+  const processedTrack = (): MediaStreamTrack => {
+    if (processor.processedTrack === undefined)
+      throw new Error(`Processor ${processor.name} produced no track`);
+    return processor.processedTrack;
+  };
+  return {
+    name: processor.name,
+    init: async (track, element) => {
+      await processor.init({ kind: Track.Kind.Video, track, element });
+      return processedTrack();
+    },
+    restart: async (track, element) => {
+      await processor.restart({ kind: Track.Kind.Video, track, element });
+      return processedTrack();
+    },
+    destroy: async () => processor.destroy(),
+  };
+}

src/App.tsx

@@ -35,7 +35,7 @@ import { ErrorPage, LoadingPage } from "./FullScreenView";
 import { Initializer } from "./initializer";
 import { type WidgetHelpers } from "./widget";
 import { useTheme } from "./useTheme";
-import { ProcessorProvider } from "./tracks/TrackProcessorContext";
+import { BackgroundEffectsProvider } from "./tracks/BackgroundEffectsContext";
 import { type AppViewModel } from "./state/AppViewModel";
 import { MediaDevicesContext } from "./MediaDevicesContext";
 import {
@@ -148,7 +148,7 @@ export const App: FC<Props> = ({ vm, widget }) => {
     loaded && clientReady ? (
       <ClientProvider client={widgetClient}>
         <MediaDevicesContext value={vm.mediaDevices}>
-          <ProcessorProvider>
+          <BackgroundEffectsProvider effects={vm.backgroundEffects}>
             <Sentry.ErrorBoundary
               fallback={(error) => <ErrorPage error={error} />}
             >
@@ -159,7 +159,7 @@ export const App: FC<Props> = ({ vm, widget }) => {
                 <SentryRoute path="*" element={<RoomPage />} />
               </Routes>
             </Sentry.ErrorBoundary>
-          </ProcessorProvider>
+          </BackgroundEffectsProvider>
         </MediaDevicesContext>
       </ClientProvider>
     ) : (

src/room/CallView.test.tsx

@@ -50,6 +50,7 @@ import {
   mockMediaDevices,
   mockRtcMembership,
   MockRTCSession,
+  testScope,
 } from "../utils/test";
 import { CallView } from "./CallView";
 import { GroupCallErrorBoundary } from "./GroupCallErrorBoundary";
@@ -61,7 +62,10 @@ import {
 } from "../HostBridge";
 import { type JoinCallData } from "../widget";
 import { MatrixRTCTransportMissingError } from "../utils/errors";
-import { ProcessorProvider } from "../tracks/TrackProcessorContext";
+import {
+  BackgroundEffectsProvider,
+  createAppBackgroundEffects,
+} from "../tracks/BackgroundEffectsContext";
 import { MediaDevicesContext } from "../MediaDevicesContext";
 import { constant } from "../state/Behavior";
 
@@ -193,7 +197,9 @@ function createCallView(
       <HostBridgeProvider value={hostBridge}>
         <TooltipProvider>
           <MediaDevicesContext value={mockMediaDevices({})}>
-            <ProcessorProvider>
+            <BackgroundEffectsProvider
+              effects={createAppBackgroundEffects(testScope())}
+            >
               {options.withErrorBoundary ? (
                 <GroupCallErrorBoundary recoveryActionHandler={vi.fn()}>
                   {callView}
@@ -201,7 +207,7 @@ function createCallView(
               ) : (
                 callView
               )}
-            </ProcessorProvider>
+            </BackgroundEffectsProvider>
           </MediaDevicesContext>
         </TooltipProvider>
       </HostBridgeProvider>

src/room/InCallView.test.tsx

@@ -28,6 +28,7 @@ import {
   type MockRTCSession,
   mockRTCSlot,
   mockRTCMember,
+  testScope,
 } from "../utils/test";
 import { E2eeType } from "../e2ee/e2eeType";
 import {
@@ -48,7 +49,10 @@ import { MediaDevicesContext } from "../MediaDevicesContext";
 import { type MediaDevices as ECMediaDevices } from "../state/MediaDevices";
 import { AppBar } from "../AppBar";
 import { type MatrixInfo } from "./VideoPreview";
-import { ProcessorProvider } from "../tracks/TrackProcessorContext";
+import {
+  BackgroundEffectsProvider,
+  createAppBackgroundEffects,
+} from "../tracks/BackgroundEffectsContext";
 import { initializeWidget } from "../widget";
 import { RootElementProvider } from "../RootElementContext";
 
@@ -246,7 +250,9 @@ describe("ActiveCall", () => {
     const { findByTestId } = render(
       <BrowserRouter>
         <MediaDevicesContext value={mediaDevices}>
-          <ProcessorProvider>
+          <BackgroundEffectsProvider
+            effects={createAppBackgroundEffects(testScope())}
+          >
             <TooltipProvider>
               <ActiveCall
                 client={matrixRoom.client}
@@ -260,7 +266,7 @@ describe("ActiveCall", () => {
                 onLeft={(): void => {}}
               />
             </TooltipProvider>
-          </ProcessorProvider>
+          </BackgroundEffectsProvider>
         </MediaDevicesContext>
       </BrowserRouter>,
     );
@@ -295,7 +301,9 @@ describe("ActiveCall", () => {
         <BrowserRouter>
           <RootElementProvider value={root}>
             <MediaDevicesContext value={mediaDevices}>
-              <ProcessorProvider>
+              <BackgroundEffectsProvider
+                effects={createAppBackgroundEffects(testScope())}
+              >
                 <TooltipProvider>
                   <ActiveCall
                     client={matrixRoom.client}
@@ -309,7 +317,7 @@ describe("ActiveCall", () => {
                     onLeft={(): void => {}}
                   />
                 </TooltipProvider>
-              </ProcessorProvider>
+              </BackgroundEffectsProvider>
             </MediaDevicesContext>
           </RootElementProvider>
         </BrowserRouter>,

src/room/InCallView.tsx

@@ -110,7 +110,7 @@ import { useBehavior } from "../useBehavior.ts";
 import { useValueBehavior } from "../useValueBehavior.ts";
 import { Toast } from "../Toast.tsx";
 import overlayStyles from "../Overlay.module.css";
-import { useTrackProcessorState$ } from "../tracks/TrackProcessorContext.tsx";
+import { useBackgroundEffectsState$ } from "../tracks/BackgroundEffectsContext";
 import { type Layout } from "../state/layout-types.ts";
 import { ObservableScope } from "../state/ObservableScope.ts";
 import { CallFooter, type FooterSnapshot } from "../components/CallFooter.tsx";
@@ -153,7 +153,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
   const urlParams = useUrlParams();
   const hostBridge = useHostBridge();
   const mediaDevices = useMediaDevices();
-  const trackProcessorState$ = useTrackProcessorState$();
+  const backgroundEffectsState$ = useBackgroundEffectsState$();
   // The element we have to draw the call in: the page, or the container a host
   // gave us. Its size, not the window's, decides how the call is laid out.
   const rootElement = useRootElement();
@@ -191,7 +191,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
         encryptionSystem: props.e2eeSystem,
         capture,
         videoProcessor$: scope.behavior(
-          trackProcessorState$.pipe(map((state) => state.processor)),
+          backgroundEffectsState$.pipe(map((state) => state.processor)),
         ),
         hostBridge,
         autoLeaveWhenOthersLeft,
@@ -221,7 +221,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
     urlParams,
     hostBridge,
     mediaDevices,
-    trackProcessorState$,
+    backgroundEffectsState$,
     props.client,
     props.media,
     rootElement,
@@ -253,7 +253,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
     props.onLeft,
     urlParams,
     mediaDevices,
-    trackProcessorState$,
+    backgroundEffectsState$,
     props.client,
     vm,
   ]);

src/room/KnockLobbyView.test.tsx

@@ -19,7 +19,7 @@ import {
 import { KnockLobbyView } from "./KnockLobbyView";
 import { LeaveToHomeProvider } from "../LeaveToHomeContext";
 import { MediaDevicesContext } from "../MediaDevicesContext";
-import { type ProcessorState } from "../tracks/TrackProcessorContext";
+import { type BackgroundEffectsState } from "../state/BackgroundEffects";
 import { mockMediaDevices } from "../utils/test";
 
 // One instance each: the lobby keys effects on them
@@ -27,12 +27,12 @@ const stable = vi.hoisted(() => ({
   processorState$: undefined as unknown,
   media: undefined as unknown,
 }));
-vi.mock("../tracks/TrackProcessorContext", () => ({
-  useTrackProcessorState$: (): Behavior<ProcessorState> =>
+vi.mock("../tracks/BackgroundEffectsContext", () => ({
+  useBackgroundEffectsState$: (): Behavior<BackgroundEffectsState> =>
     (stable.processorState$ ??= constant({
       supported: false,
       processor: undefined,
-    })) as Behavior<ProcessorState>,
+    })) as Behavior<BackgroundEffectsState>,
 }));
 // Media that opens no device: the view is what is under test
 vi.mock("../useLocalMedia", () => ({

src/room/LobbyView.test.tsx

@@ -33,7 +33,7 @@ import {
 } from "../utils/test";
 import { type MediaDevices } from "../state/MediaDevices";
 import { MediaDevicesContext } from "../MediaDevicesContext";
-import { type ProcessorState } from "../tracks/TrackProcessorContext";
+import { type BackgroundEffectsState } from "../state/BackgroundEffects";
 import { type EncryptionSystem } from "../e2ee/sharedKeyManagement";
 import lobbyStyles from "./LobbyView.module.css";
 import headerStyles from "../Header.module.css";
@@ -44,12 +44,12 @@ const leaveToHome = vi.fn();
 
 // One instance: the view keys an effect on it
 const processorState$ = vi.hoisted(() => ({ current: undefined as unknown }));
-vi.mock("../tracks/TrackProcessorContext", () => ({
-  useTrackProcessorState$: (): Behavior<ProcessorState> =>
+vi.mock("../tracks/BackgroundEffectsContext", () => ({
+  useBackgroundEffectsState$: (): Behavior<BackgroundEffectsState> =>
     (processorState$.current ??= constant({
       supported: false,
       processor: undefined,
-    })) as Behavior<ProcessorState>,
+    })) as Behavior<BackgroundEffectsState>,
 }));
 
 // Media that opens no device: the view is what is under test

src/room/LobbyView.tsx

@@ -31,7 +31,7 @@ import { LeaveToHomeLink } from "../button/LeaveToHomeLink";
 import { useLeaveToHome } from "../LeaveToHomeContext";
 import { useMediaDevices } from "../MediaDevicesContext";
 import { ObservableScope } from "../state/ObservableScope";
-import { useTrackProcessorState$ } from "../tracks/TrackProcessorContext";
+import { useBackgroundEffectsState$ } from "../tracks/BackgroundEffectsContext";
 import { useBehavior } from "../useBehavior";
 import { CallFooter, type FooterSnapshot } from "../components/CallFooter";
 import { useCallViewKeyboardShortcuts } from "../useCallViewKeyboardShortcuts";
@@ -122,7 +122,7 @@ export const LobbyView: FC<Props> = ({
 
   const devices = useMediaDevices();
   const videoInputId = useBehavior(devices.videoInput.selected$)?.id;
-  const trackProcessorState$ = useTrackProcessorState$();
+  const backgroundEffectsState$ = useBackgroundEffectsState$();
 
   const [mediaVm, setMediaVm] = useState<LobbyMediaViewModel | null>(null);
   useEffect(() => {
@@ -135,14 +135,14 @@ export const LobbyView: FC<Props> = ({
         devices,
         captureSettings(),
         mediaScope.behavior(
-          trackProcessorState$.pipe(map((state) => state.processor)),
+          backgroundEffectsState$.pipe(map((state) => state.processor)),
         ),
       ),
     );
     return (): void => {
       mediaScope.end();
     };
-  }, [media, muteStates, devices, trackProcessorState$]);
+  }, [media, muteStates, devices, backgroundEffectsState$]);
   const videoTrack = useBehavior(mediaVm?.videoTrack$ ?? noVideoTrack);
 
   useEffect(() => {

src/settings/SettingsModal.tsx

@@ -27,7 +27,7 @@ import {
 import { PreferencesSettingsTab } from "./PreferencesSettingsTab";
 import { Slider } from "../Slider";
 import { DeviceSelection } from "./DeviceSelection";
-import { useTrackProcessor } from "../tracks/TrackProcessorContext";
+import { useBackgroundEffects } from "../tracks/BackgroundEffectsContext";
 import {
   DeveloperSettingsTab,
   type DeveloperSettingsSnapshot,
@@ -78,7 +78,7 @@ export const SettingsModal: FC<Props> = ({
 
   // Generate a `Checkbox` input to turn blur on or off.
   const BlurCheckbox: React.FC = (): ReactNode => {
-    const { supported } = useTrackProcessor();
+    const { supported } = useBackgroundEffects();
 
     const [blurActive, setBlurActive] = useSetting(backgroundBlurSetting);
 

src/state/AppViewModel.ts

@@ -7,6 +7,7 @@ Please see LICENSE in the repository root for full details.
 
 import { type AudioOutputOptions, MediaDevices } from "./MediaDevices";
 import { type ObservableScope } from "./ObservableScope";
+import { createAppBackgroundEffects } from "../tracks/BackgroundEffectsContext";
 
 /**
  * The top-level state holder for the application.
@@ -16,6 +17,7 @@ export class AppViewModel {
     this.scope,
     this.audioOutputOptions,
   );
+  public readonly backgroundEffects = createAppBackgroundEffects(this.scope);
 
   // TODO: Move more application logic here. The CallViewModel, at the very
   // least, ought to be accessible from this object.

src/state/BackgroundEffects.test.ts

@@ -0,0 +1,140 @@
+/*
+Copyright 2026 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { BehaviorSubject, distinctUntilChanged, map } from "rxjs";
+import { type BackgroundProcessorWrapper } from "@livekit/track-processors";
+
+import {
+  type BackgroundEffects,
+  createBackgroundEffects,
+  type BackgroundEffectsOptions,
+  type BackgroundEffectsState,
+} from "./BackgroundEffects";
+import { flushPromises, testScope, withTestScheduler } from "../utils/test";
+
+/** A pipeline that records what it is switched to. */
+function fakePipeline(): {
+  pipeline: BackgroundProcessorWrapper;
+  switches: unknown[];
+  /** Holds the next switch until it resolves, as a picture loading does. */
+  holdNext: () => () => void;
+} {
+  const switches: unknown[] = [];
+  let slow: Promise<void> | undefined;
+  const pipeline = {
+    switchTo: vi.fn(async (options: unknown) => {
+      switches.push(options);
+      const held = slow;
+      slow = undefined;
+      await held;
+    }),
+  } as unknown as BackgroundProcessorWrapper;
+  return {
+    pipeline,
+    switches,
+    holdNext: () => {
+      let release!: () => void;
+      slow = new Promise((resolve) => (release = resolve));
+      return release;
+    },
+  };
+}
+
+/** One letter per state: idle, attached. */
+function letter(state: BackgroundEffectsState): string {
+  return state.processor === undefined ? "i" : "a";
+}
+
+describe("the pipeline's state", () => {
+  function testState({
+    effect,
+    expected,
+  }: {
+    effect: string;
+    expected: string;
+  }): void {
+    withTestScheduler(({ behavior, expectObservable }) => {
+      const effects = createBackgroundEffects(testScope(), {
+        supported: true,
+        blur$: behavior(effect, { n: false, b: true }),
+        pipeline: fakePipeline().pipeline,
+      });
+      expectObservable(
+        effects.state$.pipe(map(letter), distinctUntilChanged()),
+      ).toBe(expected);
+    });
+  }
+
+  it("attaches on first use and stays attached", () =>
+    testState({ effect: "nbn", expected: "ia" }));
+});
+
+describe("background effects", () => {
+  let blur$: BehaviorSubject<boolean>;
+  let fake: ReturnType<typeof fakePipeline>;
+
+  function build(
+    options: Partial<BackgroundEffectsOptions> = {},
+  ): BackgroundEffects {
+    return createBackgroundEffects(testScope(), {
+      supported: true,
+      blur$,
+      pipeline: fake.pipeline,
+      ...options,
+    });
+  }
+  const blur = async (on: boolean): Promise<void> => {
+    blur$.next(on);
+    await flushPromises();
+  };
+
+  beforeEach(() => {
+    blur$ = new BehaviorSubject(false);
+    fake = fakePipeline();
+  });
+
+  it("switches in place rather than reattaching", async () => {
+    const effects = build();
+    await blur(true);
+    const pipeline = effects.state$.value.processor;
+    await blur(false);
+    await blur(true);
+
+    expect(effects.state$.value.processor).toBe(pipeline);
+    expect(fake.switches).toEqual([
+      { mode: "background-blur", blurRadius: 15 },
+      { mode: "disabled" },
+      { mode: "background-blur", blurRadius: 15 },
+    ]);
+  });
+
+  it("keeps switching after a switch fails", async () => {
+    vi.mocked(fake.pipeline.switchTo).mockRejectedValueOnce(new Error("lost"));
+    build();
+    await blur(true);
+    await blur(false);
+    expect(fake.switches).toEqual([{ mode: "disabled" }]);
+  });
+
+  it("switches one at a time, skipping those overtaken", async () => {
+    const finished = fake.holdNext();
+    build();
+    await blur(true);
+    await blur(false);
+    await blur(true);
+    await blur(false);
+    expect(fake.switches).toHaveLength(1);
+
+    finished();
+    await flushPromises();
+    expect(fake.switches).toEqual([
+      { mode: "background-blur", blurRadius: 15 },
+      { mode: "disabled" },
+    ]);
+  });
+});

src/state/BackgroundEffects.ts

@@ -0,0 +1,91 @@
+/*
+Copyright 2026 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { combineLatest, distinctUntilChanged, map, scan } from "rxjs";
+import {
+  type BackgroundProcessorWrapper,
+  type SwitchBackgroundProcessorOptions,
+} from "@livekit/track-processors";
+import { logger } from "matrix-js-sdk/lib/logger";
+import { deepCompare } from "matrix-js-sdk/lib/utils";
+import {
+  fromLivekitProcessor,
+  type VideoProcessor,
+} from "@element-hq/matrixrtc-sdk";
+
+import { type Behavior } from "./Behavior";
+import { type ObservableScope } from "./ObservableScope";
+
+const blurRadius = 15;
+
+export interface BackgroundEffectsState {
+  supported: boolean | undefined;
+  processor: undefined | VideoProcessor;
+}
+
+export interface BackgroundEffectsOptions {
+  /** Whether this browser can run a pipeline at all. */
+  supported: boolean;
+  /** Whether blur is chosen. */
+  blur$: Behavior<boolean>;
+  /** The background processor pipeline to be switched between effects. */
+  pipeline: BackgroundProcessorWrapper;
+}
+
+/** The background effect pipeline, as the camera tracks and the menus see it. */
+export interface BackgroundEffects {
+  readonly state$: Behavior<BackgroundEffectsState>;
+}
+
+/** Switches the pipeline as the choice changes, for as long as the scope lasts. */
+export function createBackgroundEffects(
+  scope: ObservableScope,
+  { supported, blur$, pipeline }: BackgroundEffectsOptions,
+): BackgroundEffects {
+  // Wrapped once, so the call and the preview see one processor throughout.
+  const processor = fromLivekitProcessor(pipeline);
+  const state$ = scope.behavior(
+    blur$.pipe(
+      scan<boolean, BackgroundEffectsState>(
+        (previous, wanted) => {
+          // Attached the first time an effect is wanted and never detached
+          // after, so someone who never turns one on pays for none of it.
+          const enable =
+            previous.processor !== undefined || (supported && wanted);
+          return { supported, processor: enable ? processor : undefined };
+        },
+        { supported, processor: undefined },
+      ),
+    ),
+  );
+
+  const switchOptions$ = scope.behavior<
+    SwitchBackgroundProcessorOptions | undefined
+  >(
+    combineLatest([state$, blur$]).pipe(
+      map(([{ processor }, blur]) =>
+        processor === undefined
+          ? undefined
+          : blur
+            ? { mode: "background-blur", blurRadius }
+            : { mode: "disabled" },
+      ),
+      distinctUntilChanged(deepCompare),
+    ),
+  );
+  // In turn: a picture's switch ends once it loads, so an earlier one could land last.
+  scope.reconcile(switchOptions$, async (options) => {
+    if (options === undefined) return;
+    try {
+      await pipeline.switchTo(options);
+    } catch (e) {
+      logger.warn("Failed to switch background effect", e);
+    }
+  });
+
+  return { state$ };
+}

src/tracks/BackgroundEffectTransformer.test.ts

@@ -0,0 +1,24 @@
+/*
+Copyright 2026 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { describe, expect, it } from "vitest";
+
+import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer";
+
+describe("BackgroundEffectTransformer", () => {
+  it("passes frames on untouched once disabled, even after blurring", async () => {
+    const transformer = new BackgroundEffectTransformer({});
+    await transformer.update({ blurRadius: 15, backgroundDisabled: false });
+    await transformer.update({
+      imagePath: undefined,
+      backgroundDisabled: true,
+    });
+    // With neither, the library passes a frame on without segmenting it.
+    expect(transformer.options.blurRadius).toBeUndefined();
+    expect(transformer.options.imagePath).toBeUndefined();
+  });
+});

src/tracks/BackgroundEffectTransformer.ts

@@ -1,5 +1,6 @@
 /*
 Copyright 2024-2025 New Vector Ltd.
+Copyright 2026 Element Creations Ltd.
 
 SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
@@ -9,6 +10,7 @@ import {
   BackgroundTransformer,
   VideoTransformer,
   type VideoTransformerInitOptions,
+  type BackgroundOptions,
 } from "@livekit/track-processors";
 import { ImageSegmenter } from "@mediapipe/tasks-vision";
 
@@ -49,7 +51,18 @@ const wasmFileset: WasmFileset = {
  * loads the segmentation models from our own bundle rather than as an external
  * resource fetched from the public internet.
  */
-export class BlurBackgroundTransformer extends BackgroundTransformer {
+export class BackgroundEffectTransformer extends BackgroundTransformer {
+  /**
+   * As the library's, except that disabling also drops the blur radius: kept,
+   * it has every frame segmented and the result thrown away, where with
+   * neither a radius nor a picture frames are passed on untouched.
+   */
+  public override async update(opts: BackgroundOptions): Promise<void> {
+    await super.update(
+      opts.backgroundDisabled ? { ...opts, blurRadius: undefined } : opts,
+    );
+  }
+
   public async init({
     outputCanvas,
     inputElement: inputVideo,
@@ -73,8 +86,10 @@ export class BlurBackgroundTransformer extends BackgroundTransformer {
       outputConfidenceMasks: false,
     });
 
+    // BackgroundTransformer's own init applies these, and this one replaces it.
     if (this.options.blurRadius) {
       this.gl?.setBlurRadius(this.options.blurRadius);
     }
+    this.gl?.setBackgroundDisabled(this.options.backgroundDisabled ?? false);
   }
 }

src/tracks/BackgroundEffectsContext.test.ts

@@ -0,0 +1,115 @@
+/*
+Copyright 2026 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { act, createElement, type FC } from "react";
+import { render } from "@testing-library/react";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+import {
+  BackgroundEffectsProvider,
+  useBackgroundEffects,
+  createAppBackgroundEffects,
+} from "./BackgroundEffectsContext";
+import {
+  type BackgroundEffects,
+  type BackgroundEffectsState,
+} from "../state/BackgroundEffects";
+import { backgroundBlur } from "../settings/settings";
+import { flushPromises, testScope } from "../utils/test";
+
+const pipelines = vi.hoisted(() => ({
+  built: 0,
+  destroyed: 0,
+  switches: [] as unknown[],
+}));
+
+// The pipeline itself needs WebGL and MediaPipe; what these tests are about is
+// what the provider asks of it.
+vi.mock("@livekit/track-processors", () => ({
+  BackgroundProcessorWrapper: vi.fn(function (this: Record<string, unknown>) {
+    pipelines.built += 1;
+    this.switchTo = vi.fn(async (options: unknown) => {
+      pipelines.switches.push(options);
+      return Promise.resolve();
+    });
+    this.destroy = vi.fn(async () => {
+      pipelines.destroyed += 1;
+      return Promise.resolve();
+    });
+  }),
+  supportsBackgroundProcessors: (): boolean => true,
+}));
+const platformMock = vi.hoisted(() => ({ platform: "desktop" }));
+vi.mock("../Platform", () => platformMock);
+vi.mock("./BackgroundEffectTransformer", () => ({
+  BackgroundEffectTransformer: vi.fn(),
+}));
+
+describe("BackgroundEffectsProvider", () => {
+  let seen: BackgroundEffectsState[];
+  const Surface: FC = () => {
+    seen.push(useBackgroundEffects());
+    return null;
+  };
+  // Made on first render, once a test has set up the platform.
+  let effects: BackgroundEffects | undefined;
+  const surface = (): ReturnType<typeof createElement> =>
+    createElement(BackgroundEffectsProvider, {
+      effects: (effects ??= createAppBackgroundEffects(testScope())),
+      children: createElement(Surface),
+    });
+  const blur = async (on: boolean): Promise<void> => {
+    await act(async () => {
+      backgroundBlur.setValue(on);
+      await flushPromises();
+    });
+  };
+  const latest = (): BackgroundEffectsState => seen[seen.length - 1];
+
+  beforeEach(() => {
+    seen = [];
+    effects = undefined;
+    pipelines.built = 0;
+    pipelines.destroyed = 0;
+    pipelines.switches = [];
+    platformMock.platform = "desktop";
+    backgroundBlur.setValue(false);
+  });
+  afterEach(() => backgroundBlur.setValue(false));
+
+  // A pipeline primes itself when it is built and when it is destroyed, and a
+  // primed pipeline lets one frame through untouched: so the frame is spent
+  // once, as long as it is built once and never taken off the camera.
+  it("builds one pipeline on first use and keeps it through every change", async () => {
+    render(surface());
+    for (const on of [true, false, true, false, true]) await blur(on);
+
+    expect(pipelines.built).toBe(1);
+    expect(pipelines.destroyed).toBe(0);
+    expect(latest().processor).toBeDefined();
+  });
+
+  it("does not run on a phone, even where its browser could", async () => {
+    platformMock.platform = "ios";
+    render(surface());
+    await blur(true);
+
+    expect(latest().supported).toBe(false);
+    expect(latest().processor).toBeUndefined();
+  });
+
+  it("switches to an effect chosen while the camera is off", async () => {
+    render(surface());
+    await blur(true);
+
+    // Already switched to blur, with nothing yet to attach it to
+    expect(pipelines.switches).toEqual([
+      { mode: "background-blur", blurRadius: 15 },
+    ]);
+    expect(pipelines.built).toBe(1);
+  });
+});

src/tracks/BackgroundEffectsContext.tsx

@@ -0,0 +1,74 @@
+/*
+Copyright 2024-2026 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import {
+  BackgroundProcessorWrapper,
+  supportsBackgroundProcessors as supportsBackgroundProcessorsLivekitSdk,
+} from "@livekit/track-processors";
+import { createContext, type FC, type JSX, use } from "react";
+
+import { backgroundBlur as backgroundBlurSettings } from "../settings/settings";
+import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer";
+import { type Behavior } from "../state/Behavior";
+import { type ObservableScope } from "../state/ObservableScope";
+import { platform } from "../Platform";
+import {
+  type BackgroundEffects,
+  type BackgroundEffectsState,
+  createBackgroundEffects,
+} from "../state/BackgroundEffects";
+import { useBehavior } from "../useBehavior";
+
+const BackgroundEffectsContext = createContext<BackgroundEffects | undefined>(
+  undefined,
+);
+
+export function useBackgroundEffects(): BackgroundEffectsState {
+  return useBehavior(useBackgroundEffectsState$());
+}
+
+export function useBackgroundEffectsState$(): Behavior<BackgroundEffectsState> {
+  const effects = use(BackgroundEffectsContext);
+  if (effects === undefined)
+    throw new Error(
+      "useBackgroundEffects must be used within a BackgroundEffectsProvider",
+    );
+  return effects.state$;
+}
+
+function supportsBackgroundProcessors(): boolean {
+  return supportsBackgroundProcessorsLivekitSdk() && platform === "desktop";
+}
+
+/**
+ * The app's one pipeline, shared by every camera track it opens. Its builds
+ * and teardowns take their turns through the SDK, which runs one operation
+ * at a time on a processor.
+ */
+export function createAppBackgroundEffects(
+  scope: ObservableScope,
+): BackgroundEffects {
+  return createBackgroundEffects(scope, {
+    supported: supportsBackgroundProcessors(),
+    blur$: backgroundBlurSettings.value$,
+    pipeline: new BackgroundProcessorWrapper(
+      new BackgroundEffectTransformer({ backgroundDisabled: true }),
+      "background-effect",
+    ),
+  });
+}
+
+interface Props {
+  effects: BackgroundEffects;
+  children: JSX.Element;
+}
+
+export const BackgroundEffectsProvider: FC<Props> = ({ effects, children }) => (
+  <BackgroundEffectsContext value={effects}>
+    {children}
+  </BackgroundEffectsContext>
+);

src/tracks/TrackProcessorContext.test.ts

@@ -1,51 +0,0 @@
-/*
-Copyright 2026 Element Creations Ltd.
-
-SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
-Please see LICENSE in the repository root for full details.
-*/
-
-import { describe, expect, it, vi } from "vitest";
-import {
-  type BackgroundOptions,
-  type ProcessorWrapper,
-} from "@livekit/track-processors";
-
-import { fromLivekitProcessor } from "./TrackProcessorContext";
-
-describe("fromLivekitProcessor", () => {
-  const captured = { id: "captured" } as MediaStreamTrack;
-  const processed = { id: "processed" } as MediaStreamTrack;
-
-  function mockWrapper(
-    processedTrack: MediaStreamTrack | undefined,
-  ): ProcessorWrapper<BackgroundOptions> {
-    return {
-      name: "background-blur",
-      processedTrack,
-      init: vi.fn().mockResolvedValue(undefined),
-      restart: vi.fn().mockResolvedValue(undefined),
-      destroy: vi.fn().mockResolvedValue(undefined),
-    } as unknown as ProcessorWrapper<BackgroundOptions>;
-  }
-
-  it("resolves with the track the wrapper produced", async () => {
-    const wrapper = mockWrapper(processed);
-    const sdkProcessor = fromLivekitProcessor(wrapper);
-    expect(sdkProcessor.name).toBe("background-blur");
-    await expect(sdkProcessor.init(captured)).resolves.toBe(processed);
-    expect(wrapper.init).toHaveBeenCalledWith({
-      kind: "video",
-      track: captured,
-    });
-    await expect(sdkProcessor.restart(captured)).resolves.toBe(processed);
-    await sdkProcessor.destroy();
-    expect(wrapper.destroy).toHaveBeenCalled();
-  });
-
-  it("rejects where the wrapper produced no track", async () => {
-    await expect(
-      fromLivekitProcessor(mockWrapper(undefined)).init(captured),
-    ).rejects.toThrow("produced no track");
-  });
-});

src/tracks/TrackProcessorContext.tsx

@@ -1,110 +0,0 @@
-/*
-Copyright 2024-2025 New Vector Ltd.
-
-SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
-Please see LICENSE in the repository root for full details.
-*/
-
-import {
-  ProcessorWrapper,
-  supportsBackgroundProcessors as supportsBackgroundProcessorsLivekitSdk,
-  type BackgroundOptions,
-} from "@livekit/track-processors";
-import { createContext, type FC, type JSX, use, useMemo } from "react";
-import { Track } from "livekit-client";
-import { type VideoProcessor } from "@element-hq/matrixrtc-sdk";
-
-import {
-  backgroundBlur as backgroundBlurSettings,
-  useSetting,
-} from "../settings/settings";
-import { BlurBackgroundTransformer } from "./BlurBackgroundTransformer";
-import { type Behavior } from "../state/Behavior";
-import { platform } from "../Platform";
-import { useValueBehavior } from "../useValueBehavior";
-
-//TODO-MULTI-SFU: This is not yet fully there.
-// it is a combination of exposing observable and react hooks.
-// preferably we should not make this a context anymore and instead just a vm?
-
-export type ProcessorState = {
-  supported: boolean | undefined;
-  processor: undefined | VideoProcessor;
-};
-
-/**
- * This allows to use a LiveKit processor for the matrixRTCSDK (VideoProcessor).
- * They have slightly different apis so we need to convert them with this
- * function before we can use them in the matrixRTCSDK.
- */
-export function fromLivekitProcessor(
-  wrapper: ProcessorWrapper<BackgroundOptions>,
-): VideoProcessor {
-  const processedTrack = (): MediaStreamTrack => {
-    if (wrapper.processedTrack === undefined)
-      throw new Error(`Processor ${wrapper.name} produced no track`);
-    return wrapper.processedTrack;
-  };
-  return {
-    name: wrapper.name,
-    init: async (track: MediaStreamTrack, element?: HTMLMediaElement) => {
-      await wrapper.init({ kind: Track.Kind.Video, track, element });
-      return processedTrack();
-    },
-    restart: async (track: MediaStreamTrack, element?: HTMLMediaElement) => {
-      await wrapper.restart({ kind: Track.Kind.Video, track, element });
-      return processedTrack();
-    },
-    destroy: async () => wrapper.destroy(),
-  };
-}
-
-const ProcessorContext = createContext<ProcessorState | undefined>(undefined);
-
-export function useTrackProcessor(): ProcessorState {
-  const state = use(ProcessorContext);
-  if (state === undefined)
-    throw new Error(
-      "useTrackProcessor must be used within a ProcessorProvider",
-    );
-  return state;
-}
-
-export function useTrackProcessorState$(): Behavior<ProcessorState> {
-  return useValueBehavior(useTrackProcessor());
-}
-
-interface Props {
-  children: JSX.Element;
-}
-
-function supportsBackgroundProcessors(): boolean {
-  return supportsBackgroundProcessorsLivekitSdk() && platform === "desktop";
-}
-
-export const ProcessorProvider: FC<Props> = ({ children }) => {
-  // The setting the user wants to have
-  const [blurActivated] = useSetting(backgroundBlurSettings);
-  const supported = useMemo(() => supportsBackgroundProcessors(), []);
-  const blur = useMemo(
-    () =>
-      fromLivekitProcessor(
-        new ProcessorWrapper(
-          new BlurBackgroundTransformer({ blurRadius: 15 }),
-          "background-blur",
-        ),
-      ),
-    [],
-  );
-
-  // This is the actual state exposed through the context
-  const processorState = useMemo(
-    () => ({
-      supported,
-      processor: supported && blurActivated ? blur : undefined,
-    }),
-    [supported, blurActivated, blur],
-  );
-
-  return <ProcessorContext value={processorState}>{children}</ProcessorContext>;
-};