From b90f267009e1b5c70af374c10299140b447e51ce Mon Sep 17 00:00:00 2001 From: fkwp Date: Thu, 17 Sep 2026 16:10:47 +0200 Subject: [PATCH] Attach background effects through one switchable pipeline Replace the blur-only processor with a single wrapper that switches between no effect, blur and a background image in place. Switching in place rather than rebuilding matters: destroy() resets the transformer's first-frame flag, and that flag makes it emit one unprocessed frame when it next starts processing, so a rebuild per change would leak a frame of the real background every time. Attach the pipeline the first time an effect is chosen and keep it attached afterwards, including at no effect. Measured against a production build, the segmentation assets are about 11.4MB and take ~143ms to initialise, so always attaching would charge that to every user on every camera-on call, including the majority who never choose an effect. Keeping it attached once it is there costs about 25us per frame, which is within noise of no pipeline at all. Complete init() while moving it. The old subclass replaced the base class's init wholesale and dropped its image loading and disabled-mode setup, which went unnoticed while blur was the only effect and blurRadius was the only option that mattered. Keep loading the segmenter from our own bundle. The library's assetPaths option resolves WASM through MediaPipe's FilesetResolver from a directory, which is not the same thing, and its defaults reach jsDelivr and googleapis. Exploration for FEATURES_SPEC/2026-09_Background_Effects.md. Not for merge: the two shipped backgrounds are stand-in gradients and there is no picker yet, recorded as S1 and S2 in the spec. Co-Authored-By: Claude Opus 5 (1M context) --- ...rmer.ts => BackgroundEffectTransformer.ts} | 20 ++++- src/livekit/TrackProcessorContext.tsx | 73 ++++++++++++++++--- src/livekit/backgroundEffects.ts | 58 +++++++++++++++ src/settings/settings.ts | 9 +++ 4 files changed, 145 insertions(+), 15 deletions(-) rename src/livekit/{BlurBackgroundTransformer.ts => BackgroundEffectTransformer.ts} (72%) create mode 100644 src/livekit/backgroundEffects.ts diff --git a/src/livekit/BlurBackgroundTransformer.ts b/src/livekit/BackgroundEffectTransformer.ts similarity index 72% rename from src/livekit/BlurBackgroundTransformer.ts rename to src/livekit/BackgroundEffectTransformer.ts index f86120d33..e5a2d3854 100644 --- a/src/livekit/BlurBackgroundTransformer.ts +++ b/src/livekit/BackgroundEffectTransformer.ts @@ -43,13 +43,17 @@ const wasmFileset: WasmFileset = { }; /** - * Track processor that applies effects such as blurring to a user's background. + * Track processor that applies background effects — blur, or a replacement + * image — to a user's camera. * * This is just like LiveKit's prebuilt BackgroundTransformer except that it * loads the segmentation models from our own bundle rather than as an external - * resource fetched from the public internet. + * resource fetched from the public internet. The library's own `assetPaths` + * option cannot do this: it takes a directory and resolves the WASM through + * MediaPipe's FilesetResolver, which picks its own variants, whereas we want + * only the SIMD ones. */ -export class BlurBackgroundTransformer extends BackgroundTransformer { +export class BackgroundEffectTransformer extends BackgroundTransformer { public async init({ outputCanvas, inputElement: inputVideo, @@ -73,8 +77,16 @@ export class BlurBackgroundTransformer extends BackgroundTransformer { outputConfidenceMasks: false, }); - if (this.options.blurRadius) { + // BackgroundTransformer applies these at the end of its own init. Because + // we replace init wholesale rather than extending it, we have to repeat + // them, or an effect that was already selected when the pipeline starts is + // silently ignored. + if (this.options.imagePath) { + await this.loadAndSetBackground(this.options.imagePath); + } + if (typeof this.options.blurRadius === "number") { this.gl?.setBlurRadius(this.options.blurRadius); } + this.gl?.setBackgroundDisabled(this.options.backgroundDisabled ?? false); } } diff --git a/src/livekit/TrackProcessorContext.tsx b/src/livekit/TrackProcessorContext.tsx index 96897929f..389bf2971 100644 --- a/src/livekit/TrackProcessorContext.tsx +++ b/src/livekit/TrackProcessorContext.tsx @@ -6,9 +6,11 @@ Please see LICENSE in the repository root for full details. */ import { - ProcessorWrapper, + BackgroundProcessorWrapper, + type ProcessorWrapper, supportsBackgroundProcessors as supportsBackgroundProcessorsLivekitSdk, type BackgroundOptions, + type SwitchBackgroundProcessorOptions, } from "@livekit/track-processors"; import { createContext, @@ -17,6 +19,7 @@ import { use, useEffect, useMemo, + useState, } from "react"; import { type LocalVideoTrack } from "livekit-client"; import { logger } from "matrix-js-sdk/lib/logger"; @@ -24,10 +27,16 @@ import { combineLatest, map, type Observable } from "rxjs"; import { useObservable } from "observable-hooks"; import { - backgroundBlur as backgroundBlurSettings, + backgroundEffect as backgroundEffectSetting, useSetting, } from "../settings/settings"; -import { BlurBackgroundTransformer } from "./BlurBackgroundTransformer"; +import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer"; +import { + blurRadius, + imagePathFor, + parseEffect, + type BackgroundEffect, +} from "./backgroundEffects"; import { type Behavior } from "../state/Behavior"; import { type ObservableScope } from "../state/ObservableScope"; import { platform } from "../Platform"; @@ -129,26 +138,68 @@ function supportsBackgroundProcessors(): boolean { return supportsBackgroundProcessorsLivekitSdk() && platform === "desktop"; } +/** Translates a chosen effect into the pipeline's own vocabulary. */ +function switchOptionsFor( + effect: BackgroundEffect, +): SwitchBackgroundProcessorOptions { + switch (effect.kind) { + case "blur": + return { mode: "background-blur", blurRadius }; + case "image": { + const imagePath = imagePathFor(effect.id); + return imagePath + ? { mode: "virtual-background", imagePath } + : { mode: "disabled" }; + } + default: + return { mode: "disabled" }; + } +} + export const ProcessorProvider: FC = ({ children }) => { - // The setting the user wants to have - const [blurActivated] = useSetting(backgroundBlurSettings); + const [effectRaw] = useSetting(backgroundEffectSetting); const supported = useMemo(() => supportsBackgroundProcessors(), []); - const blur = useMemo( + + // One pipeline for the lifetime of the app, so the pre-join preview and the + // call share it and its priming frame is spent before anything is published + // (D4). + const pipeline = useMemo( () => - new ProcessorWrapper( - new BlurBackgroundTransformer({ blurRadius: 15 }), - "background-blur", + new BackgroundProcessorWrapper( + new BackgroundEffectTransformer({ backgroundDisabled: true }), + "background-effect", ), [], ); + // D5: nothing is attached until an effect is first chosen, so a user who + // never chooses one pays neither the segmentation assets nor the time to + // initialise them. Once attached it stays attached, including at no effect, + // so a later change is a switch rather than a fresh attachment and spends no + // further priming frame. + const [attached, setAttached] = useState( + () => parseEffect(effectRaw).kind !== "none", + ); + useEffect(() => { + if (parseEffect(effectRaw).kind !== "none") setAttached(true); + }, [effectRaw]); + + // D2: switch the running pipeline in place rather than tearing it down, so + // the previous effect stays in force until the new one is live. + useEffect(() => { + if (!supported || !attached) return; + pipeline + .switchTo(switchOptionsFor(parseEffect(effectRaw))) + .catch((e) => logger.warn("Failed to switch background effect", e)); + }, [pipeline, supported, attached, effectRaw]); + // This is the actual state exposed through the context const processorState = useMemo( () => ({ supported, - processor: supported && blurActivated ? blur : undefined, + processor: supported && attached ? pipeline : undefined, }), - [supported, blurActivated, blur], + [supported, attached, pipeline], ); return {children}; diff --git a/src/livekit/backgroundEffects.ts b/src/livekit/backgroundEffects.ts new file mode 100644 index 000000000..18c5b019d --- /dev/null +++ b/src/livekit/backgroundEffects.ts @@ -0,0 +1,58 @@ +/* +Copyright 2024-2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import desktopGradient from "../graphics/desktop-gradient.png?url"; +import mobileGradient from "../graphics/mobile-gradient.png?url"; + +/** How much to blur, when the chosen effect is blur. */ +export const blurRadius = 15; + +export interface ShippedBackground { + id: string; + imagePath: string; +} + +// EXPLORATION SHORTCUT (S1): the two shipped backgrounds, one indoor and one +// outdoor, do not exist yet. These stand-ins are existing gradient assets, so +// the pipeline can be exercised and measured. Never port this. +export const shippedBackgrounds: ShippedBackground[] = [ + { id: "indoor", imagePath: desktopGradient }, + { id: "outdoor", imagePath: mobileGradient }, +]; + +export type BackgroundEffect = + | { kind: "none" } + | { kind: "blur" } + | { kind: "image"; id: string }; + +export const noEffect: BackgroundEffect = { kind: "none" }; + +/** Parses the stored form of a chosen effect, falling back to no effect. */ +export function parseEffect(raw: string): BackgroundEffect { + if (raw === "blur") return { kind: "blur" }; + if (raw.startsWith("image:")) { + const id = raw.slice("image:".length); + if (shippedBackgrounds.some((b) => b.id === id)) return { kind: "image", id }; + } + return noEffect; +} + +/** The stored form of a chosen effect. */ +export function serializeEffect(effect: BackgroundEffect): string { + switch (effect.kind) { + case "blur": + return "blur"; + case "image": + return `image:${effect.id}`; + default: + return "none"; + } +} + +export function imagePathFor(id: string): string | undefined { + return shippedBackgrounds.find((b) => b.id === id)?.imagePath; +} diff --git a/src/settings/settings.ts b/src/settings/settings.ts index f108eb5f9..0194891a0 100644 --- a/src/settings/settings.ts +++ b/src/settings/settings.ts @@ -117,6 +117,15 @@ export const videoInput = new Setting( export const backgroundBlur = new Setting("background-blur", false); +/** + * The background effect applied to the local camera: "none", "blur", or + * "image:". See backgroundEffects.ts for the stored form. + */ +export const backgroundEffect = new Setting( + "background-effect", + "none", +); + export const showHandRaisedTimer = new Setting( "hand-raised-show-timer", false,