diff --git a/src/livekit/BlurBackgroundTransformer.ts b/src/livekit/BackgroundEffectTransformer.ts similarity index 72% rename from src/livekit/BlurBackgroundTransformer.ts rename to src/livekit/BackgroundEffectTransformer.ts index f86120d33..e5a2d3854 100644 --- a/src/livekit/BlurBackgroundTransformer.ts +++ b/src/livekit/BackgroundEffectTransformer.ts @@ -43,13 +43,17 @@ const wasmFileset: WasmFileset = { }; /** - * Track processor that applies effects such as blurring to a user's background. + * Track processor that applies background effects — blur, or a replacement + * image — to a user's camera. * * This is just like LiveKit's prebuilt BackgroundTransformer except that it * loads the segmentation models from our own bundle rather than as an external - * resource fetched from the public internet. + * resource fetched from the public internet. The library's own `assetPaths` + * option cannot do this: it takes a directory and resolves the WASM through + * MediaPipe's FilesetResolver, which picks its own variants, whereas we want + * only the SIMD ones. */ -export class BlurBackgroundTransformer extends BackgroundTransformer { +export class BackgroundEffectTransformer extends BackgroundTransformer { public async init({ outputCanvas, inputElement: inputVideo, @@ -73,8 +77,16 @@ export class BlurBackgroundTransformer extends BackgroundTransformer { outputConfidenceMasks: false, }); - if (this.options.blurRadius) { + // BackgroundTransformer applies these at the end of its own init. Because + // we replace init wholesale rather than extending it, we have to repeat + // them, or an effect that was already selected when the pipeline starts is + // silently ignored. + if (this.options.imagePath) { + await this.loadAndSetBackground(this.options.imagePath); + } + if (typeof this.options.blurRadius === "number") { this.gl?.setBlurRadius(this.options.blurRadius); } + this.gl?.setBackgroundDisabled(this.options.backgroundDisabled ?? false); } } diff --git a/src/livekit/TrackProcessorContext.tsx b/src/livekit/TrackProcessorContext.tsx index 96897929f..389bf2971 100644 --- a/src/livekit/TrackProcessorContext.tsx +++ b/src/livekit/TrackProcessorContext.tsx @@ -6,9 +6,11 @@ Please see LICENSE in the repository root for full details. */ import { - ProcessorWrapper, + BackgroundProcessorWrapper, + type ProcessorWrapper, supportsBackgroundProcessors as supportsBackgroundProcessorsLivekitSdk, type BackgroundOptions, + type SwitchBackgroundProcessorOptions, } from "@livekit/track-processors"; import { createContext, @@ -17,6 +19,7 @@ import { use, useEffect, useMemo, + useState, } from "react"; import { type LocalVideoTrack } from "livekit-client"; import { logger } from "matrix-js-sdk/lib/logger"; @@ -24,10 +27,16 @@ import { combineLatest, map, type Observable } from "rxjs"; import { useObservable } from "observable-hooks"; import { - backgroundBlur as backgroundBlurSettings, + backgroundEffect as backgroundEffectSetting, useSetting, } from "../settings/settings"; -import { BlurBackgroundTransformer } from "./BlurBackgroundTransformer"; +import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer"; +import { + blurRadius, + imagePathFor, + parseEffect, + type BackgroundEffect, +} from "./backgroundEffects"; import { type Behavior } from "../state/Behavior"; import { type ObservableScope } from "../state/ObservableScope"; import { platform } from "../Platform"; @@ -129,26 +138,68 @@ function supportsBackgroundProcessors(): boolean { return supportsBackgroundProcessorsLivekitSdk() && platform === "desktop"; } +/** Translates a chosen effect into the pipeline's own vocabulary. */ +function switchOptionsFor( + effect: BackgroundEffect, +): SwitchBackgroundProcessorOptions { + switch (effect.kind) { + case "blur": + return { mode: "background-blur", blurRadius }; + case "image": { + const imagePath = imagePathFor(effect.id); + return imagePath + ? { mode: "virtual-background", imagePath } + : { mode: "disabled" }; + } + default: + return { mode: "disabled" }; + } +} + export const ProcessorProvider: FC = ({ children }) => { - // The setting the user wants to have - const [blurActivated] = useSetting(backgroundBlurSettings); + const [effectRaw] = useSetting(backgroundEffectSetting); const supported = useMemo(() => supportsBackgroundProcessors(), []); - const blur = useMemo( + + // One pipeline for the lifetime of the app, so the pre-join preview and the + // call share it and its priming frame is spent before anything is published + // (D4). + const pipeline = useMemo( () => - new ProcessorWrapper( - new BlurBackgroundTransformer({ blurRadius: 15 }), - "background-blur", + new BackgroundProcessorWrapper( + new BackgroundEffectTransformer({ backgroundDisabled: true }), + "background-effect", ), [], ); + // D5: nothing is attached until an effect is first chosen, so a user who + // never chooses one pays neither the segmentation assets nor the time to + // initialise them. Once attached it stays attached, including at no effect, + // so a later change is a switch rather than a fresh attachment and spends no + // further priming frame. + const [attached, setAttached] = useState( + () => parseEffect(effectRaw).kind !== "none", + ); + useEffect(() => { + if (parseEffect(effectRaw).kind !== "none") setAttached(true); + }, [effectRaw]); + + // D2: switch the running pipeline in place rather than tearing it down, so + // the previous effect stays in force until the new one is live. + useEffect(() => { + if (!supported || !attached) return; + pipeline + .switchTo(switchOptionsFor(parseEffect(effectRaw))) + .catch((e) => logger.warn("Failed to switch background effect", e)); + }, [pipeline, supported, attached, effectRaw]); + // This is the actual state exposed through the context const processorState = useMemo( () => ({ supported, - processor: supported && blurActivated ? blur : undefined, + processor: supported && attached ? pipeline : undefined, }), - [supported, blurActivated, blur], + [supported, attached, pipeline], ); return {children}; diff --git a/src/livekit/backgroundEffects.ts b/src/livekit/backgroundEffects.ts new file mode 100644 index 000000000..18c5b019d --- /dev/null +++ b/src/livekit/backgroundEffects.ts @@ -0,0 +1,58 @@ +/* +Copyright 2024-2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import desktopGradient from "../graphics/desktop-gradient.png?url"; +import mobileGradient from "../graphics/mobile-gradient.png?url"; + +/** How much to blur, when the chosen effect is blur. */ +export const blurRadius = 15; + +export interface ShippedBackground { + id: string; + imagePath: string; +} + +// EXPLORATION SHORTCUT (S1): the two shipped backgrounds, one indoor and one +// outdoor, do not exist yet. These stand-ins are existing gradient assets, so +// the pipeline can be exercised and measured. Never port this. +export const shippedBackgrounds: ShippedBackground[] = [ + { id: "indoor", imagePath: desktopGradient }, + { id: "outdoor", imagePath: mobileGradient }, +]; + +export type BackgroundEffect = + | { kind: "none" } + | { kind: "blur" } + | { kind: "image"; id: string }; + +export const noEffect: BackgroundEffect = { kind: "none" }; + +/** Parses the stored form of a chosen effect, falling back to no effect. */ +export function parseEffect(raw: string): BackgroundEffect { + if (raw === "blur") return { kind: "blur" }; + if (raw.startsWith("image:")) { + const id = raw.slice("image:".length); + if (shippedBackgrounds.some((b) => b.id === id)) return { kind: "image", id }; + } + return noEffect; +} + +/** The stored form of a chosen effect. */ +export function serializeEffect(effect: BackgroundEffect): string { + switch (effect.kind) { + case "blur": + return "blur"; + case "image": + return `image:${effect.id}`; + default: + return "none"; + } +} + +export function imagePathFor(id: string): string | undefined { + return shippedBackgrounds.find((b) => b.id === id)?.imagePath; +} diff --git a/src/settings/settings.ts b/src/settings/settings.ts index f108eb5f9..0194891a0 100644 --- a/src/settings/settings.ts +++ b/src/settings/settings.ts @@ -117,6 +117,15 @@ export const videoInput = new Setting( export const backgroundBlur = new Setting("background-blur", false); +/** + * The background effect applied to the local camera: "none", "blur", or + * "image:". See backgroundEffects.ts for the stored form. + */ +export const backgroundEffect = new Setting( + "background-effect", + "none", +); + export const showHandRaisedTimer = new Setting( "hand-raised-show-timer", false,