From d3eefdbcf23b041ff2bf3a434f9b337df0d798cf Mon Sep 17 00:00:00 2001 From: fkwp Date: Wed, 4 Nov 2026 12:37:40 +0100 Subject: [PATCH] Attach background effects through one switchable pipeline Blur runs through one pipeline, shared by the pre-join preview and the call: attached the first time an effect is wanted and switched in place after, never detached. The logic is BackgroundEffects in src/state; BackgroundEffectsContext only builds it, from the app's view model, so the standalone app and the component hand the same instance to every camera. Disabled drops the blur radius, so no effect costs no segmentation. The SDK gains fromLivekitProcessor beside the adapter, now toLivekitProcessor. Ported from spec/background-effects/2-pipeline (#4282): a91dab8f with its review tidy-ups 445e010c, bbc73d07, 51729bcf, f52d2b79, 813f1e6f and 584bbaa0. Left out as already solved here: OneStepPipeline (the SDK runs a processor's steps in turn) and the LiveKit-track sync helpers (the lobby's camera is an SDK track, synced by followProcessor). Co-Authored-By: Claude Fable 5.1 --- component/index.tsx | 23 ++- sdk/index.ts | 5 + .../livekit/LivekitMediaTrack.ts | 8 +- sdk/src/media-backend/livekit/Publisher.ts | 7 +- .../media-backend/livekit/publishOptions.ts | 4 +- .../livekit/videoProcessor.test.ts | 53 ++++++- .../media-backend/livekit/videoProcessor.ts | 44 +++++- src/App.tsx | 6 +- src/room/CallView.test.tsx | 12 +- src/room/InCallView.test.tsx | 18 ++- src/room/InCallView.tsx | 10 +- src/room/KnockLobbyView.test.tsx | 8 +- src/room/LobbyView.test.tsx | 8 +- src/room/LobbyView.tsx | 8 +- src/settings/SettingsModal.tsx | 4 +- src/state/AppViewModel.ts | 2 + src/state/BackgroundEffects.test.ts | 140 ++++++++++++++++++ src/state/BackgroundEffects.ts | 91 ++++++++++++ .../BackgroundEffectTransformer.test.ts | 24 +++ ...rmer.ts => BackgroundEffectTransformer.ts} | 17 ++- src/tracks/BackgroundEffectsContext.test.ts | 115 ++++++++++++++ src/tracks/BackgroundEffectsContext.tsx | 74 +++++++++ src/tracks/TrackProcessorContext.test.ts | 51 ------- src/tracks/TrackProcessorContext.tsx | 110 -------------- 24 files changed, 611 insertions(+), 231 deletions(-) create mode 100644 src/state/BackgroundEffects.test.ts create mode 100644 src/state/BackgroundEffects.ts create mode 100644 src/tracks/BackgroundEffectTransformer.test.ts rename src/tracks/{BlurBackgroundTransformer.ts => BackgroundEffectTransformer.ts} (78%) create mode 100644 src/tracks/BackgroundEffectsContext.test.ts create mode 100644 src/tracks/BackgroundEffectsContext.tsx delete mode 100644 src/tracks/TrackProcessorContext.test.ts delete mode 100644 src/tracks/TrackProcessorContext.tsx diff --git a/component/index.tsx b/component/index.tsx index 4a4b4d8cf..2b96272e3 100644 --- a/component/index.tsx +++ b/component/index.tsx @@ -72,9 +72,9 @@ import { useUrlParams, } from "../src/UrlParams"; import { MediaDevicesContext } from "../src/MediaDevicesContext"; -import { MediaDevices } from "../src/state/MediaDevices"; +import { AppViewModel } from "../src/state/AppViewModel"; import { ObservableScope } from "../src/state/ObservableScope"; -import { ProcessorProvider } from "../src/tracks/TrackProcessorContext"; +import { BackgroundEffectsProvider } from "../src/tracks/BackgroundEffectsContext"; import { Config } from "../src/config/Config"; import { type ConfigOptions } from "../src/config/ConfigOptions"; import { i18n } from "../src/utils/i18n"; @@ -194,14 +194,12 @@ export const ElementCall: FC = ({ // observers running for the rest of the page's life. Null until then, which // is one render. const { controlledAudioDevices, callIntent } = params; - const [mediaDevices, setMediaDevices] = useState(null); + const [vm, setVm] = useState(null); useEffect(() => { const scope = new ObservableScope(); - setMediaDevices( - new MediaDevices(scope, { controlledAudioDevices, callIntent }), - ); + setVm(new AppViewModel(scope, { controlledAudioDevices, callIntent })); return (): void => { - setMediaDevices(null); + setVm(null); scope.end(); }; }, [controlledAudioDevices, callIntent]); @@ -222,8 +220,7 @@ export const ElementCall: FC = ({ // is told that Element Call has loaded, as the widget tells its client once // its own initialisation is over. Once per mount, however often the pieces // are later swapped out. - const ready = - container !== null && rtcSession !== null && mediaDevices !== null; + const ready = container !== null && rtcSession !== null && vm !== null; const announcedLoaded = useRef(false); useEffect(() => { if (!ready || announcedLoaded.current) return; @@ -254,8 +251,10 @@ export const ElementCall: FC = ({ - - + + = ({ preload={params.preload} skipLobby={params.skipLobby} /> - + diff --git a/sdk/index.ts b/sdk/index.ts index 0b97897b2..5818c311b 100644 --- a/sdk/index.ts +++ b/sdk/index.ts @@ -63,6 +63,11 @@ export { } from "./src/media-backend/livekit/LivekitBackend"; // Tracks for a lobby, captured ahead of the join and handed to it export { createLocalMedia } from "./src/LocalMedia"; +// For a host whose processor is a livekit-client one, such as @livekit/track-processors +export { + fromLivekitProcessor, + toLivekitProcessor, +} from "./src/media-backend/livekit/videoProcessor"; // For a developer panel to check a transport before using it export { getSFUConfig as authenticateWithTransport, diff --git a/sdk/src/media-backend/livekit/LivekitMediaTrack.ts b/sdk/src/media-backend/livekit/LivekitMediaTrack.ts index b5767daf0..0e6c1986f 100644 --- a/sdk/src/media-backend/livekit/LivekitMediaTrack.ts +++ b/sdk/src/media-backend/livekit/LivekitMediaTrack.ts @@ -50,10 +50,7 @@ import { type VideoMediaTrack, } from "../../media-api"; import { MatrixRTCError } from "../../errors"; -import { - convertToLivekitProcessor, - type LivekitProcessor, -} from "./videoProcessor"; +import { toLivekitProcessor, type LivekitProcessor } from "./videoProcessor"; export const mediaSources: Record = { [Track.Source.Microphone]: "microphone", @@ -245,8 +242,7 @@ export function createLocalLivekitMediaTrack( if (!track || track.mediaStreamTrack.readyState === "ended") return; const current = track.getProcessor() as LivekitProcessor | undefined; if (processor === current?.source) return; - if (processor) - await track.setProcessor(convertToLivekitProcessor(processor)); + if (processor) await track.setProcessor(toLivekitProcessor(processor)); else if (current) await track.stopProcessor(); }, }; diff --git a/sdk/src/media-backend/livekit/Publisher.ts b/sdk/src/media-backend/livekit/Publisher.ts index 53df00ce9..40728cef5 100644 --- a/sdk/src/media-backend/livekit/Publisher.ts +++ b/sdk/src/media-backend/livekit/Publisher.ts @@ -22,7 +22,7 @@ import { type PublishRequest, } from "../../media-api"; import { livekitSources, mediaSources } from "./LivekitMediaTrack"; -import { convertToLivekitProcessor } from "./videoProcessor"; +import { toLivekitProcessor } from "./videoProcessor"; import { audioCaptureOptions, screenShareCaptureOptions, @@ -159,10 +159,7 @@ export class Publisher { if (muted) await local.mute(); if (processor && local instanceof LocalVideoTrack) await local.setProcessor( - convertToLivekitProcessor( - processor.processor, - processor.processedTrack, - ), + toLivekitProcessor(processor.processor, processor.processedTrack), ); return this.room.localParticipant.publishTrack(local, { source: livekitSources[request.source], diff --git a/sdk/src/media-backend/livekit/publishOptions.ts b/sdk/src/media-backend/livekit/publishOptions.ts index f030c3fd4..cbdfbdc6f 100644 --- a/sdk/src/media-backend/livekit/publishOptions.ts +++ b/sdk/src/media-backend/livekit/publishOptions.ts @@ -16,7 +16,7 @@ import { type PublishRequest, type VideoCaptureSettings, } from "../../media-api"; -import { convertToLivekitProcessor } from "./videoProcessor"; +import { toLivekitProcessor } from "./videoProcessor"; export type PublishRequestFor = Extract< PublishRequest, @@ -40,7 +40,7 @@ export function videoCaptureOptions({ if (!deviceId && !processor && !capture?.resolution) return undefined; return { ...(deviceId && { deviceId }), - ...(processor && { processor: convertToLivekitProcessor(processor) }), + ...(processor && { processor: toLivekitProcessor(processor) }), ...(capture?.resolution && { resolution: capture.resolution }), }; } diff --git a/sdk/src/media-backend/livekit/videoProcessor.test.ts b/sdk/src/media-backend/livekit/videoProcessor.test.ts index e8394e484..4ac6e9283 100644 --- a/sdk/src/media-backend/livekit/videoProcessor.test.ts +++ b/sdk/src/media-backend/livekit/videoProcessor.test.ts @@ -5,13 +5,17 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { Track, type VideoProcessorOptions } from "livekit-client"; +import { + Track, + type TrackProcessor, + type VideoProcessorOptions, +} from "livekit-client"; import { describe, expect, it, vi } from "vitest"; import { type VideoProcessor } from "../../media-api"; -import { convertToLivekitProcessor } from "./videoProcessor"; +import { fromLivekitProcessor, toLivekitProcessor } from "./videoProcessor"; -describe("convertToLivekitProcessor", () => { +describe("toLivekitProcessor", () => { const captured = { id: "captured" } as MediaStreamTrack; const processed = { id: "processed" } as MediaStreamTrack; const restarted = { id: "restarted" } as MediaStreamTrack; @@ -33,7 +37,7 @@ describe("convertToLivekitProcessor", () => { it("holds the processed track where LiveKit reads it", async () => { const processor = fakeProcessor(); - const adapter = convertToLivekitProcessor(processor); + const adapter = toLivekitProcessor(processor); expect(adapter.name).toBe("blur"); expect(adapter.processedTrack).toBeUndefined(); @@ -59,8 +63,8 @@ describe("convertToLivekitProcessor", () => { finishDestroy = resolve; }), ); - const previous = convertToLivekitProcessor(processor); - const next = convertToLivekitProcessor(processor); + const previous = toLivekitProcessor(processor); + const next = toLivekitProcessor(processor); const destroying = previous.destroy(); const initialising = next.init(options); @@ -73,3 +77,40 @@ describe("convertToLivekitProcessor", () => { expect(next.processedTrack).toBe(processed); }); }); + +describe("fromLivekitProcessor", () => { + const captured = { id: "captured" } as MediaStreamTrack; + const processed = { id: "processed" } as MediaStreamTrack; + + function fakeLivekitProcessor( + processedTrack: MediaStreamTrack | undefined, + ): TrackProcessor { + return { + name: "background-blur", + processedTrack, + init: vi.fn().mockResolvedValue(undefined), + restart: vi.fn().mockResolvedValue(undefined), + destroy: vi.fn().mockResolvedValue(undefined), + }; + } + + it("resolves with the track the LiveKit processor produced", async () => { + const livekitProcessor = fakeLivekitProcessor(processed); + const processor = fromLivekitProcessor(livekitProcessor); + expect(processor.name).toBe("background-blur"); + await expect(processor.init(captured)).resolves.toBe(processed); + expect(livekitProcessor.init).toHaveBeenCalledWith({ + kind: Track.Kind.Video, + track: captured, + }); + await expect(processor.restart(captured)).resolves.toBe(processed); + await processor.destroy(); + expect(livekitProcessor.destroy).toHaveBeenCalled(); + }); + + it("rejects where the LiveKit processor produced no track", async () => { + await expect( + fromLivekitProcessor(fakeLivekitProcessor(undefined)).init(captured), + ).rejects.toThrow("produced no track"); + }); +}); diff --git a/sdk/src/media-backend/livekit/videoProcessor.ts b/sdk/src/media-backend/livekit/videoProcessor.ts index 17edd0a9b..557a2f4b3 100644 --- a/sdk/src/media-backend/livekit/videoProcessor.ts +++ b/sdk/src/media-backend/livekit/videoProcessor.ts @@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type Track, type TrackProcessor } from "livekit-client"; +import { Track, type TrackProcessor } from "livekit-client"; import { type VideoProcessor } from "../../media-api"; import { inTurn } from "../../utils/processorTurns"; @@ -16,14 +16,15 @@ export interface LivekitProcessor extends TrackProcessor { } /** - * A `VideoProcessor` as livekit-client wants it. LiveKit reads the processed - * track off the processor after `init` and `restart` rather than taking it as - * their result, so the adapter keeps it there. A processor a lobby already - * runs on the track is adopted with its output, `running`, so that the join - * does not initialise it again. Every step takes its turn on the processor, - * which a stopped track tears down without waiting for the next one's init. + * A `VideoProcessor` as a livekit-client processor. LiveKit reads the + * processed track off the processor after `init` and `restart` rather than + * taking it as their result, so the adapter keeps it there. A processor a + * lobby already runs on the track is adopted with its output, `running`, so + * that the join does not initialise it again. Every step takes its turn on + * the processor, which a stopped track tears down without waiting for the + * next one's init. */ -export function convertToLivekitProcessor( +export function toLivekitProcessor( processor: VideoProcessor, running?: MediaStreamTrack, ): LivekitProcessor { @@ -47,3 +48,30 @@ export function convertToLivekitProcessor( }; return adapter; } + +/** + * A livekit-client processor, such as one from `@livekit/track-processors`, + * as a `VideoProcessor`: `init` and `restart` resolve with the track it + * leaves on itself. + */ +export function fromLivekitProcessor( + processor: TrackProcessor, +): VideoProcessor { + const processedTrack = (): MediaStreamTrack => { + if (processor.processedTrack === undefined) + throw new Error(`Processor ${processor.name} produced no track`); + return processor.processedTrack; + }; + return { + name: processor.name, + init: async (track, element) => { + await processor.init({ kind: Track.Kind.Video, track, element }); + return processedTrack(); + }, + restart: async (track, element) => { + await processor.restart({ kind: Track.Kind.Video, track, element }); + return processedTrack(); + }, + destroy: async () => processor.destroy(), + }; +} diff --git a/src/App.tsx b/src/App.tsx index a92d56b59..e3aff9086 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -35,7 +35,7 @@ import { ErrorPage, LoadingPage } from "./FullScreenView"; import { Initializer } from "./initializer"; import { type WidgetHelpers } from "./widget"; import { useTheme } from "./useTheme"; -import { ProcessorProvider } from "./tracks/TrackProcessorContext"; +import { BackgroundEffectsProvider } from "./tracks/BackgroundEffectsContext"; import { type AppViewModel } from "./state/AppViewModel"; import { MediaDevicesContext } from "./MediaDevicesContext"; import { @@ -148,7 +148,7 @@ export const App: FC = ({ vm, widget }) => { loaded && clientReady ? ( - + } > @@ -159,7 +159,7 @@ export const App: FC = ({ vm, widget }) => { } /> - + ) : ( diff --git a/src/room/CallView.test.tsx b/src/room/CallView.test.tsx index 2c2208c56..ed747c56c 100644 --- a/src/room/CallView.test.tsx +++ b/src/room/CallView.test.tsx @@ -50,6 +50,7 @@ import { mockMediaDevices, mockRtcMembership, MockRTCSession, + testScope, } from "../utils/test"; import { CallView } from "./CallView"; import { GroupCallErrorBoundary } from "./GroupCallErrorBoundary"; @@ -61,7 +62,10 @@ import { } from "../HostBridge"; import { type JoinCallData } from "../widget"; import { MatrixRTCTransportMissingError } from "../utils/errors"; -import { ProcessorProvider } from "../tracks/TrackProcessorContext"; +import { + BackgroundEffectsProvider, + createAppBackgroundEffects, +} from "../tracks/BackgroundEffectsContext"; import { MediaDevicesContext } from "../MediaDevicesContext"; import { constant } from "../state/Behavior"; @@ -193,7 +197,9 @@ function createCallView( - + {options.withErrorBoundary ? ( {callView} @@ -201,7 +207,7 @@ function createCallView( ) : ( callView )} - + diff --git a/src/room/InCallView.test.tsx b/src/room/InCallView.test.tsx index d8b6e637a..f6bc938f6 100644 --- a/src/room/InCallView.test.tsx +++ b/src/room/InCallView.test.tsx @@ -28,6 +28,7 @@ import { type MockRTCSession, mockRTCSlot, mockRTCMember, + testScope, } from "../utils/test"; import { E2eeType } from "../e2ee/e2eeType"; import { @@ -48,7 +49,10 @@ import { MediaDevicesContext } from "../MediaDevicesContext"; import { type MediaDevices as ECMediaDevices } from "../state/MediaDevices"; import { AppBar } from "../AppBar"; import { type MatrixInfo } from "./VideoPreview"; -import { ProcessorProvider } from "../tracks/TrackProcessorContext"; +import { + BackgroundEffectsProvider, + createAppBackgroundEffects, +} from "../tracks/BackgroundEffectsContext"; import { initializeWidget } from "../widget"; import { RootElementProvider } from "../RootElementContext"; @@ -246,7 +250,9 @@ describe("ActiveCall", () => { const { findByTestId } = render( - + { onLeft={(): void => {}} /> - + , ); @@ -295,7 +301,9 @@ describe("ActiveCall", () => { - + { onLeft={(): void => {}} /> - + , diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index ff4d5f086..2ba8427a2 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -110,7 +110,7 @@ import { useBehavior } from "../useBehavior.ts"; import { useValueBehavior } from "../useValueBehavior.ts"; import { Toast } from "../Toast.tsx"; import overlayStyles from "../Overlay.module.css"; -import { useTrackProcessorState$ } from "../tracks/TrackProcessorContext.tsx"; +import { useBackgroundEffectsState$ } from "../tracks/BackgroundEffectsContext"; import { type Layout } from "../state/layout-types.ts"; import { ObservableScope } from "../state/ObservableScope.ts"; import { CallFooter, type FooterSnapshot } from "../components/CallFooter.tsx"; @@ -153,7 +153,7 @@ export const ActiveCall: FC = (props) => { const urlParams = useUrlParams(); const hostBridge = useHostBridge(); const mediaDevices = useMediaDevices(); - const trackProcessorState$ = useTrackProcessorState$(); + const backgroundEffectsState$ = useBackgroundEffectsState$(); // The element we have to draw the call in: the page, or the container a host // gave us. Its size, not the window's, decides how the call is laid out. const rootElement = useRootElement(); @@ -191,7 +191,7 @@ export const ActiveCall: FC = (props) => { encryptionSystem: props.e2eeSystem, capture, videoProcessor$: scope.behavior( - trackProcessorState$.pipe(map((state) => state.processor)), + backgroundEffectsState$.pipe(map((state) => state.processor)), ), hostBridge, autoLeaveWhenOthersLeft, @@ -221,7 +221,7 @@ export const ActiveCall: FC = (props) => { urlParams, hostBridge, mediaDevices, - trackProcessorState$, + backgroundEffectsState$, props.client, props.media, rootElement, @@ -253,7 +253,7 @@ export const ActiveCall: FC = (props) => { props.onLeft, urlParams, mediaDevices, - trackProcessorState$, + backgroundEffectsState$, props.client, vm, ]); diff --git a/src/room/KnockLobbyView.test.tsx b/src/room/KnockLobbyView.test.tsx index 6cd2a33f4..cc70047ff 100644 --- a/src/room/KnockLobbyView.test.tsx +++ b/src/room/KnockLobbyView.test.tsx @@ -19,7 +19,7 @@ import { import { KnockLobbyView } from "./KnockLobbyView"; import { LeaveToHomeProvider } from "../LeaveToHomeContext"; import { MediaDevicesContext } from "../MediaDevicesContext"; -import { type ProcessorState } from "../tracks/TrackProcessorContext"; +import { type BackgroundEffectsState } from "../state/BackgroundEffects"; import { mockMediaDevices } from "../utils/test"; // One instance each: the lobby keys effects on them @@ -27,12 +27,12 @@ const stable = vi.hoisted(() => ({ processorState$: undefined as unknown, media: undefined as unknown, })); -vi.mock("../tracks/TrackProcessorContext", () => ({ - useTrackProcessorState$: (): Behavior => +vi.mock("../tracks/BackgroundEffectsContext", () => ({ + useBackgroundEffectsState$: (): Behavior => (stable.processorState$ ??= constant({ supported: false, processor: undefined, - })) as Behavior, + })) as Behavior, })); // Media that opens no device: the view is what is under test vi.mock("../useLocalMedia", () => ({ diff --git a/src/room/LobbyView.test.tsx b/src/room/LobbyView.test.tsx index 5258377b6..75e78520b 100644 --- a/src/room/LobbyView.test.tsx +++ b/src/room/LobbyView.test.tsx @@ -33,7 +33,7 @@ import { } from "../utils/test"; import { type MediaDevices } from "../state/MediaDevices"; import { MediaDevicesContext } from "../MediaDevicesContext"; -import { type ProcessorState } from "../tracks/TrackProcessorContext"; +import { type BackgroundEffectsState } from "../state/BackgroundEffects"; import { type EncryptionSystem } from "../e2ee/sharedKeyManagement"; import lobbyStyles from "./LobbyView.module.css"; import headerStyles from "../Header.module.css"; @@ -44,12 +44,12 @@ const leaveToHome = vi.fn(); // One instance: the view keys an effect on it const processorState$ = vi.hoisted(() => ({ current: undefined as unknown })); -vi.mock("../tracks/TrackProcessorContext", () => ({ - useTrackProcessorState$: (): Behavior => +vi.mock("../tracks/BackgroundEffectsContext", () => ({ + useBackgroundEffectsState$: (): Behavior => (processorState$.current ??= constant({ supported: false, processor: undefined, - })) as Behavior, + })) as Behavior, })); // Media that opens no device: the view is what is under test diff --git a/src/room/LobbyView.tsx b/src/room/LobbyView.tsx index 88038e2c9..8fa71ada8 100644 --- a/src/room/LobbyView.tsx +++ b/src/room/LobbyView.tsx @@ -31,7 +31,7 @@ import { LeaveToHomeLink } from "../button/LeaveToHomeLink"; import { useLeaveToHome } from "../LeaveToHomeContext"; import { useMediaDevices } from "../MediaDevicesContext"; import { ObservableScope } from "../state/ObservableScope"; -import { useTrackProcessorState$ } from "../tracks/TrackProcessorContext"; +import { useBackgroundEffectsState$ } from "../tracks/BackgroundEffectsContext"; import { useBehavior } from "../useBehavior"; import { CallFooter, type FooterSnapshot } from "../components/CallFooter"; import { useCallViewKeyboardShortcuts } from "../useCallViewKeyboardShortcuts"; @@ -122,7 +122,7 @@ export const LobbyView: FC = ({ const devices = useMediaDevices(); const videoInputId = useBehavior(devices.videoInput.selected$)?.id; - const trackProcessorState$ = useTrackProcessorState$(); + const backgroundEffectsState$ = useBackgroundEffectsState$(); const [mediaVm, setMediaVm] = useState(null); useEffect(() => { @@ -135,14 +135,14 @@ export const LobbyView: FC = ({ devices, captureSettings(), mediaScope.behavior( - trackProcessorState$.pipe(map((state) => state.processor)), + backgroundEffectsState$.pipe(map((state) => state.processor)), ), ), ); return (): void => { mediaScope.end(); }; - }, [media, muteStates, devices, trackProcessorState$]); + }, [media, muteStates, devices, backgroundEffectsState$]); const videoTrack = useBehavior(mediaVm?.videoTrack$ ?? noVideoTrack); useEffect(() => { diff --git a/src/settings/SettingsModal.tsx b/src/settings/SettingsModal.tsx index 7c1e79271..f962281fd 100644 --- a/src/settings/SettingsModal.tsx +++ b/src/settings/SettingsModal.tsx @@ -27,7 +27,7 @@ import { import { PreferencesSettingsTab } from "./PreferencesSettingsTab"; import { Slider } from "../Slider"; import { DeviceSelection } from "./DeviceSelection"; -import { useTrackProcessor } from "../tracks/TrackProcessorContext"; +import { useBackgroundEffects } from "../tracks/BackgroundEffectsContext"; import { DeveloperSettingsTab, type DeveloperSettingsSnapshot, @@ -78,7 +78,7 @@ export const SettingsModal: FC = ({ // Generate a `Checkbox` input to turn blur on or off. const BlurCheckbox: React.FC = (): ReactNode => { - const { supported } = useTrackProcessor(); + const { supported } = useBackgroundEffects(); const [blurActive, setBlurActive] = useSetting(backgroundBlurSetting); diff --git a/src/state/AppViewModel.ts b/src/state/AppViewModel.ts index 3f69515b2..e8882b8c1 100644 --- a/src/state/AppViewModel.ts +++ b/src/state/AppViewModel.ts @@ -7,6 +7,7 @@ Please see LICENSE in the repository root for full details. import { type AudioOutputOptions, MediaDevices } from "./MediaDevices"; import { type ObservableScope } from "./ObservableScope"; +import { createAppBackgroundEffects } from "../tracks/BackgroundEffectsContext"; /** * The top-level state holder for the application. @@ -16,6 +17,7 @@ export class AppViewModel { this.scope, this.audioOutputOptions, ); + public readonly backgroundEffects = createAppBackgroundEffects(this.scope); // TODO: Move more application logic here. The CallViewModel, at the very // least, ought to be accessible from this object. diff --git a/src/state/BackgroundEffects.test.ts b/src/state/BackgroundEffects.test.ts new file mode 100644 index 000000000..96ae80ba0 --- /dev/null +++ b/src/state/BackgroundEffects.test.ts @@ -0,0 +1,140 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { BehaviorSubject, distinctUntilChanged, map } from "rxjs"; +import { type BackgroundProcessorWrapper } from "@livekit/track-processors"; + +import { + type BackgroundEffects, + createBackgroundEffects, + type BackgroundEffectsOptions, + type BackgroundEffectsState, +} from "./BackgroundEffects"; +import { flushPromises, testScope, withTestScheduler } from "../utils/test"; + +/** A pipeline that records what it is switched to. */ +function fakePipeline(): { + pipeline: BackgroundProcessorWrapper; + switches: unknown[]; + /** Holds the next switch until it resolves, as a picture loading does. */ + holdNext: () => () => void; +} { + const switches: unknown[] = []; + let slow: Promise | undefined; + const pipeline = { + switchTo: vi.fn(async (options: unknown) => { + switches.push(options); + const held = slow; + slow = undefined; + await held; + }), + } as unknown as BackgroundProcessorWrapper; + return { + pipeline, + switches, + holdNext: () => { + let release!: () => void; + slow = new Promise((resolve) => (release = resolve)); + return release; + }, + }; +} + +/** One letter per state: idle, attached. */ +function letter(state: BackgroundEffectsState): string { + return state.processor === undefined ? "i" : "a"; +} + +describe("the pipeline's state", () => { + function testState({ + effect, + expected, + }: { + effect: string; + expected: string; + }): void { + withTestScheduler(({ behavior, expectObservable }) => { + const effects = createBackgroundEffects(testScope(), { + supported: true, + blur$: behavior(effect, { n: false, b: true }), + pipeline: fakePipeline().pipeline, + }); + expectObservable( + effects.state$.pipe(map(letter), distinctUntilChanged()), + ).toBe(expected); + }); + } + + it("attaches on first use and stays attached", () => + testState({ effect: "nbn", expected: "ia" })); +}); + +describe("background effects", () => { + let blur$: BehaviorSubject; + let fake: ReturnType; + + function build( + options: Partial = {}, + ): BackgroundEffects { + return createBackgroundEffects(testScope(), { + supported: true, + blur$, + pipeline: fake.pipeline, + ...options, + }); + } + const blur = async (on: boolean): Promise => { + blur$.next(on); + await flushPromises(); + }; + + beforeEach(() => { + blur$ = new BehaviorSubject(false); + fake = fakePipeline(); + }); + + it("switches in place rather than reattaching", async () => { + const effects = build(); + await blur(true); + const pipeline = effects.state$.value.processor; + await blur(false); + await blur(true); + + expect(effects.state$.value.processor).toBe(pipeline); + expect(fake.switches).toEqual([ + { mode: "background-blur", blurRadius: 15 }, + { mode: "disabled" }, + { mode: "background-blur", blurRadius: 15 }, + ]); + }); + + it("keeps switching after a switch fails", async () => { + vi.mocked(fake.pipeline.switchTo).mockRejectedValueOnce(new Error("lost")); + build(); + await blur(true); + await blur(false); + expect(fake.switches).toEqual([{ mode: "disabled" }]); + }); + + it("switches one at a time, skipping those overtaken", async () => { + const finished = fake.holdNext(); + build(); + await blur(true); + await blur(false); + await blur(true); + await blur(false); + expect(fake.switches).toHaveLength(1); + + finished(); + await flushPromises(); + expect(fake.switches).toEqual([ + { mode: "background-blur", blurRadius: 15 }, + { mode: "disabled" }, + ]); + }); +}); diff --git a/src/state/BackgroundEffects.ts b/src/state/BackgroundEffects.ts new file mode 100644 index 000000000..25c4b2c7d --- /dev/null +++ b/src/state/BackgroundEffects.ts @@ -0,0 +1,91 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { combineLatest, distinctUntilChanged, map, scan } from "rxjs"; +import { + type BackgroundProcessorWrapper, + type SwitchBackgroundProcessorOptions, +} from "@livekit/track-processors"; +import { logger } from "matrix-js-sdk/lib/logger"; +import { deepCompare } from "matrix-js-sdk/lib/utils"; +import { + fromLivekitProcessor, + type VideoProcessor, +} from "@element-hq/matrixrtc-sdk"; + +import { type Behavior } from "./Behavior"; +import { type ObservableScope } from "./ObservableScope"; + +const blurRadius = 15; + +export interface BackgroundEffectsState { + supported: boolean | undefined; + processor: undefined | VideoProcessor; +} + +export interface BackgroundEffectsOptions { + /** Whether this browser can run a pipeline at all. */ + supported: boolean; + /** Whether blur is chosen. */ + blur$: Behavior; + /** The background processor pipeline to be switched between effects. */ + pipeline: BackgroundProcessorWrapper; +} + +/** The background effect pipeline, as the camera tracks and the menus see it. */ +export interface BackgroundEffects { + readonly state$: Behavior; +} + +/** Switches the pipeline as the choice changes, for as long as the scope lasts. */ +export function createBackgroundEffects( + scope: ObservableScope, + { supported, blur$, pipeline }: BackgroundEffectsOptions, +): BackgroundEffects { + // Wrapped once, so the call and the preview see one processor throughout. + const processor = fromLivekitProcessor(pipeline); + const state$ = scope.behavior( + blur$.pipe( + scan( + (previous, wanted) => { + // Attached the first time an effect is wanted and never detached + // after, so someone who never turns one on pays for none of it. + const enable = + previous.processor !== undefined || (supported && wanted); + return { supported, processor: enable ? processor : undefined }; + }, + { supported, processor: undefined }, + ), + ), + ); + + const switchOptions$ = scope.behavior< + SwitchBackgroundProcessorOptions | undefined + >( + combineLatest([state$, blur$]).pipe( + map(([{ processor }, blur]) => + processor === undefined + ? undefined + : blur + ? { mode: "background-blur", blurRadius } + : { mode: "disabled" }, + ), + distinctUntilChanged(deepCompare), + ), + ); + // In turn: a picture's switch ends once it loads, so an earlier one could land last. + scope.reconcile(switchOptions$, async (options) => { + if (options === undefined) return; + try { + await pipeline.switchTo(options); + } catch (e) { + logger.warn("Failed to switch background effect", e); + } + }); + + return { state$ }; +} diff --git a/src/tracks/BackgroundEffectTransformer.test.ts b/src/tracks/BackgroundEffectTransformer.test.ts new file mode 100644 index 000000000..b8a26b86b --- /dev/null +++ b/src/tracks/BackgroundEffectTransformer.test.ts @@ -0,0 +1,24 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { describe, expect, it } from "vitest"; + +import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer"; + +describe("BackgroundEffectTransformer", () => { + it("passes frames on untouched once disabled, even after blurring", async () => { + const transformer = new BackgroundEffectTransformer({}); + await transformer.update({ blurRadius: 15, backgroundDisabled: false }); + await transformer.update({ + imagePath: undefined, + backgroundDisabled: true, + }); + // With neither, the library passes a frame on without segmenting it. + expect(transformer.options.blurRadius).toBeUndefined(); + expect(transformer.options.imagePath).toBeUndefined(); + }); +}); diff --git a/src/tracks/BlurBackgroundTransformer.ts b/src/tracks/BackgroundEffectTransformer.ts similarity index 78% rename from src/tracks/BlurBackgroundTransformer.ts rename to src/tracks/BackgroundEffectTransformer.ts index f86120d33..965359211 100644 --- a/src/tracks/BlurBackgroundTransformer.ts +++ b/src/tracks/BackgroundEffectTransformer.ts @@ -1,5 +1,6 @@ /* Copyright 2024-2025 New Vector Ltd. +Copyright 2026 Element Creations Ltd. SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. @@ -9,6 +10,7 @@ import { BackgroundTransformer, VideoTransformer, type VideoTransformerInitOptions, + type BackgroundOptions, } from "@livekit/track-processors"; import { ImageSegmenter } from "@mediapipe/tasks-vision"; @@ -49,7 +51,18 @@ const wasmFileset: WasmFileset = { * loads the segmentation models from our own bundle rather than as an external * resource fetched from the public internet. */ -export class BlurBackgroundTransformer extends BackgroundTransformer { +export class BackgroundEffectTransformer extends BackgroundTransformer { + /** + * As the library's, except that disabling also drops the blur radius: kept, + * it has every frame segmented and the result thrown away, where with + * neither a radius nor a picture frames are passed on untouched. + */ + public override async update(opts: BackgroundOptions): Promise { + await super.update( + opts.backgroundDisabled ? { ...opts, blurRadius: undefined } : opts, + ); + } + public async init({ outputCanvas, inputElement: inputVideo, @@ -73,8 +86,10 @@ export class BlurBackgroundTransformer extends BackgroundTransformer { outputConfidenceMasks: false, }); + // BackgroundTransformer's own init applies these, and this one replaces it. if (this.options.blurRadius) { this.gl?.setBlurRadius(this.options.blurRadius); } + this.gl?.setBackgroundDisabled(this.options.backgroundDisabled ?? false); } } diff --git a/src/tracks/BackgroundEffectsContext.test.ts b/src/tracks/BackgroundEffectsContext.test.ts new file mode 100644 index 000000000..958c5477e --- /dev/null +++ b/src/tracks/BackgroundEffectsContext.test.ts @@ -0,0 +1,115 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { act, createElement, type FC } from "react"; +import { render } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { + BackgroundEffectsProvider, + useBackgroundEffects, + createAppBackgroundEffects, +} from "./BackgroundEffectsContext"; +import { + type BackgroundEffects, + type BackgroundEffectsState, +} from "../state/BackgroundEffects"; +import { backgroundBlur } from "../settings/settings"; +import { flushPromises, testScope } from "../utils/test"; + +const pipelines = vi.hoisted(() => ({ + built: 0, + destroyed: 0, + switches: [] as unknown[], +})); + +// The pipeline itself needs WebGL and MediaPipe; what these tests are about is +// what the provider asks of it. +vi.mock("@livekit/track-processors", () => ({ + BackgroundProcessorWrapper: vi.fn(function (this: Record) { + pipelines.built += 1; + this.switchTo = vi.fn(async (options: unknown) => { + pipelines.switches.push(options); + return Promise.resolve(); + }); + this.destroy = vi.fn(async () => { + pipelines.destroyed += 1; + return Promise.resolve(); + }); + }), + supportsBackgroundProcessors: (): boolean => true, +})); +const platformMock = vi.hoisted(() => ({ platform: "desktop" })); +vi.mock("../Platform", () => platformMock); +vi.mock("./BackgroundEffectTransformer", () => ({ + BackgroundEffectTransformer: vi.fn(), +})); + +describe("BackgroundEffectsProvider", () => { + let seen: BackgroundEffectsState[]; + const Surface: FC = () => { + seen.push(useBackgroundEffects()); + return null; + }; + // Made on first render, once a test has set up the platform. + let effects: BackgroundEffects | undefined; + const surface = (): ReturnType => + createElement(BackgroundEffectsProvider, { + effects: (effects ??= createAppBackgroundEffects(testScope())), + children: createElement(Surface), + }); + const blur = async (on: boolean): Promise => { + await act(async () => { + backgroundBlur.setValue(on); + await flushPromises(); + }); + }; + const latest = (): BackgroundEffectsState => seen[seen.length - 1]; + + beforeEach(() => { + seen = []; + effects = undefined; + pipelines.built = 0; + pipelines.destroyed = 0; + pipelines.switches = []; + platformMock.platform = "desktop"; + backgroundBlur.setValue(false); + }); + afterEach(() => backgroundBlur.setValue(false)); + + // A pipeline primes itself when it is built and when it is destroyed, and a + // primed pipeline lets one frame through untouched: so the frame is spent + // once, as long as it is built once and never taken off the camera. + it("builds one pipeline on first use and keeps it through every change", async () => { + render(surface()); + for (const on of [true, false, true, false, true]) await blur(on); + + expect(pipelines.built).toBe(1); + expect(pipelines.destroyed).toBe(0); + expect(latest().processor).toBeDefined(); + }); + + it("does not run on a phone, even where its browser could", async () => { + platformMock.platform = "ios"; + render(surface()); + await blur(true); + + expect(latest().supported).toBe(false); + expect(latest().processor).toBeUndefined(); + }); + + it("switches to an effect chosen while the camera is off", async () => { + render(surface()); + await blur(true); + + // Already switched to blur, with nothing yet to attach it to + expect(pipelines.switches).toEqual([ + { mode: "background-blur", blurRadius: 15 }, + ]); + expect(pipelines.built).toBe(1); + }); +}); diff --git a/src/tracks/BackgroundEffectsContext.tsx b/src/tracks/BackgroundEffectsContext.tsx new file mode 100644 index 000000000..f1b384e0a --- /dev/null +++ b/src/tracks/BackgroundEffectsContext.tsx @@ -0,0 +1,74 @@ +/* +Copyright 2024-2026 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + BackgroundProcessorWrapper, + supportsBackgroundProcessors as supportsBackgroundProcessorsLivekitSdk, +} from "@livekit/track-processors"; +import { createContext, type FC, type JSX, use } from "react"; + +import { backgroundBlur as backgroundBlurSettings } from "../settings/settings"; +import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer"; +import { type Behavior } from "../state/Behavior"; +import { type ObservableScope } from "../state/ObservableScope"; +import { platform } from "../Platform"; +import { + type BackgroundEffects, + type BackgroundEffectsState, + createBackgroundEffects, +} from "../state/BackgroundEffects"; +import { useBehavior } from "../useBehavior"; + +const BackgroundEffectsContext = createContext( + undefined, +); + +export function useBackgroundEffects(): BackgroundEffectsState { + return useBehavior(useBackgroundEffectsState$()); +} + +export function useBackgroundEffectsState$(): Behavior { + const effects = use(BackgroundEffectsContext); + if (effects === undefined) + throw new Error( + "useBackgroundEffects must be used within a BackgroundEffectsProvider", + ); + return effects.state$; +} + +function supportsBackgroundProcessors(): boolean { + return supportsBackgroundProcessorsLivekitSdk() && platform === "desktop"; +} + +/** + * The app's one pipeline, shared by every camera track it opens. Its builds + * and teardowns take their turns through the SDK, which runs one operation + * at a time on a processor. + */ +export function createAppBackgroundEffects( + scope: ObservableScope, +): BackgroundEffects { + return createBackgroundEffects(scope, { + supported: supportsBackgroundProcessors(), + blur$: backgroundBlurSettings.value$, + pipeline: new BackgroundProcessorWrapper( + new BackgroundEffectTransformer({ backgroundDisabled: true }), + "background-effect", + ), + }); +} + +interface Props { + effects: BackgroundEffects; + children: JSX.Element; +} + +export const BackgroundEffectsProvider: FC = ({ effects, children }) => ( + + {children} + +); diff --git a/src/tracks/TrackProcessorContext.test.ts b/src/tracks/TrackProcessorContext.test.ts deleted file mode 100644 index 2acb3777a..000000000 --- a/src/tracks/TrackProcessorContext.test.ts +++ /dev/null @@ -1,51 +0,0 @@ -/* -Copyright 2026 Element Creations Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { describe, expect, it, vi } from "vitest"; -import { - type BackgroundOptions, - type ProcessorWrapper, -} from "@livekit/track-processors"; - -import { fromLivekitProcessor } from "./TrackProcessorContext"; - -describe("fromLivekitProcessor", () => { - const captured = { id: "captured" } as MediaStreamTrack; - const processed = { id: "processed" } as MediaStreamTrack; - - function mockWrapper( - processedTrack: MediaStreamTrack | undefined, - ): ProcessorWrapper { - return { - name: "background-blur", - processedTrack, - init: vi.fn().mockResolvedValue(undefined), - restart: vi.fn().mockResolvedValue(undefined), - destroy: vi.fn().mockResolvedValue(undefined), - } as unknown as ProcessorWrapper; - } - - it("resolves with the track the wrapper produced", async () => { - const wrapper = mockWrapper(processed); - const sdkProcessor = fromLivekitProcessor(wrapper); - expect(sdkProcessor.name).toBe("background-blur"); - await expect(sdkProcessor.init(captured)).resolves.toBe(processed); - expect(wrapper.init).toHaveBeenCalledWith({ - kind: "video", - track: captured, - }); - await expect(sdkProcessor.restart(captured)).resolves.toBe(processed); - await sdkProcessor.destroy(); - expect(wrapper.destroy).toHaveBeenCalled(); - }); - - it("rejects where the wrapper produced no track", async () => { - await expect( - fromLivekitProcessor(mockWrapper(undefined)).init(captured), - ).rejects.toThrow("produced no track"); - }); -}); diff --git a/src/tracks/TrackProcessorContext.tsx b/src/tracks/TrackProcessorContext.tsx deleted file mode 100644 index 5f828aba3..000000000 --- a/src/tracks/TrackProcessorContext.tsx +++ /dev/null @@ -1,110 +0,0 @@ -/* -Copyright 2024-2025 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - ProcessorWrapper, - supportsBackgroundProcessors as supportsBackgroundProcessorsLivekitSdk, - type BackgroundOptions, -} from "@livekit/track-processors"; -import { createContext, type FC, type JSX, use, useMemo } from "react"; -import { Track } from "livekit-client"; -import { type VideoProcessor } from "@element-hq/matrixrtc-sdk"; - -import { - backgroundBlur as backgroundBlurSettings, - useSetting, -} from "../settings/settings"; -import { BlurBackgroundTransformer } from "./BlurBackgroundTransformer"; -import { type Behavior } from "../state/Behavior"; -import { platform } from "../Platform"; -import { useValueBehavior } from "../useValueBehavior"; - -//TODO-MULTI-SFU: This is not yet fully there. -// it is a combination of exposing observable and react hooks. -// preferably we should not make this a context anymore and instead just a vm? - -export type ProcessorState = { - supported: boolean | undefined; - processor: undefined | VideoProcessor; -}; - -/** - * This allows to use a LiveKit processor for the matrixRTCSDK (VideoProcessor). - * They have slightly different apis so we need to convert them with this - * function before we can use them in the matrixRTCSDK. - */ -export function fromLivekitProcessor( - wrapper: ProcessorWrapper, -): VideoProcessor { - const processedTrack = (): MediaStreamTrack => { - if (wrapper.processedTrack === undefined) - throw new Error(`Processor ${wrapper.name} produced no track`); - return wrapper.processedTrack; - }; - return { - name: wrapper.name, - init: async (track: MediaStreamTrack, element?: HTMLMediaElement) => { - await wrapper.init({ kind: Track.Kind.Video, track, element }); - return processedTrack(); - }, - restart: async (track: MediaStreamTrack, element?: HTMLMediaElement) => { - await wrapper.restart({ kind: Track.Kind.Video, track, element }); - return processedTrack(); - }, - destroy: async () => wrapper.destroy(), - }; -} - -const ProcessorContext = createContext(undefined); - -export function useTrackProcessor(): ProcessorState { - const state = use(ProcessorContext); - if (state === undefined) - throw new Error( - "useTrackProcessor must be used within a ProcessorProvider", - ); - return state; -} - -export function useTrackProcessorState$(): Behavior { - return useValueBehavior(useTrackProcessor()); -} - -interface Props { - children: JSX.Element; -} - -function supportsBackgroundProcessors(): boolean { - return supportsBackgroundProcessorsLivekitSdk() && platform === "desktop"; -} - -export const ProcessorProvider: FC = ({ children }) => { - // The setting the user wants to have - const [blurActivated] = useSetting(backgroundBlurSettings); - const supported = useMemo(() => supportsBackgroundProcessors(), []); - const blur = useMemo( - () => - fromLivekitProcessor( - new ProcessorWrapper( - new BlurBackgroundTransformer({ blurRadius: 15 }), - "background-blur", - ), - ), - [], - ); - - // This is the actual state exposed through the context - const processorState = useMemo( - () => ({ - supported, - processor: supported && blurActivated ? blur : undefined, - }), - [supported, blurActivated, blur], - ); - - return {children}; -};