diff --git a/component/index.tsx b/component/index.tsx index 4a4b4d8cf..2b96272e3 100644 --- a/component/index.tsx +++ b/component/index.tsx @@ -72,9 +72,9 @@ import { useUrlParams, } from "../src/UrlParams"; import { MediaDevicesContext } from "../src/MediaDevicesContext"; -import { MediaDevices } from "../src/state/MediaDevices"; +import { AppViewModel } from "../src/state/AppViewModel"; import { ObservableScope } from "../src/state/ObservableScope"; -import { ProcessorProvider } from "../src/tracks/TrackProcessorContext"; +import { BackgroundEffectsProvider } from "../src/tracks/BackgroundEffectsContext"; import { Config } from "../src/config/Config"; import { type ConfigOptions } from "../src/config/ConfigOptions"; import { i18n } from "../src/utils/i18n"; @@ -194,14 +194,12 @@ export const ElementCall: FC = ({ // observers running for the rest of the page's life. Null until then, which // is one render. const { controlledAudioDevices, callIntent } = params; - const [mediaDevices, setMediaDevices] = useState(null); + const [vm, setVm] = useState(null); useEffect(() => { const scope = new ObservableScope(); - setMediaDevices( - new MediaDevices(scope, { controlledAudioDevices, callIntent }), - ); + setVm(new AppViewModel(scope, { controlledAudioDevices, callIntent })); return (): void => { - setMediaDevices(null); + setVm(null); scope.end(); }; }, [controlledAudioDevices, callIntent]); @@ -222,8 +220,7 @@ export const ElementCall: FC = ({ // is told that Element Call has loaded, as the widget tells its client once // its own initialisation is over. Once per mount, however often the pieces // are later swapped out. - const ready = - container !== null && rtcSession !== null && mediaDevices !== null; + const ready = container !== null && rtcSession !== null && vm !== null; const announcedLoaded = useRef(false); useEffect(() => { if (!ready || announcedLoaded.current) return; @@ -254,8 +251,10 @@ export const ElementCall: FC = ({ - - + + = ({ preload={params.preload} skipLobby={params.skipLobby} /> - + diff --git a/sdk/index.ts b/sdk/index.ts index 0b97897b2..5818c311b 100644 --- a/sdk/index.ts +++ b/sdk/index.ts @@ -63,6 +63,11 @@ export { } from "./src/media-backend/livekit/LivekitBackend"; // Tracks for a lobby, captured ahead of the join and handed to it export { createLocalMedia } from "./src/LocalMedia"; +// For a host whose processor is a livekit-client one, such as @livekit/track-processors +export { + fromLivekitProcessor, + toLivekitProcessor, +} from "./src/media-backend/livekit/videoProcessor"; // For a developer panel to check a transport before using it export { getSFUConfig as authenticateWithTransport, diff --git a/sdk/src/media-backend/livekit/LivekitMediaTrack.ts b/sdk/src/media-backend/livekit/LivekitMediaTrack.ts index b5767daf0..0e6c1986f 100644 --- a/sdk/src/media-backend/livekit/LivekitMediaTrack.ts +++ b/sdk/src/media-backend/livekit/LivekitMediaTrack.ts @@ -50,10 +50,7 @@ import { type VideoMediaTrack, } from "../../media-api"; import { MatrixRTCError } from "../../errors"; -import { - convertToLivekitProcessor, - type LivekitProcessor, -} from "./videoProcessor"; +import { toLivekitProcessor, type LivekitProcessor } from "./videoProcessor"; export const mediaSources: Record = { [Track.Source.Microphone]: "microphone", @@ -245,8 +242,7 @@ export function createLocalLivekitMediaTrack( if (!track || track.mediaStreamTrack.readyState === "ended") return; const current = track.getProcessor() as LivekitProcessor | undefined; if (processor === current?.source) return; - if (processor) - await track.setProcessor(convertToLivekitProcessor(processor)); + if (processor) await track.setProcessor(toLivekitProcessor(processor)); else if (current) await track.stopProcessor(); }, }; diff --git a/sdk/src/media-backend/livekit/Publisher.ts b/sdk/src/media-backend/livekit/Publisher.ts index 53df00ce9..40728cef5 100644 --- a/sdk/src/media-backend/livekit/Publisher.ts +++ b/sdk/src/media-backend/livekit/Publisher.ts @@ -22,7 +22,7 @@ import { type PublishRequest, } from "../../media-api"; import { livekitSources, mediaSources } from "./LivekitMediaTrack"; -import { convertToLivekitProcessor } from "./videoProcessor"; +import { toLivekitProcessor } from "./videoProcessor"; import { audioCaptureOptions, screenShareCaptureOptions, @@ -159,10 +159,7 @@ export class Publisher { if (muted) await local.mute(); if (processor && local instanceof LocalVideoTrack) await local.setProcessor( - convertToLivekitProcessor( - processor.processor, - processor.processedTrack, - ), + toLivekitProcessor(processor.processor, processor.processedTrack), ); return this.room.localParticipant.publishTrack(local, { source: livekitSources[request.source], diff --git a/sdk/src/media-backend/livekit/publishOptions.ts b/sdk/src/media-backend/livekit/publishOptions.ts index f030c3fd4..cbdfbdc6f 100644 --- a/sdk/src/media-backend/livekit/publishOptions.ts +++ b/sdk/src/media-backend/livekit/publishOptions.ts @@ -16,7 +16,7 @@ import { type PublishRequest, type VideoCaptureSettings, } from "../../media-api"; -import { convertToLivekitProcessor } from "./videoProcessor"; +import { toLivekitProcessor } from "./videoProcessor"; export type PublishRequestFor = Extract< PublishRequest, @@ -40,7 +40,7 @@ export function videoCaptureOptions({ if (!deviceId && !processor && !capture?.resolution) return undefined; return { ...(deviceId && { deviceId }), - ...(processor && { processor: convertToLivekitProcessor(processor) }), + ...(processor && { processor: toLivekitProcessor(processor) }), ...(capture?.resolution && { resolution: capture.resolution }), }; } diff --git a/sdk/src/media-backend/livekit/videoProcessor.test.ts b/sdk/src/media-backend/livekit/videoProcessor.test.ts index e8394e484..4ac6e9283 100644 --- a/sdk/src/media-backend/livekit/videoProcessor.test.ts +++ b/sdk/src/media-backend/livekit/videoProcessor.test.ts @@ -5,13 +5,17 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { Track, type VideoProcessorOptions } from "livekit-client"; +import { + Track, + type TrackProcessor, + type VideoProcessorOptions, +} from "livekit-client"; import { describe, expect, it, vi } from "vitest"; import { type VideoProcessor } from "../../media-api"; -import { convertToLivekitProcessor } from "./videoProcessor"; +import { fromLivekitProcessor, toLivekitProcessor } from "./videoProcessor"; -describe("convertToLivekitProcessor", () => { +describe("toLivekitProcessor", () => { const captured = { id: "captured" } as MediaStreamTrack; const processed = { id: "processed" } as MediaStreamTrack; const restarted = { id: "restarted" } as MediaStreamTrack; @@ -33,7 +37,7 @@ describe("convertToLivekitProcessor", () => { it("holds the processed track where LiveKit reads it", async () => { const processor = fakeProcessor(); - const adapter = convertToLivekitProcessor(processor); + const adapter = toLivekitProcessor(processor); expect(adapter.name).toBe("blur"); expect(adapter.processedTrack).toBeUndefined(); @@ -59,8 +63,8 @@ describe("convertToLivekitProcessor", () => { finishDestroy = resolve; }), ); - const previous = convertToLivekitProcessor(processor); - const next = convertToLivekitProcessor(processor); + const previous = toLivekitProcessor(processor); + const next = toLivekitProcessor(processor); const destroying = previous.destroy(); const initialising = next.init(options); @@ -73,3 +77,40 @@ describe("convertToLivekitProcessor", () => { expect(next.processedTrack).toBe(processed); }); }); + +describe("fromLivekitProcessor", () => { + const captured = { id: "captured" } as MediaStreamTrack; + const processed = { id: "processed" } as MediaStreamTrack; + + function fakeLivekitProcessor( + processedTrack: MediaStreamTrack | undefined, + ): TrackProcessor { + return { + name: "background-blur", + processedTrack, + init: vi.fn().mockResolvedValue(undefined), + restart: vi.fn().mockResolvedValue(undefined), + destroy: vi.fn().mockResolvedValue(undefined), + }; + } + + it("resolves with the track the LiveKit processor produced", async () => { + const livekitProcessor = fakeLivekitProcessor(processed); + const processor = fromLivekitProcessor(livekitProcessor); + expect(processor.name).toBe("background-blur"); + await expect(processor.init(captured)).resolves.toBe(processed); + expect(livekitProcessor.init).toHaveBeenCalledWith({ + kind: Track.Kind.Video, + track: captured, + }); + await expect(processor.restart(captured)).resolves.toBe(processed); + await processor.destroy(); + expect(livekitProcessor.destroy).toHaveBeenCalled(); + }); + + it("rejects where the LiveKit processor produced no track", async () => { + await expect( + fromLivekitProcessor(fakeLivekitProcessor(undefined)).init(captured), + ).rejects.toThrow("produced no track"); + }); +}); diff --git a/sdk/src/media-backend/livekit/videoProcessor.ts b/sdk/src/media-backend/livekit/videoProcessor.ts index 17edd0a9b..557a2f4b3 100644 --- a/sdk/src/media-backend/livekit/videoProcessor.ts +++ b/sdk/src/media-backend/livekit/videoProcessor.ts @@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type Track, type TrackProcessor } from "livekit-client"; +import { Track, type TrackProcessor } from "livekit-client"; import { type VideoProcessor } from "../../media-api"; import { inTurn } from "../../utils/processorTurns"; @@ -16,14 +16,15 @@ export interface LivekitProcessor extends TrackProcessor { } /** - * A `VideoProcessor` as livekit-client wants it. LiveKit reads the processed - * track off the processor after `init` and `restart` rather than taking it as - * their result, so the adapter keeps it there. A processor a lobby already - * runs on the track is adopted with its output, `running`, so that the join - * does not initialise it again. Every step takes its turn on the processor, - * which a stopped track tears down without waiting for the next one's init. + * A `VideoProcessor` as a livekit-client processor. LiveKit reads the + * processed track off the processor after `init` and `restart` rather than + * taking it as their result, so the adapter keeps it there. A processor a + * lobby already runs on the track is adopted with its output, `running`, so + * that the join does not initialise it again. Every step takes its turn on + * the processor, which a stopped track tears down without waiting for the + * next one's init. */ -export function convertToLivekitProcessor( +export function toLivekitProcessor( processor: VideoProcessor, running?: MediaStreamTrack, ): LivekitProcessor { @@ -47,3 +48,30 @@ export function convertToLivekitProcessor( }; return adapter; } + +/** + * A livekit-client processor, such as one from `@livekit/track-processors`, + * as a `VideoProcessor`: `init` and `restart` resolve with the track it + * leaves on itself. + */ +export function fromLivekitProcessor( + processor: TrackProcessor, +): VideoProcessor { + const processedTrack = (): MediaStreamTrack => { + if (processor.processedTrack === undefined) + throw new Error(`Processor ${processor.name} produced no track`); + return processor.processedTrack; + }; + return { + name: processor.name, + init: async (track, element) => { + await processor.init({ kind: Track.Kind.Video, track, element }); + return processedTrack(); + }, + restart: async (track, element) => { + await processor.restart({ kind: Track.Kind.Video, track, element }); + return processedTrack(); + }, + destroy: async () => processor.destroy(), + }; +} diff --git a/src/App.tsx b/src/App.tsx index a92d56b59..e3aff9086 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -35,7 +35,7 @@ import { ErrorPage, LoadingPage } from "./FullScreenView"; import { Initializer } from "./initializer"; import { type WidgetHelpers } from "./widget"; import { useTheme } from "./useTheme"; -import { ProcessorProvider } from "./tracks/TrackProcessorContext"; +import { BackgroundEffectsProvider } from "./tracks/BackgroundEffectsContext"; import { type AppViewModel } from "./state/AppViewModel"; import { MediaDevicesContext } from "./MediaDevicesContext"; import { @@ -148,7 +148,7 @@ export const App: FC = ({ vm, widget }) => { loaded && clientReady ? ( - + } > @@ -159,7 +159,7 @@ export const App: FC = ({ vm, widget }) => { } /> - + ) : ( diff --git a/src/room/CallView.test.tsx b/src/room/CallView.test.tsx index 2c2208c56..ed747c56c 100644 --- a/src/room/CallView.test.tsx +++ b/src/room/CallView.test.tsx @@ -50,6 +50,7 @@ import { mockMediaDevices, mockRtcMembership, MockRTCSession, + testScope, } from "../utils/test"; import { CallView } from "./CallView"; import { GroupCallErrorBoundary } from "./GroupCallErrorBoundary"; @@ -61,7 +62,10 @@ import { } from "../HostBridge"; import { type JoinCallData } from "../widget"; import { MatrixRTCTransportMissingError } from "../utils/errors"; -import { ProcessorProvider } from "../tracks/TrackProcessorContext"; +import { + BackgroundEffectsProvider, + createAppBackgroundEffects, +} from "../tracks/BackgroundEffectsContext"; import { MediaDevicesContext } from "../MediaDevicesContext"; import { constant } from "../state/Behavior"; @@ -193,7 +197,9 @@ function createCallView( - + {options.withErrorBoundary ? ( {callView} @@ -201,7 +207,7 @@ function createCallView( ) : ( callView )} - + diff --git a/src/room/InCallView.test.tsx b/src/room/InCallView.test.tsx index d8b6e637a..f6bc938f6 100644 --- a/src/room/InCallView.test.tsx +++ b/src/room/InCallView.test.tsx @@ -28,6 +28,7 @@ import { type MockRTCSession, mockRTCSlot, mockRTCMember, + testScope, } from "../utils/test"; import { E2eeType } from "../e2ee/e2eeType"; import { @@ -48,7 +49,10 @@ import { MediaDevicesContext } from "../MediaDevicesContext"; import { type MediaDevices as ECMediaDevices } from "../state/MediaDevices"; import { AppBar } from "../AppBar"; import { type MatrixInfo } from "./VideoPreview"; -import { ProcessorProvider } from "../tracks/TrackProcessorContext"; +import { + BackgroundEffectsProvider, + createAppBackgroundEffects, +} from "../tracks/BackgroundEffectsContext"; import { initializeWidget } from "../widget"; import { RootElementProvider } from "../RootElementContext"; @@ -246,7 +250,9 @@ describe("ActiveCall", () => { const { findByTestId } = render( - + { onLeft={(): void => {}} /> - + , ); @@ -295,7 +301,9 @@ describe("ActiveCall", () => { - + { onLeft={(): void => {}} /> - + , diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index ff4d5f086..2ba8427a2 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -110,7 +110,7 @@ import { useBehavior } from "../useBehavior.ts"; import { useValueBehavior } from "../useValueBehavior.ts"; import { Toast } from "../Toast.tsx"; import overlayStyles from "../Overlay.module.css"; -import { useTrackProcessorState$ } from "../tracks/TrackProcessorContext.tsx"; +import { useBackgroundEffectsState$ } from "../tracks/BackgroundEffectsContext"; import { type Layout } from "../state/layout-types.ts"; import { ObservableScope } from "../state/ObservableScope.ts"; import { CallFooter, type FooterSnapshot } from "../components/CallFooter.tsx"; @@ -153,7 +153,7 @@ export const ActiveCall: FC = (props) => { const urlParams = useUrlParams(); const hostBridge = useHostBridge(); const mediaDevices = useMediaDevices(); - const trackProcessorState$ = useTrackProcessorState$(); + const backgroundEffectsState$ = useBackgroundEffectsState$(); // The element we have to draw the call in: the page, or the container a host // gave us. Its size, not the window's, decides how the call is laid out. const rootElement = useRootElement(); @@ -191,7 +191,7 @@ export const ActiveCall: FC = (props) => { encryptionSystem: props.e2eeSystem, capture, videoProcessor$: scope.behavior( - trackProcessorState$.pipe(map((state) => state.processor)), + backgroundEffectsState$.pipe(map((state) => state.processor)), ), hostBridge, autoLeaveWhenOthersLeft, @@ -221,7 +221,7 @@ export const ActiveCall: FC = (props) => { urlParams, hostBridge, mediaDevices, - trackProcessorState$, + backgroundEffectsState$, props.client, props.media, rootElement, @@ -253,7 +253,7 @@ export const ActiveCall: FC = (props) => { props.onLeft, urlParams, mediaDevices, - trackProcessorState$, + backgroundEffectsState$, props.client, vm, ]); diff --git a/src/room/KnockLobbyView.test.tsx b/src/room/KnockLobbyView.test.tsx index 6cd2a33f4..cc70047ff 100644 --- a/src/room/KnockLobbyView.test.tsx +++ b/src/room/KnockLobbyView.test.tsx @@ -19,7 +19,7 @@ import { import { KnockLobbyView } from "./KnockLobbyView"; import { LeaveToHomeProvider } from "../LeaveToHomeContext"; import { MediaDevicesContext } from "../MediaDevicesContext"; -import { type ProcessorState } from "../tracks/TrackProcessorContext"; +import { type BackgroundEffectsState } from "../state/BackgroundEffects"; import { mockMediaDevices } from "../utils/test"; // One instance each: the lobby keys effects on them @@ -27,12 +27,12 @@ const stable = vi.hoisted(() => ({ processorState$: undefined as unknown, media: undefined as unknown, })); -vi.mock("../tracks/TrackProcessorContext", () => ({ - useTrackProcessorState$: (): Behavior => +vi.mock("../tracks/BackgroundEffectsContext", () => ({ + useBackgroundEffectsState$: (): Behavior => (stable.processorState$ ??= constant({ supported: false, processor: undefined, - })) as Behavior, + })) as Behavior, })); // Media that opens no device: the view is what is under test vi.mock("../useLocalMedia", () => ({ diff --git a/src/room/LobbyView.test.tsx b/src/room/LobbyView.test.tsx index 5258377b6..75e78520b 100644 --- a/src/room/LobbyView.test.tsx +++ b/src/room/LobbyView.test.tsx @@ -33,7 +33,7 @@ import { } from "../utils/test"; import { type MediaDevices } from "../state/MediaDevices"; import { MediaDevicesContext } from "../MediaDevicesContext"; -import { type ProcessorState } from "../tracks/TrackProcessorContext"; +import { type BackgroundEffectsState } from "../state/BackgroundEffects"; import { type EncryptionSystem } from "../e2ee/sharedKeyManagement"; import lobbyStyles from "./LobbyView.module.css"; import headerStyles from "../Header.module.css"; @@ -44,12 +44,12 @@ const leaveToHome = vi.fn(); // One instance: the view keys an effect on it const processorState$ = vi.hoisted(() => ({ current: undefined as unknown })); -vi.mock("../tracks/TrackProcessorContext", () => ({ - useTrackProcessorState$: (): Behavior => +vi.mock("../tracks/BackgroundEffectsContext", () => ({ + useBackgroundEffectsState$: (): Behavior => (processorState$.current ??= constant({ supported: false, processor: undefined, - })) as Behavior, + })) as Behavior, })); // Media that opens no device: the view is what is under test diff --git a/src/room/LobbyView.tsx b/src/room/LobbyView.tsx index 88038e2c9..8fa71ada8 100644 --- a/src/room/LobbyView.tsx +++ b/src/room/LobbyView.tsx @@ -31,7 +31,7 @@ import { LeaveToHomeLink } from "../button/LeaveToHomeLink"; import { useLeaveToHome } from "../LeaveToHomeContext"; import { useMediaDevices } from "../MediaDevicesContext"; import { ObservableScope } from "../state/ObservableScope"; -import { useTrackProcessorState$ } from "../tracks/TrackProcessorContext"; +import { useBackgroundEffectsState$ } from "../tracks/BackgroundEffectsContext"; import { useBehavior } from "../useBehavior"; import { CallFooter, type FooterSnapshot } from "../components/CallFooter"; import { useCallViewKeyboardShortcuts } from "../useCallViewKeyboardShortcuts"; @@ -122,7 +122,7 @@ export const LobbyView: FC = ({ const devices = useMediaDevices(); const videoInputId = useBehavior(devices.videoInput.selected$)?.id; - const trackProcessorState$ = useTrackProcessorState$(); + const backgroundEffectsState$ = useBackgroundEffectsState$(); const [mediaVm, setMediaVm] = useState(null); useEffect(() => { @@ -135,14 +135,14 @@ export const LobbyView: FC = ({ devices, captureSettings(), mediaScope.behavior( - trackProcessorState$.pipe(map((state) => state.processor)), + backgroundEffectsState$.pipe(map((state) => state.processor)), ), ), ); return (): void => { mediaScope.end(); }; - }, [media, muteStates, devices, trackProcessorState$]); + }, [media, muteStates, devices, backgroundEffectsState$]); const videoTrack = useBehavior(mediaVm?.videoTrack$ ?? noVideoTrack); useEffect(() => { diff --git a/src/settings/SettingsModal.tsx b/src/settings/SettingsModal.tsx index 7c1e79271..f962281fd 100644 --- a/src/settings/SettingsModal.tsx +++ b/src/settings/SettingsModal.tsx @@ -27,7 +27,7 @@ import { import { PreferencesSettingsTab } from "./PreferencesSettingsTab"; import { Slider } from "../Slider"; import { DeviceSelection } from "./DeviceSelection"; -import { useTrackProcessor } from "../tracks/TrackProcessorContext"; +import { useBackgroundEffects } from "../tracks/BackgroundEffectsContext"; import { DeveloperSettingsTab, type DeveloperSettingsSnapshot, @@ -78,7 +78,7 @@ export const SettingsModal: FC = ({ // Generate a `Checkbox` input to turn blur on or off. const BlurCheckbox: React.FC = (): ReactNode => { - const { supported } = useTrackProcessor(); + const { supported } = useBackgroundEffects(); const [blurActive, setBlurActive] = useSetting(backgroundBlurSetting); diff --git a/src/state/AppViewModel.ts b/src/state/AppViewModel.ts index 3f69515b2..e8882b8c1 100644 --- a/src/state/AppViewModel.ts +++ b/src/state/AppViewModel.ts @@ -7,6 +7,7 @@ Please see LICENSE in the repository root for full details. import { type AudioOutputOptions, MediaDevices } from "./MediaDevices"; import { type ObservableScope } from "./ObservableScope"; +import { createAppBackgroundEffects } from "../tracks/BackgroundEffectsContext"; /** * The top-level state holder for the application. @@ -16,6 +17,7 @@ export class AppViewModel { this.scope, this.audioOutputOptions, ); + public readonly backgroundEffects = createAppBackgroundEffects(this.scope); // TODO: Move more application logic here. The CallViewModel, at the very // least, ought to be accessible from this object. diff --git a/src/state/BackgroundEffects.test.ts b/src/state/BackgroundEffects.test.ts new file mode 100644 index 000000000..96ae80ba0 --- /dev/null +++ b/src/state/BackgroundEffects.test.ts @@ -0,0 +1,140 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { BehaviorSubject, distinctUntilChanged, map } from "rxjs"; +import { type BackgroundProcessorWrapper } from "@livekit/track-processors"; + +import { + type BackgroundEffects, + createBackgroundEffects, + type BackgroundEffectsOptions, + type BackgroundEffectsState, +} from "./BackgroundEffects"; +import { flushPromises, testScope, withTestScheduler } from "../utils/test"; + +/** A pipeline that records what it is switched to. */ +function fakePipeline(): { + pipeline: BackgroundProcessorWrapper; + switches: unknown[]; + /** Holds the next switch until it resolves, as a picture loading does. */ + holdNext: () => () => void; +} { + const switches: unknown[] = []; + let slow: Promise | undefined; + const pipeline = { + switchTo: vi.fn(async (options: unknown) => { + switches.push(options); + const held = slow; + slow = undefined; + await held; + }), + } as unknown as BackgroundProcessorWrapper; + return { + pipeline, + switches, + holdNext: () => { + let release!: () => void; + slow = new Promise((resolve) => (release = resolve)); + return release; + }, + }; +} + +/** One letter per state: idle, attached. */ +function letter(state: BackgroundEffectsState): string { + return state.processor === undefined ? "i" : "a"; +} + +describe("the pipeline's state", () => { + function testState({ + effect, + expected, + }: { + effect: string; + expected: string; + }): void { + withTestScheduler(({ behavior, expectObservable }) => { + const effects = createBackgroundEffects(testScope(), { + supported: true, + blur$: behavior(effect, { n: false, b: true }), + pipeline: fakePipeline().pipeline, + }); + expectObservable( + effects.state$.pipe(map(letter), distinctUntilChanged()), + ).toBe(expected); + }); + } + + it("attaches on first use and stays attached", () => + testState({ effect: "nbn", expected: "ia" })); +}); + +describe("background effects", () => { + let blur$: BehaviorSubject; + let fake: ReturnType; + + function build( + options: Partial = {}, + ): BackgroundEffects { + return createBackgroundEffects(testScope(), { + supported: true, + blur$, + pipeline: fake.pipeline, + ...options, + }); + } + const blur = async (on: boolean): Promise => { + blur$.next(on); + await flushPromises(); + }; + + beforeEach(() => { + blur$ = new BehaviorSubject(false); + fake = fakePipeline(); + }); + + it("switches in place rather than reattaching", async () => { + const effects = build(); + await blur(true); + const pipeline = effects.state$.value.processor; + await blur(false); + await blur(true); + + expect(effects.state$.value.processor).toBe(pipeline); + expect(fake.switches).toEqual([ + { mode: "background-blur", blurRadius: 15 }, + { mode: "disabled" }, + { mode: "background-blur", blurRadius: 15 }, + ]); + }); + + it("keeps switching after a switch fails", async () => { + vi.mocked(fake.pipeline.switchTo).mockRejectedValueOnce(new Error("lost")); + build(); + await blur(true); + await blur(false); + expect(fake.switches).toEqual([{ mode: "disabled" }]); + }); + + it("switches one at a time, skipping those overtaken", async () => { + const finished = fake.holdNext(); + build(); + await blur(true); + await blur(false); + await blur(true); + await blur(false); + expect(fake.switches).toHaveLength(1); + + finished(); + await flushPromises(); + expect(fake.switches).toEqual([ + { mode: "background-blur", blurRadius: 15 }, + { mode: "disabled" }, + ]); + }); +}); diff --git a/src/state/BackgroundEffects.ts b/src/state/BackgroundEffects.ts new file mode 100644 index 000000000..25c4b2c7d --- /dev/null +++ b/src/state/BackgroundEffects.ts @@ -0,0 +1,91 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { combineLatest, distinctUntilChanged, map, scan } from "rxjs"; +import { + type BackgroundProcessorWrapper, + type SwitchBackgroundProcessorOptions, +} from "@livekit/track-processors"; +import { logger } from "matrix-js-sdk/lib/logger"; +import { deepCompare } from "matrix-js-sdk/lib/utils"; +import { + fromLivekitProcessor, + type VideoProcessor, +} from "@element-hq/matrixrtc-sdk"; + +import { type Behavior } from "./Behavior"; +import { type ObservableScope } from "./ObservableScope"; + +const blurRadius = 15; + +export interface BackgroundEffectsState { + supported: boolean | undefined; + processor: undefined | VideoProcessor; +} + +export interface BackgroundEffectsOptions { + /** Whether this browser can run a pipeline at all. */ + supported: boolean; + /** Whether blur is chosen. */ + blur$: Behavior; + /** The background processor pipeline to be switched between effects. */ + pipeline: BackgroundProcessorWrapper; +} + +/** The background effect pipeline, as the camera tracks and the menus see it. */ +export interface BackgroundEffects { + readonly state$: Behavior; +} + +/** Switches the pipeline as the choice changes, for as long as the scope lasts. */ +export function createBackgroundEffects( + scope: ObservableScope, + { supported, blur$, pipeline }: BackgroundEffectsOptions, +): BackgroundEffects { + // Wrapped once, so the call and the preview see one processor throughout. + const processor = fromLivekitProcessor(pipeline); + const state$ = scope.behavior( + blur$.pipe( + scan( + (previous, wanted) => { + // Attached the first time an effect is wanted and never detached + // after, so someone who never turns one on pays for none of it. + const enable = + previous.processor !== undefined || (supported && wanted); + return { supported, processor: enable ? processor : undefined }; + }, + { supported, processor: undefined }, + ), + ), + ); + + const switchOptions$ = scope.behavior< + SwitchBackgroundProcessorOptions | undefined + >( + combineLatest([state$, blur$]).pipe( + map(([{ processor }, blur]) => + processor === undefined + ? undefined + : blur + ? { mode: "background-blur", blurRadius } + : { mode: "disabled" }, + ), + distinctUntilChanged(deepCompare), + ), + ); + // In turn: a picture's switch ends once it loads, so an earlier one could land last. + scope.reconcile(switchOptions$, async (options) => { + if (options === undefined) return; + try { + await pipeline.switchTo(options); + } catch (e) { + logger.warn("Failed to switch background effect", e); + } + }); + + return { state$ }; +} diff --git a/src/tracks/BackgroundEffectTransformer.test.ts b/src/tracks/BackgroundEffectTransformer.test.ts new file mode 100644 index 000000000..b8a26b86b --- /dev/null +++ b/src/tracks/BackgroundEffectTransformer.test.ts @@ -0,0 +1,24 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { describe, expect, it } from "vitest"; + +import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer"; + +describe("BackgroundEffectTransformer", () => { + it("passes frames on untouched once disabled, even after blurring", async () => { + const transformer = new BackgroundEffectTransformer({}); + await transformer.update({ blurRadius: 15, backgroundDisabled: false }); + await transformer.update({ + imagePath: undefined, + backgroundDisabled: true, + }); + // With neither, the library passes a frame on without segmenting it. + expect(transformer.options.blurRadius).toBeUndefined(); + expect(transformer.options.imagePath).toBeUndefined(); + }); +}); diff --git a/src/tracks/BlurBackgroundTransformer.ts b/src/tracks/BackgroundEffectTransformer.ts similarity index 78% rename from src/tracks/BlurBackgroundTransformer.ts rename to src/tracks/BackgroundEffectTransformer.ts index f86120d33..965359211 100644 --- a/src/tracks/BlurBackgroundTransformer.ts +++ b/src/tracks/BackgroundEffectTransformer.ts @@ -1,5 +1,6 @@ /* Copyright 2024-2025 New Vector Ltd. +Copyright 2026 Element Creations Ltd. SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. @@ -9,6 +10,7 @@ import { BackgroundTransformer, VideoTransformer, type VideoTransformerInitOptions, + type BackgroundOptions, } from "@livekit/track-processors"; import { ImageSegmenter } from "@mediapipe/tasks-vision"; @@ -49,7 +51,18 @@ const wasmFileset: WasmFileset = { * loads the segmentation models from our own bundle rather than as an external * resource fetched from the public internet. */ -export class BlurBackgroundTransformer extends BackgroundTransformer { +export class BackgroundEffectTransformer extends BackgroundTransformer { + /** + * As the library's, except that disabling also drops the blur radius: kept, + * it has every frame segmented and the result thrown away, where with + * neither a radius nor a picture frames are passed on untouched. + */ + public override async update(opts: BackgroundOptions): Promise { + await super.update( + opts.backgroundDisabled ? { ...opts, blurRadius: undefined } : opts, + ); + } + public async init({ outputCanvas, inputElement: inputVideo, @@ -73,8 +86,10 @@ export class BlurBackgroundTransformer extends BackgroundTransformer { outputConfidenceMasks: false, }); + // BackgroundTransformer's own init applies these, and this one replaces it. if (this.options.blurRadius) { this.gl?.setBlurRadius(this.options.blurRadius); } + this.gl?.setBackgroundDisabled(this.options.backgroundDisabled ?? false); } } diff --git a/src/tracks/BackgroundEffectsContext.test.ts b/src/tracks/BackgroundEffectsContext.test.ts new file mode 100644 index 000000000..958c5477e --- /dev/null +++ b/src/tracks/BackgroundEffectsContext.test.ts @@ -0,0 +1,115 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { act, createElement, type FC } from "react"; +import { render } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { + BackgroundEffectsProvider, + useBackgroundEffects, + createAppBackgroundEffects, +} from "./BackgroundEffectsContext"; +import { + type BackgroundEffects, + type BackgroundEffectsState, +} from "../state/BackgroundEffects"; +import { backgroundBlur } from "../settings/settings"; +import { flushPromises, testScope } from "../utils/test"; + +const pipelines = vi.hoisted(() => ({ + built: 0, + destroyed: 0, + switches: [] as unknown[], +})); + +// The pipeline itself needs WebGL and MediaPipe; what these tests are about is +// what the provider asks of it. +vi.mock("@livekit/track-processors", () => ({ + BackgroundProcessorWrapper: vi.fn(function (this: Record) { + pipelines.built += 1; + this.switchTo = vi.fn(async (options: unknown) => { + pipelines.switches.push(options); + return Promise.resolve(); + }); + this.destroy = vi.fn(async () => { + pipelines.destroyed += 1; + return Promise.resolve(); + }); + }), + supportsBackgroundProcessors: (): boolean => true, +})); +const platformMock = vi.hoisted(() => ({ platform: "desktop" })); +vi.mock("../Platform", () => platformMock); +vi.mock("./BackgroundEffectTransformer", () => ({ + BackgroundEffectTransformer: vi.fn(), +})); + +describe("BackgroundEffectsProvider", () => { + let seen: BackgroundEffectsState[]; + const Surface: FC = () => { + seen.push(useBackgroundEffects()); + return null; + }; + // Made on first render, once a test has set up the platform. + let effects: BackgroundEffects | undefined; + const surface = (): ReturnType => + createElement(BackgroundEffectsProvider, { + effects: (effects ??= createAppBackgroundEffects(testScope())), + children: createElement(Surface), + }); + const blur = async (on: boolean): Promise => { + await act(async () => { + backgroundBlur.setValue(on); + await flushPromises(); + }); + }; + const latest = (): BackgroundEffectsState => seen[seen.length - 1]; + + beforeEach(() => { + seen = []; + effects = undefined; + pipelines.built = 0; + pipelines.destroyed = 0; + pipelines.switches = []; + platformMock.platform = "desktop"; + backgroundBlur.setValue(false); + }); + afterEach(() => backgroundBlur.setValue(false)); + + // A pipeline primes itself when it is built and when it is destroyed, and a + // primed pipeline lets one frame through untouched: so the frame is spent + // once, as long as it is built once and never taken off the camera. + it("builds one pipeline on first use and keeps it through every change", async () => { + render(surface()); + for (const on of [true, false, true, false, true]) await blur(on); + + expect(pipelines.built).toBe(1); + expect(pipelines.destroyed).toBe(0); + expect(latest().processor).toBeDefined(); + }); + + it("does not run on a phone, even where its browser could", async () => { + platformMock.platform = "ios"; + render(surface()); + await blur(true); + + expect(latest().supported).toBe(false); + expect(latest().processor).toBeUndefined(); + }); + + it("switches to an effect chosen while the camera is off", async () => { + render(surface()); + await blur(true); + + // Already switched to blur, with nothing yet to attach it to + expect(pipelines.switches).toEqual([ + { mode: "background-blur", blurRadius: 15 }, + ]); + expect(pipelines.built).toBe(1); + }); +}); diff --git a/src/tracks/BackgroundEffectsContext.tsx b/src/tracks/BackgroundEffectsContext.tsx new file mode 100644 index 000000000..f1b384e0a --- /dev/null +++ b/src/tracks/BackgroundEffectsContext.tsx @@ -0,0 +1,74 @@ +/* +Copyright 2024-2026 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + BackgroundProcessorWrapper, + supportsBackgroundProcessors as supportsBackgroundProcessorsLivekitSdk, +} from "@livekit/track-processors"; +import { createContext, type FC, type JSX, use } from "react"; + +import { backgroundBlur as backgroundBlurSettings } from "../settings/settings"; +import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer"; +import { type Behavior } from "../state/Behavior"; +import { type ObservableScope } from "../state/ObservableScope"; +import { platform } from "../Platform"; +import { + type BackgroundEffects, + type BackgroundEffectsState, + createBackgroundEffects, +} from "../state/BackgroundEffects"; +import { useBehavior } from "../useBehavior"; + +const BackgroundEffectsContext = createContext( + undefined, +); + +export function useBackgroundEffects(): BackgroundEffectsState { + return useBehavior(useBackgroundEffectsState$()); +} + +export function useBackgroundEffectsState$(): Behavior { + const effects = use(BackgroundEffectsContext); + if (effects === undefined) + throw new Error( + "useBackgroundEffects must be used within a BackgroundEffectsProvider", + ); + return effects.state$; +} + +function supportsBackgroundProcessors(): boolean { + return supportsBackgroundProcessorsLivekitSdk() && platform === "desktop"; +} + +/** + * The app's one pipeline, shared by every camera track it opens. Its builds + * and teardowns take their turns through the SDK, which runs one operation + * at a time on a processor. + */ +export function createAppBackgroundEffects( + scope: ObservableScope, +): BackgroundEffects { + return createBackgroundEffects(scope, { + supported: supportsBackgroundProcessors(), + blur$: backgroundBlurSettings.value$, + pipeline: new BackgroundProcessorWrapper( + new BackgroundEffectTransformer({ backgroundDisabled: true }), + "background-effect", + ), + }); +} + +interface Props { + effects: BackgroundEffects; + children: JSX.Element; +} + +export const BackgroundEffectsProvider: FC = ({ effects, children }) => ( + + {children} + +); diff --git a/src/tracks/TrackProcessorContext.test.ts b/src/tracks/TrackProcessorContext.test.ts deleted file mode 100644 index 2acb3777a..000000000 --- a/src/tracks/TrackProcessorContext.test.ts +++ /dev/null @@ -1,51 +0,0 @@ -/* -Copyright 2026 Element Creations Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { describe, expect, it, vi } from "vitest"; -import { - type BackgroundOptions, - type ProcessorWrapper, -} from "@livekit/track-processors"; - -import { fromLivekitProcessor } from "./TrackProcessorContext"; - -describe("fromLivekitProcessor", () => { - const captured = { id: "captured" } as MediaStreamTrack; - const processed = { id: "processed" } as MediaStreamTrack; - - function mockWrapper( - processedTrack: MediaStreamTrack | undefined, - ): ProcessorWrapper { - return { - name: "background-blur", - processedTrack, - init: vi.fn().mockResolvedValue(undefined), - restart: vi.fn().mockResolvedValue(undefined), - destroy: vi.fn().mockResolvedValue(undefined), - } as unknown as ProcessorWrapper; - } - - it("resolves with the track the wrapper produced", async () => { - const wrapper = mockWrapper(processed); - const sdkProcessor = fromLivekitProcessor(wrapper); - expect(sdkProcessor.name).toBe("background-blur"); - await expect(sdkProcessor.init(captured)).resolves.toBe(processed); - expect(wrapper.init).toHaveBeenCalledWith({ - kind: "video", - track: captured, - }); - await expect(sdkProcessor.restart(captured)).resolves.toBe(processed); - await sdkProcessor.destroy(); - expect(wrapper.destroy).toHaveBeenCalled(); - }); - - it("rejects where the wrapper produced no track", async () => { - await expect( - fromLivekitProcessor(mockWrapper(undefined)).init(captured), - ).rejects.toThrow("produced no track"); - }); -}); diff --git a/src/tracks/TrackProcessorContext.tsx b/src/tracks/TrackProcessorContext.tsx deleted file mode 100644 index 5f828aba3..000000000 --- a/src/tracks/TrackProcessorContext.tsx +++ /dev/null @@ -1,110 +0,0 @@ -/* -Copyright 2024-2025 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - ProcessorWrapper, - supportsBackgroundProcessors as supportsBackgroundProcessorsLivekitSdk, - type BackgroundOptions, -} from "@livekit/track-processors"; -import { createContext, type FC, type JSX, use, useMemo } from "react"; -import { Track } from "livekit-client"; -import { type VideoProcessor } from "@element-hq/matrixrtc-sdk"; - -import { - backgroundBlur as backgroundBlurSettings, - useSetting, -} from "../settings/settings"; -import { BlurBackgroundTransformer } from "./BlurBackgroundTransformer"; -import { type Behavior } from "../state/Behavior"; -import { platform } from "../Platform"; -import { useValueBehavior } from "../useValueBehavior"; - -//TODO-MULTI-SFU: This is not yet fully there. -// it is a combination of exposing observable and react hooks. -// preferably we should not make this a context anymore and instead just a vm? - -export type ProcessorState = { - supported: boolean | undefined; - processor: undefined | VideoProcessor; -}; - -/** - * This allows to use a LiveKit processor for the matrixRTCSDK (VideoProcessor). - * They have slightly different apis so we need to convert them with this - * function before we can use them in the matrixRTCSDK. - */ -export function fromLivekitProcessor( - wrapper: ProcessorWrapper, -): VideoProcessor { - const processedTrack = (): MediaStreamTrack => { - if (wrapper.processedTrack === undefined) - throw new Error(`Processor ${wrapper.name} produced no track`); - return wrapper.processedTrack; - }; - return { - name: wrapper.name, - init: async (track: MediaStreamTrack, element?: HTMLMediaElement) => { - await wrapper.init({ kind: Track.Kind.Video, track, element }); - return processedTrack(); - }, - restart: async (track: MediaStreamTrack, element?: HTMLMediaElement) => { - await wrapper.restart({ kind: Track.Kind.Video, track, element }); - return processedTrack(); - }, - destroy: async () => wrapper.destroy(), - }; -} - -const ProcessorContext = createContext(undefined); - -export function useTrackProcessor(): ProcessorState { - const state = use(ProcessorContext); - if (state === undefined) - throw new Error( - "useTrackProcessor must be used within a ProcessorProvider", - ); - return state; -} - -export function useTrackProcessorState$(): Behavior { - return useValueBehavior(useTrackProcessor()); -} - -interface Props { - children: JSX.Element; -} - -function supportsBackgroundProcessors(): boolean { - return supportsBackgroundProcessorsLivekitSdk() && platform === "desktop"; -} - -export const ProcessorProvider: FC = ({ children }) => { - // The setting the user wants to have - const [blurActivated] = useSetting(backgroundBlurSettings); - const supported = useMemo(() => supportsBackgroundProcessors(), []); - const blur = useMemo( - () => - fromLivekitProcessor( - new ProcessorWrapper( - new BlurBackgroundTransformer({ blurRadius: 15 }), - "background-blur", - ), - ), - [], - ); - - // This is the actual state exposed through the context - const processorState = useMemo( - () => ({ - supported, - processor: supported && blurActivated ? blur : undefined, - }), - [supported, blurActivated, blur], - ); - - return {children}; -};