From 5fdb920ea8a61613143836a4af5b72ff4eb3977d Mon Sep 17 00:00:00 2001 From: fkwp Date: Thu, 24 Sep 2026 15:17:21 +0200 Subject: [PATCH 1/3] Say where background effects run slowly - Where the browser has no MediaStreamTrackProcessor, effects draw each frame through a canvas: slower, and the first build stalls the page. They stay offered and choosable there, with a notice under the grid that says so and describes the group. - Decided in the view model, beside whether effects can run at all, so the lobby and the call give the same answer. - The notice stays within the menu's width however long the sentence, which the story checks. Co-Authored-By: Claude Opus 5.5 (1M context) --- locales/en/app.json | 1 + src/components/CallFooter.tsx | 6 ++- src/components/CallFooterViewModel.test.ts | 12 ++++++ src/components/CallFooterViewModel.tsx | 12 +++++- .../MediaMuteAndSwitchButton.stories.tsx | 40 +++++++++++++++++++ src/tracks/backgroundProcessing.ts | 14 ++++++- 6 files changed, 80 insertions(+), 5 deletions(-) diff --git a/locales/en/app.json b/locales/en/app.json index 981f41324..3b08e8be9 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -26,6 +26,7 @@ "blur": "Blur", "none": "None", "numbered": "Background {{n}}", + "slow": "Background effects run slowly on this platform, which may cause your video to stutter.", "unavailable": "Background effects are not supported on this platform." }, "call_ended_view": { diff --git a/src/components/CallFooter.tsx b/src/components/CallFooter.tsx index 2a4d6de18..f3625f02b 100644 --- a/src/components/CallFooter.tsx +++ b/src/components/CallFooter.tsx @@ -78,7 +78,7 @@ export interface FooterState { /** Every effect on offer, in the order they are shown. */ backgroundEffects: BackgroundEffectGridItem[]; /** What the user is told about effects here, if anything. */ - backgroundEffectNotice: "unavailable" | undefined; + backgroundEffectNotice: "unavailable" | "slow" | undefined; showFooter: boolean; /* This is needed for WindowMode = "flat" */ @@ -233,7 +233,9 @@ export const CallFooter: FC = ({ backgroundEffectNotice={ backgroundEffectNotice === "unavailable" ? t("background_effects.unavailable") - : undefined + : backgroundEffectNotice === "slow" + ? t("background_effects.slow") + : undefined } />, ); diff --git a/src/components/CallFooterViewModel.test.ts b/src/components/CallFooterViewModel.test.ts index e47dd444c..5b098c013 100644 --- a/src/components/CallFooterViewModel.test.ts +++ b/src/components/CallFooterViewModel.test.ts @@ -30,8 +30,10 @@ vi.mock("../Platform", () => ({ // The SDK's own check needs WebGL and canvas APIs jsdom does not have. const sdkSupportMock = vi.hoisted(() => vi.fn(() => false)); +const modernRouteMock = vi.hoisted(() => vi.fn(() => true)); vi.mock("@livekit/track-processors", () => ({ supportsBackgroundProcessors: (): boolean => sdkSupportMock(), + supportsModernBackgroundProcessors: (): boolean => modernRouteMock(), })); const outputSelectionMock = vi.hoisted(() => vi.fn(() => true)); @@ -275,6 +277,16 @@ describe("createCallFooterViewModel", () => { expect(backgroundEffectSetting.getValue()).toBe("none"); }); + it("says they run slowly where only the slower route exists", () => { + sdkSupportMock.mockReturnValue(true); + modernRouteMock.mockReturnValue(false); + const slow = lobbyOn("desktop"); + expect(slow.backgroundEffectNotice$.value).toBe("slow"); + expect(slow.selectBackgroundEffect$.value).toBeDefined(); + modernRouteMock.mockReturnValue(true); + expect(lobbyOn("desktop").backgroundEffectNotice$.value).toBeUndefined(); + }); + it("offers every effect in order", () => { sdkSupportMock.mockReturnValue(true); expect(lobbyOn("desktop").backgroundEffects$.value).toEqual([ diff --git a/src/components/CallFooterViewModel.tsx b/src/components/CallFooterViewModel.tsx index 16f82ee97..e698f9d29 100644 --- a/src/components/CallFooterViewModel.tsx +++ b/src/components/CallFooterViewModel.tsx @@ -8,7 +8,10 @@ Please see LICENSE in the repository root for full details. import { combineLatest, map, type Observable, switchMap } from "rxjs"; import { supportsAudioOutputSelection } from "livekit-client"; -import { supportsBackgroundProcessors } from "../tracks/backgroundProcessing"; +import { + supportsBackgroundProcessors, + usesFallbackProcessing, +} from "../tracks/backgroundProcessing"; import { parseEffect, serializeEffect, @@ -102,6 +105,7 @@ function buildDeviceBehaviors( ); const supported = supportsBackgroundProcessors(); + const slow = usesFallbackProcessing(); const offered$ = disableSwitcher$.pipe( map((switcherDisabled) => !switcherDisabled && supported), ); @@ -148,7 +152,11 @@ function buildDeviceBehaviors( ), backgroundEffects$: constant(backgroundEffectChoices()), backgroundEffectNotice$: scope.behavior( - offered$.pipe(map((offered) => (offered ? undefined : "unavailable"))), + offered$.pipe( + map((offered) => + !offered ? "unavailable" : slow ? "slow" : undefined, + ), + ), ), }; } diff --git a/src/components/MediaMuteAndSwitchButton.stories.tsx b/src/components/MediaMuteAndSwitchButton.stories.tsx index 9be73e1da..2f507cb93 100644 --- a/src/components/MediaMuteAndSwitchButton.stories.tsx +++ b/src/components/MediaMuteAndSwitchButton.stories.tsx @@ -274,6 +274,46 @@ export const BackgroundEffectsUnavailable: Story = { }, }; +/** Where only the slower route exists: offered and choosable, with the cost said. */ +export const BackgroundEffectsSlowInThisBrowser: Story = { + args: { + ...VideoUnmute.args, + backgroundEffectNotice: + "Background effects run slowly on this platform, which may cause your video to stutter.", + }, + play: async ({ args, canvasElement }) => { + const canvas = within(canvasElement); + await userEvent.click(canvas.getByRole("button", { name: "Camera" })); + const menu = within(document.body); + const section = await menu.findByRole("group", { + name: "Background effects", + }); + // On the tiles' own group: the section's can't carry a description. + await expect( + within(section).getByRole("group"), + ).toHaveAccessibleDescription( + "Background effects run slowly on this platform, which may cause your video to stutter.", + ); + for (const tile of within(section).getAllByRole("menuitemradio")) + await expect(tile).not.toHaveAttribute("aria-disabled"); + await userEvent.click( + within(section).getByRole("menuitemradio", { name: "Background 1" }), + ); + await expect(args.onSelectBackgroundEffect).toHaveBeenCalledWith( + "image:arc", + ); + + const notice = document.body.querySelector(`.${styles.notice}`)!; + const frame = document.body + .querySelector("[role='menu']")! + .getBoundingClientRect(); + await expect(frame.width).toBeLessThanOrEqual(296); + await expect(notice.getBoundingClientRect().right).toBeLessThanOrEqual( + frame.right, + ); + }, +}; + /** In a short call the effects scroll into view with the list. */ export const BackgroundEffectsScrollWhenTheyDoNotFit: Story = { args: { diff --git a/src/tracks/backgroundProcessing.ts b/src/tracks/backgroundProcessing.ts index 8b6168dad..4463be3af 100644 --- a/src/tracks/backgroundProcessing.ts +++ b/src/tracks/backgroundProcessing.ts @@ -5,7 +5,10 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { supportsBackgroundProcessors as supportsBackgroundProcessorsLivekitSdk } from "@livekit/track-processors"; +import { + supportsBackgroundProcessors as supportsBackgroundProcessorsLivekitSdk, + supportsModernBackgroundProcessors, +} from "@livekit/track-processors"; import { platform } from "../Platform"; @@ -13,3 +16,12 @@ import { platform } from "../Platform"; export function supportsBackgroundProcessors(): boolean { return supportsBackgroundProcessorsLivekitSdk() && platform === "desktop"; } + +/** + * Whether effects, where they run at all, draw every frame through a canvas: + * the slower route, which costs frame rate and stalls the page while the + * first one is built. + */ +export function usesFallbackProcessing(): boolean { + return !supportsModernBackgroundProcessors(); +} From 7fb222ce606413af139fc6e5de3c5406cbf5d930 Mon Sep 17 00:00:00 2001 From: fkwp Date: Thu, 24 Sep 2026 15:30:56 +0200 Subject: [PATCH 2/3] Show the wait for the first background effect on its tile - The first effect chosen in a session shows a spinner on its own tile, in place of the tick, until a frame carrying it is drawn. Later changes show nothing, having nothing to prepare. - Ended by the frame, not the promises, which resolve while the segmenter is still being built. The transformer reports the first frame that went through the segmenter; one passed through untouched doesn't count. - The pipeline publishes the wait with its state, and both footer view models take it as an input. - The lobby tests' provider mock gains the observable hook, built once: a fresh observable per render rebuilt the footer on every render. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/components/CallFooter.stories.tsx | 1 + src/components/CallFooter.tsx | 4 +++ src/components/CallFooterViewModel.test.ts | 24 +++++++++++++ src/components/CallFooterViewModel.tsx | 34 ++++++++++++++++-- .../MediaMuteAndSwitchButton.stories.tsx | 27 ++++++++++++++ src/components/MediaMuteAndSwitchButton.tsx | 4 +++ src/room/InCallView.tsx | 1 + src/room/LobbyView.tsx | 10 +++++- src/state/BackgroundEffects.test.ts | 24 ++++++++++--- src/state/BackgroundEffects.ts | 36 +++++++++++++++---- .../BackgroundEffectTransformer.test.ts | 29 +++++++++++++++ src/tracks/BackgroundEffectTransformer.ts | 14 ++++++++ src/tracks/BackgroundEffectsContext.tsx | 9 ++--- src/utils/test-viewmodel.ts | 1 + 14 files changed, 200 insertions(+), 18 deletions(-) diff --git a/src/components/CallFooter.stories.tsx b/src/components/CallFooter.stories.tsx index d332ad757..b587cc40e 100644 --- a/src/components/CallFooter.stories.tsx +++ b/src/components/CallFooter.stories.tsx @@ -136,6 +136,7 @@ export const Default: Story = { })), ], backgroundEffectNotice: undefined, + backgroundEffectSettling: false, hangup: fn(), buttonSize: "lg", showFooter: true, diff --git a/src/components/CallFooter.tsx b/src/components/CallFooter.tsx index f3625f02b..8606d3a47 100644 --- a/src/components/CallFooter.tsx +++ b/src/components/CallFooter.tsx @@ -79,6 +79,8 @@ export interface FooterState { backgroundEffects: BackgroundEffectGridItem[]; /** What the user is told about effects here, if anything. */ backgroundEffectNotice: "unavailable" | "slow" | undefined; + /** Whether the first effect chosen is still being prepared. */ + backgroundEffectSettling: boolean; showFooter: boolean; /* This is needed for WindowMode = "flat" */ @@ -165,6 +167,7 @@ export const CallFooter: FC = ({ const backgroundEffect = useBehavior(vm.backgroundEffect$); const selectBackgroundEffect = useBehavior(vm.selectBackgroundEffect$); const backgroundEffectNotice = useBehavior(vm.backgroundEffectNotice$); + const backgroundEffectSettling = useBehavior(vm.backgroundEffectSettling$); const backgroundEffects = useBehavior(vm.backgroundEffects$); const buttonSize = useBehavior(vm.buttonSize$); const showLogo = useBehavior(vm.showLogo$); @@ -230,6 +233,7 @@ export const CallFooter: FC = ({ backgroundEffects={backgroundEffects} selectedBackgroundEffect={backgroundEffect} onSelectBackgroundEffect={selectBackgroundEffect} + backgroundEffectSettling={backgroundEffectSettling} backgroundEffectNotice={ backgroundEffectNotice === "unavailable" ? t("background_effects.unavailable") diff --git a/src/components/CallFooterViewModel.test.ts b/src/components/CallFooterViewModel.test.ts index 5b098c013..52004185e 100644 --- a/src/components/CallFooterViewModel.test.ts +++ b/src/components/CallFooterViewModel.test.ts @@ -6,6 +6,7 @@ Please see LICENSE in the repository root for full details. */ import { afterEach, describe, expect, it, vi } from "vitest"; +import { BehaviorSubject } from "rxjs"; import { testScope, mockMuteStates, mockMediaDevices } from "../utils/test"; import { constant } from "../state/Behavior"; @@ -111,6 +112,7 @@ describe("createCallFooterViewModel", () => { buildMinimalCallViewModel(gridLayout), mockMuteStates(), twoMicsAndOneCamMediaDevices, + constant(false), /* reactionIdentifier */ undefined, { showControls: true, header: HeaderStyle.Standard }, ); @@ -153,6 +155,7 @@ describe("createCallFooterViewModel", () => { select: vi.fn(), }, }), + constant(false), /* reactionIdentifier */ undefined, { showControls: true, header: HeaderStyle.Standard }, ); @@ -171,6 +174,7 @@ describe("createCallFooterViewModel", () => { buildMinimalCallViewModel(layout), mockMuteStates(), twoMicsAndOneCamMediaDevices, + constant(false), /* reactionIdentifier */ undefined, { showControls: true, header: HeaderStyle.Standard }, ); @@ -193,6 +197,7 @@ describe("createCallFooterViewModel", () => { buildMinimalCallViewModel(gridLayout), mockMuteStates(), twoMicsAndOneCamMediaDevices, + constant(false), /* reactionIdentifier */ undefined, { showControls: true, header: HeaderStyle.Standard }, ); @@ -238,6 +243,7 @@ describe("createCallFooterViewModel", () => { testScope(), mockMuteStates(), twoMicsAndOneCamMediaDevices, + constant(false), /* openSettings */ undefined, /* hangup */ undefined, /* showLogo */ false, @@ -307,6 +313,23 @@ describe("createCallFooterViewModel", () => { ); }); + it("passes on the wait for the first effect", () => { + platformMock.mockReturnValue("desktop"); + const settling$ = new BehaviorSubject(true); + const vm = createLobbyFooterViewModel( + testScope(), + mockMuteStates(), + twoMicsAndOneCamMediaDevices, + settling$, + /* openSettings */ undefined, + /* hangup */ undefined, + /* showLogo */ false, + ); + expect(vm.backgroundEffectSettling$.value).toBe(true); + settling$.next(false); + expect(vm.backgroundEffectSettling$.value).toBe(false); + }); + it("availability is the same before and during a call", () => { for (const supported of [true, false]) { sdkSupportMock.mockReturnValue(supported); @@ -316,6 +339,7 @@ describe("createCallFooterViewModel", () => { buildMinimalCallViewModel(gridLayout), mockMuteStates(), twoMicsAndOneCamMediaDevices, + constant(false), /* reactionIdentifier */ undefined, { showControls: true, header: HeaderStyle.Standard }, ); diff --git a/src/components/CallFooterViewModel.tsx b/src/components/CallFooterViewModel.tsx index e698f9d29..183ebff8d 100644 --- a/src/components/CallFooterViewModel.tsx +++ b/src/components/CallFooterViewModel.tsx @@ -5,7 +5,13 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { combineLatest, map, type Observable, switchMap } from "rxjs"; +import { + combineLatest, + distinctUntilChanged, + map, + type Observable, + switchMap, +} from "rxjs"; import { supportsAudioOutputSelection } from "livekit-client"; import { @@ -73,6 +79,7 @@ function buildDeviceBehaviors( mediaDevices: MediaDevices, /** return empty arrays for audioOptions and videoOptions*/ disableSwitcher$: Behavior, + backgroundEffectSettling$: Observable, ): Pick< ViewModel, | "audioOptions$" @@ -88,6 +95,7 @@ function buildDeviceBehaviors( | "selectBackgroundEffect$" | "backgroundEffects$" | "backgroundEffectNotice$" + | "backgroundEffectSettling$" > { const options$ = ( available$: Behavior>, @@ -158,6 +166,10 @@ function buildDeviceBehaviors( ), ), ), + backgroundEffectSettling$: scope.behavior( + backgroundEffectSettling$.pipe(distinctUntilChanged()), + false, + ), }; } @@ -168,6 +180,8 @@ function buildDeviceBehaviors( * @param callModel - The root CallViewModel; provides layout, grid mode, reactions, etc. * @param muteStates - Audio and video mute state + toggles. * @param mediaDevices - Available and selected input devices. + * @param backgroundEffectSettling$ - Whether the first effect chosen is still + * being prepared. * @param reactionIdentifier - The local user's reaction identifier string, or * undefined when reactions are not supported (hides the reaction button). * @param options - `showControls`: whether the call controls should be shown. @@ -178,6 +192,7 @@ export function createCallFooterViewModel( callModel: CallViewModel, muteStates: MuteStates, mediaDevices: MediaDevices, + backgroundEffectSettling$: Observable, reactionIdentifier: string | undefined, options: { showControls: boolean; header: HeaderStyle }, ): ViewModel { @@ -192,7 +207,12 @@ export function createCallFooterViewModel( ); return { ...buildMuteBehaviors(scope, muteStates), - ...buildDeviceBehaviors(scope, mediaDevices, disableDeviceSwitcher$), + ...buildDeviceBehaviors( + scope, + mediaDevices, + disableDeviceSwitcher$, + backgroundEffectSettling$, + ), // candidat to move into the FooterViewModel showFooter$: callModel.showFooter$, hideControls$: constant(!showControls), @@ -254,6 +274,8 @@ export function createCallFooterViewModel( * @param scope - ObservableScope that bounds the lifetime of derived behaviors. * @param muteStates - Audio and video mute state + toggles. * @param mediaDevices - Available and selected input devices. + * @param backgroundEffectSettling$ - Whether the first effect chosen is still + * being prepared. * @param openSettings - Callback to open the settings modal, or undefined. * @param hangup - Callback to leave/cancel, or undefined (hides the button). * @param showLogo - Whether to show the Element Call logo. @@ -262,6 +284,7 @@ export function createLobbyFooterViewModel( scope: ObservableScope, muteStates: MuteStates, mediaDevices: MediaDevices, + backgroundEffectSettling$: Observable, openSettings: (() => void) | undefined, hangup: (() => void) | undefined, showLogo: boolean, @@ -305,7 +328,12 @@ export function createLobbyFooterViewModel( selectVideoButtonOption: undefined, }), ...buildMuteBehaviors(scope, muteStates), - ...buildDeviceBehaviors(scope, mediaDevices, constant(false)), + ...buildDeviceBehaviors( + scope, + mediaDevices, + constant(false), + backgroundEffectSettling$, + ), }; } diff --git a/src/components/MediaMuteAndSwitchButton.stories.tsx b/src/components/MediaMuteAndSwitchButton.stories.tsx index 2f507cb93..ba9163234 100644 --- a/src/components/MediaMuteAndSwitchButton.stories.tsx +++ b/src/components/MediaMuteAndSwitchButton.stories.tsx @@ -314,6 +314,33 @@ export const BackgroundEffectsSlowInThisBrowser: Story = { }, }; +/** The first effect of a session shows its wait on its own tile. */ +export const BackgroundEffectsSettling: Story = { + args: { + ...VideoUnmute.args, + selectedBackgroundEffect: "blur", + backgroundEffectSettling: true, + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await userEvent.click(canvas.getByRole("button", { name: "Camera" })); + const menu = within(document.body); + const chosen = await menu.findByRole("menuitemradio", { + name: "Blur", + checked: true, + }); + await expect(chosen).toHaveAttribute("aria-busy", "true"); + const marks = chosen.querySelectorAll("svg"); + await expect(marks).toHaveLength(1); + await expect(getComputedStyle(marks[0]).animationName).not.toBe("none"); + const section = menu.getByRole("group", { name: "Background effects" }); + for (const other of within(section).getAllByRole("menuitemradio", { + checked: false, + })) + await expect(other).toHaveAttribute("aria-busy", "false"); + }, +}; + /** In a short call the effects scroll into view with the list. */ export const BackgroundEffectsScrollWhenTheyDoNotFit: Story = { args: { diff --git a/src/components/MediaMuteAndSwitchButton.tsx b/src/components/MediaMuteAndSwitchButton.tsx index 6ad77c8f1..aa6d7afe6 100644 --- a/src/components/MediaMuteAndSwitchButton.tsx +++ b/src/components/MediaMuteAndSwitchButton.tsx @@ -77,6 +77,8 @@ export interface MediaMuteAndSwitchButtonProps { onSelectBackgroundEffect?: (id: string) => void; /** What the user should know before choosing an effect, if anything. */ backgroundEffectNotice?: string; + /** Whether the first effect chosen is still being prepared. */ + backgroundEffectSettling?: boolean; /** * For any toggle and option this method will be called. * So toggles need to be implemented by listening here and setting the right toggle item to `enabled` @@ -110,6 +112,7 @@ export const MediaMuteAndSwitchButton: FC = ({ selectedBackgroundEffect, onSelectBackgroundEffect, backgroundEffectNotice, + backgroundEffectSettling, onSelect, }) => { // Requested but not yet selected. Keyed by kind too, since Chrome uses @@ -387,6 +390,7 @@ export const MediaMuteAndSwitchButton: FC = ({ effects={backgroundEffects ?? []} selected={selectedBackgroundEffect} onSelect={onSelectBackgroundEffect} + settling={backgroundEffectSettling} describedBy={ backgroundEffectNotice === undefined ? undefined : noticeId } diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index b6279b107..3459d5565 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -241,6 +241,7 @@ export const ActiveCall: FC = (props) => { vm, props.muteStates, mediaDevices, + backgroundEffectsState$.pipe(map(({ settling }) => settling ?? false)), `${props.client.getUserId()}:${props.client.getDeviceId()}`, { showControls: urlParams.showControls, header: urlParams.header }, ); diff --git a/src/room/LobbyView.tsx b/src/room/LobbyView.tsx index 8fa71ada8..1a197b8d0 100644 --- a/src/room/LobbyView.tsx +++ b/src/room/LobbyView.tsx @@ -164,6 +164,7 @@ export const LobbyView: FC = ({ footerScope, muteStates, devices, + backgroundEffectsState$.pipe(map(({ settling }) => settling ?? false)), openSettings, hangup, // Logo and header are connected: only show the logo in SPA with header. @@ -173,7 +174,14 @@ export const LobbyView: FC = ({ return (): void => { footerScope.end(); }; - }, [devices, hangup, hideHeader, muteStates, openSettings]); + }, [ + devices, + backgroundEffectsState$, + hangup, + hideHeader, + muteStates, + openSettings, + ]); // TODO: Unify this component with InCallView, so we can get slick joining // animations and don't have to feel bad about reusing its CSS diff --git a/src/state/BackgroundEffects.test.ts b/src/state/BackgroundEffects.test.ts index d00238dde..74856f82e 100644 --- a/src/state/BackgroundEffects.test.ts +++ b/src/state/BackgroundEffects.test.ts @@ -46,25 +46,33 @@ function fakePipeline(): { }; } -/** One letter per state: idle, attached. */ +/** One letter per state: idle, waiting for a frame, attached. */ function letter(state: BackgroundEffectsState): string { - return state.processor === undefined ? "i" : "a"; + if (state.processor === undefined) return "i"; + return state.settling ? "w" : "a"; } describe("the pipeline's state", () => { function testState({ effect, + firstFrame = "", expected, }: { effect: string; + firstFrame?: string; expected: string; }): void { - withTestScheduler(({ behavior, expectObservable }) => { + withTestScheduler(({ behavior, schedule, expectObservable }) => { + const transformer = { + onFirstFrame: undefined as (() => void) | undefined, + }; const effects = createBackgroundEffects(testScope(), { supported: true, effect$: behavior(effect, { n: "none", b: "blur" }), pipeline: fakePipeline().pipeline, + transformer, }); + schedule(firstFrame, { f: () => transformer.onFirstFrame?.() }); expectObservable( effects.state$.pipe(map(letter), distinctUntilChanged()), ).toBe(expected); @@ -73,8 +81,15 @@ describe("the pipeline's state", () => { it("defaults to no effect", () => testState({ effect: "n", expected: "i" })); + it("waits from the first effect until a frame carries it", () => + testState({ + effect: " nb-n-b", + firstFrame: " --f", + expected: " iwa", + })); + it("attaches on first use and stays attached", () => - testState({ effect: "nbn", expected: "ia" })); + testState({ effect: "nb-n", firstFrame: "--f", expected: "iwa" })); }); describe("background effects", () => { @@ -88,6 +103,7 @@ describe("background effects", () => { supported: true, effect$, pipeline: fake.pipeline, + transformer: { onFirstFrame: undefined }, ...options, }); } diff --git a/src/state/BackgroundEffects.ts b/src/state/BackgroundEffects.ts index df3ce6a95..a00cf3432 100644 --- a/src/state/BackgroundEffects.ts +++ b/src/state/BackgroundEffects.ts @@ -5,7 +5,13 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { combineLatest, distinctUntilChanged, map, scan } from "rxjs"; +import { + combineLatest, + distinctUntilChanged, + map, + Observable, + scan, +} from "rxjs"; import { type BackgroundProcessorWrapper, type SwitchBackgroundProcessorOptions, @@ -30,6 +36,8 @@ const blurRadius = 15; export interface BackgroundEffectsState { supported: boolean | undefined; processor: undefined | VideoProcessor; + /** From the first effect chosen until a frame carrying it is drawn. */ + settling?: boolean; } export interface BackgroundEffectsOptions { @@ -39,6 +47,8 @@ export interface BackgroundEffectsOptions { effect$: Behavior; /** The background processor pipeline to be switched between effects. */ pipeline: BackgroundProcessorWrapper; + /** Tells, once, that a frame carrying an effect has been drawn. */ + transformer: { onFirstFrame: (() => void) | undefined }; } /** The background effect pipeline, as the camera tracks and the menus see it. */ @@ -49,7 +59,7 @@ export interface BackgroundEffects { /** Switches the pipeline as the choice changes, for as long as the scope lasts. */ export function createBackgroundEffects( scope: ObservableScope, - { supported, effect$, pipeline }: BackgroundEffectsOptions, + { supported, effect$, pipeline, transformer }: BackgroundEffectsOptions, ): BackgroundEffects { // Wrapped once, so the call and the preview see one processor throughout. const processor = fromLivekitProcessor(pipeline); @@ -59,15 +69,29 @@ export function createBackgroundEffects( distinctUntilChanged(), ); + const drewAFrame$ = scope.behavior( + new Observable((subscriber) => { + subscriber.next(false); + transformer.onFirstFrame = (): void => subscriber.next(true); + return (): void => { + transformer.onFirstFrame = undefined; + }; + }), + ); + const state$ = scope.behavior( - wanted$.pipe( - scan( - (previous, wanted) => { + combineLatest([wanted$, drewAFrame$]).pipe( + scan<[boolean, boolean], BackgroundEffectsState>( + (previous, [wanted, drewAFrame]) => { // Attached the first time an effect is wanted and never detached // after, so someone who never turns one on pays for none of it. const enable = previous.processor !== undefined || (supported && wanted); - return { supported, processor: enable ? processor : undefined }; + return { + supported, + processor: enable ? processor : undefined, + settling: enable && !drewAFrame, + }; }, { supported, processor: undefined }, ), diff --git a/src/tracks/BackgroundEffectTransformer.test.ts b/src/tracks/BackgroundEffectTransformer.test.ts index fae7a2af1..400a50416 100644 --- a/src/tracks/BackgroundEffectTransformer.test.ts +++ b/src/tracks/BackgroundEffectTransformer.test.ts @@ -15,6 +15,9 @@ import { ImageSegmenter } from "@mediapipe/tasks-vision"; import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer"; +const frame = {} as VideoFrame; +const controller = {} as TransformStreamDefaultController; + describe("BackgroundEffectTransformer", () => { afterEach(() => vi.restoreAllMocks()); @@ -38,6 +41,32 @@ describe("BackgroundEffectTransformer", () => { expect(load).toHaveBeenCalledWith("/background.jpg"); }); + it("reports the first frame carrying an effect, once", async () => { + vi.spyOn(BackgroundTransformer.prototype, "transform").mockImplementation( + async function (this: BackgroundTransformer) { + this.isFirstFrame = false; + return Promise.resolve(); + }, + ); + const transformer = new BackgroundEffectTransformer({}); + transformer.onFirstFrame = vi.fn(); + + await transformer.transform(frame, controller); + await transformer.transform(frame, controller); + expect(transformer.onFirstFrame).toHaveBeenCalledTimes(1); + }); + + it("does not report a frame passed through untouched", async () => { + vi.spyOn(BackgroundTransformer.prototype, "transform").mockResolvedValue(); + const transformer = new BackgroundEffectTransformer({ + backgroundDisabled: true, + }); + transformer.onFirstFrame = vi.fn(); + + await transformer.transform(frame, controller); + expect(transformer.onFirstFrame).not.toHaveBeenCalled(); + }); + it("passes frames on untouched once disabled, even after blurring", async () => { const transformer = new BackgroundEffectTransformer({}); await transformer.update({ blurRadius: 15, backgroundDisabled: false }); diff --git a/src/tracks/BackgroundEffectTransformer.ts b/src/tracks/BackgroundEffectTransformer.ts index 65ca75f8f..e3c0a9183 100644 --- a/src/tracks/BackgroundEffectTransformer.ts +++ b/src/tracks/BackgroundEffectTransformer.ts @@ -53,6 +53,20 @@ const wasmFileset: WasmFileset = { * resource fetched from the public internet. */ export class BackgroundEffectTransformer extends BackgroundTransformer { + /** Called once, when the first frame carrying an effect has been drawn. */ + public onFirstFrame: (() => void) | undefined; + + public override async transform( + frame: VideoFrame, + controller: TransformStreamDefaultController, + ): Promise { + const priming = this.isFirstFrame; + await super.transform(frame, controller); + // Cleared only once a frame has been through the segmenter; a frame + // passed through untouched leaves it set. + if (priming && !this.isFirstFrame) this.onFirstFrame?.(); + } + /** * As the library's, except that disabling also drops the blur radius. The * library skips a frame on `isDisabled ?? backgroundDisabled`, and diff --git a/src/tracks/BackgroundEffectsContext.tsx b/src/tracks/BackgroundEffectsContext.tsx index 225b7fdae..78f47db3f 100644 --- a/src/tracks/BackgroundEffectsContext.tsx +++ b/src/tracks/BackgroundEffectsContext.tsx @@ -45,13 +45,14 @@ export function useBackgroundEffectsState$(): Behavior { export function createAppBackgroundEffects( scope: ObservableScope, ): BackgroundEffects { + const transformer = new BackgroundEffectTransformer({ + backgroundDisabled: true, + }); return createBackgroundEffects(scope, { supported: supportsBackgroundProcessors(), effect$: backgroundEffectSetting.value$, - pipeline: new BackgroundProcessorWrapper( - new BackgroundEffectTransformer({ backgroundDisabled: true }), - "background-effect", - ), + pipeline: new BackgroundProcessorWrapper(transformer, "background-effect"), + transformer, }); } diff --git a/src/utils/test-viewmodel.ts b/src/utils/test-viewmodel.ts index c30d08cda..0cf2a8b76 100644 --- a/src/utils/test-viewmodel.ts +++ b/src/utils/test-viewmodel.ts @@ -217,6 +217,7 @@ export function getBasicCallViewModelEnvironment( vm, muteStates, mediaDevices, + constant(false), "reactionId", { showControls: true, header: HeaderStyle.Standard }, ); From 4e779ac4f5d179e412ce20b6b8d8fe50fcdc3343 Mon Sep 17 00:00:00 2001 From: fkwp Date: Thu, 8 Oct 2026 09:06:53 +0200 Subject: [PATCH 3/3] Offer background effects on any platform whose browser can run them - The desktop rule from #4059 goes: now that the menu says where effects run slowly, using them there is the user's choice. A phone on the fast route held 99% of its frame rate and was refused; a desktop on the slow one held 73% and was allowed. - Asks @livekit/track-processors directly; the wrapper module goes. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/components/CallFooterViewModel.test.ts | 6 ++--- src/components/CallFooterViewModel.tsx | 8 +++--- src/tracks/BackgroundEffectsContext.test.ts | 6 ++--- src/tracks/BackgroundEffectsContext.tsx | 6 +++-- src/tracks/backgroundProcessing.ts | 27 --------------------- 5 files changed, 14 insertions(+), 39 deletions(-) delete mode 100644 src/tracks/backgroundProcessing.ts diff --git a/src/components/CallFooterViewModel.test.ts b/src/components/CallFooterViewModel.test.ts index 52004185e..e061f4ef2 100644 --- a/src/components/CallFooterViewModel.test.ts +++ b/src/components/CallFooterViewModel.test.ts @@ -250,12 +250,12 @@ describe("createCallFooterViewModel", () => { ); } - it("offers nothing on a phone, where the pipeline would refuse them", () => { + it("offers them on a phone whose browser can run them", () => { sdkSupportMock.mockReturnValue(true); - expect(lobbyOn("ios").selectBackgroundEffect$.value).toBeUndefined(); + expect(lobbyOn("ios").selectBackgroundEffect$.value).toBeDefined(); }); - it("offers them where the pipeline will honour them", () => { + it("offers them on a desktop whose browser can run them", () => { sdkSupportMock.mockReturnValue(true); expect(lobbyOn("desktop").selectBackgroundEffect$.value).toBeDefined(); }); diff --git a/src/components/CallFooterViewModel.tsx b/src/components/CallFooterViewModel.tsx index 183ebff8d..e9a262335 100644 --- a/src/components/CallFooterViewModel.tsx +++ b/src/components/CallFooterViewModel.tsx @@ -13,11 +13,11 @@ import { switchMap, } from "rxjs"; import { supportsAudioOutputSelection } from "livekit-client"; - import { supportsBackgroundProcessors, - usesFallbackProcessing, -} from "../tracks/backgroundProcessing"; + supportsModernBackgroundProcessors, +} from "@livekit/track-processors"; + import { parseEffect, serializeEffect, @@ -113,7 +113,7 @@ function buildDeviceBehaviors( ); const supported = supportsBackgroundProcessors(); - const slow = usesFallbackProcessing(); + const slow = !supportsModernBackgroundProcessors(); const offered$ = disableSwitcher$.pipe( map((switcherDisabled) => !switcherDisabled && supported), ); diff --git a/src/tracks/BackgroundEffectsContext.test.ts b/src/tracks/BackgroundEffectsContext.test.ts index acb7a9e47..71d9d7152 100644 --- a/src/tracks/BackgroundEffectsContext.test.ts +++ b/src/tracks/BackgroundEffectsContext.test.ts @@ -93,13 +93,13 @@ describe("BackgroundEffectsProvider", () => { expect(latest().processor).toBeDefined(); }); - it("does not run on a phone, even where its browser could", async () => { + it("runs on a phone whose browser can run it", async () => { platformMock.platform = "ios"; render(surface()); await blur(true); - expect(latest().supported).toBe(false); - expect(latest().processor).toBeUndefined(); + expect(latest().supported).toBe(true); + expect(latest().processor).toBeDefined(); }); it("switches to an effect chosen while the camera is off", async () => { diff --git a/src/tracks/BackgroundEffectsContext.tsx b/src/tracks/BackgroundEffectsContext.tsx index 78f47db3f..d582bc0b3 100644 --- a/src/tracks/BackgroundEffectsContext.tsx +++ b/src/tracks/BackgroundEffectsContext.tsx @@ -5,12 +5,14 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { BackgroundProcessorWrapper } from "@livekit/track-processors"; +import { + BackgroundProcessorWrapper, + supportsBackgroundProcessors, +} from "@livekit/track-processors"; import { createContext, type FC, type JSX, use } from "react"; import { backgroundEffect as backgroundEffectSetting } from "../settings/settings"; import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer"; -import { supportsBackgroundProcessors } from "./backgroundProcessing"; import { type Behavior } from "../state/Behavior"; import { type ObservableScope } from "../state/ObservableScope"; import { diff --git a/src/tracks/backgroundProcessing.ts b/src/tracks/backgroundProcessing.ts deleted file mode 100644 index 4463be3af..000000000 --- a/src/tracks/backgroundProcessing.ts +++ /dev/null @@ -1,27 +0,0 @@ -/* -Copyright 2026 Element Creations Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - supportsBackgroundProcessors as supportsBackgroundProcessorsLivekitSdk, - supportsModernBackgroundProcessors, -} from "@livekit/track-processors"; - -import { platform } from "../Platform"; - -/** One answer for the controls and the pipeline, so neither offers what the other refuses. */ -export function supportsBackgroundProcessors(): boolean { - return supportsBackgroundProcessorsLivekitSdk() && platform === "desktop"; -} - -/** - * Whether effects, where they run at all, draw every frame through a canvas: - * the slower route, which costs frame rate and stalls the page while the - * first one is built. - */ -export function usesFallbackProcessing(): boolean { - return !supportsModernBackgroundProcessors(); -}