diff --git a/locales/en/app.json b/locales/en/app.json index 981f41324..3b08e8be9 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -26,6 +26,7 @@ "blur": "Blur", "none": "None", "numbered": "Background {{n}}", + "slow": "Background effects run slowly on this platform, which may cause your video to stutter.", "unavailable": "Background effects are not supported on this platform." }, "call_ended_view": { diff --git a/src/components/CallFooter.stories.tsx b/src/components/CallFooter.stories.tsx index d332ad757..b587cc40e 100644 --- a/src/components/CallFooter.stories.tsx +++ b/src/components/CallFooter.stories.tsx @@ -136,6 +136,7 @@ export const Default: Story = { })), ], backgroundEffectNotice: undefined, + backgroundEffectSettling: false, hangup: fn(), buttonSize: "lg", showFooter: true, diff --git a/src/components/CallFooter.tsx b/src/components/CallFooter.tsx index 2a4d6de18..8606d3a47 100644 --- a/src/components/CallFooter.tsx +++ b/src/components/CallFooter.tsx @@ -78,7 +78,9 @@ export interface FooterState { /** Every effect on offer, in the order they are shown. */ backgroundEffects: BackgroundEffectGridItem[]; /** What the user is told about effects here, if anything. */ - backgroundEffectNotice: "unavailable" | undefined; + backgroundEffectNotice: "unavailable" | "slow" | undefined; + /** Whether the first effect chosen is still being prepared. */ + backgroundEffectSettling: boolean; showFooter: boolean; /* This is needed for WindowMode = "flat" */ @@ -165,6 +167,7 @@ export const CallFooter: FC = ({ const backgroundEffect = useBehavior(vm.backgroundEffect$); const selectBackgroundEffect = useBehavior(vm.selectBackgroundEffect$); const backgroundEffectNotice = useBehavior(vm.backgroundEffectNotice$); + const backgroundEffectSettling = useBehavior(vm.backgroundEffectSettling$); const backgroundEffects = useBehavior(vm.backgroundEffects$); const buttonSize = useBehavior(vm.buttonSize$); const showLogo = useBehavior(vm.showLogo$); @@ -230,10 +233,13 @@ export const CallFooter: FC = ({ backgroundEffects={backgroundEffects} selectedBackgroundEffect={backgroundEffect} onSelectBackgroundEffect={selectBackgroundEffect} + backgroundEffectSettling={backgroundEffectSettling} backgroundEffectNotice={ backgroundEffectNotice === "unavailable" ? t("background_effects.unavailable") - : undefined + : backgroundEffectNotice === "slow" + ? t("background_effects.slow") + : undefined } />, ); diff --git a/src/components/CallFooterViewModel.test.ts b/src/components/CallFooterViewModel.test.ts index e47dd444c..e061f4ef2 100644 --- a/src/components/CallFooterViewModel.test.ts +++ b/src/components/CallFooterViewModel.test.ts @@ -6,6 +6,7 @@ Please see LICENSE in the repository root for full details. */ import { afterEach, describe, expect, it, vi } from "vitest"; +import { BehaviorSubject } from "rxjs"; import { testScope, mockMuteStates, mockMediaDevices } from "../utils/test"; import { constant } from "../state/Behavior"; @@ -30,8 +31,10 @@ vi.mock("../Platform", () => ({ // The SDK's own check needs WebGL and canvas APIs jsdom does not have. const sdkSupportMock = vi.hoisted(() => vi.fn(() => false)); +const modernRouteMock = vi.hoisted(() => vi.fn(() => true)); vi.mock("@livekit/track-processors", () => ({ supportsBackgroundProcessors: (): boolean => sdkSupportMock(), + supportsModernBackgroundProcessors: (): boolean => modernRouteMock(), })); const outputSelectionMock = vi.hoisted(() => vi.fn(() => true)); @@ -109,6 +112,7 @@ describe("createCallFooterViewModel", () => { buildMinimalCallViewModel(gridLayout), mockMuteStates(), twoMicsAndOneCamMediaDevices, + constant(false), /* reactionIdentifier */ undefined, { showControls: true, header: HeaderStyle.Standard }, ); @@ -151,6 +155,7 @@ describe("createCallFooterViewModel", () => { select: vi.fn(), }, }), + constant(false), /* reactionIdentifier */ undefined, { showControls: true, header: HeaderStyle.Standard }, ); @@ -169,6 +174,7 @@ describe("createCallFooterViewModel", () => { buildMinimalCallViewModel(layout), mockMuteStates(), twoMicsAndOneCamMediaDevices, + constant(false), /* reactionIdentifier */ undefined, { showControls: true, header: HeaderStyle.Standard }, ); @@ -191,6 +197,7 @@ describe("createCallFooterViewModel", () => { buildMinimalCallViewModel(gridLayout), mockMuteStates(), twoMicsAndOneCamMediaDevices, + constant(false), /* reactionIdentifier */ undefined, { showControls: true, header: HeaderStyle.Standard }, ); @@ -236,18 +243,19 @@ describe("createCallFooterViewModel", () => { testScope(), mockMuteStates(), twoMicsAndOneCamMediaDevices, + constant(false), /* openSettings */ undefined, /* hangup */ undefined, /* showLogo */ false, ); } - it("offers nothing on a phone, where the pipeline would refuse them", () => { + it("offers them on a phone whose browser can run them", () => { sdkSupportMock.mockReturnValue(true); - expect(lobbyOn("ios").selectBackgroundEffect$.value).toBeUndefined(); + expect(lobbyOn("ios").selectBackgroundEffect$.value).toBeDefined(); }); - it("offers them where the pipeline will honour them", () => { + it("offers them on a desktop whose browser can run them", () => { sdkSupportMock.mockReturnValue(true); expect(lobbyOn("desktop").selectBackgroundEffect$.value).toBeDefined(); }); @@ -275,6 +283,16 @@ describe("createCallFooterViewModel", () => { expect(backgroundEffectSetting.getValue()).toBe("none"); }); + it("says they run slowly where only the slower route exists", () => { + sdkSupportMock.mockReturnValue(true); + modernRouteMock.mockReturnValue(false); + const slow = lobbyOn("desktop"); + expect(slow.backgroundEffectNotice$.value).toBe("slow"); + expect(slow.selectBackgroundEffect$.value).toBeDefined(); + modernRouteMock.mockReturnValue(true); + expect(lobbyOn("desktop").backgroundEffectNotice$.value).toBeUndefined(); + }); + it("offers every effect in order", () => { sdkSupportMock.mockReturnValue(true); expect(lobbyOn("desktop").backgroundEffects$.value).toEqual([ @@ -295,6 +313,23 @@ describe("createCallFooterViewModel", () => { ); }); + it("passes on the wait for the first effect", () => { + platformMock.mockReturnValue("desktop"); + const settling$ = new BehaviorSubject(true); + const vm = createLobbyFooterViewModel( + testScope(), + mockMuteStates(), + twoMicsAndOneCamMediaDevices, + settling$, + /* openSettings */ undefined, + /* hangup */ undefined, + /* showLogo */ false, + ); + expect(vm.backgroundEffectSettling$.value).toBe(true); + settling$.next(false); + expect(vm.backgroundEffectSettling$.value).toBe(false); + }); + it("availability is the same before and during a call", () => { for (const supported of [true, false]) { sdkSupportMock.mockReturnValue(supported); @@ -304,6 +339,7 @@ describe("createCallFooterViewModel", () => { buildMinimalCallViewModel(gridLayout), mockMuteStates(), twoMicsAndOneCamMediaDevices, + constant(false), /* reactionIdentifier */ undefined, { showControls: true, header: HeaderStyle.Standard }, ); diff --git a/src/components/CallFooterViewModel.tsx b/src/components/CallFooterViewModel.tsx index 16f82ee97..e9a262335 100644 --- a/src/components/CallFooterViewModel.tsx +++ b/src/components/CallFooterViewModel.tsx @@ -5,10 +5,19 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { combineLatest, map, type Observable, switchMap } from "rxjs"; +import { + combineLatest, + distinctUntilChanged, + map, + type Observable, + switchMap, +} from "rxjs"; import { supportsAudioOutputSelection } from "livekit-client"; +import { + supportsBackgroundProcessors, + supportsModernBackgroundProcessors, +} from "@livekit/track-processors"; -import { supportsBackgroundProcessors } from "../tracks/backgroundProcessing"; import { parseEffect, serializeEffect, @@ -70,6 +79,7 @@ function buildDeviceBehaviors( mediaDevices: MediaDevices, /** return empty arrays for audioOptions and videoOptions*/ disableSwitcher$: Behavior, + backgroundEffectSettling$: Observable, ): Pick< ViewModel, | "audioOptions$" @@ -85,6 +95,7 @@ function buildDeviceBehaviors( | "selectBackgroundEffect$" | "backgroundEffects$" | "backgroundEffectNotice$" + | "backgroundEffectSettling$" > { const options$ = ( available$: Behavior>, @@ -102,6 +113,7 @@ function buildDeviceBehaviors( ); const supported = supportsBackgroundProcessors(); + const slow = !supportsModernBackgroundProcessors(); const offered$ = disableSwitcher$.pipe( map((switcherDisabled) => !switcherDisabled && supported), ); @@ -148,7 +160,15 @@ function buildDeviceBehaviors( ), backgroundEffects$: constant(backgroundEffectChoices()), backgroundEffectNotice$: scope.behavior( - offered$.pipe(map((offered) => (offered ? undefined : "unavailable"))), + offered$.pipe( + map((offered) => + !offered ? "unavailable" : slow ? "slow" : undefined, + ), + ), + ), + backgroundEffectSettling$: scope.behavior( + backgroundEffectSettling$.pipe(distinctUntilChanged()), + false, ), }; } @@ -160,6 +180,8 @@ function buildDeviceBehaviors( * @param callModel - The root CallViewModel; provides layout, grid mode, reactions, etc. * @param muteStates - Audio and video mute state + toggles. * @param mediaDevices - Available and selected input devices. + * @param backgroundEffectSettling$ - Whether the first effect chosen is still + * being prepared. * @param reactionIdentifier - The local user's reaction identifier string, or * undefined when reactions are not supported (hides the reaction button). * @param options - `showControls`: whether the call controls should be shown. @@ -170,6 +192,7 @@ export function createCallFooterViewModel( callModel: CallViewModel, muteStates: MuteStates, mediaDevices: MediaDevices, + backgroundEffectSettling$: Observable, reactionIdentifier: string | undefined, options: { showControls: boolean; header: HeaderStyle }, ): ViewModel { @@ -184,7 +207,12 @@ export function createCallFooterViewModel( ); return { ...buildMuteBehaviors(scope, muteStates), - ...buildDeviceBehaviors(scope, mediaDevices, disableDeviceSwitcher$), + ...buildDeviceBehaviors( + scope, + mediaDevices, + disableDeviceSwitcher$, + backgroundEffectSettling$, + ), // candidat to move into the FooterViewModel showFooter$: callModel.showFooter$, hideControls$: constant(!showControls), @@ -246,6 +274,8 @@ export function createCallFooterViewModel( * @param scope - ObservableScope that bounds the lifetime of derived behaviors. * @param muteStates - Audio and video mute state + toggles. * @param mediaDevices - Available and selected input devices. + * @param backgroundEffectSettling$ - Whether the first effect chosen is still + * being prepared. * @param openSettings - Callback to open the settings modal, or undefined. * @param hangup - Callback to leave/cancel, or undefined (hides the button). * @param showLogo - Whether to show the Element Call logo. @@ -254,6 +284,7 @@ export function createLobbyFooterViewModel( scope: ObservableScope, muteStates: MuteStates, mediaDevices: MediaDevices, + backgroundEffectSettling$: Observable, openSettings: (() => void) | undefined, hangup: (() => void) | undefined, showLogo: boolean, @@ -297,7 +328,12 @@ export function createLobbyFooterViewModel( selectVideoButtonOption: undefined, }), ...buildMuteBehaviors(scope, muteStates), - ...buildDeviceBehaviors(scope, mediaDevices, constant(false)), + ...buildDeviceBehaviors( + scope, + mediaDevices, + constant(false), + backgroundEffectSettling$, + ), }; } diff --git a/src/components/MediaMuteAndSwitchButton.stories.tsx b/src/components/MediaMuteAndSwitchButton.stories.tsx index 9be73e1da..ba9163234 100644 --- a/src/components/MediaMuteAndSwitchButton.stories.tsx +++ b/src/components/MediaMuteAndSwitchButton.stories.tsx @@ -274,6 +274,73 @@ export const BackgroundEffectsUnavailable: Story = { }, }; +/** Where only the slower route exists: offered and choosable, with the cost said. */ +export const BackgroundEffectsSlowInThisBrowser: Story = { + args: { + ...VideoUnmute.args, + backgroundEffectNotice: + "Background effects run slowly on this platform, which may cause your video to stutter.", + }, + play: async ({ args, canvasElement }) => { + const canvas = within(canvasElement); + await userEvent.click(canvas.getByRole("button", { name: "Camera" })); + const menu = within(document.body); + const section = await menu.findByRole("group", { + name: "Background effects", + }); + // On the tiles' own group: the section's can't carry a description. + await expect( + within(section).getByRole("group"), + ).toHaveAccessibleDescription( + "Background effects run slowly on this platform, which may cause your video to stutter.", + ); + for (const tile of within(section).getAllByRole("menuitemradio")) + await expect(tile).not.toHaveAttribute("aria-disabled"); + await userEvent.click( + within(section).getByRole("menuitemradio", { name: "Background 1" }), + ); + await expect(args.onSelectBackgroundEffect).toHaveBeenCalledWith( + "image:arc", + ); + + const notice = document.body.querySelector(`.${styles.notice}`)!; + const frame = document.body + .querySelector("[role='menu']")! + .getBoundingClientRect(); + await expect(frame.width).toBeLessThanOrEqual(296); + await expect(notice.getBoundingClientRect().right).toBeLessThanOrEqual( + frame.right, + ); + }, +}; + +/** The first effect of a session shows its wait on its own tile. */ +export const BackgroundEffectsSettling: Story = { + args: { + ...VideoUnmute.args, + selectedBackgroundEffect: "blur", + backgroundEffectSettling: true, + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await userEvent.click(canvas.getByRole("button", { name: "Camera" })); + const menu = within(document.body); + const chosen = await menu.findByRole("menuitemradio", { + name: "Blur", + checked: true, + }); + await expect(chosen).toHaveAttribute("aria-busy", "true"); + const marks = chosen.querySelectorAll("svg"); + await expect(marks).toHaveLength(1); + await expect(getComputedStyle(marks[0]).animationName).not.toBe("none"); + const section = menu.getByRole("group", { name: "Background effects" }); + for (const other of within(section).getAllByRole("menuitemradio", { + checked: false, + })) + await expect(other).toHaveAttribute("aria-busy", "false"); + }, +}; + /** In a short call the effects scroll into view with the list. */ export const BackgroundEffectsScrollWhenTheyDoNotFit: Story = { args: { diff --git a/src/components/MediaMuteAndSwitchButton.tsx b/src/components/MediaMuteAndSwitchButton.tsx index 6ad77c8f1..aa6d7afe6 100644 --- a/src/components/MediaMuteAndSwitchButton.tsx +++ b/src/components/MediaMuteAndSwitchButton.tsx @@ -77,6 +77,8 @@ export interface MediaMuteAndSwitchButtonProps { onSelectBackgroundEffect?: (id: string) => void; /** What the user should know before choosing an effect, if anything. */ backgroundEffectNotice?: string; + /** Whether the first effect chosen is still being prepared. */ + backgroundEffectSettling?: boolean; /** * For any toggle and option this method will be called. * So toggles need to be implemented by listening here and setting the right toggle item to `enabled` @@ -110,6 +112,7 @@ export const MediaMuteAndSwitchButton: FC = ({ selectedBackgroundEffect, onSelectBackgroundEffect, backgroundEffectNotice, + backgroundEffectSettling, onSelect, }) => { // Requested but not yet selected. Keyed by kind too, since Chrome uses @@ -387,6 +390,7 @@ export const MediaMuteAndSwitchButton: FC = ({ effects={backgroundEffects ?? []} selected={selectedBackgroundEffect} onSelect={onSelectBackgroundEffect} + settling={backgroundEffectSettling} describedBy={ backgroundEffectNotice === undefined ? undefined : noticeId } diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index b6279b107..3459d5565 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -241,6 +241,7 @@ export const ActiveCall: FC = (props) => { vm, props.muteStates, mediaDevices, + backgroundEffectsState$.pipe(map(({ settling }) => settling ?? false)), `${props.client.getUserId()}:${props.client.getDeviceId()}`, { showControls: urlParams.showControls, header: urlParams.header }, ); diff --git a/src/room/LobbyView.tsx b/src/room/LobbyView.tsx index 8fa71ada8..1a197b8d0 100644 --- a/src/room/LobbyView.tsx +++ b/src/room/LobbyView.tsx @@ -164,6 +164,7 @@ export const LobbyView: FC = ({ footerScope, muteStates, devices, + backgroundEffectsState$.pipe(map(({ settling }) => settling ?? false)), openSettings, hangup, // Logo and header are connected: only show the logo in SPA with header. @@ -173,7 +174,14 @@ export const LobbyView: FC = ({ return (): void => { footerScope.end(); }; - }, [devices, hangup, hideHeader, muteStates, openSettings]); + }, [ + devices, + backgroundEffectsState$, + hangup, + hideHeader, + muteStates, + openSettings, + ]); // TODO: Unify this component with InCallView, so we can get slick joining // animations and don't have to feel bad about reusing its CSS diff --git a/src/state/BackgroundEffects.test.ts b/src/state/BackgroundEffects.test.ts index d00238dde..74856f82e 100644 --- a/src/state/BackgroundEffects.test.ts +++ b/src/state/BackgroundEffects.test.ts @@ -46,25 +46,33 @@ function fakePipeline(): { }; } -/** One letter per state: idle, attached. */ +/** One letter per state: idle, waiting for a frame, attached. */ function letter(state: BackgroundEffectsState): string { - return state.processor === undefined ? "i" : "a"; + if (state.processor === undefined) return "i"; + return state.settling ? "w" : "a"; } describe("the pipeline's state", () => { function testState({ effect, + firstFrame = "", expected, }: { effect: string; + firstFrame?: string; expected: string; }): void { - withTestScheduler(({ behavior, expectObservable }) => { + withTestScheduler(({ behavior, schedule, expectObservable }) => { + const transformer = { + onFirstFrame: undefined as (() => void) | undefined, + }; const effects = createBackgroundEffects(testScope(), { supported: true, effect$: behavior(effect, { n: "none", b: "blur" }), pipeline: fakePipeline().pipeline, + transformer, }); + schedule(firstFrame, { f: () => transformer.onFirstFrame?.() }); expectObservable( effects.state$.pipe(map(letter), distinctUntilChanged()), ).toBe(expected); @@ -73,8 +81,15 @@ describe("the pipeline's state", () => { it("defaults to no effect", () => testState({ effect: "n", expected: "i" })); + it("waits from the first effect until a frame carries it", () => + testState({ + effect: " nb-n-b", + firstFrame: " --f", + expected: " iwa", + })); + it("attaches on first use and stays attached", () => - testState({ effect: "nbn", expected: "ia" })); + testState({ effect: "nb-n", firstFrame: "--f", expected: "iwa" })); }); describe("background effects", () => { @@ -88,6 +103,7 @@ describe("background effects", () => { supported: true, effect$, pipeline: fake.pipeline, + transformer: { onFirstFrame: undefined }, ...options, }); } diff --git a/src/state/BackgroundEffects.ts b/src/state/BackgroundEffects.ts index df3ce6a95..a00cf3432 100644 --- a/src/state/BackgroundEffects.ts +++ b/src/state/BackgroundEffects.ts @@ -5,7 +5,13 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { combineLatest, distinctUntilChanged, map, scan } from "rxjs"; +import { + combineLatest, + distinctUntilChanged, + map, + Observable, + scan, +} from "rxjs"; import { type BackgroundProcessorWrapper, type SwitchBackgroundProcessorOptions, @@ -30,6 +36,8 @@ const blurRadius = 15; export interface BackgroundEffectsState { supported: boolean | undefined; processor: undefined | VideoProcessor; + /** From the first effect chosen until a frame carrying it is drawn. */ + settling?: boolean; } export interface BackgroundEffectsOptions { @@ -39,6 +47,8 @@ export interface BackgroundEffectsOptions { effect$: Behavior; /** The background processor pipeline to be switched between effects. */ pipeline: BackgroundProcessorWrapper; + /** Tells, once, that a frame carrying an effect has been drawn. */ + transformer: { onFirstFrame: (() => void) | undefined }; } /** The background effect pipeline, as the camera tracks and the menus see it. */ @@ -49,7 +59,7 @@ export interface BackgroundEffects { /** Switches the pipeline as the choice changes, for as long as the scope lasts. */ export function createBackgroundEffects( scope: ObservableScope, - { supported, effect$, pipeline }: BackgroundEffectsOptions, + { supported, effect$, pipeline, transformer }: BackgroundEffectsOptions, ): BackgroundEffects { // Wrapped once, so the call and the preview see one processor throughout. const processor = fromLivekitProcessor(pipeline); @@ -59,15 +69,29 @@ export function createBackgroundEffects( distinctUntilChanged(), ); + const drewAFrame$ = scope.behavior( + new Observable((subscriber) => { + subscriber.next(false); + transformer.onFirstFrame = (): void => subscriber.next(true); + return (): void => { + transformer.onFirstFrame = undefined; + }; + }), + ); + const state$ = scope.behavior( - wanted$.pipe( - scan( - (previous, wanted) => { + combineLatest([wanted$, drewAFrame$]).pipe( + scan<[boolean, boolean], BackgroundEffectsState>( + (previous, [wanted, drewAFrame]) => { // Attached the first time an effect is wanted and never detached // after, so someone who never turns one on pays for none of it. const enable = previous.processor !== undefined || (supported && wanted); - return { supported, processor: enable ? processor : undefined }; + return { + supported, + processor: enable ? processor : undefined, + settling: enable && !drewAFrame, + }; }, { supported, processor: undefined }, ), diff --git a/src/tracks/BackgroundEffectTransformer.test.ts b/src/tracks/BackgroundEffectTransformer.test.ts index fae7a2af1..400a50416 100644 --- a/src/tracks/BackgroundEffectTransformer.test.ts +++ b/src/tracks/BackgroundEffectTransformer.test.ts @@ -15,6 +15,9 @@ import { ImageSegmenter } from "@mediapipe/tasks-vision"; import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer"; +const frame = {} as VideoFrame; +const controller = {} as TransformStreamDefaultController; + describe("BackgroundEffectTransformer", () => { afterEach(() => vi.restoreAllMocks()); @@ -38,6 +41,32 @@ describe("BackgroundEffectTransformer", () => { expect(load).toHaveBeenCalledWith("/background.jpg"); }); + it("reports the first frame carrying an effect, once", async () => { + vi.spyOn(BackgroundTransformer.prototype, "transform").mockImplementation( + async function (this: BackgroundTransformer) { + this.isFirstFrame = false; + return Promise.resolve(); + }, + ); + const transformer = new BackgroundEffectTransformer({}); + transformer.onFirstFrame = vi.fn(); + + await transformer.transform(frame, controller); + await transformer.transform(frame, controller); + expect(transformer.onFirstFrame).toHaveBeenCalledTimes(1); + }); + + it("does not report a frame passed through untouched", async () => { + vi.spyOn(BackgroundTransformer.prototype, "transform").mockResolvedValue(); + const transformer = new BackgroundEffectTransformer({ + backgroundDisabled: true, + }); + transformer.onFirstFrame = vi.fn(); + + await transformer.transform(frame, controller); + expect(transformer.onFirstFrame).not.toHaveBeenCalled(); + }); + it("passes frames on untouched once disabled, even after blurring", async () => { const transformer = new BackgroundEffectTransformer({}); await transformer.update({ blurRadius: 15, backgroundDisabled: false }); diff --git a/src/tracks/BackgroundEffectTransformer.ts b/src/tracks/BackgroundEffectTransformer.ts index 65ca75f8f..e3c0a9183 100644 --- a/src/tracks/BackgroundEffectTransformer.ts +++ b/src/tracks/BackgroundEffectTransformer.ts @@ -53,6 +53,20 @@ const wasmFileset: WasmFileset = { * resource fetched from the public internet. */ export class BackgroundEffectTransformer extends BackgroundTransformer { + /** Called once, when the first frame carrying an effect has been drawn. */ + public onFirstFrame: (() => void) | undefined; + + public override async transform( + frame: VideoFrame, + controller: TransformStreamDefaultController, + ): Promise { + const priming = this.isFirstFrame; + await super.transform(frame, controller); + // Cleared only once a frame has been through the segmenter; a frame + // passed through untouched leaves it set. + if (priming && !this.isFirstFrame) this.onFirstFrame?.(); + } + /** * As the library's, except that disabling also drops the blur radius. The * library skips a frame on `isDisabled ?? backgroundDisabled`, and diff --git a/src/tracks/BackgroundEffectsContext.test.ts b/src/tracks/BackgroundEffectsContext.test.ts index acb7a9e47..71d9d7152 100644 --- a/src/tracks/BackgroundEffectsContext.test.ts +++ b/src/tracks/BackgroundEffectsContext.test.ts @@ -93,13 +93,13 @@ describe("BackgroundEffectsProvider", () => { expect(latest().processor).toBeDefined(); }); - it("does not run on a phone, even where its browser could", async () => { + it("runs on a phone whose browser can run it", async () => { platformMock.platform = "ios"; render(surface()); await blur(true); - expect(latest().supported).toBe(false); - expect(latest().processor).toBeUndefined(); + expect(latest().supported).toBe(true); + expect(latest().processor).toBeDefined(); }); it("switches to an effect chosen while the camera is off", async () => { diff --git a/src/tracks/BackgroundEffectsContext.tsx b/src/tracks/BackgroundEffectsContext.tsx index 225b7fdae..d582bc0b3 100644 --- a/src/tracks/BackgroundEffectsContext.tsx +++ b/src/tracks/BackgroundEffectsContext.tsx @@ -5,12 +5,14 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { BackgroundProcessorWrapper } from "@livekit/track-processors"; +import { + BackgroundProcessorWrapper, + supportsBackgroundProcessors, +} from "@livekit/track-processors"; import { createContext, type FC, type JSX, use } from "react"; import { backgroundEffect as backgroundEffectSetting } from "../settings/settings"; import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer"; -import { supportsBackgroundProcessors } from "./backgroundProcessing"; import { type Behavior } from "../state/Behavior"; import { type ObservableScope } from "../state/ObservableScope"; import { @@ -45,13 +47,14 @@ export function useBackgroundEffectsState$(): Behavior { export function createAppBackgroundEffects( scope: ObservableScope, ): BackgroundEffects { + const transformer = new BackgroundEffectTransformer({ + backgroundDisabled: true, + }); return createBackgroundEffects(scope, { supported: supportsBackgroundProcessors(), effect$: backgroundEffectSetting.value$, - pipeline: new BackgroundProcessorWrapper( - new BackgroundEffectTransformer({ backgroundDisabled: true }), - "background-effect", - ), + pipeline: new BackgroundProcessorWrapper(transformer, "background-effect"), + transformer, }); } diff --git a/src/tracks/backgroundProcessing.ts b/src/tracks/backgroundProcessing.ts deleted file mode 100644 index 8b6168dad..000000000 --- a/src/tracks/backgroundProcessing.ts +++ /dev/null @@ -1,15 +0,0 @@ -/* -Copyright 2026 Element Creations Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { supportsBackgroundProcessors as supportsBackgroundProcessorsLivekitSdk } from "@livekit/track-processors"; - -import { platform } from "../Platform"; - -/** One answer for the controls and the pipeline, so neither offers what the other refuses. */ -export function supportsBackgroundProcessors(): boolean { - return supportsBackgroundProcessorsLivekitSdk() && platform === "desktop"; -} diff --git a/src/utils/test-viewmodel.ts b/src/utils/test-viewmodel.ts index c30d08cda..0cf2a8b76 100644 --- a/src/utils/test-viewmodel.ts +++ b/src/utils/test-viewmodel.ts @@ -217,6 +217,7 @@ export function getBasicCallViewModelEnvironment( vm, muteStates, mediaDevices, + constant(false), "reactionId", { showControls: true, header: HeaderStyle.Standard }, );