From 74ac3a129d3ba83be37de4d5cd57a14939ea8e51 Mon Sep 17 00:00:00 2001 From: fkwp Date: Fri, 25 Sep 2026 11:14:08 +0200 Subject: [PATCH] Offer no effects for the session where the pipeline cannot be built - A segmenter that can't be built, with the model unreachable or a GPU its delegate refuses, took the camera down with the pipeline: the SDK fails the track it was attaching to, and the lobby muted camera and microphone, on every load while the choice was kept. - One is now built on trial when an effect is first wanted, before the pipeline is attached to anything, on the CPU where the GPU refuses. Where it can't be built, the options are disabled with the existing notice for the rest of the session. The choice is kept: a browser update may fix what failed. - The view model takes that verdict from the pipeline's state rather than asking the SDK, so both say the same, and it no longer builds a WebGL context of its own to ask. - Until the trial is over, the lobby opens no camera and the call makes no camera track, so neither starts without the effect. Co-Authored-By: Claude Opus 5.5 (1M context) --- playwright/background-effects.spec.ts | 40 +++++++++++++ src/components/CallFooterViewModel.test.ts | 36 +++++++++++- src/components/CallFooterViewModel.tsx | 21 ++++--- .../BackgroundEffectTransformer.test.ts | 39 ++++++++++++- src/livekit/BackgroundEffectTransformer.ts | 51 ++++++++++++---- src/livekit/BackgroundEffectsContext.test.ts | 3 +- src/livekit/BackgroundEffectsContext.tsx | 8 ++- src/room/LobbyView.tsx | 28 ++++++--- src/state/BackgroundEffects.test.ts | 58 +++++++++++++------ src/state/BackgroundEffects.ts | 45 +++++++++++--- .../localMember/Publisher.test.ts | 12 ++++ .../CallViewModel/localMember/Publisher.ts | 37 +++++++++--- 12 files changed, 308 insertions(+), 70 deletions(-) diff --git a/playwright/background-effects.spec.ts b/playwright/background-effects.spec.ts index f47c59cba..ba2901c05 100644 --- a/playwright/background-effects.spec.ts +++ b/playwright/background-effects.spec.ts @@ -178,6 +178,46 @@ test.describe("background effects section", () => { }); }); +test.describe("a pipeline that fails to build", () => { + test("offers no effects for the session, and keeps the choice", async ({ + page, + }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall(page, "Failing user", "Pipeline fails"); + const preview = page.locator("video").first(); + const camera = await cameraColour(preview); + await page.route("**/*.tflite*", async (route) => route.abort()); + + await page.getByRole("button", { name: "Camera", exact: true }).click(); + const tile = page.getByRole("menuitemradio", { name: "Background 1" }); + await tile.click(); + await expect( + page.getByText("Background effects are not supported on this platform."), + ).toBeVisible({ timeout: 30_000 }); + await expect(tile).toBeDisabled(); + await page.keyboard.press("Escape"); + // The camera and microphone carry on without it. + expect(distance(await cameraColour(preview), camera)).toBeLessThan(30); + await expect(page.getByTestId("incall_mute")).toHaveAttribute( + "aria-checked", + "true", + ); + + await page.unroute("**/*.tflite*"); + await page.reload(); + await expect(page.getByTestId("lobby_joinCall")).toBeVisible(); + await page.getByRole("button", { name: "Camera", exact: true }).click(); + await expect(tile).toHaveAttribute("aria-checked", "true"); + const picture = await averageColour(tile.locator("img")); + await expect + .poll(async () => distance(await averageColour(preview), picture), { + timeout: 60_000, + }) + .toBeLessThan(distance(camera, picture) / 2); + }); +}); + test.describe("the chosen effect", () => { test("chosen effect survives leaving and rejoining", async ({ page }) => { test.setTimeout(120_000); diff --git a/src/components/CallFooterViewModel.test.ts b/src/components/CallFooterViewModel.test.ts index a4cf9b8b3..cb1915b45 100644 --- a/src/components/CallFooterViewModel.test.ts +++ b/src/components/CallFooterViewModel.test.ts @@ -15,7 +15,7 @@ import { mockMuteStates, mockMediaDevices, } from "../utils/test"; -import { constant } from "../state/Behavior"; +import { type Behavior, constant } from "../state/Behavior"; import type { CallViewModel } from "../state/CallViewModel/CallViewModel"; import type { Alignment, Layout } from "../state/layout-types"; import type { SpotlightTileViewModel } from "../state/TileViewModel"; @@ -122,6 +122,11 @@ const noPipeline = constant({ processor: undefined, }); +/** The provider's verdict, which the SDK's check stands for here. */ +function pipelineAsTheSdkSays(): Behavior { + return constant({ supported: sdkSupportMock(), processor: undefined }); +} + const twoMicsAndOneCamMediaDevices = mockMediaDevices({ audioInput: { available$: constant( @@ -284,7 +289,7 @@ describe("createCallFooterViewModel", () => { testScope(), mockMuteStates(), twoMicsAndOneCamMediaDevices, - noPipeline, + pipelineAsTheSdkSays(), /* openSettings */ undefined, /* hangup */ undefined, /* showLogo */ false, @@ -388,6 +393,31 @@ describe("createCallFooterViewModel", () => { expect(vm.backgroundEffectSettling$.value).toBe(false); }); + it("offers nothing once the pipeline fails to build, keeping the choice", () => { + platformMock.mockReturnValue("desktop"); + backgroundEffectSetting.setValue("blur"); + const pipeline$ = new BehaviorSubject({ + supported: true, + processor: undefined, + }); + const vm = createLobbyFooterViewModel( + testScope(), + mockMuteStates(), + twoMicsAndOneCamMediaDevices, + pipeline$, + /* openSettings */ undefined, + /* hangup */ undefined, + /* showLogo */ false, + ); + expect(vm.backgroundEffect$.value).toBe("blur"); + + pipeline$.next({ supported: false, processor: undefined }); + expect(vm.selectBackgroundEffect$.value).toBeUndefined(); + expect(vm.addBackgroundImage$.value).toBeUndefined(); + expect(vm.backgroundEffect$.value).toBe("none"); + expect(backgroundEffectSetting.getValue()).toBe("blur"); + }); + it("adding is unavailable at the limit", () => { sdkSupportMock.mockReturnValue(true); const added$ = addedBackgrounds.added$ as BehaviorSubject< @@ -480,7 +510,7 @@ describe("createCallFooterViewModel", () => { buildMinimalCallViewModel(gridLayout), mockMuteStates(), twoMicsAndOneCamMediaDevices, - noPipeline, + pipelineAsTheSdkSays(), /* reactionIdentifier */ undefined, { showControls: true, header: HeaderStyle.Standard }, ); diff --git a/src/components/CallFooterViewModel.tsx b/src/components/CallFooterViewModel.tsx index ca9f780de..4e7659901 100644 --- a/src/components/CallFooterViewModel.tsx +++ b/src/components/CallFooterViewModel.tsx @@ -15,10 +15,7 @@ import { switchMap, } from "rxjs"; import { supportsAudioOutputSelection } from "livekit-client"; -import { - supportsBackgroundProcessors, - supportsModernBackgroundProcessors, -} from "@livekit/track-processors"; +import { supportsModernBackgroundProcessors } from "@livekit/track-processors"; import { logger } from "matrix-js-sdk/lib/logger"; import { @@ -130,10 +127,14 @@ function buildDeviceBehaviors( ), ); - const supported = supportsBackgroundProcessors(); + // Also false once the pipeline has failed to build this session. + const supported$ = backgroundEffectsState$.pipe( + map(({ supported }) => supported === true), + distinctUntilChanged(), + ); const slow = !supportsModernBackgroundProcessors(); - const offered$ = disableSwitcher$.pipe( - map((switcherDisabled) => !switcherDisabled && supported), + const offered$ = combineLatest([disableSwitcher$, supported$]).pipe( + map(([switcherDisabled, supported]) => !switcherDisabled && supported), ); // A new object for each refusal, so the same one twice is shown twice. const refusal$ = new BehaviorSubject( @@ -164,8 +165,10 @@ function buildDeviceBehaviors( ), selectVideoButtonOption$: constant(mediaDevices.videoInput.select), backgroundEffect$: scope.behavior( - backgroundEffectSetting.value$.pipe( - map((raw) => (supported ? serializeEffect(parseEffect(raw)) : "none")), + combineLatest([backgroundEffectSetting.value$, supported$]).pipe( + map(([raw, supported]) => + supported ? serializeEffect(parseEffect(raw)) : "none", + ), ), ), selectBackgroundEffect$: scope.behavior( diff --git a/src/livekit/BackgroundEffectTransformer.test.ts b/src/livekit/BackgroundEffectTransformer.test.ts index 400a50416..09de8ef5f 100644 --- a/src/livekit/BackgroundEffectTransformer.test.ts +++ b/src/livekit/BackgroundEffectTransformer.test.ts @@ -13,13 +13,19 @@ import { } from "@livekit/track-processors"; import { ImageSegmenter } from "@mediapipe/tasks-vision"; -import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer"; +import { + BackgroundEffectTransformer, + canSegment, +} from "./BackgroundEffectTransformer"; const frame = {} as VideoFrame; const controller = {} as TransformStreamDefaultController; describe("BackgroundEffectTransformer", () => { - afterEach(() => vi.restoreAllMocks()); + afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + }); it("loads a background chosen before it was built", async () => { vi.spyOn(VideoTransformer.prototype, "init").mockResolvedValue(); @@ -78,4 +84,33 @@ describe("BackgroundEffectTransformer", () => { expect(transformer.options.blurRadius).toBeUndefined(); expect(transformer.options.imagePath).toBeUndefined(); }); + + it("segments on the CPU where the GPU refuses", async () => { + vi.spyOn(VideoTransformer.prototype, "init").mockResolvedValue(); + const build = vi + .spyOn(ImageSegmenter, "createFromOptions") + .mockRejectedValueOnce(new Error("blocklisted")) + .mockResolvedValue({} as ImageSegmenter); + + await new BackgroundEffectTransformer({}).init( + {} as VideoTransformerInitOptions, + ); + expect(build.mock.calls.map(([, o]) => o.baseOptions?.delegate)).toEqual([ + "GPU", + "CPU", + ]); + }); + + it("says whether a segmenter can be built at all", async () => { + vi.stubGlobal("OffscreenCanvas", class {}); + const close = vi.fn(); + const build = vi + .spyOn(ImageSegmenter, "createFromOptions") + .mockResolvedValue({ close } as unknown as ImageSegmenter); + expect(await canSegment()).toBe(true); + expect(close).toHaveBeenCalled(); + + build.mockRejectedValue(new Error("no model")); + expect(await canSegment()).toBe(false); + }); }); diff --git a/src/livekit/BackgroundEffectTransformer.ts b/src/livekit/BackgroundEffectTransformer.ts index ce0a1e5e6..a2231ba3e 100644 --- a/src/livekit/BackgroundEffectTransformer.ts +++ b/src/livekit/BackgroundEffectTransformer.ts @@ -11,6 +11,7 @@ import { VideoTransformer, type VideoTransformerInitOptions, type BackgroundOptions, + type SegmenterOptions, } from "@livekit/track-processors"; import { logger } from "matrix-js-sdk/lib/logger"; import { ImageSegmenter } from "@mediapipe/tasks-vision"; @@ -89,17 +90,10 @@ export class BackgroundEffectTransformer extends BackgroundTransformer { inputElement: inputVideo, }); - this.imageSegmenter = await ImageSegmenter.createFromOptions(wasmFileset, { - baseOptions: { - modelAssetPath, - delegate: "GPU", - ...this.options.segmenterOptions, - }, - canvas: this.canvas, - runningMode: "VIDEO", - outputCategoryMask: true, - outputConfidenceMasks: false, - }); + this.imageSegmenter = await createSegmenter( + this.canvas, + this.options.segmenterOptions, + ); // BackgroundTransformer's own init applies these, and this one replaces it. if (this.options.imagePath) { @@ -113,3 +107,38 @@ export class BackgroundEffectTransformer extends BackgroundTransformer { this.gl?.setBackgroundDisabled(this.options.backgroundDisabled ?? false); } } + +/** + * Whether a segmenter can be built here, asked once before any camera depends + * on one: a pipeline that fails to build takes the camera down with it. + */ +export async function canSegment(): Promise { + try { + (await createSegmenter(new OffscreenCanvas(1, 1))).close(); + return true; + } catch (e) { + logger.warn("Background effects cannot run here", e); + return false; + } +} + +/** On the GPU, or on the CPU where the GPU refuses, as a blocklisted one does. */ +async function createSegmenter( + canvas: HTMLCanvasElement | OffscreenCanvas | undefined, + segmenterOptions?: SegmenterOptions, +): Promise { + const build = async (delegate: "GPU" | "CPU"): Promise => + ImageSegmenter.createFromOptions(wasmFileset, { + baseOptions: { modelAssetPath, delegate, ...segmenterOptions }, + canvas, + runningMode: "VIDEO", + outputCategoryMask: true, + outputConfidenceMasks: false, + }); + try { + return await build("GPU"); + } catch (e) { + logger.warn("Segmenting on the CPU, as the GPU refused", e); + return build("CPU"); + } +} diff --git a/src/livekit/BackgroundEffectsContext.test.ts b/src/livekit/BackgroundEffectsContext.test.ts index c5cac860c..c95133efe 100644 --- a/src/livekit/BackgroundEffectsContext.test.ts +++ b/src/livekit/BackgroundEffectsContext.test.ts @@ -65,7 +65,8 @@ vi.mock("./backgroundImages", async () => { }; }); vi.mock("./BackgroundEffectTransformer", () => ({ - BackgroundEffectTransformer: vi.fn(), + BackgroundEffectTransformer: vi.fn(function () {}), + canSegment: async (): Promise => Promise.resolve(true), })); const processor = {} as ProcessorWrapper; diff --git a/src/livekit/BackgroundEffectsContext.tsx b/src/livekit/BackgroundEffectsContext.tsx index 274b4297a..4bf2bf42a 100644 --- a/src/livekit/BackgroundEffectsContext.tsx +++ b/src/livekit/BackgroundEffectsContext.tsx @@ -24,7 +24,10 @@ import { } from "rxjs"; import { backgroundEffect as backgroundEffectSetting } from "../settings/settings"; -import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer"; +import { + BackgroundEffectTransformer, + canSegment, +} from "./BackgroundEffectTransformer"; import { type SyncedCameraTrack } from "./cameraTrack"; import { OneStepPipeline } from "./OneStepPipeline"; import { addedBackgrounds } from "./backgroundImages"; @@ -47,6 +50,8 @@ export type BackgroundEffectsState = { settling?: boolean; /** The camera this provider's pipeline is synced to. */ cameraTrack?: SyncedCameraTrack; + /** While asking whether the pipeline can be built, before it is attached. */ + preparing?: boolean; }; const BackgroundEffectsContext = createContext( @@ -157,6 +162,7 @@ export function createAppBackgroundEffects( effect$: backgroundEffectSetting.value$, setEffect: backgroundEffectSetting.setValue, added$: addedBackgrounds.added$, + canSegment, pipeline: new OneStepPipeline(transformer, "background-effect"), transformer, }); diff --git a/src/room/LobbyView.tsx b/src/room/LobbyView.tsx index cb7bb6fed..a6c9a285a 100644 --- a/src/room/LobbyView.tsx +++ b/src/room/LobbyView.tsx @@ -9,6 +9,7 @@ import { type FC, useCallback, useMemo, + useRef, useState, type JSX, useEffect, @@ -137,10 +138,15 @@ export const LobbyView: FC = ({ }, ); - const { processor } = useBackgroundEffects(); + const { processor, preparing } = useBackgroundEffects(); const backgroundEffectsState$ = useBackgroundEffectsState$(); - const initialProcessor = useInitial(() => processor); + // Taken once it is known whether the pipeline builds, and then kept, as + // useInitial keeps a value: one attached later is synced below. + const initialProcessorRef = useRef<{ value: typeof processor }>(undefined); + if (!preparing) initialProcessorRef.current ??= { value: processor }; + const initialProcessor = initialProcessorRef.current?.value; + const processorTaken = initialProcessorRef.current !== undefined; const localTrackOptions = useMemo( () => ({ // The only reason we request audio here is to get the audio permission @@ -150,12 +156,20 @@ export const LobbyView: FC = ({ // We also pass in a clone because livekit mutates the object passed in, // which would cause the devices to be re-opened on the next render. audio: Object.assign({}, initialAudioOptions), - video: videoEnabled && { - deviceId: videoInputId, - processor: initialProcessor, - }, + // Not opened before then, so that the preview never shows the room. + video: videoEnabled && + processorTaken && { + deviceId: videoInputId, + processor: initialProcessor, + }, }), - [initialAudioOptions, videoEnabled, videoInputId, initialProcessor], + [ + initialAudioOptions, + videoEnabled, + processorTaken, + videoInputId, + initialProcessor, + ], ); const onError = useCallback( diff --git a/src/state/BackgroundEffects.test.ts b/src/state/BackgroundEffects.test.ts index e9d0e4aa7..73ae8239f 100644 --- a/src/state/BackgroundEffects.test.ts +++ b/src/state/BackgroundEffects.test.ts @@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details. */ import { beforeEach, describe, expect, it, vi } from "vitest"; -import { BehaviorSubject, distinctUntilChanged, map } from "rxjs"; +import { BehaviorSubject, distinctUntilChanged, map, of } from "rxjs"; import { type BackgroundProcessorWrapper } from "@livekit/track-processors"; import { @@ -47,8 +47,10 @@ function fakePipeline(): { }; } -/** One letter per state: idle, waiting for a frame, attached. */ +/** One letter per state: idle, preparing, waiting for a frame, attached. */ function letter(state: BackgroundEffectsState): string { + if (state.supported === false) return "x"; + if (state.preparing) return "p"; if (state.processor === undefined) return "i"; return state.settling ? "w" : "a"; } @@ -56,14 +58,17 @@ function letter(state: BackgroundEffectsState): string { describe("the pipeline's state", () => { function testState({ effect, + answer = "-y", firstFrame = "", expected, }: { effect: string; + /** When the trial build answers, from being asked. */ + answer?: string; firstFrame?: string; expected: string; }): void { - withTestScheduler(({ behavior, schedule, expectObservable }) => { + withTestScheduler(({ behavior, cold, schedule, expectObservable }) => { const transformer = { onFirstFrame: undefined as (() => void) | undefined, }; @@ -72,6 +77,7 @@ describe("the pipeline's state", () => { effect$: behavior(effect, { n: "none", b: "blur" }), setEffect: vi.fn(), added$: new BehaviorSubject([]), + canSegment: () => cold(answer, { y: true, n: false }), pipeline: fakePipeline().pipeline, transformer, }); @@ -84,15 +90,21 @@ describe("the pipeline's state", () => { it("defaults to no effect", () => testState({ effect: "n", expected: "i" })); + it("attaches only once it is known the pipeline builds", () => + testState({ effect: "nb", answer: "--y", expected: "ip-w" })); + + it("offers none where the pipeline fails to build", () => + testState({ effect: "nb", answer: "-n", expected: "ipx" })); + it("waits from the first effect until a frame carries it", () => testState({ - effect: " nb-n-b", - firstFrame: " --f", - expected: " iwa", + effect: " nb---n-b", + firstFrame: "---f", + expected: " ipwa", })); it("attaches on first use and stays attached", () => - testState({ effect: "nb-n", firstFrame: "--f", expected: "iwa" })); + testState({ effect: "nb--n", firstFrame: "---f", expected: "ipwa" })); }); describe("background effects", () => { @@ -109,6 +121,7 @@ describe("background effects", () => { effect$, setEffect, added$, + canSegment: () => of(true), pipeline: fake.pipeline, transformer: { onFirstFrame: undefined }, ...options, @@ -118,8 +131,6 @@ describe("background effects", () => { effect$.next(raw); await flushPromises(); }; - const blur = async (on: boolean): Promise => - choose(on ? "blur" : "none"); beforeEach(() => { effect$ = new BehaviorSubject("none"); @@ -128,6 +139,17 @@ describe("background effects", () => { fake = fakePipeline(); }); + it("offers none for the session where the pipeline fails to build", async () => { + const effects = build({ canSegment: () => of(false) }); + await choose("blur"); + + expect(effects.state$.value.supported).toBe(false); + expect(effects.state$.value.processor).toBeUndefined(); + expect(effects.state$.value.settling).toBe(false); + // A browser update may fix it, so the next session asks again. + expect(effect$.value).toBe("blur"); + }); + it("puts a shipped background on as that picture", async () => { build(); await choose(`image:${shippedBackgrounds[0].id}`); @@ -141,10 +163,10 @@ describe("background effects", () => { it("switches in place rather than reattaching", async () => { const effects = build(); - await blur(true); + await choose("blur"); const pipeline = effects.state$.value.processor; - await blur(false); - await blur(true); + await choose("none"); + await choose("blur"); expect(effects.state$.value.processor).toBe(pipeline); expect(fake.switches).toEqual([ @@ -157,18 +179,18 @@ describe("background effects", () => { it("keeps switching after a switch fails", async () => { vi.mocked(fake.pipeline.switchTo).mockRejectedValueOnce(new Error("lost")); build(); - await blur(true); - await blur(false); + await choose("blur"); + await choose("none"); expect(fake.switches).toEqual([{ mode: "disabled" }]); }); it("switches one at a time, skipping those overtaken", async () => { const finished = fake.holdNext(); build(); - await blur(true); - await blur(false); - await blur(true); - await blur(false); + await choose("blur"); + await choose("none"); + await choose("blur"); + await choose("none"); expect(fake.switches).toHaveLength(1); finished(); diff --git a/src/state/BackgroundEffects.ts b/src/state/BackgroundEffects.ts index 84cc77359..b29fffb30 100644 --- a/src/state/BackgroundEffects.ts +++ b/src/state/BackgroundEffects.ts @@ -8,9 +8,15 @@ Please see LICENSE in the repository root for full details. import { combineLatest, distinctUntilChanged, + first, + from, map, Observable, + type ObservableInput, + of, scan, + startWith, + switchMap, } from "rxjs"; import { type BackgroundProcessorWrapper, @@ -41,6 +47,8 @@ export interface BackgroundEffectsOptions { setEffect: (id: EffectId) => void; /** The backgrounds the device keeps, undefined until it has said. */ added$: Behavior; + /** Builds a segmenter on trial, and answers whether it could. */ + canSegment: () => ObservableInput; /** The background processor pipeline to be switched between effects. */ pipeline: BackgroundProcessorWrapper; /** Tells, once, that a frame carrying an effect has been drawn. */ @@ -60,6 +68,7 @@ export function createBackgroundEffects( effect$, setEffect, added$, + canSegment, pipeline, transformer, }: BackgroundEffectsOptions, @@ -67,9 +76,11 @@ export function createBackgroundEffects( // The camera the pipeline is synced to. const cameraTrack = new SyncedCameraTrack(); const choice$ = effect$.pipe(map(parseEffect)); - const wanted$ = choice$.pipe( - map((effect) => effect.kind !== "none"), - distinctUntilChanged(), + const wanted$ = scope.behavior( + choice$.pipe( + map((effect) => effect.kind !== "none"), + distinctUntilChanged(), + ), ); combineLatest([choice$, added$]) @@ -85,6 +96,18 @@ export function createBackgroundEffects( setEffect("none"); }); + // Asked once an effect is first wanted. A browser update may fix what + // fails, so the choice is kept for the next session rather than cleared. + const buildable$ = scope.behavior( + supported + ? wanted$.pipe( + first(Boolean), + switchMap(() => from(canSegment())), + startWith(undefined), + ) + : of(undefined), + ); + const drewAFrame$ = scope.behavior( new Observable((subscriber) => { subscriber.next(false); @@ -95,19 +118,23 @@ export function createBackgroundEffects( }), ); + // In one step, so that nothing sees the pipeline neither preparing nor + // attached. const state$ = scope.behavior( - combineLatest([wanted$, drewAFrame$]).pipe( - scan<[boolean, boolean], BackgroundEffectsState>( - (previous, [wanted, drewAFrame]) => { + combineLatest([wanted$, buildable$, drewAFrame$]).pipe( + scan<[boolean, boolean | undefined, boolean], BackgroundEffectsState>( + (previous, [wanted, buildable, drewAFrame]) => { // Attached the first time an effect is wanted and never detached // after, so someone who never turns one on pays for none of it. const enable = - previous.processor !== undefined || (supported && wanted); + previous.processor !== undefined || (wanted && buildable === true); + const preparing = supported && wanted && buildable === undefined; return { - supported, + supported: supported && buildable !== false, processor: enable ? pipeline : undefined, - settling: enable && !drewAFrame, + settling: (preparing || enable) && !drewAFrame, cameraTrack, + preparing, }; }, { supported, processor: undefined }, diff --git a/src/state/CallViewModel/localMember/Publisher.test.ts b/src/state/CallViewModel/localMember/Publisher.test.ts index b91b5f160..837c48166 100644 --- a/src/state/CallViewModel/localMember/Publisher.test.ts +++ b/src/state/CallViewModel/localMember/Publisher.test.ts @@ -451,6 +451,18 @@ describe("turning the camera on with an effect chosen", () => { expect(madeWith).toEqual([processor]); }); + it("waits to make a camera track until it is known the pipeline builds", async () => { + state$.next({ supported: true, processor: undefined, preparing: true }); + await publisher.createAndSetupTracks(); + videoEnabled$.next(true); + await flushPromises(); + expect(madeWith).toEqual([]); + + state$.next({ supported: true, processor, preparing: false }); + await flushPromises(); + expect(madeWith).toEqual([processor]); + }); + it("replaces a camera track turned off before the effect was chosen", async () => { state$.next({ supported: true, processor }); const stopped = Object.assign(Object.create(LocalVideoTrack.prototype), { diff --git a/src/state/CallViewModel/localMember/Publisher.ts b/src/state/CallViewModel/localMember/Publisher.ts index 439971f9e..a24c841e0 100644 --- a/src/state/CallViewModel/localMember/Publisher.ts +++ b/src/state/CallViewModel/localMember/Publisher.ts @@ -14,6 +14,8 @@ import { Track, } from "livekit-client"; import { + filter, + firstValueFrom, map, NEVER, type Observable, @@ -192,14 +194,19 @@ export class Publisher { // We are using the `ParticipantEvent.LocalTrackPublished` to be notified // when tracks are actually published, and at that point // we can pause upstream if needed (depending on if startPublishing has been called). - if (audio && video) { - // Enable both at once in order to have a single permission prompt! - void lkRoom.localParticipant.enableCameraAndMicrophone(); - } else if (audio) { - void lkRoom.localParticipant.setMicrophoneEnabled(true); - } else if (video) { - void lkRoom.localParticipant.setCameraEnabled(true); - } + const enableTracks = (): void => { + if (audio && video) { + // Enable both at once in order to have a single permission prompt! + void lkRoom.localParticipant.enableCameraAndMicrophone(); + } else if (audio) { + void lkRoom.localParticipant.setMicrophoneEnabled(true); + } else if (video) { + void lkRoom.localParticipant.setCameraEnabled(true); + } + }; + if (video && this.backgroundEffectsState$.value.preparing) + void this.pipelineKnown().then(enableTracks); + else enableTracks(); return Promise.resolve(); } @@ -415,7 +422,9 @@ export class Publisher { this.muteStates.video.setHandler(async (enable) => { try { this.logger.debug(`handler: Setting LiveKit camera enabled: ${enable}`); - const { processor } = this.backgroundEffectsState$.value; + const { processor } = enable + ? await this.pipelineKnown() + : this.backgroundEffectsState$.value; if (enable && processor) await this.dropCameraWithoutEffect(lkRoom); await lkRoom.localParticipant.setCameraEnabled(enable); // Unmute will restart the track if it was paused upstream, @@ -444,6 +453,16 @@ export class Publisher { await lkRoom.localParticipant.unpublishTrack(track); } + /** + * The pipeline's state once it is known whether it builds: a camera track + * made before then would start without the effect. + */ + private async pipelineKnown(): Promise { + return firstValueFrom( + this.backgroundEffectsState$.pipe(filter((s) => !s.preparing)), + ); + } + private observeBackgroundEffects( scope: ObservableScope, room: LivekitRoom,