Merge pull request #4292 from element-hq/spec/background-effects/12-unbuildable-pipeline

Offer no effects for the session where the pipeline cannot be built

playwright/background-effects.spec.ts

@@ -211,6 +211,46 @@ test.describe("background effects section", () => {
   });
 });
 
+test.describe("a pipeline that fails to build", () => {
+  test("offers no effects for the session, and keeps the choice", async ({
+    page,
+  }) => {
+    test.setTimeout(120_000);
+    await page.goto("/");
+    await SpaHelpers.createCall(page, "Failing user", "Pipeline fails");
+    const preview = page.locator("video").first();
+    const camera = await cameraColour(preview);
+    await page.route("**/*.tflite*", async (route) => route.abort());
+
+    await page.getByRole("button", { name: "Camera", exact: true }).click();
+    const tile = page.getByRole("menuitemradio", { name: "Background 1" });
+    await tile.click();
+    await expect(
+      page.getByText("Background effects are not supported on this platform."),
+    ).toBeVisible({ timeout: 30_000 });
+    await expect(tile).toBeDisabled();
+    await page.keyboard.press("Escape");
+    // The camera and microphone carry on without it.
+    expect(distance(await cameraColour(preview), camera)).toBeLessThan(30);
+    await expect(page.getByTestId("incall_mute")).toHaveAttribute(
+      "aria-checked",
+      "true",
+    );
+
+    await page.unroute("**/*.tflite*");
+    await page.reload();
+    await expect(page.getByTestId("lobby_joinCall")).toBeVisible();
+    await page.getByRole("button", { name: "Camera", exact: true }).click();
+    await expect(tile).toHaveAttribute("aria-checked", "true");
+    const picture = await averageColour(tile.locator("img"));
+    await expect
+      .poll(async () => distance(await averageColour(preview), picture), {
+        timeout: 60_000,
+      })
+      .toBeLessThan(distance(camera, picture) / 2);
+  });
+});
+
 test.describe("the chosen effect", () => {
   test("chosen effect survives leaving and rejoining", async ({ page }) => {
     test.setTimeout(120_000);

src/components/CallFooterViewModel.test.ts

@@ -16,7 +16,8 @@ import {
   mockMediaDevices,
   mockVideoTrack,
 } from "../utils/test";
-import { constant } from "../state/Behavior";
+import { type Behavior, constant } from "../state/Behavior";
+import { type BackgroundEffectsState } from "../state/BackgroundEffects";
 import type { CallViewModel } from "../state/CallViewModel/CallViewModel";
 import type { Layout } from "../state/layout-types";
 import type { SpotlightTileViewModel } from "../state/TileViewModel";
@@ -112,6 +113,16 @@ const pipLayout: Layout = {
   spotlight: {} as SpotlightTileViewModel,
 };
 
+const noPipeline = constant<BackgroundEffectsState>({
+  supported: false,
+  processor: undefined,
+});
+
+/** The provider's verdict, which the SDK's check stands for here. */
+function pipelineAsTheSdkSays(): Behavior<BackgroundEffectsState> {
+  return constant({ supported: sdkSupportMock(), processor: undefined });
+}
+
 const twoMicsAndOneCamMediaDevices = mockMediaDevices({
   audioInput: {
     available$: constant(
@@ -143,7 +154,7 @@ describe("createCallFooterViewModel", () => {
         buildMinimalCallViewModel(gridLayout),
         mockMuteStates(),
         twoMicsAndOneCamMediaDevices,
-        constant(false),
+        noPipeline,
         /* reactionIdentifier */ undefined,
         { showControls: true, header: HeaderStyle.Standard },
       );
@@ -186,7 +197,7 @@ describe("createCallFooterViewModel", () => {
             select: vi.fn(),
           },
         }),
-        constant(false),
+        noPipeline,
         /* reactionIdentifier */ undefined,
         { showControls: true, header: HeaderStyle.Standard },
       );
@@ -205,7 +216,7 @@ describe("createCallFooterViewModel", () => {
         buildMinimalCallViewModel(layout),
         mockMuteStates(),
         twoMicsAndOneCamMediaDevices,
-        constant(false),
+        noPipeline,
         /* reactionIdentifier */ undefined,
         { showControls: true, header: HeaderStyle.Standard },
       );
@@ -228,7 +239,7 @@ describe("createCallFooterViewModel", () => {
         buildMinimalCallViewModel(gridLayout),
         mockMuteStates(),
         twoMicsAndOneCamMediaDevices,
-        constant(false),
+        noPipeline,
         /* reactionIdentifier */ undefined,
         { showControls: true, header: HeaderStyle.Standard },
       );
@@ -274,7 +285,7 @@ describe("createCallFooterViewModel", () => {
         testScope(),
         mockMuteStates(),
         twoMicsAndOneCamMediaDevices,
-        constant(false),
+        pipelineAsTheSdkSays(),
         /* camera$ */ constant(null),
         /* openSettings */ undefined,
         /* hangup */ undefined,
@@ -360,22 +371,52 @@ describe("createCallFooterViewModel", () => {
 
     it("passes on the wait for the first effect", () => {
       platformMock.mockReturnValue("desktop");
-      const settling$ = new BehaviorSubject(true);
+      const pipeline$ = new BehaviorSubject<BackgroundEffectsState>({
+        supported: true,
+        processor: undefined,
+        wait: "settling",
+      });
       const vm = createLobbyFooterViewModel(
         testScope(),
         mockMuteStates(),
         twoMicsAndOneCamMediaDevices,
-        settling$,
+        pipeline$,
         /* camera$ */ constant(null),
         /* openSettings */ undefined,
         /* hangup */ undefined,
         /* showLogo */ false,
       );
       expect(vm.backgroundEffectSettling$.value).toBe(true);
-      settling$.next(false);
+      pipeline$.next({ ...pipeline$.value, wait: undefined });
       expect(vm.backgroundEffectSettling$.value).toBe(false);
     });
 
+    it("offers nothing once the pipeline fails to build, keeping the choice", () => {
+      platformMock.mockReturnValue("desktop");
+      backgroundEffectSetting.setValue("blur");
+      const pipeline$ = new BehaviorSubject<BackgroundEffectsState>({
+        supported: true,
+        processor: undefined,
+      });
+      const vm = createLobbyFooterViewModel(
+        testScope(),
+        mockMuteStates(),
+        twoMicsAndOneCamMediaDevices,
+        pipeline$,
+        /* camera$ */ constant(null),
+        /* openSettings */ undefined,
+        /* hangup */ undefined,
+        /* showLogo */ false,
+      );
+      expect(vm.backgroundEffect$.value).toBe("blur");
+
+      pipeline$.next({ supported: false, processor: undefined });
+      expect(vm.selectBackgroundEffect$.value).toBeUndefined();
+      expect(vm.addBackgroundImage$.value).toBeUndefined();
+      expect(vm.backgroundEffect$.value).toBe("none");
+      expect(backgroundEffectSetting.getValue()).toBe("blur");
+    });
+
     it("adding is unavailable at the limit", () => {
       sdkSupportMock.mockReturnValue(true);
       const added$ = addedBackgrounds.added$ as BehaviorSubject<
@@ -439,7 +480,7 @@ describe("createCallFooterViewModel", () => {
         testScope(),
         mockMuteStates(),
         twoMicsAndOneCamMediaDevices,
-        constant(false),
+        noPipeline,
         camera$,
         /* openSettings */ undefined,
         /* hangup */ undefined,
@@ -459,7 +500,7 @@ describe("createCallFooterViewModel", () => {
         { ...buildMinimalCallViewModel(gridLayout), camera$: constant(track) },
         mockMuteStates(),
         twoMicsAndOneCamMediaDevices,
-        constant(false),
+        noPipeline,
         /* reactionIdentifier */ undefined,
         { showControls: true, header: HeaderStyle.Standard },
       );
@@ -479,7 +520,7 @@ describe("createCallFooterViewModel", () => {
           buildMinimalCallViewModel(gridLayout),
           mockMuteStates(),
           twoMicsAndOneCamMediaDevices,
-          constant(false),
+          pipelineAsTheSdkSays(),
           /* reactionIdentifier */ undefined,
           { showControls: true, header: HeaderStyle.Standard },
         );

src/components/CallFooterViewModel.tsx

@@ -16,10 +16,7 @@ import {
 } from "rxjs";
 import { supportsAudioOutputSelection } from "livekit-client";
 import { type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk";
-import {
-  supportsBackgroundProcessors,
-  supportsModernBackgroundProcessors,
-} from "@livekit/track-processors";
+import { supportsModernBackgroundProcessors } from "@livekit/track-processors";
 import { logger } from "matrix-js-sdk/lib/logger";
 
 import {
@@ -35,6 +32,7 @@ import {
 } from "../tracks/backgroundImages";
 
 import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
+import { type BackgroundEffectsState } from "../state/BackgroundEffects";
 import { type MenuOptions } from "./MediaMuteAndSwitchButton";
 import { type MediaDevices } from "../state/MediaDevices";
 import {
@@ -89,7 +87,7 @@ function buildDeviceBehaviors(
   mediaDevices: MediaDevices,
   /** return empty arrays for  audioOptions and videoOptions*/
   disableSwitcher$: Behavior<boolean>,
-  backgroundEffectSettling$: Observable<boolean>,
+  backgroundEffectsState$: Observable<BackgroundEffectsState>,
   /** The camera as the pipeline has it, for the menu's preview. */
   camera$: Observable<LocalVideoMediaTrack | null | undefined>,
 ): Pick<
@@ -128,10 +126,14 @@ function buildDeviceBehaviors(
       ),
     );
 
-  const supported = supportsBackgroundProcessors();
+  // Also false once the pipeline has failed to build this session.
+  const supported$ = backgroundEffectsState$.pipe(
+    map(({ supported }) => supported === true),
+    distinctUntilChanged(),
+  );
   const slow = !supportsModernBackgroundProcessors();
-  const offered$ = disableSwitcher$.pipe(
-    map((switcherDisabled) => !switcherDisabled && supported),
+  const offered$ = combineLatest([disableSwitcher$, supported$]).pipe(
+    map(([switcherDisabled, supported]) => !switcherDisabled && supported),
   );
   // A new object for each refusal, so the same one twice is shown twice.
   const refusal$ = new BehaviorSubject<BackgroundImageRefusal | undefined>(
@@ -162,8 +164,10 @@ function buildDeviceBehaviors(
     ),
     selectVideoButtonOption$: constant(mediaDevices.videoInput.select),
     backgroundEffect$: scope.behavior(
-      backgroundEffectSetting.value$.pipe(
-        map((raw) => (supported ? serializeEffect(parseEffect(raw)) : "none")),
+      combineLatest([backgroundEffectSetting.value$, supported$]).pipe(
+        map(([raw, supported]) =>
+          supported ? serializeEffect(parseEffect(raw)) : "none",
+        ),
       ),
     ),
     selectBackgroundEffect$: scope.behavior(
@@ -191,7 +195,10 @@ function buildDeviceBehaviors(
       ),
     ),
     backgroundEffectSettling$: scope.behavior(
-      backgroundEffectSettling$.pipe(distinctUntilChanged()),
+      backgroundEffectsState$.pipe(
+        map(({ wait }) => wait !== undefined),
+        distinctUntilChanged(),
+      ),
       false,
     ),
     // Kept and offered, not put on: that waits for the user to choose it.
@@ -253,8 +260,8 @@ function buildDeviceBehaviors(
  * @param callModel - The root CallViewModel; provides layout, grid mode, reactions, etc.
  * @param muteStates - Audio and video mute state + toggles.
  * @param mediaDevices - Available and selected input devices.
- * @param backgroundEffectSettling$ - Whether the first effect chosen is still
- *   being prepared.
+ * @param backgroundEffectsState$ - The background pipeline's state: whether
+ *   effects run here, and the wait for the first.
  * @param reactionIdentifier - The local user's reaction identifier string, or
  *   undefined when reactions are not supported (hides the reaction button).
  * @param options - `showControls`: whether the call controls should be shown.
@@ -265,7 +272,7 @@ export function createCallFooterViewModel(
   callModel: CallViewModel,
   muteStates: MuteStates,
   mediaDevices: MediaDevices,
-  backgroundEffectSettling$: Observable<boolean>,
+  backgroundEffectsState$: Observable<BackgroundEffectsState>,
   reactionIdentifier: string | undefined,
   options: { showControls: boolean; header: HeaderStyle },
 ): ViewModel<FooterSnapshot> {
@@ -284,7 +291,7 @@ export function createCallFooterViewModel(
       scope,
       mediaDevices,
       disableDeviceSwitcher$,
-      backgroundEffectSettling$,
+      backgroundEffectsState$,
       callModel.camera$,
     ),
     // candidat to move into the FooterViewModel
@@ -348,8 +355,8 @@ export function createCallFooterViewModel(
  * @param scope - ObservableScope that bounds the lifetime of derived behaviors.
  * @param muteStates - Audio and video mute state + toggles.
  * @param mediaDevices - Available and selected input devices.
- * @param backgroundEffectSettling$ - Whether the first effect chosen is still
- *   being prepared.
+ * @param backgroundEffectsState$ - The background pipeline's state: whether
+ *   effects run here, and the wait for the first.
  * @param camera$ - The lobby's camera, for the camera menu's preview.
  * @param openSettings - Callback to open the settings modal, or undefined.
  * @param hangup - Callback to leave/cancel, or undefined (hides the button).
@@ -359,7 +366,7 @@ export function createLobbyFooterViewModel(
   scope: ObservableScope,
   muteStates: MuteStates,
   mediaDevices: MediaDevices,
-  backgroundEffectSettling$: Observable<boolean>,
+  backgroundEffectsState$: Observable<BackgroundEffectsState>,
   camera$: Behavior<LocalVideoMediaTrack | null>,
   openSettings: (() => void) | undefined,
   hangup: (() => void) | undefined,
@@ -408,7 +415,7 @@ export function createLobbyFooterViewModel(
       scope,
       mediaDevices,
       constant(false),
-      backgroundEffectSettling$,
+      backgroundEffectsState$,
       camera$,
     ),
   };

src/room/InCallView.tsx

@@ -75,7 +75,6 @@ import {
   type RTCSlotOptions,
   constant,
 } from "@element-hq/matrixrtc-sdk";
-import { map } from "rxjs";
 import {
   advancedCamera,
   advancedScreenShare,
@@ -181,7 +180,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
               props.muteStates,
               mediaDevices,
               capture,
-              backgroundEffectsState$.value.processor,
+              backgroundEffectsState$.value,
             ),
     });
 
@@ -195,9 +194,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
         ...callViewModelOptionsFromParams(urlParams),
         encryptionSystem: props.e2eeSystem,
         capture,
-        videoProcessor$: scope.behavior(
-          backgroundEffectsState$.pipe(map((state) => state.processor)),
-        ),
+        backgroundEffectsState$,
         hostBridge,
         autoLeaveWhenOthersLeft,
         waitForCallPickup: waitForCallPickup && sendNotificationType === "ring",
@@ -241,7 +238,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
       vm,
       props.muteStates,
       mediaDevices,
-      backgroundEffectsState$.pipe(map(({ settling }) => settling ?? false)),
+      backgroundEffectsState$,
       `${props.client.getUserId()}:${props.client.getDeviceId()}`,
       { showControls: urlParams.showControls, header: urlParams.header },
     );

src/room/LobbyView.tsx

@@ -16,7 +16,6 @@ import {
   type LocalMedia,
   type LocalVideoMediaTrack,
 } from "@element-hq/matrixrtc-sdk";
-import { map } from "rxjs";
 
 import inCallStyles from "./InCallView.module.css";
 import styles from "./LobbyView.module.css";
@@ -134,9 +133,7 @@ export const LobbyView: FC<Props> = ({
         muteStates,
         devices,
         captureSettings(),
-        mediaScope.behavior(
-          backgroundEffectsState$.pipe(map((state) => state.processor)),
-        ),
+        backgroundEffectsState$,
       ),
     );
     return (): void => {
@@ -165,7 +162,7 @@ export const LobbyView: FC<Props> = ({
         footerScope,
         muteStates,
         devices,
-        backgroundEffectsState$.pipe(map(({ settling }) => settling ?? false)),
+        backgroundEffectsState$,
         mediaVm.videoTrack$,
         openSettings,
         hangup,

src/state/BackgroundEffects.test.ts

@@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details.
 */
 
 import { beforeEach, describe, expect, it, vi } from "vitest";
-import { BehaviorSubject, distinctUntilChanged, map } from "rxjs";
+import { BehaviorSubject, distinctUntilChanged, map, of } from "rxjs";
 import { type BackgroundProcessorWrapper } from "@livekit/track-processors";
 
 import {
@@ -47,23 +47,28 @@ function fakePipeline(): {
   };
 }
 
-/** One letter per state: idle, waiting for a frame, attached. */
+/** One letter per state: idle, preparing, waiting for a frame, attached. */
 function letter(state: BackgroundEffectsState): string {
+  if (state.supported === false) return "x";
+  if (state.wait === "preparing") return "p";
   if (state.processor === undefined) return "i";
-  return state.settling ? "w" : "a";
+  return state.wait === "settling" ? "w" : "a";
 }
 
 describe("the pipeline's state", () => {
   function testState({
     effect,
+    answer = "-y",
     firstFrame = "",
     expected,
   }: {
     effect: string;
+    /** When the trial build answers, from being asked. */
+    answer?: string;
     firstFrame?: string;
     expected: string;
   }): void {
-    withTestScheduler(({ behavior, schedule, expectObservable }) => {
+    withTestScheduler(({ behavior, cold, schedule, expectObservable }) => {
       const transformer = {
         onFirstFrame: undefined as (() => void) | undefined,
       };
@@ -72,6 +77,7 @@ describe("the pipeline's state", () => {
         effect$: behavior(effect, { n: "none", b: "blur" }),
         setEffect: vi.fn(),
         added$: new BehaviorSubject<AddedBackground[] | undefined>([]),
+        canSegment: () => cold(answer, { y: true, n: false }),
         pipeline: fakePipeline().pipeline,
         transformer,
       });
@@ -84,15 +90,21 @@ describe("the pipeline's state", () => {
 
   it("defaults to no effect", () => testState({ effect: "n", expected: "i" }));
 
+  it("attaches only once it is known the pipeline builds", () =>
+    testState({ effect: "nb", answer: "--y", expected: "ip-w" }));
+
+  it("offers none where the pipeline fails to build", () =>
+    testState({ effect: "nb", answer: "-n", expected: "ipx" }));
+
   it("waits from the first effect until a frame carries it", () =>
     testState({
-      effect: "    nb-n-b",
-      firstFrame: "  --f",
-      expected: "  iwa",
+      effect: "    nb---n-b",
+      firstFrame: "---f",
+      expected: "  ipwa",
     }));
 
   it("attaches on first use and stays attached", () =>
-    testState({ effect: "nb-n", firstFrame: "--f", expected: "iwa" }));
+    testState({ effect: "nb--n", firstFrame: "---f", expected: "ipwa" }));
 });
 
 describe("background effects", () => {
@@ -109,6 +121,7 @@ describe("background effects", () => {
       effect$,
       setEffect,
       added$,
+      canSegment: () => of(true),
       pipeline: fake.pipeline,
       transformer: { onFirstFrame: undefined },
       ...options,
@@ -118,8 +131,6 @@ describe("background effects", () => {
     effect$.next(raw);
     await flushPromises();
   };
-  const blur = async (on: boolean): Promise<void> =>
-    choose(on ? "blur" : "none");
 
   beforeEach(() => {
     effect$ = new BehaviorSubject("none");
@@ -128,6 +139,17 @@ describe("background effects", () => {
     fake = fakePipeline();
   });
 
+  it("offers none for the session where the pipeline fails to build", async () => {
+    const effects = build({ canSegment: () => of(false) });
+    await choose("blur");
+
+    expect(effects.state$.value.supported).toBe(false);
+    expect(effects.state$.value.processor).toBeUndefined();
+    expect(effects.state$.value.wait).toBeUndefined();
+    // A browser update may fix it, so the next session asks again.
+    expect(effect$.value).toBe("blur");
+  });
+
   it("puts a shipped background on as that picture", async () => {
     build();
     await choose(`image:${shippedBackgrounds[0].id}`);
@@ -141,10 +163,10 @@ describe("background effects", () => {
 
   it("switches in place rather than reattaching", async () => {
     const effects = build();
-    await blur(true);
+    await choose("blur");
     const pipeline = effects.state$.value.processor;
-    await blur(false);
-    await blur(true);
+    await choose("none");
+    await choose("blur");
 
     expect(effects.state$.value.processor).toBe(pipeline);
     expect(fake.switches).toEqual([
@@ -157,18 +179,18 @@ describe("background effects", () => {
   it("keeps switching after a switch fails", async () => {
     vi.mocked(fake.pipeline.switchTo).mockRejectedValueOnce(new Error("lost"));
     build();
-    await blur(true);
-    await blur(false);
+    await choose("blur");
+    await choose("none");
     expect(fake.switches).toEqual([{ mode: "disabled" }]);
   });
 
   it("switches one at a time, skipping those overtaken", async () => {
     const finished = fake.holdNext();
     build();
-    await blur(true);
-    await blur(false);
-    await blur(true);
-    await blur(false);
+    await choose("blur");
+    await choose("none");
+    await choose("blur");
+    await choose("none");
     expect(fake.switches).toHaveLength(1);
 
     finished();

src/state/BackgroundEffects.ts

@@ -8,9 +8,15 @@ Please see LICENSE in the repository root for full details.
 import {
   combineLatest,
   distinctUntilChanged,
+  first,
+  from,
   map,
   Observable,
+  type ObservableInput,
+  of,
   scan,
+  startWith,
+  switchMap,
 } from "rxjs";
 import {
   type BackgroundProcessorWrapper,
@@ -38,8 +44,12 @@ const blurRadius = 15;
 export interface BackgroundEffectsState {
   supported: boolean | undefined;
   processor: undefined | VideoProcessor;
-  /** From the first effect chosen until a frame carrying it is drawn. */
-  settling?: boolean;
+  /**
+   * The effect is not ready:
+   * preparing, asking whether the pipeline can be built.
+   * settling, once it is attached until a frame carrying the effect
+   */
+  wait?: "preparing" | "settling";
 }
 
 export interface BackgroundEffectsOptions {
@@ -51,6 +61,8 @@ export interface BackgroundEffectsOptions {
   setEffect: (id: EffectId) => void;
   /** The backgrounds the device keeps, undefined until it has said. */
   added$: Behavior<AddedBackground[] | undefined>;
+  /** Builds a segmenter on trial, and answers whether it could. */
+  canSegment: () => ObservableInput<boolean>;
   /** The background processor pipeline to be switched between effects. */
   pipeline: BackgroundProcessorWrapper;
   /** Tells, once, that a frame carrying an effect has been drawn. */
@@ -70,6 +82,7 @@ export function createBackgroundEffects(
     effect$,
     setEffect,
     added$,
+    canSegment,
     pipeline,
     transformer,
   }: BackgroundEffectsOptions,
@@ -77,9 +90,11 @@ export function createBackgroundEffects(
   // Wrapped once, so the call and the preview see one processor throughout.
   const processor = fromLivekitProcessor(pipeline);
   const choice$ = effect$.pipe(map(parseEffect));
-  const wanted$ = choice$.pipe(
-    map((effect) => effect.kind !== "none"),
-    distinctUntilChanged(),
+  const wanted$ = scope.behavior(
+    choice$.pipe(
+      map((effect) => effect.kind !== "none"),
+      distinctUntilChanged(),
+    ),
   );
 
   combineLatest([choice$, added$])
@@ -95,6 +110,18 @@ export function createBackgroundEffects(
         setEffect("none");
     });
 
+  // Asked once an effect is first wanted. A browser update may fix what
+  // fails, so the choice is kept for the next session rather than cleared.
+  const buildable$ = scope.behavior<boolean | undefined>(
+    supported
+      ? wanted$.pipe(
+          first(Boolean),
+          switchMap(() => from(canSegment())),
+          startWith(undefined),
+        )
+      : of(undefined),
+  );
+
   const drewAFrame$ = scope.behavior(
     new Observable<boolean>((subscriber) => {
       subscriber.next(false);
@@ -105,18 +132,25 @@ export function createBackgroundEffects(
     }),
   );
 
+  // In one step, so that nothing sees the pipeline neither preparing nor
+  // attached.
   const state$ = scope.behavior(
-    combineLatest([wanted$, drewAFrame$]).pipe(
-      scan<[boolean, boolean], BackgroundEffectsState>(
-        (previous, [wanted, drewAFrame]) => {
+    combineLatest([wanted$, buildable$, drewAFrame$]).pipe(
+      scan<[boolean, boolean | undefined, boolean], BackgroundEffectsState>(
+        (previous, [wanted, buildable, drewAFrame]) => {
           // Attached the first time an effect is wanted and never detached
           // after, so someone who never turns one on pays for none of it.
           const enable =
-            previous.processor !== undefined || (supported && wanted);
+            previous.processor !== undefined || (wanted && buildable === true);
+          const preparing = supported && wanted && buildable === undefined;
           return {
-            supported,
+            supported: supported && buildable !== false,
             processor: enable ? processor : undefined,
-            settling: enable && !drewAFrame,
+            wait: preparing
+              ? "preparing"
+              : enable && !drewAFrame
+                ? "settling"
+                : undefined,
           };
         },
         { supported, processor: undefined },

src/state/CallViewModel/CallViewModel.test.ts

@@ -70,9 +70,11 @@ import {
 } from "./CallViewModel.ts";
 import { type Handler, MuteState } from "../MuteStates.ts";
 import {
+  type LocalVideoMediaTrack,
   type RTCParticipation,
   type VideoProcessor,
 } from "@element-hq/matrixrtc-sdk";
+import { type BackgroundEffectsState } from "../BackgroundEffects";
 
 initializeWidget();
 
@@ -1692,28 +1694,113 @@ describe("a camera with a video processor", () => {
         muteStates,
         mockMediaDevices({}),
         {},
-        processor,
+        { supported: true, processor },
       ).find((request) => request.source === "camera"),
     ).toMatchObject({ processor });
   });
 
+  it("leaves the camera out of the join while the processor is prepared", () => {
+    const muteStates = mockMuteStates();
+    vi.spyOn(muteStates.video.enabled$, "value", "get").mockReturnValue(true);
+    const sources = (wait?: "preparing"): string[] =>
+      initialPublishRequests(
+        muteStates,
+        mockMediaDevices({}),
+        {},
+        {
+          supported: true,
+          processor: undefined,
+          wait,
+        },
+      ).map((request) => request.source);
+    expect(sources("preparing")).not.toContain("camera");
+    expect(sources()).toContain("camera");
+  });
+
+  it("waits to make a camera track until it is known the processor builds", async () => {
+    const setHandler = vi.spyOn(MuteState.prototype, "setHandler");
+    onTestFinished(() => setHandler.mockRestore());
+    const effects$ = new BehaviorSubject<BackgroundEffectsState>({
+      supported: true,
+      processor: undefined,
+      wait: "preparing",
+    });
+    let participation: RTCParticipation | undefined;
+    withCallViewModel(
+      {},
+      (_vm, _session, { rtcParticipation }) => {
+        participation = rtcParticipation;
+      },
+      { backgroundEffectsState$: effects$ },
+    );
+    const camera = participation!.localMember$.value?.tracks$.value?.find(
+      (track) => track.source === "camera",
+    ) as LocalVideoMediaTrack;
+    const turnedOn = cameraHandler(setHandler)(true);
+    await flushPromises();
+    expect(camera.setEnabled).not.toHaveBeenCalledWith(true);
+
+    effects$.next({ supported: true, processor });
+    await turnedOn;
+    expect(camera.setEnabled).toHaveBeenCalledWith(true);
+    expect(participation!.publish).not.toHaveBeenCalled();
+  });
+
+  it("turns the camera on only once its processor is on the track", async () => {
+    const setHandler = vi.spyOn(MuteState.prototype, "setHandler");
+    onTestFinished(() => setHandler.mockRestore());
+    const effects$ = new BehaviorSubject<BackgroundEffectsState>({
+      supported: true,
+      processor: undefined,
+      wait: "preparing",
+    });
+    let participation: RTCParticipation | undefined;
+    withCallViewModel(
+      {},
+      (_vm, _session, { rtcParticipation }) => {
+        participation = rtcParticipation;
+      },
+      { backgroundEffectsState$: effects$ },
+    );
+    const camera = participation!.localMember$.value?.tracks$.value?.find(
+      (track) => track.source === "camera",
+    ) as LocalVideoMediaTrack;
+    // Attaching takes its time, as the pipeline is built on the track
+    let attached!: () => void;
+    vi.mocked(camera.setProcessor).mockReturnValueOnce(
+      new Promise<void>((resolve) => {
+        attached = resolve;
+      }),
+    );
+    const turnedOn = cameraHandler(setHandler)(true);
+    effects$.next({ supported: true, processor });
+    await flushPromises();
+    expect(camera.setProcessor).toHaveBeenCalledWith(processor);
+    expect(camera.setEnabled).not.toHaveBeenCalledWith(true);
+
+    attached();
+    await turnedOn;
+    expect(camera.setEnabled).toHaveBeenCalledWith(true);
+  });
+
   // The SDK builds the processor on the camera's restart, before the room
   // sees a frame, so the track is unmuted rather than published anew
   it("turns on a camera turned off before the processor was chosen", async () => {
     const setHandler = vi.spyOn(MuteState.prototype, "setHandler");
     onTestFinished(() => setHandler.mockRestore());
-    const videoProcessor$ = new BehaviorSubject<VideoProcessor | undefined>(
-      undefined,
-    );
+    const effects$ = new BehaviorSubject<BackgroundEffectsState>({
+      supported: true,
+      processor: undefined,
+    });
     let participation: RTCParticipation | undefined;
     withCallViewModel(
       {},
       (_vm, _session, { rtcParticipation }) => {
         participation = rtcParticipation;
       },
-      { videoProcessor$ },
+      { backgroundEffectsState$: effects$ },
     );
-    videoProcessor$.next(processor);
+    effects$.next({ supported: true, processor });
     await cameraHandler(setHandler)(true);
     await flushPromises();
 

src/state/CallViewModel/CallViewModel.ts

@@ -37,6 +37,7 @@ import {
   combineLatest,
   distinctUntilChanged,
   filter,
+  firstValueFrom,
   fromEvent,
   map,
   merge,
@@ -77,6 +78,7 @@ import {
   type ReactionInfo,
   type ReactionOption,
 } from "../../reactions";
+import { type BackgroundEffectsState } from "../BackgroundEffects";
 import { shallowEquals as arrayShallowEquals } from "../../utils/array";
 import { type MediaDevices } from "../MediaDevices";
 import { type MuteStates } from "../MuteStates";
@@ -170,13 +172,16 @@ export function initialPublishRequests(
   muteStates: MuteStates,
   mediaDevices: MediaDevices,
   capture: CaptureSettings,
-  processor?: VideoProcessor,
+  effects?: BackgroundEffectsState,
 ): PublishRequest[] {
   const requests: PublishRequest[] = [];
   if (muteStates.audio.enabled$.value)
     requests.push(publishRequest("microphone", mediaDevices, capture));
-  if (muteStates.video.enabled$.value)
-    requests.push(publishRequest("camera", mediaDevices, capture, processor));
+  // Left out while the processor is prepared; the call publishes it after
+  if (muteStates.video.enabled$.value && effects?.wait !== "preparing")
+    requests.push(
+      publishRequest("camera", mediaDevices, capture, effects?.processor),
+    );
   return requests;
 }
 
@@ -187,8 +192,8 @@ export interface CallViewModelOptions {
    * defaults otherwise.
    */
   capture?: CaptureSettings;
-  /** Background blur and the like, applied to the camera as it changes. */
-  videoProcessor$?: Behavior<VideoProcessor | undefined>;
+  /** The background effects state (preparing, supported...) including the processor itself. */
+  backgroundEffectsState$?: Behavior<BackgroundEffectsState>;
   /**
    * The application hosting Element Call, which can ask it to hang up and wants
    * to know when the user joins or leaves. Defaults to no host.
@@ -490,7 +495,21 @@ export function createCallViewModel$(
     sentCallNotification$ = of(null),
   } = options;
 
-  const { capture = {}, videoProcessor$ = constant(undefined) } = options;
+  const {
+    capture = {},
+    backgroundEffectsState$ = constant<BackgroundEffectsState>({
+      supported: false,
+      processor: undefined,
+    }),
+  } = options;
+  const videoProcessor$ = scope.behavior(
+    backgroundEffectsState$.pipe(map(({ processor }) => processor)),
+  );
+  // Resolves once it is known whether the processor builds
+  const processorKnown = async (): Promise<unknown> =>
+    firstValueFrom(
+      backgroundEffectsState$.pipe(filter(({ wait }) => wait !== "preparing")),
+    );
   const localTracks$ = scope.behavior<
     (LocalAudioMediaTrack | LocalVideoMediaTrack)[]
   >(
@@ -514,6 +533,19 @@ export function createCallViewModel$(
     "screenShare",
   ) as Behavior<LocalVideoMediaTrack | undefined>;
 
+  const processorAttached = followProcessor(
+    scope,
+    camera$,
+    videoProcessor$,
+    logger,
+  );
+
+  // A camera turned on waits for its processor to be known, then to be on
+  // the track: LiveKit restarts the track with whatever processor it has then.
+  const cameraReady = async (): Promise<void> => {
+    await processorKnown();
+    await processorAttached();
+  };
   const supply = (source: "microphone" | "camera"): TrackSupply => ({
     publish: async () =>
       rtcParticipation.publish(
@@ -524,6 +556,7 @@ export function createCallViewModel$(
           source === "camera" ? videoProcessor$.value : undefined,
         ),
       ),
+    ready: source === "camera" ? cameraReady : undefined,
   });
   followMuteState(
     scope,
@@ -550,6 +583,29 @@ export function createCallViewModel$(
   );
   followDevice(scope, camera$, mediaDevices.videoInput, "camera", logger);
 
+  // A camera wanted at the join was left out of it while the processor was
+  // prepared (see initialPublishRequests), and is published once that is known
+  if (
+    backgroundEffectsState$.value.wait === "preparing" &&
+    muteStates.video.enabled$.value
+  )
+    processorKnown()
+      .then(async () => {
+        if (camera$.value !== undefined || !muteStates.video.enabled$.value)
+          return;
+        await rtcParticipation.publish(
+          publishRequest(
+            "camera",
+            mediaDevices,
+            capture,
+            videoProcessor$.value,
+          ),
+        );
+      })
+      .catch((e) => {
+        logger.error("Failed to publish the camera", e);
+      });
+
   // The output device follows the selection onto the client the same way. A
   // host that routes audio itself picks the output outside the browser, so the
   // client is left alone
@@ -563,8 +619,6 @@ export function createCallViewModel$(
         });
       });
 
-  followProcessor(scope, camera$, videoProcessor$, logger);
-
   const members$ = scope.behavior<(LocalRTCMember | RemoteRTCMember)[]>(
     combineLatest(
       [rtcParticipation.localMember$, rtcParticipation.remoteMembers$],

src/state/LobbyMediaViewModel.test.ts

@@ -21,6 +21,7 @@ import { type MediaDevices } from "./MediaDevices";
 import { MuteStates } from "./MuteStates";
 import { nullHostBridge } from "../HostBridge";
 import { flushPromises, mockMediaDevices, testScope } from "../utils/test";
+import { type BackgroundEffectsState } from "./BackgroundEffects";
 
 describe("LobbyMediaViewModel", () => {
   it("opens what starts enabled in one capture and follows the devices", async () => {
@@ -135,7 +136,7 @@ function setup(
     muteStates,
     mediaDevices,
     {},
-    constant(processor),
+    constant<BackgroundEffectsState>({ supported: true, processor }),
   );
   return { media, vm, muteStates, mediaDevices };
 }

src/state/LobbyMediaViewModel.ts

@@ -13,7 +13,6 @@ import {
   type ObservableScope,
   type PublishRequest,
   trackBySource$,
-  type VideoProcessor,
 } from "@element-hq/matrixrtc-sdk";
 import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
 import { combineLatest, debounceTime, map } from "rxjs";
@@ -25,6 +24,7 @@ import {
 import { type MediaDevices } from "./MediaDevices";
 import { type MuteStates } from "./MuteStates";
 import { followDevice, followProcessor } from "./localMediaControls";
+import { type BackgroundEffectsState } from "./BackgroundEffects";
 
 export interface LobbyMediaViewModel {
   /** The camera, once open, for the preview. */
@@ -42,9 +42,13 @@ export function createLobbyMediaViewModel(
   muteStates: MuteStates,
   mediaDevices: MediaDevices,
   capture: CaptureSettings,
-  videoProcessor$: Behavior<VideoProcessor | undefined>,
+  /** No camera is opened while its pipeline is prepared: it would show the room without the processor. */
+  backgroundEffectsState$: Behavior<BackgroundEffectsState>,
 ): LobbyMediaViewModel {
   const logger = rootLogger.getChild("[LobbyMedia]");
+  const videoProcessor$ = scope.behavior(
+    backgroundEffectsState$.pipe(map(({ processor }) => processor)),
+  );
   const microphone$ = trackBySource$(
     scope,
     media.tracks$,
@@ -64,13 +68,19 @@ export function createLobbyMediaViewModel(
     muteStates.video.enabled$,
     microphone$,
     camera$,
+    backgroundEffectsState$,
   ])
     .pipe(debounceTime(0), scope.bind())
-    .subscribe(([audio, video, microphone, camera]) => {
+    .subscribe(([audio, video, microphone, camera, { wait }]) => {
       const open: PublishRequest[] = [];
       if (audio && microphone === undefined && !opening.has("microphone"))
         open.push(publishRequest("microphone", mediaDevices, capture));
-      if (video && camera === undefined && !opening.has("camera"))
+      if (
+        video &&
+        camera === undefined &&
+        !opening.has("camera") &&
+        wait !== "preparing"
+      )
         open.push(
           publishRequest(
             "camera",

src/state/localMediaControls.ts

@@ -21,6 +21,8 @@ import { type Handler } from "./MuteStates";
 /** How a mute switch gets a track where there is none. */
 export interface TrackSupply {
   publish: () => Promise<unknown>;
+  /** Awaited before a switch turned on acts: a camera waits for its processor to be known. */
+  ready?: () => Promise<unknown>;
 }
 
 /**
@@ -45,6 +47,7 @@ export function followMuteState(
   let desired = muteState.enabled$.value;
   muteState.setHandler(async (enabled) => {
     desired = enabled;
+    if (enabled) await supply.ready?.();
     const track = track$.value;
     if (track !== undefined) {
       desired = await track.setEnabled(enabled);
@@ -94,18 +97,23 @@ export function followDevice(
 /**
  * Attached once the camera track exists rather than carried in the publish
  * request, so that a new track and a change of processor take the same path.
+ * Answers with a wait for the latest attachment to have gone through: a
+ * camera turned on before then would restart without its processor.
  */
 export function followProcessor(
   scope: ObservableScope,
   camera$: Behavior<LocalVideoMediaTrack | undefined>,
   videoProcessor$: Behavior<VideoProcessor | undefined>,
   logger: Logger,
-): void {
+): () => Promise<void> {
+  let attached = Promise.resolve();
   combineLatest([camera$, videoProcessor$])
     .pipe(scope.bind())
     .subscribe(([camera, processor]) => {
-      camera?.setProcessor(processor).catch((e) => {
-        logger.error("Failed to set the video processor", e);
-      });
+      attached =
+        camera?.setProcessor(processor).catch((e) => {
+          logger.error("Failed to set the video processor", e);
+        }) ?? Promise.resolve();
     });
+  return async () => attached;
 }

src/tracks/BackgroundEffectTransformer.test.ts

@@ -13,13 +13,19 @@ import {
 } from "@livekit/track-processors";
 import { ImageSegmenter } from "@mediapipe/tasks-vision";
 
-import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer";
+import {
+  BackgroundEffectTransformer,
+  canSegment,
+} from "./BackgroundEffectTransformer";
 
 const frame = {} as VideoFrame;
 const controller = {} as TransformStreamDefaultController<VideoFrame>;
 
 describe("BackgroundEffectTransformer", () => {
-  afterEach(() => vi.restoreAllMocks());
+  afterEach(() => {
+    vi.restoreAllMocks();
+    vi.unstubAllGlobals();
+  });
 
   it("loads a background chosen before it was built", async () => {
     vi.spyOn(VideoTransformer.prototype, "init").mockResolvedValue();
@@ -78,4 +84,33 @@ describe("BackgroundEffectTransformer", () => {
     expect(transformer.options.blurRadius).toBeUndefined();
     expect(transformer.options.imagePath).toBeUndefined();
   });
+
+  it("segments on the CPU where the GPU refuses", async () => {
+    vi.spyOn(VideoTransformer.prototype, "init").mockResolvedValue();
+    const build = vi
+      .spyOn(ImageSegmenter, "createFromOptions")
+      .mockRejectedValueOnce(new Error("blocklisted"))
+      .mockResolvedValue({} as ImageSegmenter);
+
+    await new BackgroundEffectTransformer({}).init(
+      {} as VideoTransformerInitOptions,
+    );
+    expect(build.mock.calls.map(([, o]) => o.baseOptions?.delegate)).toEqual([
+      "GPU",
+      "CPU",
+    ]);
+  });
+
+  it("says whether a segmenter can be built at all", async () => {
+    vi.stubGlobal("OffscreenCanvas", class {});
+    const close = vi.fn();
+    const build = vi
+      .spyOn(ImageSegmenter, "createFromOptions")
+      .mockResolvedValue({ close } as unknown as ImageSegmenter);
+    expect(await canSegment()).toBe(true);
+    expect(close).toHaveBeenCalled();
+
+    build.mockRejectedValue(new Error("no model"));
+    expect(await canSegment()).toBe(false);
+  });
 });

src/tracks/BackgroundEffectTransformer.ts

@@ -11,6 +11,7 @@ import {
   VideoTransformer,
   type VideoTransformerInitOptions,
   type BackgroundOptions,
+  type SegmenterOptions,
 } from "@livekit/track-processors";
 import { logger } from "matrix-js-sdk/lib/logger";
 import { ImageSegmenter } from "@mediapipe/tasks-vision";
@@ -92,17 +93,10 @@ export class BackgroundEffectTransformer extends BackgroundTransformer {
       inputElement: inputVideo,
     });
 
-    this.imageSegmenter = await ImageSegmenter.createFromOptions(wasmFileset, {
-      baseOptions: {
-        modelAssetPath,
-        delegate: "GPU",
-        ...this.options.segmenterOptions,
-      },
-      canvas: this.canvas,
-      runningMode: "VIDEO",
-      outputCategoryMask: true,
-      outputConfidenceMasks: false,
-    });
+    this.imageSegmenter = await createSegmenter(
+      this.canvas,
+      this.options.segmenterOptions,
+    );
 
     // BackgroundTransformer's own init applies these, and this one replaces it.
     if (this.options.imagePath) {
@@ -116,3 +110,38 @@ export class BackgroundEffectTransformer extends BackgroundTransformer {
     this.gl?.setBackgroundDisabled(this.options.backgroundDisabled ?? false);
   }
 }
+
+/**
+ * Whether a segmenter can be built here, asked once before any camera depends
+ * on one: a pipeline that fails to build takes the camera down with it.
+ */
+export async function canSegment(): Promise<boolean> {
+  try {
+    (await createSegmenter(new OffscreenCanvas(1, 1))).close();
+    return true;
+  } catch (e) {
+    logger.warn("Background effects cannot run here", e);
+    return false;
+  }
+}
+
+/** On the GPU, or on the CPU where the GPU refuses, as a blocklisted one does. */
+async function createSegmenter(
+  canvas: HTMLCanvasElement | OffscreenCanvas | undefined,
+  segmenterOptions?: SegmenterOptions,
+): Promise<ImageSegmenter> {
+  const build = async (delegate: "GPU" | "CPU"): Promise<ImageSegmenter> =>
+    ImageSegmenter.createFromOptions(wasmFileset, {
+      baseOptions: { modelAssetPath, delegate, ...segmenterOptions },
+      canvas,
+      runningMode: "VIDEO",
+      outputCategoryMask: true,
+      outputConfidenceMasks: false,
+    });
+  try {
+    return await build("GPU");
+  } catch (e) {
+    logger.warn("Segmenting on the CPU, as the GPU refused", e);
+    return build("CPU");
+  }
+}

src/tracks/BackgroundEffectsContext.test.ts

@@ -56,7 +56,8 @@ vi.mock("./backgroundImages", async () => {
   };
 });
 vi.mock("./BackgroundEffectTransformer", () => ({
-  BackgroundEffectTransformer: vi.fn(),
+  BackgroundEffectTransformer: vi.fn(function () {}),
+  canSegment: async (): Promise<boolean> => Promise.resolve(true),
 }));
 
 describe("BackgroundEffectsProvider", () => {

src/tracks/BackgroundEffectsContext.tsx

@@ -12,7 +12,10 @@ import {
 import { createContext, type FC, type JSX, use } from "react";
 
 import { backgroundEffect as backgroundEffectSetting } from "../settings/settings";
-import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer";
+import {
+  BackgroundEffectTransformer,
+  canSegment,
+} from "./BackgroundEffectTransformer";
 import { addedBackgrounds } from "./backgroundImages";
 import { type Behavior } from "../state/Behavior";
 import { type ObservableScope } from "../state/ObservableScope";
@@ -56,6 +59,7 @@ export function createAppBackgroundEffects(
     effect$: backgroundEffectSetting.value$,
     setEffect: backgroundEffectSetting.setValue,
     added$: addedBackgrounds.added$,
+    canSegment,
     pipeline: new BackgroundProcessorWrapper(transformer, "background-effect"),
     transformer,
   });

src/utils/test-viewmodel.ts

@@ -217,7 +217,7 @@ export function getBasicCallViewModelEnvironment(
     vm,
     muteStates,
     mediaDevices,
-    constant(false),
+    constant({ supported: false, processor: undefined }),
     "reactionId",
     { showControls: true, header: HeaderStyle.Standard },
   );