Offer effects wherever they run, and say what they cost

- Drops the desktop test. Measured, it sorted devices by the wrong thing:
  a phone on the fast path held 99% of its frame rate and was refused, a
  desktop on the slow one held 73% and was allowed.
- What the cost tracks is the processing path, and Safari and Firefox take
  the slow one on every platform. That is told, not enforced: the cost is
  smoothness, the reason to pay it is privacy, and only the user knows
  whether they would rather show the room they are sitting in.
- The first effect of a session takes a while to prepare, so its own tile
  spins where its tick will go, until a frame has actually come out of the
  pipeline. The promises say nothing useful about when that is: attaching
  resolves in about three seconds and switching in none at all, and the
  slow path then holds the page still for twelve to fifteen more. The
  spinner turns by rotating, which a browser composites, so it keeps moving
  through a pause that has stopped everything else.
- Copy is the exploration's and needs an owner (S3).

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

locales/en/app.json

@@ -27,6 +27,9 @@
     "upload_file": "Upload file"
   },
   "analytics_notice": "By participating in this beta, you consent to the collection of anonymous data, which we use to improve the product. You can find more information about which data we track in our <2>Privacy Policy</2> and our <6>Cookie Policy</6>.",
+  "background_effects": {
+    "slow_in_this_browser": "This browser runs background effects slowly, so other people may see your video stutter."
+  },
   "call_ended_view": {
     "create_account_button": "Create account",
     "create_account_prompt": "<0>Why not finish by setting up a password to keep your account?</0><1>You'll be able to keep your name and set an avatar for use on future calls</1>",

src/components/CallFooter.tsx

@@ -45,7 +45,11 @@ import {
   maxAddedBackgrounds,
   UnusableImage,
 } from "../livekit/backgroundImages";
-import { useAddedBackgrounds } from "../livekit/TrackProcessorContext";
+import {
+  useAddedBackgrounds,
+  useBackgroundProcessing,
+} from "../livekit/TrackProcessorContext";
+import { usesFallbackProcessing } from "../livekit/backgroundProcessing";
 import { type Behavior } from "../state/Behavior";
 import { type ViewModel } from "../state/ViewModel";
 import { useBehavior } from "../useBehavior";
@@ -186,6 +190,20 @@ export const CallFooter: FC<FooterProps> = ({
   const selectBackgroundEffect = useBehavior(vm.selectBackgroundEffect$);
   const beforeJoining = useBehavior(vm.beforeJoining$);
   const { added, addBackground, removeBackground } = useAddedBackgrounds();
+  const { settling } = useBackgroundProcessing();
+
+  // Said, not decided. Where only the slow path exists the effect still works,
+  // it costs frames — and the user is the only one who knows whether they would
+  // rather show the room they are sitting in. It stays put while the pipeline
+  // is being built: that wait is shown on the tile that was pressed, where a
+  // message that came and went would only flash once the assets are cached.
+  const backgroundEffectNotice = useMemo(
+    () =>
+      usesFallbackProcessing()
+        ? t("background_effects.slow_in_this_browser")
+        : undefined,
+    [t],
+  );
   const [backgroundEffectError, setBackgroundEffectError] = useState<
     string | undefined
   >(undefined);
@@ -343,6 +361,8 @@ export const CallFooter: FC<FooterProps> = ({
         }
         onRemoveBackgroundEffect={onRemoveBackgroundEffect}
         backgroundEffectError={backgroundEffectError}
+        backgroundEffectNotice={backgroundEffectNotice}
+        backgroundEffectSettling={settling}
       />,
     );
   } else {

src/components/CallFooterViewModel.test.ts

@@ -252,12 +252,16 @@ describe("createCallFooterViewModel", () => {
       );
     }
 
-    it("offers nothing the pipeline would refuse to honour", () => {
+    // A phone is offered them, and that is the point of asking the browser
+    // rather than the platform: measured, a phone on the fast path held 99% of
+    // its frame rate while a desktop on the slow one held 73%, so refusing by
+    // platform refused the better device and allowed the worse.
+    it("offers them on a phone whose browser can run them", () => {
       sdkSupportMock.mockReturnValue(true);
       const vm = lobbyFor("ios");
 
-      expect(vm.selectBackgroundEffect$.value).toBeUndefined();
-      expect(vm.toggleBlur$.value).toBeUndefined();
+      expect(vm.selectBackgroundEffect$.value).toBeDefined();
+      expect(vm.toggleBlur$.value).toBeDefined();
     });
 
     it("offers them where the pipeline will honour them", () => {

src/components/MediaMuteAndSwitchButton.module.css

@@ -387,3 +387,13 @@ Please see LICENSE in the repository root for full details.
      with a gap of its own, which makes up the difference above. */
   padding: var(--cpd-space-3x) var(--cpd-space-4x) 0;
 }
+
+/* The spinner stands exactly where the tick will, so the tile does not shift
+   when one becomes the other. */
+.effectBusy {
+  display: inline-flex;
+  align-items: center;
+  justify-content: center;
+  inline-size: 20px;
+  block-size: 20px;
+}

src/components/MediaMuteAndSwitchButton.stories.tsx

@@ -887,6 +887,133 @@ export const BackgroundImageChosen: Story = {
   },
 };
 
+/**
+ * Where the browser runs effects the slow way. They are offered and they work
+ * — the cost is smoothness, and the reason to pay it is privacy, so the user
+ * is told rather than refused.
+ */
+export const BackgroundEffectsSlowInThisBrowser: Story = {
+  args: {
+    ...BackgroundEffects.args,
+    backgroundEffectNotice:
+      "This browser runs background effects slowly, so other people may see your video stutter.",
+  },
+  play: async ({ canvasElement }) => {
+    const canvas = within(canvasElement);
+    await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
+    const body = within(document.body);
+
+    // Said, not enforced: everything is still choosable.
+    const blur = await body.findByRole("menuitemradio", { name: "Blur" });
+    await expect(blur).not.toHaveAttribute("aria-disabled", "true");
+    await expect(
+      await body.findByText(/runs background effects slowly/),
+    ).toBeInTheDocument();
+  },
+};
+
+/**
+ * The sequence a user sees on the first effect of a session.
+ *
+ * The tile that was pressed spins where its tick will go, until a frame has
+ * actually come out of the pipeline: about three seconds of building, and on
+ * Safari another twelve to fifteen while the page holds still. The spinner
+ * turns by rotating, which a browser composites, so it keeps moving through a
+ * pause that has stopped everything else.
+ *
+ * On the tile rather than in a message, because once the assets are cached the
+ * wait is a fraction of a second, and a message that appears and vanishes that
+ * fast is noise. A spinner that brief just looks like a control responding.
+ *
+ * Shortened here to a second and a half. Later effects skip it entirely:
+ * switching a built pipeline costs nothing.
+ */
+export const BackgroundEffectsSettling: Story = {
+  args: {
+    ...BackgroundEffects.args,
+    backgroundEffectNotice:
+      "This browser runs background effects slowly, so other people may see your video stutter.",
+  },
+  render: function Settling(args): JSX.Element {
+    const [settling, setSettling] = useState(false);
+    const [built, setBuilt] = useState(false);
+    const [selected, setSelected] = useState(args.selectedBackgroundEffect);
+    return (
+      <MediaMuteAndSwitchButton
+        {...args}
+        selectedBackgroundEffect={selected}
+        backgroundEffectSettling={settling}
+        onSelectBackgroundEffect={(id): void => {
+          args.onSelectBackgroundEffect?.(id);
+          setSelected(id);
+          if (built || id === "none") return;
+          setSettling(true);
+          setBuilt(true);
+          window.setTimeout(() => setSettling(false), 1500);
+        }}
+      />
+    );
+  },
+  play: async ({ canvasElement }) => {
+    const canvas = within(canvasElement);
+    await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
+    const body = within(document.body);
+
+    // What the browser costs stays put throughout — it is true before, during
+    // and after.
+    await expect(
+      await body.findByText(/runs background effects slowly/),
+    ).toBeInTheDocument();
+
+    const blur = await body.findByRole("menuitemradio", { name: "Blur" });
+    await userEvent.click(blur);
+
+    // The pressed tile is busy, and has no tick yet.
+    await waitFor(async () =>
+      expect(blur.querySelector(`.${styles.effectBusy}`)).toBeInTheDocument(),
+    );
+    await expect(
+      await body.findByText(/runs background effects slowly/),
+    ).toBeInTheDocument();
+
+    // Then the tick, once a frame has come out.
+    await waitFor(
+      async () =>
+        expect(
+          blur.querySelector(`.${styles.effectBusy}`),
+        ).not.toBeInTheDocument(),
+      { timeout: 5000 },
+    );
+    await expect(blur).toHaveAttribute("aria-checked", "true");
+  },
+};
+
+/**
+ * A refusal and a notice at once. The refusal is about what the user just did
+ * and takes the space; the notice is about the browser and will still be true
+ * the next time they open the menu.
+ */
+export const BackgroundImageRefusedOverANotice: Story = {
+  args: {
+    ...BackgroundEffects.args,
+    backgroundEffectNotice:
+      "This browser runs background effects slowly, so other people may see your video stutter.",
+    backgroundEffectError: "That file is not an image.",
+  },
+  play: async ({ canvasElement }) => {
+    const canvas = within(canvasElement);
+    await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
+    const body = within(document.body);
+
+    await expect(
+      await body.findByText("That file is not an image."),
+    ).toBeInTheDocument();
+    await expect(
+      body.queryByText(/runs background effects slowly/),
+    ).not.toBeInTheDocument();
+  },
+};
+
 /**
  * Where the browser or device cannot run background processing. The section
  * keeps its shape and its tiles, and none of them can be chosen.

src/components/MediaMuteAndSwitchButton.tsx

@@ -19,6 +19,7 @@ import {
   Button,
   Menu,
   MenuItem,
+  InlineSpinner,
   MenuTitle,
   RadioInput,
   Tooltip,
@@ -127,6 +128,20 @@ export interface MediaMuteAndSwitchButtonProps {
    * Shown with the grid, where they chose it.
    */
   backgroundEffectError?: string;
+  /**
+   * What the user should know before choosing, if anything — that effects run
+   * slowly in this browser, say. Told, not decided for them: the cost is
+   * smoothness and the reason to pay it is privacy, and only they know which
+   * they would rather have.
+   */
+  backgroundEffectNotice?: string;
+  /**
+   * Whether the pipeline is still being built, which happens once a session on
+   * the first effect chosen. Shown on that effect's own tile, in place of its
+   * tick, rather than as a message: it belongs to what the user just pressed,
+   * and where the wait is short a message would only flash.
+   */
+  backgroundEffectSettling?: boolean;
   /**
    * For any toggle and option this method will be called.
    * So toggles need to be implemented by listening here and setting the right toggle item to `enabled`
@@ -180,6 +195,8 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
   onAddBackgroundImage,
   onRemoveBackgroundEffect,
   backgroundEffectError,
+  backgroundEffectNotice,
+  backgroundEffectSettling,
   onSelect,
 }) => {
   // Which device we have asked for but not yet been given. Carries the kind as
@@ -542,7 +559,21 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
                   alt=""
                 />
               )}
-              {selectedBackgroundEffect === effect.id ? (
+              {selectedBackgroundEffect === effect.id &&
+              backgroundEffectSettling ? (
+                // Before the tick, on the tile that was pressed: the feedback
+                // belongs to the thing the user acted on. It spins by rotating,
+                // which a browser composites, so it keeps turning through the
+                // pause where the page itself has stopped — the difference
+                // between looking busy and looking crashed.
+                <span
+                  className={classNames(styles.effectCheck, styles.effectBusy, {
+                    [styles.effectCheckOnImage]: effect.kind === "image",
+                  })}
+                >
+                  <InlineSpinner size={20} />
+                </span>
+              ) : selectedBackgroundEffect === effect.id ? (
                 // The tick stands where the glyph would, and on a picture it
                 // carries its own ground so it reads against whatever is behind
                 // it.
@@ -779,6 +810,16 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
               />
             </div>
           )}
+        {backgroundEffectError === undefined &&
+          backgroundEffectNotice !== undefined && (
+            // Never both at once: a refusal is about what the user just did and
+            // wins the space, where this is about the browser and will still be
+            // true next time. Not dismissible for the same reason — there is
+            // nothing to dismiss, only something to know.
+            <div role="none" className={styles.effectError}>
+              <Alert type="info" title={backgroundEffectNotice} />
+            </div>
+          )}
         {toggles.length > 0 && <hr />}
         {toggles.map((toggle) => (
           <ToggleMenuItem

src/livekit/BackgroundEffectTransformer.ts

@@ -54,6 +54,29 @@ const wasmFileset: WasmFileset = {
  * only the SIMD ones.
  */
 export class BackgroundEffectTransformer extends BackgroundTransformer {
+  /**
+   * Called once, when a frame has actually come out of the pipeline.
+   *
+   * The promises say nothing useful about when that happens. Measured on four
+   * devices, attaching resolves in about three seconds and switching in none
+   * at all, and then a browser on the slow path spends another twelve to
+   * fifteen seconds before the first frame appears. Only the frame itself
+   * marks the end of the wait.
+   */
+  public onFirstFrame: (() => void) | undefined;
+  private produced = false;
+
+  public override async transform(
+    frame: VideoFrame,
+    controller: TransformStreamDefaultController<VideoFrame>,
+  ): Promise<void> {
+    await super.transform(frame, controller);
+    if (!this.produced) {
+      this.produced = true;
+      this.onFirstFrame?.();
+    }
+  }
+
   public async init({
     outputCanvas,
     inputElement: inputVideo,

src/livekit/TrackProcessorContext.tsx

@@ -77,6 +77,30 @@ export interface AddedBackgrounds {
   removeBackground: (id: string) => Promise<void>;
 }
 
+/** What the camera menu needs to know about the pipeline itself. */
+export interface BackgroundProcessing {
+  /**
+   * Whether the pipeline is still being built. Only ever true once a session,
+   * and only the first time an effect is chosen: building it fetches, compiles
+   * and sets up the segmenter, which on some browsers holds the page still for
+   * long enough that saying nothing looks like a crash.
+   */
+  settling: boolean;
+}
+
+const BackgroundProcessingContext = createContext<
+  BackgroundProcessing | undefined
+>(undefined);
+
+export function useBackgroundProcessing(): BackgroundProcessing {
+  const value = use(BackgroundProcessingContext);
+  if (value === undefined)
+    throw new Error(
+      "useBackgroundProcessing must be used within a ProcessorProvider",
+    );
+  return value;
+}
+
 const AddedBackgroundsContext = createContext<AddedBackgrounds | undefined>(
   undefined,
 );
@@ -207,14 +231,14 @@ export const ProcessorProvider: FC<Props> = ({ children }) => {
   // One pipeline for the lifetime of the app, so the pre-join preview and the
   // call share it and its priming frame is spent before anything is published
   // (D4).
-  const pipeline = useMemo(
-    () =>
-      new BackgroundProcessorWrapper(
-        new BackgroundEffectTransformer({ backgroundDisabled: true }),
-        "background-effect",
-      ),
+  const transformer = useMemo(
+    () => new BackgroundEffectTransformer({ backgroundDisabled: true }),
     [],
   );
+  const pipeline = useMemo(
+    () => new BackgroundProcessorWrapper(transformer, "background-effect"),
+    [transformer],
+  );
 
   // The backgrounds this device keeps. Their URLs live as long as the provider
   // does, and are replaced wholesale whenever the set changes: an object URL
@@ -284,6 +308,20 @@ export const ProcessorProvider: FC<Props> = ({ children }) => {
       .catch((e) => logger.warn("Failed to switch background effect", e));
   }, [pipeline, supported, attached, options]);
 
+  // The wait runs from deciding to attach until a frame actually comes out,
+  // which is the only thing that marks the end of it: the promises resolve
+  // while the segmenter is still being built, and on the slow path the page
+  // then holds still for another twelve to fifteen seconds. Bounded by the
+  // frame rather than by a timer, so it cannot end early and claim to be ready.
+  const [producedAFrame, setProducedAFrame] = useState(false);
+  useEffect(() => {
+    transformer.onFirstFrame = (): void => setProducedAFrame(true);
+    return (): void => {
+      transformer.onFirstFrame = undefined;
+    };
+  }, [transformer]);
+  const settling = supported === true && attached && !producedAFrame;
+
   // This is the actual state exposed through the context
   const processorState = useMemo(
     () => ({
@@ -298,10 +336,14 @@ export const ProcessorProvider: FC<Props> = ({ children }) => {
     [added, addBackground, removeBackground],
   );
 
+  const backgroundProcessing = useMemo(() => ({ settling }), [settling]);
+
   return (
     <ProcessorContext value={processorState}>
       <AddedBackgroundsContext value={addedBackgrounds}>
-        {children}
+        <BackgroundProcessingContext value={backgroundProcessing}>
+          {children}
+        </BackgroundProcessingContext>
       </AddedBackgroundsContext>
     </ProcessorContext>
   );

src/livekit/backgroundProcessing.ts

@@ -5,9 +5,10 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import { supportsBackgroundProcessors as supportsBackgroundProcessorsLivekitSdk } from "@livekit/track-processors";
-
-import { platform } from "../Platform";
+import {
+  supportsBackgroundProcessors as supportsBackgroundProcessorsLivekitSdk,
+  supportsModernBackgroundProcessors,
+} from "@livekit/track-processors";
 
 /**
  * Whether this device can run background effects at all.
@@ -21,7 +22,33 @@ import { platform } from "../Platform";
  *
  * Anything that offers a background effect must ask this, so that what is
  * offered and what can be delivered cannot drift apart again.
+ *
+ * It no longer asks for a desktop. Measured, that test sorted devices by the
+ * wrong thing: a phone on the fast path held 99% of its frame rate and was
+ * refused, while a desktop on the slow one held 73% and was allowed. What the
+ * cost tracks is the path, which {@link usesFallbackProcessing} names — and a
+ * cost is something to tell someone about, not to decide for them, because the
+ * user is the only one who knows whether they would rather show the room.
  */
 export function supportsBackgroundProcessors(): boolean {
-  return supportsBackgroundProcessorsLivekitSdk() && platform === "desktop";
+  return supportsBackgroundProcessorsLivekitSdk();
+}
+
+/**
+ * Whether effects here run the slow way.
+ *
+ * Browsers without `MediaStreamTrackProcessor` — Safari and Firefox, on every
+ * platform — fall back to drawing each frame through a canvas. Measured across
+ * four of them, that costs between a sixth and a quarter of the frame rate,
+ * phone or desktop, and on Safari it also blocks the page for twelve to
+ * fifteen seconds the first time the segmenter is built.
+ *
+ * Not a reason to withhold the feature: that cost ships today wherever the
+ * fallback runs, and it is a cost in smoothness while the feature is about
+ * privacy. It is a reason to say so.
+ */
+export function usesFallbackProcessing(): boolean {
+  return (
+    supportsBackgroundProcessors() && !supportsModernBackgroundProcessors()
+  );
 }

src/room/KnockLobbyView.test.tsx

@@ -27,6 +27,7 @@ vi.mock("../livekit/TrackProcessorContext", () => ({
     processor: undefined,
   }),
   useTrackProcessorSync: (): void => {},
+  useBackgroundProcessing: (): { settling: boolean } => ({ settling: false }),
   useAddedBackgrounds: (): {
     added: [];
     addBackground: () => Promise<void>;

src/room/LobbyView.test.tsx

@@ -42,6 +42,7 @@ vi.mock("../livekit/TrackProcessorContext", () => ({
     processor: undefined,
   }),
   useTrackProcessorSync: (): void => {},
+  useBackgroundProcessing: (): { settling: boolean } => ({ settling: false }),
   useAddedBackgrounds: (): {
     added: [];
     addBackground: () => Promise<void>;