From 1e445a4d14efbfabcdf61a030c6f40e4f9eaba15 Mon Sep 17 00:00:00 2001 From: fkwp Date: Fri, 18 Sep 2026 11:35:42 +0200 Subject: [PATCH] Measure whether a device can run background effects - The camera and the pipeline, no call in the way: if a device cannot hold a frame rate here it cannot hold one in a call either. - Reports mean and worst five-second frame rate, startup, and the worst stall on the main thread. Run no effect first; every number is only meaningful beside that device's own baseline. - Ignores the desktop gate on purpose. Measuring what it forbids is the point, and the gate was written from reports rather than numbers. - Exploration only, on an unlisted route, and one path segment deep: the config is fetched relative to the page, so a nested path never starts. Co-Authored-By: Claude Opus 5 (1M context) --- src/App.tsx | 11 + src/livekit/BackgroundEffectsBench.tsx | 335 +++++++++++++++++++++++++ 2 files changed, 346 insertions(+) create mode 100644 src/livekit/BackgroundEffectsBench.tsx diff --git a/src/App.tsx b/src/App.tsx index c07553afc..10035346b 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -22,6 +22,8 @@ import { } from "react-router-dom"; import * as Sentry from "@sentry/react"; import { TooltipProvider } from "@vector-im/compound-web"; + +import { BackgroundEffectsBench } from "./livekit/BackgroundEffectsBench"; import { logger } from "matrix-js-sdk/lib/logger"; import { type MatrixClient } from "matrix-js-sdk"; import { I18nextProvider } from "react-i18next"; @@ -156,6 +158,15 @@ export const App: FC = ({ vm, widget }) => { } /> } /> } /> + {/* Exploration only: measures whether this device can run + background effects, on a device the app refuses to run them + on. One path segment, not two: the config is fetched + relative to the page, so a nested path looks for it in a + directory that does not exist and the app never starts. */} + } + /> } /> diff --git a/src/livekit/BackgroundEffectsBench.tsx b/src/livekit/BackgroundEffectsBench.tsx new file mode 100644 index 000000000..d67c01325 --- /dev/null +++ b/src/livekit/BackgroundEffectsBench.tsx @@ -0,0 +1,335 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type FC, useCallback, useRef, useState } from "react"; +import { + createLocalVideoTrack, + type LocalVideoTrack, + VideoPresets, +} from "livekit-client"; +import { + BackgroundProcessorWrapper, + supportsBackgroundProcessors as supportsBackgroundProcessorsLivekitSdk, + supportsModernBackgroundProcessors, + type SwitchBackgroundProcessorOptions, +} from "@livekit/track-processors"; +import { logger } from "matrix-js-sdk/lib/logger"; + +import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer"; +import { blurRadius, imagePathFor } from "./backgroundEffects"; +import { supportsBackgroundProcessors } from "./backgroundProcessing"; +import { platform } from "../Platform"; + +/** + * Whether background effects can hold their frame rate on this device, asked + * without a call in the way. + * + * The camera and the pipeline only: no SFU, no encoder, no second participant. + * If a device cannot hold a frame rate here it cannot hold one in a call + * either, so this is the cheap half of the question — and it is the half that + * has never been asked. The gate that keeps effects off phones was added from + * reports, not measurements, and nothing in the app will run the pipeline + * there, so there has been nothing to measure. + * + * Deliberately ignores that gate: measuring what it forbids is the point. It + * reports what the gate would have said, and runs anyway. + * + * Exploration only, and never ported: an unlisted route, its text not + * translated, its numbers meant for whoever is holding the phone. + */ + +type Effect = "none" | "blur" | "image"; + +interface Bucket { + /** Seconds since the first frame. */ + at: number; + fps: number; +} + +interface Run { + effect: Effect; + device: string; + platform: string; + /** Whether the browser has the API that avoids the canvas fallback. */ + modernApi: boolean; + /** What the app's own verdict would have said about this device. */ + appWouldAllow: boolean; + sdkWouldAllow: boolean; + capture: { width?: number; height?: number; frameRate?: number }; + processed: { width?: number; height?: number; frameRate?: number }; + /** Milliseconds from asking for the effect to the first frame carrying it. */ + startupMs: number | null; + meanFps: number; + /** The worst five-second stretch — where throttling shows up first. */ + worstFps: number; + buckets: Bucket[]; + /** Worst observed delay on a 100ms timer: the main thread's own stutter. */ + worstTimerLagMs: number; + frames: number; + seconds: number; +} + +const bucketSeconds = 5; + +function optionsFor(effect: Effect): SwitchBackgroundProcessorOptions { + switch (effect) { + case "blur": + return { mode: "background-blur", blurRadius }; + case "image": { + const imagePath = imagePathFor("indoor"); + return imagePath + ? { mode: "virtual-background", imagePath } + : { mode: "disabled" }; + } + default: + return { mode: "disabled" }; + } +} + +export const BackgroundEffectsBench: FC = () => { + const video = useRef(null); + const [seconds, setSeconds] = useState(60); + const [busy, setBusy] = useState(false); + const [status, setStatus] = useState("Ready."); + const [runs, setRuns] = useState([]); + + const measure = useCallback( + async (effect: Effect): Promise => { + setBusy(true); + setStatus(`Starting the camera for "${effect}"…`); + let track: LocalVideoTrack | undefined; + try { + // The resolution and frame rate a real call captures at, so the cost + // measured here is the cost a call would pay. + track = await createLocalVideoTrack({ + resolution: VideoPresets.h720.resolution, + }); + const capture = track.mediaStreamTrack.getSettings(); + + let startupMs: number | null = null; + if (effect !== "none") { + setStatus("Starting the pipeline…"); + const askedAt = performance.now(); + const pipeline = new BackgroundProcessorWrapper( + new BackgroundEffectTransformer({ backgroundDisabled: true }), + "background-effect-bench", + ); + await track.setProcessor(pipeline); + await pipeline.switchTo(optionsFor(effect)); + startupMs = Math.round(performance.now() - askedAt); + } + + const element = video.current!; + track.attach(element); + await element.play().catch(() => undefined); + + setStatus(`Measuring "${effect}" for ${seconds}s…`); + const result = await countFrames(element, seconds, (done) => + setStatus(`Measuring "${effect}": ${done}s of ${seconds}s`), + ); + + const processed = track.mediaStreamTrack.getSettings(); + setRuns((previous) => [ + ...previous, + { + effect, + device: navigator.userAgent, + platform, + modernApi: supportsModernBackgroundProcessors(), + appWouldAllow: supportsBackgroundProcessors(), + sdkWouldAllow: supportsBackgroundProcessorsLivekitSdk(), + capture: { + width: capture.width, + height: capture.height, + frameRate: capture.frameRate, + }, + processed: { + width: processed.width, + height: processed.height, + frameRate: processed.frameRate, + }, + startupMs, + ...result, + }, + ]); + setStatus(`Done: ${effect}.`); + } catch (e) { + logger.error("Background effects bench failed", e); + setStatus(`Failed: ${e instanceof Error ? e.message : String(e)}`); + } finally { + track?.stop(); + setBusy(false); + } + }, + [seconds], + ); + + return ( +
+

Background effects: can this device?

+

+ The camera and the pipeline, with no call in the way. Run{" "} + No effect first — every other number is only meaningful + beside it, on this device, in this light. +

+

+ This device: {platform}, the app would{" "} + {supportsBackgroundProcessors() ? "allow" : "refuse"}{" "} + effects here, the browser has{" "} + + {supportsModernBackgroundProcessors() + ? "the fast path" + : "only the canvas fallback"} + + . +

+ +
+ {(["none", "blur", "image"] as const).map((effect) => ( + + ))} +
+

+ {status} +

+