Let a user add a background of their own

- Added backgrounds are kept on the device in IndexedDB; nothing uploads
  them and no identifier would let anything fetch one
- Refuses what cannot be used and reduces what is larger than we keep:
  a 4000x3000 photo became 1920x1440 and 238KB became 5KB
- Animation is read from the file, not guessed from its type: an
  animated WebP and a still one share a type, and a .png may be an APNG
- Does not crop. The pipeline already covers the camera's frame with
  whatever it is given, recomputed as the frame changes, so cropping on
  the way in would bake in one shape and lose the rest for good
- The add tile is withheld once four are kept, rather than failing
- Added backgrounds live in their own context, not in ProcessorState:
  the publisher has no use for them and every stub of it would carry them

Not done: removing an added background, which the design gives no
affordance for, and telling the user why a file was refused, which has no
surface in the menu yet. Both are logged for now.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
fkwp
2026-09-17 23:01:54 +02:00
co-authored by Claude Opus 5
parent 8a7eb5c7fd
commit 81fe5d355f
10 changed files with 436 additions and 41 deletions
+10
View File
@@ -12,6 +12,7 @@ import { Link } from "@vector-im/compound-web";
import type { Meta, StoryObj } from "@storybook/react-vite";
import { CallFooter, type FooterSnapshot } from "./CallFooter";
import { ProcessorProvider } from "../livekit/TrackProcessorContext";
import inCallViewStyles from "../room/InCallView.module.css";
import { useStaticViewModel } from "../state/ViewModel";
import { ReactionsSenderContext } from "../reactions/useReactionsSender";
@@ -82,6 +83,15 @@ const fnArgType = {
const meta = {
component: CallFooterStoryWrapper,
// The footer reads the backgrounds this device keeps from the processor
// provider, the same way the lobby and the call do.
decorators: [
(Story): JSX.Element => (
<ProcessorProvider>
<Story />
</ProcessorProvider>
),
],
argTypes: {
layout: {
control: "radio",
+85 -17
View File
@@ -5,9 +5,18 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type FC, type JSX, type Ref, useMemo } from "react";
import {
Fragment,
type FC,
type JSX,
type Ref,
useCallback,
useMemo,
useRef,
} from "react";
import classNames from "classnames";
import { useTranslation } from "react-i18next";
import { logger } from "matrix-js-sdk/lib/logger";
import LogoMark from "../icons/LogoMark.svg?react";
import LogoType from "../icons/LogoType.svg?react";
@@ -28,7 +37,15 @@ import {
MediaMuteAndSwitchButton,
type MenuOptions,
} from "./MediaMuteAndSwitchButton";
import { shippedBackgrounds } from "../livekit/backgroundEffects";
import {
serializeEffect,
shippedBackgrounds,
} from "../livekit/backgroundEffects";
import {
maxAddedBackgrounds,
UnusableImage,
} from "../livekit/backgroundImages";
import { useAddedBackgrounds } from "../livekit/TrackProcessorContext";
import { type Behavior } from "../state/Behavior";
import { type ViewModel } from "../state/ViewModel";
import { useBehavior } from "../useBehavior";
@@ -161,6 +178,32 @@ export const CallFooter: FC<FooterProps> = ({
const selectVideoButtonOption = useBehavior(vm.selectVideoButtonOption$);
const backgroundEffect = useBehavior(vm.backgroundEffect$);
const selectBackgroundEffect = useBehavior(vm.selectBackgroundEffect$);
const { added, addBackground } = useAddedBackgrounds();
const chooseFile = useRef<HTMLInputElement>(null);
const onAddBackgroundImage = useCallback((): void => {
chooseFile.current?.click();
}, []);
const onFileChosen = useCallback(
(event: React.ChangeEvent<HTMLInputElement>): void => {
const file = event.target.files?.[0];
// Cleared so choosing the same file twice in a row still counts.
event.target.value = "";
if (!file) return;
addBackground(file).catch((e) => {
// TODO: FR-021 wants the user told what went wrong. There is no
// surface for that in the menu yet, and inventing one is design's
// call, so for now this is only logged.
logger.warn(
e instanceof UnusableImage
? `Cannot use that file as a background: ${e.reason}`
: "Could not keep that background",
e,
);
});
},
[addBackground],
);
// The catalogue is named here rather than in the view model: the names are
// for reading, and a view model has no business holding translated text.
@@ -169,15 +212,23 @@ export const CallFooter: FC<FooterProps> = ({
{ id: "none", kind: "none", label: t("action.background_effect_none") },
{ id: "blur", kind: "blur", label: t("action.background_effect_blur") },
...shippedBackgrounds.map((background, i) => ({
id: `image:${background.id}`,
id: serializeEffect({ kind: "shipped", id: background.id }),
kind: "image" as const,
// Numbered rather than named: the images are stand-ins, and naming
// them here would invent names the design has not given them.
label: t("action.background_effect_numbered", { n: i + 1 }),
imageUrl: background.imagePath,
})),
...added.map((background, i) => ({
id: serializeEffect({ kind: "added", id: background.id }),
kind: "image" as const,
label: t("action.background_effect_numbered", {
n: shippedBackgrounds.length + i + 1,
}),
imageUrl: background.url,
})),
],
[t],
[t, added],
);
const buttonSize = useBehavior(vm.buttonSize$);
const showLogo = useBehavior(vm.showLogo$);
@@ -231,19 +282,36 @@ export const CallFooter: FC<FooterProps> = ({
if ((videoOptions?.length ?? 0) > 0) {
buttons.push(
<MediaMuteAndSwitchButton
key="video"
iconsAndLabels="video"
enabled={videoEnabled ?? false}
busy={videoBusy ?? false}
onMuteClick={toggleVideo}
options={videoOptions}
selectedOption={selectedVideo}
onSelect={selectVideoButtonOption}
backgroundEffects={backgroundEffects}
selectedBackgroundEffect={backgroundEffect}
onSelectBackgroundEffect={selectBackgroundEffect}
/>,
<Fragment key="video">
{/* The picker the add tile opens. Hidden, and driven from the tile,
because a file input cannot be styled into one. */}
<input
ref={chooseFile}
type="file"
accept="image/*"
hidden
onChange={onFileChosen}
/>
<MediaMuteAndSwitchButton
iconsAndLabels="video"
enabled={videoEnabled ?? false}
busy={videoBusy ?? false}
onMuteClick={toggleVideo}
options={videoOptions}
selectedOption={selectedVideo}
onSelect={selectVideoButtonOption}
backgroundEffects={backgroundEffects}
selectedBackgroundEffect={backgroundEffect}
onSelectBackgroundEffect={selectBackgroundEffect}
// Withheld once the device keeps as many as it will, which is what
// renders the add tile unavailable rather than letting it fail.
onAddBackgroundImage={
selectBackgroundEffect && added.length < maxAddedBackgrounds
? onAddBackgroundImage
: undefined
}
/>
</Fragment>,
);
} else {
buttons.push(
@@ -866,7 +866,9 @@ export const BackgroundEffectsUnavailable: Story = {
const blur = await within(document.body).findByRole("menuitemradio", {
name: "Blur",
});
await waitFor(() => expect(blur).toHaveAttribute("aria-disabled", "true"));
await waitFor(async () =>
expect(blur).toHaveAttribute("aria-disabled", "true"),
);
// No effect needs no background processing, so it stays choosable.
const none = await within(document.body).findByRole("menuitemradio", {
+111 -9
View File
@@ -17,8 +17,10 @@ import {
type FC,
type JSX,
use,
useCallback,
useEffect,
useMemo,
useRef,
useState,
} from "react";
import { type LocalVideoTrack } from "livekit-client";
@@ -37,6 +39,7 @@ import {
parseEffect,
type BackgroundEffect,
} from "./backgroundEffects";
import { BackgroundImageStore } from "./backgroundImages";
import { type Behavior } from "../state/Behavior";
import { type ObservableScope } from "../state/ObservableScope";
import { platform } from "../Platform";
@@ -45,11 +48,49 @@ import { platform } from "../Platform";
// it is a combination of exposing observable and react hooks.
// preferably we should not make this a context anymore and instead just a vm?
/** A background the user added, as the view needs it: an id and a picture. */
export interface AddedBackgroundImage {
id: string;
/** Lives as long as the provider does. */
url: string;
}
/**
* What the publishing side needs: whether effects can run at all, and the
* pipeline to attach. Deliberately not the backgrounds a user has added — the
* publisher has no use for those, and every test that stands in for this would
* have to carry them.
*/
export type ProcessorState = {
supported: boolean | undefined;
processor: undefined | ProcessorWrapper<BackgroundOptions>;
};
/** What the camera menu needs: the backgrounds this device keeps. */
export interface AddedBackgrounds {
/** Oldest first. */
added: AddedBackgroundImage[];
/**
* Keeps a file as a background. Rejects with `UnusableImage` for a file that
* cannot be used, and `RangeError` once the device keeps as many as it will.
*/
addBackground: (file: Blob) => Promise<void>;
removeBackground: (id: string) => Promise<void>;
}
const AddedBackgroundsContext = createContext<AddedBackgrounds | undefined>(
undefined,
);
export function useAddedBackgrounds(): AddedBackgrounds {
const value = use(AddedBackgroundsContext);
if (value === undefined)
throw new Error(
"useAddedBackgrounds must be used within a ProcessorProvider",
);
return value;
}
const ProcessorContext = createContext<ProcessorState | undefined>(undefined);
export function useTrackProcessor(): ProcessorState {
@@ -141,16 +182,22 @@ function supportsBackgroundProcessors(): boolean {
/** Translates a chosen effect into the pipeline's own vocabulary. */
function switchOptionsFor(
effect: BackgroundEffect,
added: AddedBackgroundImage[],
): SwitchBackgroundProcessorOptions {
const withImage = (
imagePath: string | undefined,
): SwitchBackgroundProcessorOptions =>
// A background the device no longer has — removed, or storage cleared —
// leaves the user with no effect rather than a pipeline drawing nothing.
imagePath ? { mode: "virtual-background", imagePath } : { mode: "disabled" };
switch (effect.kind) {
case "blur":
return { mode: "background-blur", blurRadius };
case "image": {
const imagePath = imagePathFor(effect.id);
return imagePath
? { mode: "virtual-background", imagePath }
: { mode: "disabled" };
}
case "shipped":
return withImage(imagePathFor(effect.id));
case "added":
return withImage(added.find((a) => a.id === effect.id)?.url);
default:
return { mode: "disabled" };
}
@@ -172,6 +219,46 @@ export const ProcessorProvider: FC<Props> = ({ children }) => {
[],
);
// The backgrounds this device keeps. Their URLs live as long as the provider
// does, and are replaced wholesale whenever the set changes: an object URL
// outlives the blob it names unless it is revoked, and the alternative is
// tracking one lifetime per image.
const store = useMemo(() => new BackgroundImageStore(), []);
const [added, setAdded] = useState<AddedBackgroundImage[]>([]);
const urls = useRef<string[]>([]);
const reread = useCallback(async (): Promise<void> => {
const kept = await store.list();
urls.current.forEach((url) => URL.revokeObjectURL(url));
urls.current = kept.map((background) => URL.createObjectURL(background.image));
setAdded(kept.map(({ id }, i) => ({ id, url: urls.current[i] })));
}, [store]);
useEffect(() => {
reread().catch((e) => logger.warn("Could not read added backgrounds", e));
const opened = urls;
return (): void => {
opened.current.forEach((url) => URL.revokeObjectURL(url));
opened.current = [];
};
}, [reread]);
const addBackground = useCallback(
async (file: Blob): Promise<void> => {
await store.add(file);
await reread();
},
[store, reread],
);
const removeBackground = useCallback(
async (id: string): Promise<void> => {
await store.remove(id);
await reread();
},
[store, reread],
);
// D5: nothing is attached until an effect is first chosen, so a user who
// never chooses one pays neither the segmentation assets nor the time to
// initialise them. Once attached it stays attached, including at no effect,
@@ -186,12 +273,16 @@ export const ProcessorProvider: FC<Props> = ({ children }) => {
// D2: switch the running pipeline in place rather than tearing it down, so
// the previous effect stays in force until the new one is live.
const options = useMemo(
() => switchOptionsFor(parseEffect(effectRaw), added),
[effectRaw, added],
);
useEffect(() => {
if (!supported || !attached) return;
pipeline
.switchTo(switchOptionsFor(parseEffect(effectRaw)))
.switchTo(options)
.catch((e) => logger.warn("Failed to switch background effect", e));
}, [pipeline, supported, attached, effectRaw]);
}, [pipeline, supported, attached, options]);
// This is the actual state exposed through the context
const processorState = useMemo(
@@ -202,5 +293,16 @@ export const ProcessorProvider: FC<Props> = ({ children }) => {
[supported, attached, pipeline],
);
return <ProcessorContext value={processorState}>{children}</ProcessorContext>;
const addedBackgrounds = useMemo(
() => ({ added, addBackground, removeBackground }),
[added, addBackground, removeBackground],
);
return (
<ProcessorContext value={processorState}>
<AddedBackgroundsContext value={addedBackgrounds}>
{children}
</AddedBackgroundsContext>
</ProcessorContext>
);
};
+4 -2
View File
@@ -19,7 +19,8 @@ describe("the chosen background effect", () => {
for (const effect of [
{ kind: "none" } as const,
{ kind: "blur" } as const,
{ kind: "image", id: shippedBackgrounds[0].id } as const,
{ kind: "shipped", id: shippedBackgrounds[0].id } as const,
{ kind: "added", id: "a-uuid" } as const,
])
expect(parseEffect(serializeEffect(effect))).toEqual(effect);
});
@@ -48,7 +49,8 @@ describe("the blur control in settings", () => {
serializeEffect(on ? { kind: "blur" } : { kind: "none" });
test("reads as off while an image background is in force", () => {
expect(shows(serializeEffect({ kind: "image", id: "indoor" }))).toBe(false);
expect(shows(serializeEffect({ kind: "shipped", id: "indoor" }))).toBe(false);
expect(shows(serializeEffect({ kind: "added", id: "a-uuid" }))).toBe(false);
});
test("reads as on while blur is in force", () => {
+21 -4
View File
@@ -27,16 +27,31 @@ export const shippedBackgrounds: ShippedBackground[] = [
export type BackgroundEffect =
| { kind: "none" }
| { kind: "blur" }
| { kind: "image"; id: string };
/** One of the images that come with the application. */
| { kind: "shipped"; id: string }
/** One the user added, kept on this device. */
| { kind: "added"; id: string };
export const noEffect: BackgroundEffect = { kind: "none" };
/** Parses the stored form of a chosen effect, falling back to no effect. */
/**
* Parses the stored form of a chosen effect, falling back to no effect.
*
* A shipped background is checked against what we ship, so a stored one that a
* later release no longer carries falls back rather than leaving the user with
* an effect that cannot be drawn. An added one cannot be checked here — only
* the device knows what it keeps — so that check belongs where it is resolved.
*/
export function parseEffect(raw: string): BackgroundEffect {
if (raw === "blur") return { kind: "blur" };
if (raw.startsWith("image:")) {
const id = raw.slice("image:".length);
if (shippedBackgrounds.some((b) => b.id === id)) return { kind: "image", id };
if (shippedBackgrounds.some((b) => b.id === id))
return { kind: "shipped", id };
}
if (raw.startsWith("added:")) {
const id = raw.slice("added:".length);
if (id) return { kind: "added", id };
}
return noEffect;
}
@@ -46,8 +61,10 @@ export function serializeEffect(effect: BackgroundEffect): string {
switch (effect.kind) {
case "blur":
return "blur";
case "image":
case "shipped":
return `image:${effect.id}`;
case "added":
return `added:${effect.id}`;
default:
return "none";
}
+174
View File
@@ -0,0 +1,174 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { logger } from "matrix-js-sdk/lib/logger";
/** The most backgrounds of their own a device keeps. */
export const maxAddedBackgrounds = 4;
/**
* The longest edge an added background is kept at.
*
* Not a display constraint: the pipeline crops and scales the image to the
* camera on every use, so a larger one buys nothing. It is what stops a
* forty-megapixel photograph sitting in storage and being decoded on every
* call.
*/
export const maxStoredEdge = 1920;
/** A background the user added, as it is kept on the device. */
export interface AddedBackground {
id: string;
image: Blob;
addedAt: number;
}
/** Why a file cannot be used as a background. */
export type UnusableReason = "not-an-image" | "animated" | "undecodable";
export class UnusableImage extends Error {
public constructor(public readonly reason: UnusableReason) {
super(reason);
this.name = "UnusableImage";
}
}
/**
* Whether the file holds more than one frame.
*
* Read from the file rather than guessed from its type: an animated WebP and a
* still one are both `image/webp`, and a `.png` may be an APNG. Where the
* browser cannot tell us, the types that are usually animated are refused
* rather than accepted and left to move behind someone's head.
*/
async function isAnimated(file: Blob): Promise<boolean> {
const decoder = (
globalThis as { ImageDecoder?: new (init: unknown) => unknown }
).ImageDecoder;
if (!decoder) return /gif|apng/.test(file.type);
try {
const d = new decoder({ data: await file.arrayBuffer(), type: file.type });
await (d as { completed: Promise<void> }).completed;
const track = (
d as { tracks: { selectedTrack?: { frameCount: number } } }
).tracks.selectedTrack;
return (track?.frameCount ?? 1) > 1;
} catch (e) {
logger.debug("Could not read frame count, judging by type", e);
return /gif|apng/.test(file.type);
}
}
/**
* Prepares a file to be kept as a background: refuses what cannot be used, and
* reduces what is larger than we keep.
*
* Does not crop. The pipeline covers the camera's frame with whatever it is
* given, recomputed as the frame changes, so cropping here would bake in one
* shape and lose the rest of the picture for good.
*/
export async function prepareImage(file: Blob): Promise<Blob> {
if (!file.type.startsWith("image/"))
throw new UnusableImage("not-an-image");
if (await isAnimated(file)) throw new UnusableImage("animated");
let bitmap: ImageBitmap;
try {
bitmap = await createImageBitmap(file);
} catch (e) {
logger.debug("Could not decode the chosen file", e);
throw new UnusableImage("undecodable");
}
try {
const longest = Math.max(bitmap.width, bitmap.height);
if (longest <= maxStoredEdge) return file;
const scale = maxStoredEdge / longest;
const canvas = new OffscreenCanvas(
Math.round(bitmap.width * scale),
Math.round(bitmap.height * scale),
);
const context = canvas.getContext("2d");
if (!context) throw new UnusableImage("undecodable");
context.drawImage(bitmap, 0, 0, canvas.width, canvas.height);
return await canvas.convertToBlob({ type: "image/webp", quality: 0.9 });
} finally {
bitmap.close();
}
}
const DB_NAME = "element-call-background-images";
const STORE = "backgrounds";
/**
* The backgrounds a user added, kept on this device.
*
* They never leave it: there is no upload here and no identifier that would
* let anything fetch one.
*/
export class BackgroundImageStore {
private db?: Promise<IDBDatabase>;
/** The factory is injected so a test can hand over its own. */
public constructor(
private readonly indexedDB: IDBFactory = globalThis.indexedDB,
) {}
private async open(): Promise<IDBDatabase> {
return (this.db ??= new Promise((resolve, reject) => {
const request = this.indexedDB.open(DB_NAME, 1);
request.onupgradeneeded = (): void => {
request.result.createObjectStore(STORE, { keyPath: "id" });
};
request.onsuccess = (): void => resolve(request.result);
request.onerror = (): void => reject(request.error);
}));
}
private async run<T>(
mode: IDBTransactionMode,
body: (store: IDBObjectStore) => IDBRequest<T>,
): Promise<T> {
const db = await this.open();
return new Promise((resolve, reject) => {
const request = body(db.transaction(STORE, mode).objectStore(STORE));
request.onsuccess = (): void => resolve(request.result);
request.onerror = (): void => reject(request.error);
});
}
public async list(): Promise<AddedBackground[]> {
const all = await this.run<AddedBackground[]>("readonly", (s) =>
s.getAll(),
);
return all.sort((a, b) => a.addedAt - b.addedAt);
}
/**
* Keeps a file as a background, and answers with it.
*
* Throws {@link UnusableImage} for a file that cannot be used, and
* {@link RangeError} once the device is already keeping as many as it will.
*/
public async add(file: Blob): Promise<AddedBackground> {
const image = await prepareImage(file);
if ((await this.list()).length >= maxAddedBackgrounds)
throw new RangeError("Already keeping the most backgrounds we keep");
const background: AddedBackground = {
id: crypto.randomUUID(),
image,
addedAt: Date.now(),
};
await this.run("readwrite", (s) => s.add(background));
return background;
}
public async remove(id: string): Promise<void> {
await this.run("readwrite", (s) => s.delete(id));
}
}
+10 -8
View File
@@ -170,14 +170,16 @@ function createInCallView(args: CreateInCallViewArgs = {}): RenderResult & {
const renderResult = render(
<BrowserRouter>
<MediaDevicesContext value={mediaDevices}>
<ReactionsSenderProvider
vm={vm}
rtcSession={rtcSession.asMockedSession()}
>
<TooltipProvider>
<RoomContext value={livekitRoom}>{content}</RoomContext>
</TooltipProvider>
</ReactionsSenderProvider>
<ProcessorProvider>
<ReactionsSenderProvider
vm={vm}
rtcSession={rtcSession.asMockedSession()}
>
<TooltipProvider>
<RoomContext value={livekitRoom}>{content}</RoomContext>
</TooltipProvider>
</ReactionsSenderProvider>
</ProcessorProvider>
</MediaDevicesContext>
</BrowserRouter>,
);
+9
View File
@@ -27,6 +27,15 @@ vi.mock("../livekit/TrackProcessorContext", () => ({
processor: undefined,
}),
useTrackProcessorSync: (): void => {},
useAddedBackgrounds: (): {
added: [];
addBackground: () => Promise<void>;
removeBackground: () => Promise<void>;
} => ({
added: [],
addBackground: async (): Promise<void> => {},
removeBackground: async (): Promise<void> => {},
}),
}));
vi.mock("react-use-measure", () => ({
+9
View File
@@ -42,6 +42,15 @@ vi.mock("../livekit/TrackProcessorContext", () => ({
processor: undefined,
}),
useTrackProcessorSync: (): void => {},
useAddedBackgrounds: (): {
added: [];
addBackground: () => Promise<void>;
removeBackground: () => Promise<void>;
} => ({
added: [],
addBackground: async (): Promise<void> => {},
removeBackground: async (): Promise<void> => {},
}),
}));
vi.mock("react-use-measure", () => ({