diff --git a/src/components/CallFooter.stories.tsx b/src/components/CallFooter.stories.tsx index 8ce9d4993..3dcfe48ce 100644 --- a/src/components/CallFooter.stories.tsx +++ b/src/components/CallFooter.stories.tsx @@ -12,6 +12,7 @@ import { Link } from "@vector-im/compound-web"; import type { Meta, StoryObj } from "@storybook/react-vite"; import { CallFooter, type FooterSnapshot } from "./CallFooter"; +import { ProcessorProvider } from "../livekit/TrackProcessorContext"; import inCallViewStyles from "../room/InCallView.module.css"; import { useStaticViewModel } from "../state/ViewModel"; import { ReactionsSenderContext } from "../reactions/useReactionsSender"; @@ -82,6 +83,15 @@ const fnArgType = { const meta = { component: CallFooterStoryWrapper, + // The footer reads the backgrounds this device keeps from the processor + // provider, the same way the lobby and the call do. + decorators: [ + (Story): JSX.Element => ( + + + + ), + ], argTypes: { layout: { control: "radio", diff --git a/src/components/CallFooter.tsx b/src/components/CallFooter.tsx index 7c04090b3..86d67db73 100644 --- a/src/components/CallFooter.tsx +++ b/src/components/CallFooter.tsx @@ -5,9 +5,18 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type FC, type JSX, type Ref, useMemo } from "react"; +import { + Fragment, + type FC, + type JSX, + type Ref, + useCallback, + useMemo, + useRef, +} from "react"; import classNames from "classnames"; import { useTranslation } from "react-i18next"; +import { logger } from "matrix-js-sdk/lib/logger"; import LogoMark from "../icons/LogoMark.svg?react"; import LogoType from "../icons/LogoType.svg?react"; @@ -28,7 +37,15 @@ import { MediaMuteAndSwitchButton, type MenuOptions, } from "./MediaMuteAndSwitchButton"; -import { shippedBackgrounds } from "../livekit/backgroundEffects"; +import { + serializeEffect, + shippedBackgrounds, +} from "../livekit/backgroundEffects"; +import { + maxAddedBackgrounds, + UnusableImage, +} from "../livekit/backgroundImages"; +import { useAddedBackgrounds } from "../livekit/TrackProcessorContext"; import { type Behavior } from "../state/Behavior"; import { type ViewModel } from "../state/ViewModel"; import { useBehavior } from "../useBehavior"; @@ -161,6 +178,32 @@ export const CallFooter: FC = ({ const selectVideoButtonOption = useBehavior(vm.selectVideoButtonOption$); const backgroundEffect = useBehavior(vm.backgroundEffect$); const selectBackgroundEffect = useBehavior(vm.selectBackgroundEffect$); + const { added, addBackground } = useAddedBackgrounds(); + + const chooseFile = useRef(null); + const onAddBackgroundImage = useCallback((): void => { + chooseFile.current?.click(); + }, []); + const onFileChosen = useCallback( + (event: React.ChangeEvent): void => { + const file = event.target.files?.[0]; + // Cleared so choosing the same file twice in a row still counts. + event.target.value = ""; + if (!file) return; + addBackground(file).catch((e) => { + // TODO: FR-021 wants the user told what went wrong. There is no + // surface for that in the menu yet, and inventing one is design's + // call, so for now this is only logged. + logger.warn( + e instanceof UnusableImage + ? `Cannot use that file as a background: ${e.reason}` + : "Could not keep that background", + e, + ); + }); + }, + [addBackground], + ); // The catalogue is named here rather than in the view model: the names are // for reading, and a view model has no business holding translated text. @@ -169,15 +212,23 @@ export const CallFooter: FC = ({ { id: "none", kind: "none", label: t("action.background_effect_none") }, { id: "blur", kind: "blur", label: t("action.background_effect_blur") }, ...shippedBackgrounds.map((background, i) => ({ - id: `image:${background.id}`, + id: serializeEffect({ kind: "shipped", id: background.id }), kind: "image" as const, // Numbered rather than named: the images are stand-ins, and naming // them here would invent names the design has not given them. label: t("action.background_effect_numbered", { n: i + 1 }), imageUrl: background.imagePath, })), + ...added.map((background, i) => ({ + id: serializeEffect({ kind: "added", id: background.id }), + kind: "image" as const, + label: t("action.background_effect_numbered", { + n: shippedBackgrounds.length + i + 1, + }), + imageUrl: background.url, + })), ], - [t], + [t, added], ); const buttonSize = useBehavior(vm.buttonSize$); const showLogo = useBehavior(vm.showLogo$); @@ -231,19 +282,36 @@ export const CallFooter: FC = ({ if ((videoOptions?.length ?? 0) > 0) { buttons.push( - , + + {/* The picker the add tile opens. Hidden, and driven from the tile, + because a file input cannot be styled into one. */} + + + , ); } else { buttons.push( diff --git a/src/components/MediaMuteAndSwitchButton.stories.tsx b/src/components/MediaMuteAndSwitchButton.stories.tsx index b4b77c337..61d866640 100644 --- a/src/components/MediaMuteAndSwitchButton.stories.tsx +++ b/src/components/MediaMuteAndSwitchButton.stories.tsx @@ -866,7 +866,9 @@ export const BackgroundEffectsUnavailable: Story = { const blur = await within(document.body).findByRole("menuitemradio", { name: "Blur", }); - await waitFor(() => expect(blur).toHaveAttribute("aria-disabled", "true")); + await waitFor(async () => + expect(blur).toHaveAttribute("aria-disabled", "true"), + ); // No effect needs no background processing, so it stays choosable. const none = await within(document.body).findByRole("menuitemradio", { diff --git a/src/livekit/TrackProcessorContext.tsx b/src/livekit/TrackProcessorContext.tsx index 389bf2971..77055206f 100644 --- a/src/livekit/TrackProcessorContext.tsx +++ b/src/livekit/TrackProcessorContext.tsx @@ -17,8 +17,10 @@ import { type FC, type JSX, use, + useCallback, useEffect, useMemo, + useRef, useState, } from "react"; import { type LocalVideoTrack } from "livekit-client"; @@ -37,6 +39,7 @@ import { parseEffect, type BackgroundEffect, } from "./backgroundEffects"; +import { BackgroundImageStore } from "./backgroundImages"; import { type Behavior } from "../state/Behavior"; import { type ObservableScope } from "../state/ObservableScope"; import { platform } from "../Platform"; @@ -45,11 +48,49 @@ import { platform } from "../Platform"; // it is a combination of exposing observable and react hooks. // preferably we should not make this a context anymore and instead just a vm? +/** A background the user added, as the view needs it: an id and a picture. */ +export interface AddedBackgroundImage { + id: string; + /** Lives as long as the provider does. */ + url: string; +} + +/** + * What the publishing side needs: whether effects can run at all, and the + * pipeline to attach. Deliberately not the backgrounds a user has added — the + * publisher has no use for those, and every test that stands in for this would + * have to carry them. + */ export type ProcessorState = { supported: boolean | undefined; processor: undefined | ProcessorWrapper; }; +/** What the camera menu needs: the backgrounds this device keeps. */ +export interface AddedBackgrounds { + /** Oldest first. */ + added: AddedBackgroundImage[]; + /** + * Keeps a file as a background. Rejects with `UnusableImage` for a file that + * cannot be used, and `RangeError` once the device keeps as many as it will. + */ + addBackground: (file: Blob) => Promise; + removeBackground: (id: string) => Promise; +} + +const AddedBackgroundsContext = createContext( + undefined, +); + +export function useAddedBackgrounds(): AddedBackgrounds { + const value = use(AddedBackgroundsContext); + if (value === undefined) + throw new Error( + "useAddedBackgrounds must be used within a ProcessorProvider", + ); + return value; +} + const ProcessorContext = createContext(undefined); export function useTrackProcessor(): ProcessorState { @@ -141,16 +182,22 @@ function supportsBackgroundProcessors(): boolean { /** Translates a chosen effect into the pipeline's own vocabulary. */ function switchOptionsFor( effect: BackgroundEffect, + added: AddedBackgroundImage[], ): SwitchBackgroundProcessorOptions { + const withImage = ( + imagePath: string | undefined, + ): SwitchBackgroundProcessorOptions => + // A background the device no longer has — removed, or storage cleared — + // leaves the user with no effect rather than a pipeline drawing nothing. + imagePath ? { mode: "virtual-background", imagePath } : { mode: "disabled" }; + switch (effect.kind) { case "blur": return { mode: "background-blur", blurRadius }; - case "image": { - const imagePath = imagePathFor(effect.id); - return imagePath - ? { mode: "virtual-background", imagePath } - : { mode: "disabled" }; - } + case "shipped": + return withImage(imagePathFor(effect.id)); + case "added": + return withImage(added.find((a) => a.id === effect.id)?.url); default: return { mode: "disabled" }; } @@ -172,6 +219,46 @@ export const ProcessorProvider: FC = ({ children }) => { [], ); + // The backgrounds this device keeps. Their URLs live as long as the provider + // does, and are replaced wholesale whenever the set changes: an object URL + // outlives the blob it names unless it is revoked, and the alternative is + // tracking one lifetime per image. + const store = useMemo(() => new BackgroundImageStore(), []); + const [added, setAdded] = useState([]); + const urls = useRef([]); + + const reread = useCallback(async (): Promise => { + const kept = await store.list(); + urls.current.forEach((url) => URL.revokeObjectURL(url)); + urls.current = kept.map((background) => URL.createObjectURL(background.image)); + setAdded(kept.map(({ id }, i) => ({ id, url: urls.current[i] }))); + }, [store]); + + useEffect(() => { + reread().catch((e) => logger.warn("Could not read added backgrounds", e)); + const opened = urls; + return (): void => { + opened.current.forEach((url) => URL.revokeObjectURL(url)); + opened.current = []; + }; + }, [reread]); + + const addBackground = useCallback( + async (file: Blob): Promise => { + await store.add(file); + await reread(); + }, + [store, reread], + ); + + const removeBackground = useCallback( + async (id: string): Promise => { + await store.remove(id); + await reread(); + }, + [store, reread], + ); + // D5: nothing is attached until an effect is first chosen, so a user who // never chooses one pays neither the segmentation assets nor the time to // initialise them. Once attached it stays attached, including at no effect, @@ -186,12 +273,16 @@ export const ProcessorProvider: FC = ({ children }) => { // D2: switch the running pipeline in place rather than tearing it down, so // the previous effect stays in force until the new one is live. + const options = useMemo( + () => switchOptionsFor(parseEffect(effectRaw), added), + [effectRaw, added], + ); useEffect(() => { if (!supported || !attached) return; pipeline - .switchTo(switchOptionsFor(parseEffect(effectRaw))) + .switchTo(options) .catch((e) => logger.warn("Failed to switch background effect", e)); - }, [pipeline, supported, attached, effectRaw]); + }, [pipeline, supported, attached, options]); // This is the actual state exposed through the context const processorState = useMemo( @@ -202,5 +293,16 @@ export const ProcessorProvider: FC = ({ children }) => { [supported, attached, pipeline], ); - return {children}; + const addedBackgrounds = useMemo( + () => ({ added, addBackground, removeBackground }), + [added, addBackground, removeBackground], + ); + + return ( + + + {children} + + + ); }; diff --git a/src/livekit/backgroundEffects.test.ts b/src/livekit/backgroundEffects.test.ts index c70981df8..a01ff36d4 100644 --- a/src/livekit/backgroundEffects.test.ts +++ b/src/livekit/backgroundEffects.test.ts @@ -19,7 +19,8 @@ describe("the chosen background effect", () => { for (const effect of [ { kind: "none" } as const, { kind: "blur" } as const, - { kind: "image", id: shippedBackgrounds[0].id } as const, + { kind: "shipped", id: shippedBackgrounds[0].id } as const, + { kind: "added", id: "a-uuid" } as const, ]) expect(parseEffect(serializeEffect(effect))).toEqual(effect); }); @@ -48,7 +49,8 @@ describe("the blur control in settings", () => { serializeEffect(on ? { kind: "blur" } : { kind: "none" }); test("reads as off while an image background is in force", () => { - expect(shows(serializeEffect({ kind: "image", id: "indoor" }))).toBe(false); + expect(shows(serializeEffect({ kind: "shipped", id: "indoor" }))).toBe(false); + expect(shows(serializeEffect({ kind: "added", id: "a-uuid" }))).toBe(false); }); test("reads as on while blur is in force", () => { diff --git a/src/livekit/backgroundEffects.ts b/src/livekit/backgroundEffects.ts index 18c5b019d..e43dc9837 100644 --- a/src/livekit/backgroundEffects.ts +++ b/src/livekit/backgroundEffects.ts @@ -27,16 +27,31 @@ export const shippedBackgrounds: ShippedBackground[] = [ export type BackgroundEffect = | { kind: "none" } | { kind: "blur" } - | { kind: "image"; id: string }; + /** One of the images that come with the application. */ + | { kind: "shipped"; id: string } + /** One the user added, kept on this device. */ + | { kind: "added"; id: string }; export const noEffect: BackgroundEffect = { kind: "none" }; -/** Parses the stored form of a chosen effect, falling back to no effect. */ +/** + * Parses the stored form of a chosen effect, falling back to no effect. + * + * A shipped background is checked against what we ship, so a stored one that a + * later release no longer carries falls back rather than leaving the user with + * an effect that cannot be drawn. An added one cannot be checked here — only + * the device knows what it keeps — so that check belongs where it is resolved. + */ export function parseEffect(raw: string): BackgroundEffect { if (raw === "blur") return { kind: "blur" }; if (raw.startsWith("image:")) { const id = raw.slice("image:".length); - if (shippedBackgrounds.some((b) => b.id === id)) return { kind: "image", id }; + if (shippedBackgrounds.some((b) => b.id === id)) + return { kind: "shipped", id }; + } + if (raw.startsWith("added:")) { + const id = raw.slice("added:".length); + if (id) return { kind: "added", id }; } return noEffect; } @@ -46,8 +61,10 @@ export function serializeEffect(effect: BackgroundEffect): string { switch (effect.kind) { case "blur": return "blur"; - case "image": + case "shipped": return `image:${effect.id}`; + case "added": + return `added:${effect.id}`; default: return "none"; } diff --git a/src/livekit/backgroundImages.ts b/src/livekit/backgroundImages.ts new file mode 100644 index 000000000..7141b5763 --- /dev/null +++ b/src/livekit/backgroundImages.ts @@ -0,0 +1,174 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { logger } from "matrix-js-sdk/lib/logger"; + +/** The most backgrounds of their own a device keeps. */ +export const maxAddedBackgrounds = 4; + +/** + * The longest edge an added background is kept at. + * + * Not a display constraint: the pipeline crops and scales the image to the + * camera on every use, so a larger one buys nothing. It is what stops a + * forty-megapixel photograph sitting in storage and being decoded on every + * call. + */ +export const maxStoredEdge = 1920; + +/** A background the user added, as it is kept on the device. */ +export interface AddedBackground { + id: string; + image: Blob; + addedAt: number; +} + +/** Why a file cannot be used as a background. */ +export type UnusableReason = "not-an-image" | "animated" | "undecodable"; + +export class UnusableImage extends Error { + public constructor(public readonly reason: UnusableReason) { + super(reason); + this.name = "UnusableImage"; + } +} + +/** + * Whether the file holds more than one frame. + * + * Read from the file rather than guessed from its type: an animated WebP and a + * still one are both `image/webp`, and a `.png` may be an APNG. Where the + * browser cannot tell us, the types that are usually animated are refused + * rather than accepted and left to move behind someone's head. + */ +async function isAnimated(file: Blob): Promise { + const decoder = ( + globalThis as { ImageDecoder?: new (init: unknown) => unknown } + ).ImageDecoder; + if (!decoder) return /gif|apng/.test(file.type); + try { + const d = new decoder({ data: await file.arrayBuffer(), type: file.type }); + await (d as { completed: Promise }).completed; + const track = ( + d as { tracks: { selectedTrack?: { frameCount: number } } } + ).tracks.selectedTrack; + return (track?.frameCount ?? 1) > 1; + } catch (e) { + logger.debug("Could not read frame count, judging by type", e); + return /gif|apng/.test(file.type); + } +} + +/** + * Prepares a file to be kept as a background: refuses what cannot be used, and + * reduces what is larger than we keep. + * + * Does not crop. The pipeline covers the camera's frame with whatever it is + * given, recomputed as the frame changes, so cropping here would bake in one + * shape and lose the rest of the picture for good. + */ +export async function prepareImage(file: Blob): Promise { + if (!file.type.startsWith("image/")) + throw new UnusableImage("not-an-image"); + if (await isAnimated(file)) throw new UnusableImage("animated"); + + let bitmap: ImageBitmap; + try { + bitmap = await createImageBitmap(file); + } catch (e) { + logger.debug("Could not decode the chosen file", e); + throw new UnusableImage("undecodable"); + } + + try { + const longest = Math.max(bitmap.width, bitmap.height); + if (longest <= maxStoredEdge) return file; + + const scale = maxStoredEdge / longest; + const canvas = new OffscreenCanvas( + Math.round(bitmap.width * scale), + Math.round(bitmap.height * scale), + ); + const context = canvas.getContext("2d"); + if (!context) throw new UnusableImage("undecodable"); + context.drawImage(bitmap, 0, 0, canvas.width, canvas.height); + return await canvas.convertToBlob({ type: "image/webp", quality: 0.9 }); + } finally { + bitmap.close(); + } +} + +const DB_NAME = "element-call-background-images"; +const STORE = "backgrounds"; + +/** + * The backgrounds a user added, kept on this device. + * + * They never leave it: there is no upload here and no identifier that would + * let anything fetch one. + */ +export class BackgroundImageStore { + private db?: Promise; + + /** The factory is injected so a test can hand over its own. */ + public constructor( + private readonly indexedDB: IDBFactory = globalThis.indexedDB, + ) {} + + private async open(): Promise { + return (this.db ??= new Promise((resolve, reject) => { + const request = this.indexedDB.open(DB_NAME, 1); + request.onupgradeneeded = (): void => { + request.result.createObjectStore(STORE, { keyPath: "id" }); + }; + request.onsuccess = (): void => resolve(request.result); + request.onerror = (): void => reject(request.error); + })); + } + + private async run( + mode: IDBTransactionMode, + body: (store: IDBObjectStore) => IDBRequest, + ): Promise { + const db = await this.open(); + return new Promise((resolve, reject) => { + const request = body(db.transaction(STORE, mode).objectStore(STORE)); + request.onsuccess = (): void => resolve(request.result); + request.onerror = (): void => reject(request.error); + }); + } + + public async list(): Promise { + const all = await this.run("readonly", (s) => + s.getAll(), + ); + return all.sort((a, b) => a.addedAt - b.addedAt); + } + + /** + * Keeps a file as a background, and answers with it. + * + * Throws {@link UnusableImage} for a file that cannot be used, and + * {@link RangeError} once the device is already keeping as many as it will. + */ + public async add(file: Blob): Promise { + const image = await prepareImage(file); + if ((await this.list()).length >= maxAddedBackgrounds) + throw new RangeError("Already keeping the most backgrounds we keep"); + const background: AddedBackground = { + id: crypto.randomUUID(), + image, + addedAt: Date.now(), + }; + await this.run("readwrite", (s) => s.add(background)); + return background; + } + + public async remove(id: string): Promise { + await this.run("readwrite", (s) => s.delete(id)); + } +} diff --git a/src/room/InCallView.test.tsx b/src/room/InCallView.test.tsx index 357bc186e..e949cddb6 100644 --- a/src/room/InCallView.test.tsx +++ b/src/room/InCallView.test.tsx @@ -170,14 +170,16 @@ function createInCallView(args: CreateInCallViewArgs = {}): RenderResult & { const renderResult = render( - - - {content} - - + + + + {content} + + + , ); diff --git a/src/room/KnockLobbyView.test.tsx b/src/room/KnockLobbyView.test.tsx index a513952b7..5dc527287 100644 --- a/src/room/KnockLobbyView.test.tsx +++ b/src/room/KnockLobbyView.test.tsx @@ -27,6 +27,15 @@ vi.mock("../livekit/TrackProcessorContext", () => ({ processor: undefined, }), useTrackProcessorSync: (): void => {}, + useAddedBackgrounds: (): { + added: []; + addBackground: () => Promise; + removeBackground: () => Promise; + } => ({ + added: [], + addBackground: async (): Promise => {}, + removeBackground: async (): Promise => {}, + }), })); vi.mock("react-use-measure", () => ({ diff --git a/src/room/LobbyView.test.tsx b/src/room/LobbyView.test.tsx index a75dc1c79..f29a6d9a5 100644 --- a/src/room/LobbyView.test.tsx +++ b/src/room/LobbyView.test.tsx @@ -42,6 +42,15 @@ vi.mock("../livekit/TrackProcessorContext", () => ({ processor: undefined, }), useTrackProcessorSync: (): void => {}, + useAddedBackgrounds: (): { + added: []; + addBackground: () => Promise; + removeBackground: () => Promise; + } => ({ + added: [], + addBackground: async (): Promise => {}, + removeBackground: async (): Promise => {}, + }), })); vi.mock("react-use-measure", () => ({