From 11e143c8abe3fe896adbd7a62014a0191be1ed15 Mon Sep 17 00:00:00 2001 From: "Timo K." Date: Thu, 5 Nov 2026 13:21:42 +0100 Subject: [PATCH 01/34] fix camera restart due to device id mismatch --- .../livekit/LivekitMediaTrack.test.ts | 45 +++++++++++++++++++ .../livekit/LivekitMediaTrack.ts | 6 ++- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/sdk/src/media-backend/livekit/LivekitMediaTrack.test.ts b/sdk/src/media-backend/livekit/LivekitMediaTrack.test.ts index fc11f300d..790eddd1a 100644 --- a/sdk/src/media-backend/livekit/LivekitMediaTrack.test.ts +++ b/sdk/src/media-backend/livekit/LivekitMediaTrack.test.ts @@ -225,6 +225,51 @@ describe("createLivekitMediaTrack", () => { ); }); + // LiveKit's media stream track is the processed one while a processor runs, + // and that one names no device; the camera behind it does + it.each([ + { deviceId: "cam-1", switches: false }, + { deviceId: "cam-2", switches: true }, + ])( + "leaves a processed camera alone unless its device changes: $deviceId", + async ({ deviceId, switches }) => { + vi.stubGlobal("MediaStreamTrack", class {}); + onTestFinished(() => { + vi.unstubAllGlobals(); + }); + const camera = Object.assign(Object.create(LocalVideoTrack.prototype), { + source: Track.Source.Camera, + _mediaStreamTrack: { + readyState: "live", + label: "", + getSettings: () => ({ deviceId: "cam-1" }), + }, + // What LiveKit shows as the track, with no device of its own + processor: { + processedTrack: { label: "", getSettings: () => ({}) }, + }, + }) as LocalVideoTrack; + const switchActiveDevice = vi.fn().mockResolvedValue(true); + const { participant, publication, room } = fakes({ + track: camera as unknown as FakeTrack, + }); + const track = createLocalLivekitMediaTrack( + scope, + Object.assign(participant, { + isLocal: true, + }) as unknown as LocalParticipant, + publication as unknown as TrackPublication, + Object.assign(room, { switchActiveDevice }), + vi.fn(), + ); + + await track.setDevice(deviceId); + expect(switchActiveDevice.mock.calls).toEqual( + switches ? [["videoinput", deviceId, false]] : [], + ); + }, + ); + // LiveKit stops a camera that is turned off; the processor is built when it // is turned on again it.each([ diff --git a/sdk/src/media-backend/livekit/LivekitMediaTrack.ts b/sdk/src/media-backend/livekit/LivekitMediaTrack.ts index 588c55761..c32fc2d52 100644 --- a/sdk/src/media-backend/livekit/LivekitMediaTrack.ts +++ b/sdk/src/media-backend/livekit/LivekitMediaTrack.ts @@ -199,11 +199,13 @@ export function createLocalLivekitMediaTrack( throw new MatrixRTCError(`A ${base.source} track has no device`); // LiveKit decides whether to restart by the constraints it keeps, which // a browser that does not echo the device id back (Firefox) leaves - // without one; a track already on the device is left alone here + // without one; a track already on the device is left alone here. Asked + // of the camera track itself: with a processor on, LiveKit's media + // stream track is the processed one, which names no device const current = publication.track; if ( current instanceof LocalTrack && - current.mediaStreamTrack.getSettings().deviceId === deviceId + (await current.getDeviceId(false)) === deviceId ) return; await room.switchActiveDevice( From fb38a1e6e0fa7df71cc4c07005b3771b52959cd7 Mon Sep 17 00:00:00 2001 From: fkwp Date: Thu, 24 Sep 2026 17:33:29 +0200 Subject: [PATCH 02/34] Check that no camera start publishes a frame of the room - A host in the call records every frame of every video track it receives from the moment the track arrives. A guest with a picture chosen joins, or turns the camera on in the call from off at joining or from off in the call; none of the guest's frames may be closer to the camera image than to the picture. A guest with no effect fails it. - The preview's track stopping destroys the shared pipeline, so the call's track builds it again. That build is held back as long as the slower route takes, and publishing still waits for it; each check that holds the model back also checks that it did. - The first build survives the preview camera turned off and on while it runs, which is what running builds one at a time is for. - The camera's colour is taken once the preview shows it; taken earlier it was black, which let frames of the room pass as the picture. The camera checks wait for the call itself, as the lobby has a camera button of its own. Co-Authored-By: Claude Opus 5.5 (1M context) --- playwright/background-effects.spec.ts | 338 +++++++++++++++++++++++++- 1 file changed, 336 insertions(+), 2 deletions(-) diff --git a/playwright/background-effects.spec.ts b/playwright/background-effects.spec.ts index 81d7a3ee4..39bfb5990 100644 --- a/playwright/background-effects.spec.ts +++ b/playwright/background-effects.spec.ts @@ -5,7 +5,13 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { expect, test, type Locator, type Page } from "@playwright/test"; +import { + expect, + test, + type Browser, + type Locator, + type Page, +} from "@playwright/test"; import { SpaHelpers } from "./spa-helpers.ts"; @@ -24,7 +30,7 @@ test.describe("background effects", () => { await page.goto("/"); await SpaHelpers.createCall(page, "Effect user", "Background effects"); const preview = page.locator("video").first(); - const camera = await averageColour(preview); + const camera = await cameraColour(preview); await page.getByRole("button", { name: "Camera", exact: true }).click(); const tile = page @@ -102,6 +108,334 @@ async function drawsPerSecond(page: Page): Promise { }); } +test.describe("joining with a background chosen", () => { + test("publishes no frame of the room", async ({ browser }) => { + // Two first builds of the pipeline, one of them held back. + test.setTimeout(180_000); + const hostContext = await browser.newContext(); + const host = await hostContext.newPage(); + await recordIncomingFrames(host); + await host.goto("/"); + await SpaHelpers.createCall(host, "Host", "Join with a background", true); + await expect(host.getByTestId("incall_videomute")).toBeEnabled({ + timeout: 10_000, + }); + const inviteLink = await SpaHelpers.getCallInviteLink(host); + await host.keyboard.press("Escape"); + + const guestContext = await browser.newContext(); + const guest = await guestContext.newPage(); + await guest.goto(inviteLink); + await guest.getByTestId("joincall_displayName").fill("Guest"); + await guest.getByTestId("joincall_joincall").click(); + const preview = guest.locator("video").first(); + await expect(preview).toBeVisible(); + const camera = await cameraColour(preview); + await guest.getByRole("button", { name: "Camera", exact: true }).click(); + const tile = guest.getByRole("menuitemradio", { name: "Background 1" }); + await tile.click(); + const picture = await averageColour(tile.locator("img")); + await expect + .poll(async () => distance(await averageColour(preview), picture), { + timeout: 60_000, + }) + .toBeLessThan(distance(camera, picture) / 2); + // A pipeline built again for the call's track now takes as long as it + // does on the slower route. Routing bypasses the cache, so it waits. + let heldBack = 0; + await guest.route("**/*.tflite*", async (route) => { + heldBack += 1; + await new Promise((resolve) => setTimeout(resolve, 15_000)); + await route.continue(); + }); + await guest.keyboard.press("Escape"); + await guest.getByTestId("lobby_joinCall").click(); + + const guestTile = host + .getByTestId("videoTile") + .filter({ has: host.getByTestId("name_tag").getByText("Guest") }); + await expect(guestTile.locator("video")).toBeVisible({ timeout: 60_000 }); + // Visible can come before the stream is attached. + let trackId: string | undefined; + await expect + .poll(async () => { + trackId = await guestTile + .locator("video") + .evaluate( + (video: HTMLVideoElement) => + (video.srcObject as MediaStream | null)?.getVideoTracks()[0]?.id, + ); + return trackId; + }) + .toBeDefined(); + await expectNoFrameOfTheRoom(host, trackId!, 0, camera, picture); + // Otherwise nothing was waited for. + expect(heldBack).toBeGreaterThan(0); + + await hostContext.close(); + await guestContext.close(); + }); +}); + +test.describe("the first build", () => { + test("survives the camera turned off and on while it runs", async ({ + page, + }) => { + test.setTimeout(120_000); + const failures: string[] = []; + page.on("console", (m) => { + if (m.text().includes("Failed to attach video processor")) + failures.push(m.text()); + }); + await page.goto("/"); + await SpaHelpers.createCall(page, "Toggling user", "Toggle while building"); + const preview = page.locator("video").first(); + const camera = await cameraColour(preview); + + const held = await holdBackTheModel(page); + await page.getByRole("button", { name: "Camera", exact: true }).click(); + const tile = page.getByRole("menuitemradio", { name: "Background 1" }); + await tile.click(); + const picture = await averageColour(tile.locator("img")); + await page.keyboard.press("Escape"); + // New preview tracks, each attached while the first is still building. + for (let i = 0; i < 2; i++) { + await page.getByTestId("incall_videomute").click(); + await page.getByTestId("incall_videomute").click(); + } + + await expect + .poll(async () => distance(await averageColour(preview), picture), { + timeout: 60_000, + }) + .toBeLessThan(distance(camera, picture) / 2); + expect(failures).toEqual([]); + expect(held()).toBeGreaterThan(0); + }); +}); + +test.describe("turning the camera on with a background chosen", () => { + test("publishes no frame of the room when the camera started off", async ({ + browser, + }) => { + test.setTimeout(120_000); + const { host, guest, close } = await hostAndGuest(browser); + const camera = await cameraColour(guest.locator("video").first()); + await guest.getByTestId("incall_videomute").click(); + await guest.getByTestId("lobby_joinCall").click(); + + const picture = await chooseBackground(guest); + const held = await holdBackTheModel(guest); + await guest.getByTestId("incall_videomute").click(); + + const trackId = await incomingTrackId(host); + await expectNoFrameOfTheRoom(host, trackId, 0, camera, picture); + expect(held()).toBeGreaterThan(0); + await close(); + }); + + test("publishes no frame of the room when the camera was turned off", async ({ + browser, + }) => { + test.setTimeout(120_000); + const { host, guest, close } = await hostAndGuest(browser); + const camera = await cameraColour(guest.locator("video").first()); + await guest.getByTestId("lobby_joinCall").click(); + const trackId = await incomingTrackId(host); + await expect + .poll(async () => (await framesOf(host, trackId)).length, { + timeout: 20_000, + }) + .toBeGreaterThan(0); + await guest.getByTestId("incall_videomute").click(); + + const picture = await chooseBackground(guest); + const held = await holdBackTheModel(guest); + const before = (await framesOf(host, trackId)).length; + await guest.getByTestId("incall_videomute").click(); + + await expectNoFrameOfTheRoom(host, trackId, before, camera, picture); + expect(held()).toBeGreaterThan(0); + await close(); + }); +}); + +/** A host in a call, recording what it receives, and a guest in its lobby. */ +async function hostAndGuest(browser: Browser): Promise<{ + host: Page; + guest: Page; + close: () => Promise; +}> { + const hostContext = await browser.newContext(); + const host = await hostContext.newPage(); + await recordIncomingFrames(host); + await host.goto("/"); + await SpaHelpers.createCall(host, "Host", "Camera with a background", true); + await expect(host.getByTestId("incall_videomute")).toBeEnabled({ + timeout: 10_000, + }); + const inviteLink = await SpaHelpers.getCallInviteLink(host); + await host.keyboard.press("Escape"); + + const guestContext = await browser.newContext(); + const guest = await guestContext.newPage(); + await guest.goto(inviteLink); + await guest.getByTestId("joincall_displayName").fill("Guest"); + await guest.getByTestId("joincall_joincall").click(); + await expect(guest.locator("video").first()).toBeVisible(); + return { + host, + guest, + close: async (): Promise => { + await hostContext.close(); + await guestContext.close(); + }, + }; +} + +/** Chooses the first shipped picture in the call, and answers its colour. */ +async function chooseBackground(guest: Page): Promise { + // The lobby has a camera button of its own, so wait for the call's. + await expect(guest.getByTestId("incall_leave")).toBeVisible({ + timeout: 30_000, + }); + await expect(guest.getByTestId("incall_videomute")).toBeEnabled({ + timeout: 10_000, + }); + await guest.getByRole("button", { name: "Camera", exact: true }).click(); + const tile = guest.getByRole("menuitemradio", { name: "Background 1" }); + await tile.click(); + await expect(tile).toHaveAttribute("aria-checked", "true"); + const picture = await averageColour(tile.locator("img")); + await guest.keyboard.press("Escape"); + return picture; +} + +/** Makes the pipeline's first build as slow as on the slower route. */ +/** Answers how often it held the model back, which a check should see. */ +async function holdBackTheModel(page: Page): Promise<() => number> { + let held = 0; + // Routing bypasses the cache, so it waits. + await page.route("**/*.tflite*", async (route) => { + held += 1; + await new Promise((resolve) => setTimeout(resolve, 5_000)); + await route.continue(); + }); + return () => held; +} + +/** The id of the guest's camera track, as the host receives it. */ +async function incomingTrackId(host: Page): Promise { + const guestTile = host + .getByTestId("videoTile") + .filter({ has: host.getByTestId("name_tag").getByText("Guest") }); + await expect(guestTile.locator("video")).toBeVisible({ timeout: 60_000 }); + // Visible can come before the stream is attached. + let trackId: string | undefined; + await expect + .poll(async () => { + trackId = await guestTile + .locator("video") + .evaluate( + (video: HTMLVideoElement) => + (video.srcObject as MediaStream | null)?.getVideoTracks()[0]?.id, + ); + return trackId; + }) + .toBeDefined(); + return trackId!; +} + +/** Waits for the picture to arrive, then finds no frame of the room since. */ +async function expectNoFrameOfTheRoom( + host: Page, + trackId: string, + since: number, + camera: Colour, + picture: Colour, +): Promise { + await expect + .poll( + async () => { + const frames = (await framesOf(host, trackId)).slice(since); + // A black frame, as a track starts, is nearer the picture than the + // camera, but shows neither. + return frames.some( + (f) => + distance(f, picture) < + Math.min(distance(f, camera), distance(f, [0, 0, 0])), + ); + }, + { timeout: 60_000 }, + ) + .toBe(true); + const frames = (await framesOf(host, trackId)).slice(since); + expect( + frames.filter((f) => distance(f, camera) < distance(f, picture)), + ).toEqual([]); +} + +/** + * Records the mean colour of every frame of every video track the page + * receives, from the moment it arrives: a tile may attach it later than its + * first frame. + */ +async function recordIncomingFrames(page: Page): Promise { + await page.addInitScript(() => { + const frames: Record = {}; + (window as unknown as { receivedFrames: typeof frames }).receivedFrames = + frames; + const record = (track: MediaStreamTrack): void => { + if (track.kind !== "video" || frames[track.id]) return; + const seen: [number, number, number][] = (frames[track.id] = []); + const video = document.createElement("video"); + video.muted = true; + video.srcObject = new MediaStream([track]); + const canvas = document.createElement("canvas"); + canvas.width = canvas.height = 16; + const context = canvas.getContext("2d", { willReadFrequently: true })!; + const step = (): void => { + context.drawImage(video, 0, 0, 16, 16); + const { data } = context.getImageData(0, 0, 16, 16); + const sum = [0, 0, 0]; + for (let i = 0; i < data.length; i += 4) + for (let c = 0; c < 3; c++) sum[c] += data[i + c]; + seen.push(sum.map((v) => v / (data.length / 4)) as Colour); + video.requestVideoFrameCallback(step); + }; + video.requestVideoFrameCallback(step); + void video.play(); + }; + const Connection = window.RTCPeerConnection; + window.RTCPeerConnection = class extends Connection { + public constructor(...args: ConstructorParameters) { + super(...args); + this.addEventListener("track", (e) => record(e.track)); + } + }; + }); +} + +async function framesOf(page: Page, trackId: string): Promise { + return page.evaluate( + (id) => + ( + window as unknown as { + receivedFrames: Record; + } + ).receivedFrames[id] ?? [], + trackId, + ); +} + +/** The camera's colour, once the preview is showing it rather than nothing. */ +async function cameraColour(preview: Locator): Promise { + await expect + .poll(async () => Math.max(...(await averageColour(preview)))) + .toBeGreaterThan(40); + return averageColour(preview); +} + /** The mean colour of a video's current frame or an image, drawn small. */ async function averageColour(element: Locator): Promise { return element.evaluate( From ecf2cbc71eda4f0de0e20c525b98ab6f3266a1e9 Mon Sep 17 00:00:00 2001 From: fkwp Date: Thu, 24 Sep 2026 17:46:55 +0200 Subject: [PATCH 03/34] Check the background effects stay inside the call everywhere - The standalone app at five window sizes, from a large desktop to a phone-shaped 360x640, on Chromium and Firefox: the camera menu stays inside the window, which is the call area there, and the last effect can still be scrolled to and chosen. - As a component in a host page, and as a widget in Element Web: a picture chosen reaches the preview, so the pipeline's model and WebAssembly load from those builds, and in a small container the last effect is reached and chosen. - Compound portals the menu to the page and takes no container for it, so there it has none of the component's scoped styles, as the microphone menu has none. - The colour checks the three share live in playwright/utils. Co-Authored-By: Claude Opus 5.5 (1M context) --- playwright/background-effects.spec.ts | 92 +++++++++++-------- .../component/background-effects.spec.ts | 85 +++++++++++++++++ playwright/utils/colour.ts | 55 +++++++++++ playwright/widget/background-effects.spec.ts | 43 +++++++++ 4 files changed, 235 insertions(+), 40 deletions(-) create mode 100644 playwright/component/background-effects.spec.ts create mode 100644 playwright/utils/colour.ts create mode 100644 playwright/widget/background-effects.spec.ts diff --git a/playwright/background-effects.spec.ts b/playwright/background-effects.spec.ts index 39bfb5990..4be24c1a5 100644 --- a/playwright/background-effects.spec.ts +++ b/playwright/background-effects.spec.ts @@ -5,15 +5,15 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { - expect, - test, - type Browser, - type Locator, - type Page, -} from "@playwright/test"; +import { expect, test, type Browser, type Page } from "@playwright/test"; import { SpaHelpers } from "./spa-helpers.ts"; +import { + averageColour, + cameraColour, + type Colour, + distance, +} from "./utils/colour.ts"; // Background effects need WebGL2, and headless Firefox on a CI runner has none, // so it rightly offers none of them. @@ -22,8 +22,6 @@ test.skip( "Background effects need WebGL2, which headless Firefox on CI does not have", ); -type Colour = [number, number, number]; - test.describe("background effects", () => { test("pre-join preview shows the chosen effect", async ({ page }) => { test.slow(); @@ -108,6 +106,51 @@ async function drawsPerSecond(page: Page): Promise { }); } +test.describe("background effects section", () => { + test("section stays within the call area at every size", async ({ page }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall(page, "Sizes user", "Background effect sizes"); + + for (const size of [ + { width: 1280, height: 800 }, + { width: 800, height: 600 }, + { width: 600, height: 420 }, + { width: 480, height: 360 }, + { width: 360, height: 640 }, + ]) { + await page.setViewportSize(size); + await page.getByRole("button", { name: "Camera", exact: true }).click(); + const menu = page.getByRole("menu"); + await expect(menu).toBeVisible(); + + const box = (await menu.boundingBox())!; + const where = `at ${size.width}x${size.height}`; + expect(box.x, where).toBeGreaterThanOrEqual(0); + expect(box.y, where).toBeGreaterThanOrEqual(0); + expect(box.x + box.width, where).toBeLessThanOrEqual(size.width); + expect(box.y + box.height, where).toBeLessThanOrEqual(size.height); + + const last = page + .getByRole("group", { name: "Background effects" }) + .getByRole("menuitemradio") + .last(); + await last.scrollIntoViewIfNeeded(); + await expect(last, where).toBeInViewport({ ratio: 1 }); + await last.click(); + await expect(last, where).toHaveAttribute("aria-checked", "true"); + + // Back to no effect for the next size, and close. + await page + .getByRole("group", { name: "Background effects" }) + .getByRole("menuitemradio", { name: "None" }) + .click(); + await page.keyboard.press("Escape"); + await expect(menu).toBeHidden(); + } + }); +}); + test.describe("joining with a background chosen", () => { test("publishes no frame of the room", async ({ browser }) => { // Two first builds of the pipeline, one of them held back. @@ -427,34 +470,3 @@ async function framesOf(page: Page, trackId: string): Promise { trackId, ); } - -/** The camera's colour, once the preview is showing it rather than nothing. */ -async function cameraColour(preview: Locator): Promise { - await expect - .poll(async () => Math.max(...(await averageColour(preview)))) - .toBeGreaterThan(40); - return averageColour(preview); -} - -/** The mean colour of a video's current frame or an image, drawn small. */ -async function averageColour(element: Locator): Promise { - return element.evaluate( - async (source: HTMLVideoElement | HTMLImageElement) => { - if (source instanceof HTMLImageElement) await source.decode(); - const canvas = document.createElement("canvas"); - canvas.width = canvas.height = 32; - const context = canvas.getContext("2d")!; - context.drawImage(source, 0, 0, 32, 32); - const { data } = context.getImageData(0, 0, 32, 32); - const sum = [0, 0, 0]; - for (let i = 0; i < data.length; i += 4) - for (let c = 0; c < 3; c++) sum[c] += data[i + c]; - const pixels = data.length / 4; - return sum.map((s) => s / pixels) as Colour; - }, - ); -} - -function distance(a: Colour, b: Colour): number { - return Math.hypot(a[0] - b[0], a[1] - b[1], a[2] - b[2]); -} diff --git a/playwright/component/background-effects.spec.ts b/playwright/component/background-effects.spec.ts new file mode 100644 index 000000000..9e1a0c5c1 --- /dev/null +++ b/playwright/component/background-effects.spec.ts @@ -0,0 +1,85 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { expect, type Locator, type Page, test } from "@playwright/test"; + +import { createUserAndRoom, resizeContainer, startHarness } from "./harness.ts"; +import { averageColour, cameraColour, expectWearing } from "../utils/colour.ts"; + +// Background effects need WebGL2, and headless Firefox on a CI runner has none, +// so it rightly offers none of them. +test.skip( + ({ browserName }) => browserName === "firefox", + "Background effects need WebGL2, which headless Firefox on CI does not have", +); + +/** + * Background effects with Element Call as a component in a host page: the + * pipeline's model and WebAssembly have to load from the component's bundle, + * and the portalled menu has to be sized against the call, not the window. + * Driven from the lobby, which builds the same menu and pipeline. + */ + +// Sign-in, crypto setup and sync happen twice before anything shows. +test.describe.configure({ timeout: 180_000 }); + +test("puts a picture on the call's preview", async ({ page }) => { + const { username, roomId } = await createUserAndRoom("effects"); + const panes = await startHarness(page, username, roomId); + const pane = panes.first(); + await resizeContainer(pane.getByTestId("call-container"), { + width: 900, + height: 480, + }); + await expect(pane.getByTestId("lobby_joinCall")).toBeVisible({ + timeout: 60_000, + }); + const preview = pane.locator("video").first(); + const camera = await cameraColour(preview); + + const tile = await openEffect(page, pane, "Background 1"); + const picture = await averageColour(tile.locator("img")); + await tile.click(); + await expect(tile).toHaveAttribute("aria-checked", "true"); + await expectWearing(preview, picture, camera); +}); + +test("keeps every effect reachable in a small container", async ({ page }) => { + const { username, roomId } = await createUserAndRoom("effectsreach"); + const panes = await startHarness(page, username, roomId); + const pane = panes.first(); + const container = pane.getByTestId("call-container"); + await resizeContainer(container, { width: 400, height: 360 }); + await expect(pane.getByTestId("lobby_joinCall")).toBeVisible({ + timeout: 60_000, + }); + + const last = await openEffect(page, pane, "Background 2"); + const list = page.locator("[role='menu'] div[role='none']").first(); + expect((await list.boundingBox())!.height).toBeLessThanOrEqual( + (await container.boundingBox())!.height, + ); + await last.scrollIntoViewIfNeeded(); + await expect(last).toBeInViewport({ ratio: 1 }); + await last.click(); + await expect(last).toHaveAttribute("aria-checked", "true"); +}); + +/** Opens a component's camera menu and returns one of its effects. */ +async function openEffect( + page: Page, + pane: Locator, + name: string, +): Promise { + await pane + .getByRole("button", { name: "Camera", exact: true }) + .click({ timeout: 60_000 }); + // Portalled out of the pane, so found on the page. + return page + .getByRole("group", { name: "Background effects" }) + .getByRole("menuitemradio", { name }); +} diff --git a/playwright/utils/colour.ts b/playwright/utils/colour.ts new file mode 100644 index 000000000..cb096e15c --- /dev/null +++ b/playwright/utils/colour.ts @@ -0,0 +1,55 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { expect, type Locator } from "@playwright/test"; + +export type Colour = [number, number, number]; + +/** The camera's colour, once the preview is showing it rather than nothing. */ +export async function cameraColour(preview: Locator): Promise { + await expect + .poll(async () => Math.max(...(await averageColour(preview)))) + .toBeGreaterThan(40); + return averageColour(preview); +} + +/** The mean colour of a video's current frame or an image, drawn small. */ +export async function averageColour(element: Locator): Promise { + return element.evaluate( + async (source: HTMLVideoElement | HTMLImageElement) => { + if (source instanceof HTMLImageElement) await source.decode(); + const canvas = document.createElement("canvas"); + canvas.width = canvas.height = 32; + const context = canvas.getContext("2d")!; + context.drawImage(source, 0, 0, 32, 32); + const { data } = context.getImageData(0, 0, 32, 32); + const sum = [0, 0, 0]; + for (let i = 0; i < data.length; i += 4) + for (let c = 0; c < 3; c++) sum[c] += data[i + c]; + const pixels = data.length / 4; + return sum.map((s) => s / pixels) as Colour; + }, + ); +} + +export function distance(a: Colour, b: Colour): number { + return Math.hypot(a[0] - b[0], a[1] - b[1], a[2] - b[2]); +} + +/** Waits until a preview shows a picture rather than the camera. */ +export async function expectWearing( + preview: Locator, + picture: Colour, + camera: Colour, +): Promise { + // Long enough for the first effect of a session to be built. + await expect + .poll(async () => distance(await averageColour(preview), picture), { + timeout: 60_000, + }) + .toBeLessThan(distance(camera, picture) / 2); +} diff --git a/playwright/widget/background-effects.spec.ts b/playwright/widget/background-effects.spec.ts new file mode 100644 index 000000000..32c499bc0 --- /dev/null +++ b/playwright/widget/background-effects.spec.ts @@ -0,0 +1,43 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { expect, test } from "@playwright/test"; + +import { widgetTest } from "../fixtures/widget-user.ts"; +import { TestHelpers } from "./test-helpers.ts"; +import { averageColour, cameraColour, expectWearing } from "../utils/colour.ts"; + +// Background effects need WebGL2, and headless Firefox on a CI runner has none, +// so it rightly offers none of them. +test.skip( + ({ browserName }) => browserName === "firefox", + "Background effects need WebGL2, which headless Firefox on CI does not have", +); + +widgetTest( + "puts a picture on the preview inside Element Web", + async ({ asWidget }) => { + test.slow(); + const { brooks } = asWidget; + await TestHelpers.startCallInCurrentRoom(brooks.page, false); + const frame = brooks.page + .locator('iframe[title="Element Call"]') + .contentFrame(); + await expect(frame.getByTestId("lobby_joinCall")).toBeVisible(); + const preview = frame.locator("video").first(); + const camera = await cameraColour(preview); + + await frame.getByRole("button", { name: "Camera", exact: true }).click(); + const tile = frame + .getByRole("group", { name: "Background effects" }) + .getByRole("menuitemradio", { name: "Background 1" }); + const picture = await averageColour(tile.locator("img")); + await tile.click(); + await expect(tile).toHaveAttribute("aria-checked", "true"); + await expectWearing(preview, picture, camera); + }, +); From 4f184f504d4bd6a20a4c397e751c91cb3bc56cee Mon Sep 17 00:00:00 2001 From: "Timo K." Date: Thu, 5 Nov 2026 15:30:25 +0100 Subject: [PATCH 04/34] Expect the join to fetch no model, as it adopts the lobby's pipeline The e2e check held back the model download at the join and required one, since the pipeline used to be built again for the call's track. The join now takes the lobby's running pipeline over, so no download happens, and the check says so. Co-Authored-By: Claude Fable 5.1 --- playwright/background-effects.spec.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/playwright/background-effects.spec.ts b/playwright/background-effects.spec.ts index 4be24c1a5..c77ea5faf 100644 --- a/playwright/background-effects.spec.ts +++ b/playwright/background-effects.spec.ts @@ -183,8 +183,8 @@ test.describe("joining with a background chosen", () => { timeout: 60_000, }) .toBeLessThan(distance(camera, picture) / 2); - // A pipeline built again for the call's track now takes as long as it - // does on the slower route. Routing bypasses the cache, so it waits. + // Were the pipeline built again for the call's track, it would take as + // long as on the slower route. Routing bypasses the cache, so it waits. let heldBack = 0; await guest.route("**/*.tflite*", async (route) => { heldBack += 1; @@ -212,8 +212,8 @@ test.describe("joining with a background chosen", () => { }) .toBeDefined(); await expectNoFrameOfTheRoom(host, trackId!, 0, camera, picture); - // Otherwise nothing was waited for. - expect(heldBack).toBeGreaterThan(0); + // The join adopts the lobby's pipeline, so the model is never fetched again. + expect(heldBack).toBe(0); await hostContext.close(); await guestContext.close(); From 741dab6dbbd2fef1c50ad2b7e2704d2ce7d8bad1 Mon Sep 17 00:00:00 2001 From: fkwp Date: Thu, 24 Sep 2026 18:09:16 +0200 Subject: [PATCH 05/34] Hold the chosen effect across calls, and forget one that can't run - A remembered effect this browser can't run is cleared rather than kept, as the menu already showed no effect in force there. - A camera turned off and on again with an effect chosen gets the same pipeline as its track arrives, neither rebuilt nor switched again. - End to end, on Chromium and Firefox: the effect chosen before a call is still chosen, and worn, in the next call after leaving; and a second tab still shuts the first down with the existing "Opened in another tab" error while an effect runs. - Switching camera and sharing a screen are checked by hand: they need two cameras and a screen, which the fake devices don't give yet. Co-Authored-By: Claude Opus 5.5 (1M context) --- playwright/background-effects.spec.ts | 63 +++++++++++++++++++++ src/state/BackgroundEffects.test.ts | 11 ++++ src/state/BackgroundEffects.ts | 16 +++++- src/tracks/BackgroundEffectsContext.test.ts | 4 +- src/tracks/BackgroundEffectsContext.tsx | 1 + 5 files changed, 93 insertions(+), 2 deletions(-) diff --git a/playwright/background-effects.spec.ts b/playwright/background-effects.spec.ts index c77ea5faf..3947466ae 100644 --- a/playwright/background-effects.spec.ts +++ b/playwright/background-effects.spec.ts @@ -151,6 +151,69 @@ test.describe("background effects section", () => { }); }); +test.describe("the chosen effect", () => { + test("chosen effect survives leaving and rejoining", async ({ page }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall(page, "Returning user", "Effect remembered"); + const { camera, picture } = await chooseAndWear(page, "Background 1"); + await page.getByTestId("lobby_joinCall").click(); + await page.getByTestId("incall_leave").click(); + await expect(page.getByRole("heading")).toContainText( + "your call has ended", + ); + + await page.goto("/"); + await page.getByTestId("home_callName").fill("Effect remembered again"); + await page.getByTestId("home_go").click(); + await expect(page.getByTestId("lobby_joinCall")).toBeVisible(); + await page.getByRole("button", { name: "Camera", exact: true }).click(); + const tile = page + .getByRole("group", { name: "Background effects" }) + .getByRole("menuitemradio", { name: "Background 1" }); + await expect(tile).toHaveAttribute("aria-checked", "true"); + const preview = page.locator("video").first(); + await expect + .poll(async () => distance(await averageColour(preview), picture), { + timeout: 60_000, + }) + .toBeLessThan(distance(camera, picture) / 2); + }); + + test("second session still shows the existing error", async ({ page }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall(page, "Two tabs", "Effect in two tabs"); + await chooseAndWear(page, "Background 1"); + + const second = await page.context().newPage(); + await second.goto("/"); + await expect( + page.getByRole("heading", { name: "Opened in another tab" }), + ).toBeVisible(); + }); +}); + +/** Chooses an effect in the lobby, and waits until the preview wears it. */ +async function chooseAndWear( + page: Page, + name: string, +): Promise<{ camera: Colour; picture: Colour }> { + const preview = page.locator("video").first(); + const camera = await cameraColour(preview); + await page.getByRole("button", { name: "Camera", exact: true }).click(); + const tile = page.getByRole("menuitemradio", { name }); + await tile.click(); + const picture = await averageColour(tile.locator("img")); + await expect + .poll(async () => distance(await averageColour(preview), picture), { + timeout: 60_000, + }) + .toBeLessThan(distance(camera, picture) / 2); + await page.keyboard.press("Escape"); + return { camera, picture }; +} + test.describe("joining with a background chosen", () => { test("publishes no frame of the room", async ({ browser }) => { // Two first builds of the pipeline, one of them held back. diff --git a/src/state/BackgroundEffects.test.ts b/src/state/BackgroundEffects.test.ts index 74856f82e..375a22384 100644 --- a/src/state/BackgroundEffects.test.ts +++ b/src/state/BackgroundEffects.test.ts @@ -69,6 +69,7 @@ describe("the pipeline's state", () => { const effects = createBackgroundEffects(testScope(), { supported: true, effect$: behavior(effect, { n: "none", b: "blur" }), + setEffect: vi.fn(), pipeline: fakePipeline().pipeline, transformer, }); @@ -94,6 +95,7 @@ describe("the pipeline's state", () => { describe("background effects", () => { let effect$: BehaviorSubject; + let setEffect: (raw: string) => void; let fake: ReturnType; function build( @@ -102,6 +104,7 @@ describe("background effects", () => { return createBackgroundEffects(testScope(), { supported: true, effect$, + setEffect, pipeline: fake.pipeline, transformer: { onFirstFrame: undefined }, ...options, @@ -116,6 +119,7 @@ describe("background effects", () => { beforeEach(() => { effect$ = new BehaviorSubject("none"); + setEffect = vi.fn((raw: string) => effect$.next(raw)); fake = fakePipeline(); }); @@ -169,4 +173,11 @@ describe("background effects", () => { { mode: "disabled" }, ]); }); + + it("clears a remembered effect when it cannot be honoured", () => { + effect$.next(`image:${shippedBackgrounds[0].id}`); + const effects = build({ supported: false }); + expect(effect$.value).toBe("none"); + expect(effects.state$.value.processor).toBeUndefined(); + }); }); diff --git a/src/state/BackgroundEffects.ts b/src/state/BackgroundEffects.ts index a00cf3432..ecfd8b25a 100644 --- a/src/state/BackgroundEffects.ts +++ b/src/state/BackgroundEffects.ts @@ -27,6 +27,7 @@ import { type Behavior } from "./Behavior"; import { type ObservableScope } from "./ObservableScope"; import { type BackgroundEffect, + type EffectId, imagePathFor, parseEffect, } from "../tracks/backgroundEffectCatalog"; @@ -45,6 +46,8 @@ export interface BackgroundEffectsOptions { supported: boolean; /** The effect chosen, as the setting stores it. */ effect$: Behavior; + /** Stores a choice, to forget one that can't be honoured. */ + setEffect: (id: EffectId) => void; /** The background processor pipeline to be switched between effects. */ pipeline: BackgroundProcessorWrapper; /** Tells, once, that a frame carrying an effect has been drawn. */ @@ -59,7 +62,13 @@ export interface BackgroundEffects { /** Switches the pipeline as the choice changes, for as long as the scope lasts. */ export function createBackgroundEffects( scope: ObservableScope, - { supported, effect$, pipeline, transformer }: BackgroundEffectsOptions, + { + supported, + effect$, + setEffect, + pipeline, + transformer, + }: BackgroundEffectsOptions, ): BackgroundEffects { // Wrapped once, so the call and the preview see one processor throughout. const processor = fromLivekitProcessor(pipeline); @@ -69,6 +78,11 @@ export function createBackgroundEffects( distinctUntilChanged(), ); + choice$.pipe(scope.bind()).subscribe((effect) => { + if (effect.kind === "none") return; + if (!supported) setEffect("none"); + }); + const drewAFrame$ = scope.behavior( new Observable((subscriber) => { subscriber.next(false); diff --git a/src/tracks/BackgroundEffectsContext.test.ts b/src/tracks/BackgroundEffectsContext.test.ts index 71d9d7152..449bac4e5 100644 --- a/src/tracks/BackgroundEffectsContext.test.ts +++ b/src/tracks/BackgroundEffectsContext.test.ts @@ -21,6 +21,7 @@ import { import { backgroundEffect } from "../settings/settings"; import { flushPromises, testScope } from "../utils/test"; +const sdkSupportMock = vi.hoisted(() => vi.fn(() => true)); const pipelines = vi.hoisted(() => ({ built: 0, destroyed: 0, @@ -41,7 +42,7 @@ vi.mock("@livekit/track-processors", () => ({ return Promise.resolve(); }); }), - supportsBackgroundProcessors: (): boolean => true, + supportsBackgroundProcessors: (): boolean => sdkSupportMock(), })); const platformMock = vi.hoisted(() => ({ platform: "desktop" })); vi.mock("../Platform", () => platformMock); @@ -77,6 +78,7 @@ describe("BackgroundEffectsProvider", () => { pipelines.destroyed = 0; pipelines.switches = []; platformMock.platform = "desktop"; + sdkSupportMock.mockReturnValue(true); backgroundEffect.setValue("none"); }); afterEach(() => backgroundEffect.setValue("none")); diff --git a/src/tracks/BackgroundEffectsContext.tsx b/src/tracks/BackgroundEffectsContext.tsx index d582bc0b3..d5a00d3bb 100644 --- a/src/tracks/BackgroundEffectsContext.tsx +++ b/src/tracks/BackgroundEffectsContext.tsx @@ -53,6 +53,7 @@ export function createAppBackgroundEffects( return createBackgroundEffects(scope, { supported: supportsBackgroundProcessors(), effect$: backgroundEffectSetting.value$, + setEffect: backgroundEffectSetting.setValue, pipeline: new BackgroundProcessorWrapper(transformer, "background-effect"), transformer, }); From e9cffb60dd3a4e2494b46f7a78b38fbdd3ec462a Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 5 Nov 2026 16:08:41 +0100 Subject: [PATCH 06/34] Fix: Uncaught exception in downloadConfig --- src/config/Config.test.ts | 13 +++++++++++++ src/config/Config.ts | 17 ++++++++++++----- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/config/Config.test.ts b/src/config/Config.test.ts index 5f8b09a27..6df037483 100644 --- a/src/config/Config.test.ts +++ b/src/config/Config.test.ts @@ -95,6 +95,19 @@ describe("Config.initWith", () => { expect(Config.get().ssla).toBe("https://example.invalid/ssla"); }); + it("falls back to the defaults when fetching config.json rejects", async () => { + // A rejected promise is what a webview gives for a missing local file. + vi.spyOn(globalThis, "fetch").mockRejectedValue( + new TypeError("Load failed"), + ); + vi.spyOn(logger, "info").mockImplementation(() => {}); + Reflect.set(Config, "internalInstance", undefined); + + await expect(Config.init()).resolves.toBeUndefined(); + + expect(Config.get().media_quality).toEqual(DEFAULT_CONFIG.media_quality); + }); + it("replaces a config initialized earlier", () => { Config.initWith({ ssla: "https://first.invalid/ssla" }); Config.initWith({ ssla: "https://second.invalid/ssla" }); diff --git a/src/config/Config.ts b/src/config/Config.ts index e2c5eb8fa..3c0f17bc9 100644 --- a/src/config/Config.ts +++ b/src/config/Config.ts @@ -140,13 +140,20 @@ export function validateConfig(config: ConfigOptions): ConfigOptions { } async function downloadConfig(fetchTarget: string): Promise { - const response = await fetch(fetchTarget); + let response: Response; + try { + response = await fetch(fetchTarget); + } catch (e) { + // The embedded package ships no config.json, and some webviews (Element X + // iOS) reject the fetch of a missing local file rather than answering 404. + logger.info(`No config at ${fetchTarget}, using defaults`, e); + // resolveConfig merges in the defaults. + return {}; + } if (isFailure(response)) { - // Lack of a config isn't an error, we should just use the defaults. - // Also treat a blank config as no config, assuming the status code is 0, because we don't get 404s from file: - // URIs so this is the only way we can not fail if the file doesn't exist when loading from a file:// URI. - return DEFAULT_CONFIG; + // resolveConfig merges in the defaults. + return {}; } return response.json(); From 1bc372b587af8bba3606c1cff69961badaf7a734 Mon Sep 17 00:00:00 2001 From: fkwp Date: Thu, 24 Sep 2026 18:21:57 +0200 Subject: [PATCH 07/34] Add a background of one's own, kept on this device - An add tile after the pictures opens the file picker, and the image chosen joins the backgrounds on offer without being put on; it is in force once chosen, like a shipped one. The tile is a command rather than a choice, so a menu item beside the radios, and it isn't offered where effects can't run. - Kept in IndexedDB on this device and never sent anywhere, behind a small interface, so the unit tests use an in-memory store; the IndexedDB store itself is tested against fake-indexeddb, a new dev dependency. - The picker takes the focus, which the menu reads as a click elsewhere, so the menu holds itself open until a file is chosen or the picker is cancelled. - A chosen image the device no longer keeps, as after its storage is cleared, falls back to no effect once the device has said what it keeps. - Switching is checked between any two kinds of effect. End to end on Chromium and Firefox: an added image shows in the preview once chosen; once it is in force, no request carries the file or the copy kept, which a deliberate upload fails; chosen, it is in force again after a reload, gone once storage is cleared, and can be added again. - Kept as given here; what can be added is checked in the commit after. Co-Authored-By: Claude Opus 5.5 (1M context) --- locales/en/app.json | 1 + package.json | 1 + playwright/background-effects.spec.ts | 166 ++++++++++++++++++ pnpm-lock.yaml | 9 + src/components/BackgroundEffectGrid.tsx | 26 +++ src/components/CallFooter.stories.tsx | 1 + src/components/CallFooter.tsx | 4 + src/components/CallFooterViewModel.test.ts | 11 ++ src/components/CallFooterViewModel.tsx | 37 +++- .../MediaMuteAndSwitchButton.stories.tsx | 6 + .../MediaMuteAndSwitchButton.test.tsx | 70 ++++++++ src/components/MediaMuteAndSwitchButton.tsx | 46 +++++ src/state/BackgroundEffects.test.ts | 57 ++++++ src/state/BackgroundEffects.ts | 51 ++++-- src/tracks/BackgroundEffectsContext.test.ts | 9 + src/tracks/BackgroundEffectsContext.tsx | 2 + src/tracks/backgroundEffectCatalog.test.ts | 2 + src/tracks/backgroundEffectCatalog.ts | 16 +- src/tracks/backgroundImages.test.ts | 95 ++++++++++ src/tracks/backgroundImages.ts | 126 +++++++++++++ 20 files changed, 717 insertions(+), 19 deletions(-) create mode 100644 src/tracks/backgroundImages.test.ts create mode 100644 src/tracks/backgroundImages.ts diff --git a/locales/en/app.json b/locales/en/app.json index 3b08e8be9..63a9b8b02 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -3,6 +3,7 @@ "user_menu": "User menu" }, "action": { + "add_background_image": "Add image", "close": "Close", "copy_link": "Copy link", "edit": "Edit", diff --git a/package.json b/package.json index efcfb2b25..d867be176 100644 --- a/package.json +++ b/package.json @@ -97,6 +97,7 @@ "copy-to-clipboard": "^3.3.3", "eslint-plugin-element-call": "link:eslint", "eslint-plugin-storybook": "^10.3.6", + "fake-indexeddb": "^6.2.5", "fetch-mock": "11.1.5", "i18next": "^26.0.0", "i18next-browser-languagedetector": "^8.0.0", diff --git a/playwright/background-effects.spec.ts b/playwright/background-effects.spec.ts index 3947466ae..579955cde 100644 --- a/playwright/background-effects.spec.ts +++ b/playwright/background-effects.spec.ts @@ -214,6 +214,172 @@ async function chooseAndWear( return { camera, picture }; } +test.describe("a background of one's own", () => { + test("adds an image from the device", async ({ page }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall(page, "Adding user", "Own background"); + const preview = page.locator("video").first(); + const camera = await cameraColour(preview); + const red = await redImage(page); + + await addImage(page, red); + const mine = page + .getByRole("group", { name: "Background effects" }) + .getByRole("menuitemradio", { name: "Background 3" }); + await expect(mine).toBeVisible(); + await expect(mine).toHaveAttribute("aria-checked", "false"); + + await mine.click(); + await expect + .poll(async () => distance(await averageColour(preview), RED), { + timeout: 60_000, + }) + .toBeLessThan(distance(camera, RED) / 2); + }); + + test("adding an image makes no upload", async ({ page }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall(page, "Private user", "Own background kept"); + const preview = page.locator("video").first(); + const camera = await cameraColour(preview); + const red = await redImage(page); + const sent: { request: string; body: Buffer | null }[] = []; + page.on("request", (request) => + sent.push({ + request: `${request.method()} ${request.url()}`, + body: request.postDataBuffer(), + }), + ); + + await addImage(page, red); + await page + .getByRole("group", { name: "Background effects" }) + .getByRole("menuitemradio", { name: "Background 3" }) + .click(); + // Kept, chosen and in force: anything that would send it has had its turn. + await expect + .poll(async () => distance(await averageColour(preview), RED), { + timeout: 60_000, + }) + .toBeLessThan(distance(camera, RED) / 2); + // The file as given, and the copy kept, which is encoded anew. + const samples = [red, await keptBytes(page)].map((bytes) => + bytes.subarray(bytes.length - 64), + ); + const carriesIt = sent.filter( + ({ request, body }) => + request.includes("/_matrix/media/") || + samples.some((sample) => body?.includes(sample)), + ); + expect(carriesIt.map(({ request }) => request)).toEqual([]); + }); + + test("added image survives rejoining and is gone after storage is cleared", async ({ + page, + }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall(page, "Keeping user", "Own background again"); + const preview = page.locator("video").first(); + const camera = await cameraColour(preview); + const red = await redImage(page); + await addImage(page, red); + const mine = page + .getByRole("group", { name: "Background effects" }) + .getByRole("menuitemradio", { name: "Background 3" }); + await mine.click(); + await page.keyboard.press("Escape"); + + await page.reload(); + await expect(page.getByTestId("lobby_joinCall")).toBeVisible(); + await page.getByRole("button", { name: "Camera", exact: true }).click(); + await expect(mine).toHaveAttribute("aria-checked", "true"); + await page.keyboard.press("Escape"); + await expect + .poll(async () => distance(await averageColour(preview), RED), { + timeout: 60_000, + }) + .toBeLessThan(distance(camera, RED) / 2); + + await page.evaluate( + async () => + new Promise((resolve, reject) => { + const request = indexedDB.deleteDatabase( + "element-call-background-images", + ); + request.onsuccess = (): void => resolve(); + request.onerror = (): void => reject(request.error); + request.onblocked = (): void => resolve(); + }), + ); + await page.reload(); + await expect(page.getByTestId("lobby_joinCall")).toBeVisible(); + await page.getByRole("button", { name: "Camera", exact: true }).click(); + await expect( + page.getByRole("group", { name: "Background effects" }), + ).toBeVisible(); + await expect(mine).toHaveCount(0); + await page.keyboard.press("Escape"); + + await addImage(page, red); + await expect(mine).toBeVisible(); + }); +}); + +/** The bytes of the one image the device keeps. */ +async function keptBytes(page: Page): Promise { + const base64 = await page.evaluate( + async () => + new Promise((resolve, reject) => { + const open = indexedDB.open("element-call-background-images"); + open.onerror = (): void => reject(open.error); + open.onsuccess = (): void => { + const all = open.result + .transaction("backgrounds") + .objectStore("backgrounds") + .getAll(); + all.onerror = (): void => reject(all.error); + all.onsuccess = (): void => { + const reader = new FileReader(); + reader.onload = (): void => + resolve((reader.result as string).split(",")[1]); + reader.readAsDataURL((all.result[0] as { image: Blob }).image); + }; + }; + }), + ); + return Buffer.from(base64, "base64"); +} + +const RED: Colour = [255, 0, 0]; + +/** A solid red PNG, unlike the camera and the shipped pictures. */ +async function redImage(page: Page): Promise { + const dataUrl = await page.evaluate(() => { + const canvas = document.createElement("canvas"); + canvas.width = 320; + canvas.height = 180; + const context = canvas.getContext("2d")!; + context.fillStyle = "rgb(255, 0, 0)"; + context.fillRect(0, 0, canvas.width, canvas.height); + return canvas.toDataURL("image/png"); + }); + return Buffer.from(dataUrl.split(",")[1], "base64"); +} + +/** Adds an image from the camera menu's add tile, leaving the menu open. */ +async function addImage(page: Page, image: Buffer): Promise { + if (!(await page.getByRole("menu").isVisible())) + await page.getByRole("button", { name: "Camera", exact: true }).click(); + const chooser = page.waitForEvent("filechooser"); + await page.getByRole("menuitem", { name: "Add image" }).click(); + await ( + await chooser + ).setFiles({ name: "mine.png", mimeType: "image/png", buffer: image }); +} + test.describe("joining with a background chosen", () => { test("publishes no frame of the room", async ({ browser }) => { // Two first builds of the pipeline, one of them held back. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index dc9fb426b..2ced4cb90 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -160,6 +160,9 @@ importers: eslint-plugin-storybook: specifier: ^10.3.6 version: 10.6.2(eslint@8.57.1(supports-color@7.2.0))(supports-color@7.2.0)(typescript@5.9.3) + fake-indexeddb: + specifier: ^6.2.5 + version: 6.2.5 fetch-mock: specifier: 11.1.5 version: 11.1.5(node-fetch@2.7.0) @@ -4048,6 +4051,10 @@ packages: resolution: {integrity: sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==} engines: {node: '>=12.0.0'} + fake-indexeddb@6.2.5: + resolution: {integrity: sha512-CGnyrvbhPlWYMngksqrSSUT1BAVP49dZocrHuK0SvtR0D5TMs5wP0o3j7jexDJW01KSadjBp1M/71o/KR3nD1w==} + engines: {node: '>=18'} + fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} @@ -9712,6 +9719,8 @@ snapshots: expect-type@1.4.0: {} + fake-indexeddb@6.2.5: {} + fast-deep-equal@3.1.3: {} fast-glob@3.3.3: diff --git a/src/components/BackgroundEffectGrid.tsx b/src/components/BackgroundEffectGrid.tsx index c75c79455..ef5944fb1 100644 --- a/src/components/BackgroundEffectGrid.tsx +++ b/src/components/BackgroundEffectGrid.tsx @@ -13,6 +13,7 @@ import { BlockIcon, BlurIcon, CheckCircleSolidIcon, + PlusIcon, } from "@vector-im/compound-design-tokens/assets/web/icons"; import styles from "./BackgroundEffectGrid.module.css"; @@ -38,6 +39,9 @@ interface Props { settling?: boolean; /** Id of the text saying what the user should know before choosing. */ describedBy?: string; + /** Offers to add a background of one's own. Omit to leave that tile out. */ + onAdd?: () => void; + addLabel?: string; } /** @@ -51,6 +55,8 @@ export const BackgroundEffectGrid: FC = ({ onSelect, settling = false, describedBy, + onAdd, + addLabel, }) => { const { t } = useTranslation(); const choose = (id: string): void => { @@ -99,10 +105,30 @@ export const BackgroundEffectGrid: FC = ({ ); }); + // A command rather than a choice, so an item beside the radios. + const addTile = + onAdd === undefined ? null : ( + { + e.preventDefault(); + onAdd(); + }} + > + + + + {addLabel} + + ); + return ( // A group of its own, as the section's can't carry the description.
{tiles} + {addTile}
); }; diff --git a/src/components/CallFooter.stories.tsx b/src/components/CallFooter.stories.tsx index b587cc40e..9e9e6168c 100644 --- a/src/components/CallFooter.stories.tsx +++ b/src/components/CallFooter.stories.tsx @@ -137,6 +137,7 @@ export const Default: Story = { ], backgroundEffectNotice: undefined, backgroundEffectSettling: false, + addBackgroundImage: fn(), hangup: fn(), buttonSize: "lg", showFooter: true, diff --git a/src/components/CallFooter.tsx b/src/components/CallFooter.tsx index 8606d3a47..652f67979 100644 --- a/src/components/CallFooter.tsx +++ b/src/components/CallFooter.tsx @@ -60,6 +60,8 @@ export interface FooterActions { toggleVideo: (() => void) | undefined; /** Undefined where background effects can't be chosen. */ selectBackgroundEffect: ((id: string) => void) | undefined; + /** Undefined where no background can be added. */ + addBackgroundImage: ((file: File) => void) | undefined; toggleScreenSharing: (() => void) | undefined; /** Also controls if the settings button is visible */ openSettings: (() => void) | undefined; @@ -168,6 +170,7 @@ export const CallFooter: FC = ({ const selectBackgroundEffect = useBehavior(vm.selectBackgroundEffect$); const backgroundEffectNotice = useBehavior(vm.backgroundEffectNotice$); const backgroundEffectSettling = useBehavior(vm.backgroundEffectSettling$); + const addBackgroundImage = useBehavior(vm.addBackgroundImage$); const backgroundEffects = useBehavior(vm.backgroundEffects$); const buttonSize = useBehavior(vm.buttonSize$); const showLogo = useBehavior(vm.showLogo$); @@ -234,6 +237,7 @@ export const CallFooter: FC = ({ selectedBackgroundEffect={backgroundEffect} onSelectBackgroundEffect={selectBackgroundEffect} backgroundEffectSettling={backgroundEffectSettling} + onAddBackgroundImage={addBackgroundImage} backgroundEffectNotice={ backgroundEffectNotice === "unavailable" ? t("background_effects.unavailable") diff --git a/src/components/CallFooterViewModel.test.ts b/src/components/CallFooterViewModel.test.ts index e061f4ef2..59376184e 100644 --- a/src/components/CallFooterViewModel.test.ts +++ b/src/components/CallFooterViewModel.test.ts @@ -21,6 +21,10 @@ import { import { HeaderStyle } from "../UrlParams"; import { backgroundEffect as backgroundEffectSetting } from "../settings/settings"; import { shippedBackgrounds } from "../tracks/backgroundEffectCatalog"; +import { + type AddedBackground, + addedBackgrounds, +} from "../tracks/backgroundImages"; const platformMock = vi.hoisted(() => vi.fn(() => "desktop")); vi.mock("../Platform", () => ({ @@ -263,6 +267,7 @@ describe("createCallFooterViewModel", () => { it("offers nothing where the browser itself cannot run them", () => { sdkSupportMock.mockReturnValue(false); expect(lobbyOn("desktop").selectBackgroundEffect$.value).toBeUndefined(); + expect(lobbyOn("desktop").addBackgroundImage$.value).toBeUndefined(); }); it("puts no effect in force where the browser cannot run them", () => { @@ -295,6 +300,10 @@ describe("createCallFooterViewModel", () => { it("offers every effect in order", () => { sdkSupportMock.mockReturnValue(true); + const added$ = addedBackgrounds.added$ as BehaviorSubject< + AddedBackground[] | undefined + >; + added$.next([{ id: "mine", url: "blob:mine" }]); expect(lobbyOn("desktop").backgroundEffects$.value).toEqual([ { id: "none", kind: "none" }, { id: "blur", kind: "blur" }, @@ -303,7 +312,9 @@ describe("createCallFooterViewModel", () => { kind: "image", imageUrl: background.imagePath, })), + { id: "added:mine", kind: "image", imageUrl: "blob:mine" }, ]); + added$.next([]); }); it("says they are unavailable where they cannot be chosen", () => { diff --git a/src/components/CallFooterViewModel.tsx b/src/components/CallFooterViewModel.tsx index e9a262335..7dc4f412d 100644 --- a/src/components/CallFooterViewModel.tsx +++ b/src/components/CallFooterViewModel.tsx @@ -17,12 +17,17 @@ import { supportsBackgroundProcessors, supportsModernBackgroundProcessors, } from "@livekit/track-processors"; +import { logger } from "matrix-js-sdk/lib/logger"; import { parseEffect, serializeEffect, shippedBackgrounds, } from "../tracks/backgroundEffectCatalog"; +import { + type AddedBackground, + addedBackgrounds, +} from "../tracks/backgroundImages"; import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; import { type MenuOptions } from "./MediaMuteAndSwitchButton"; @@ -96,6 +101,7 @@ function buildDeviceBehaviors( | "backgroundEffects$" | "backgroundEffectNotice$" | "backgroundEffectSettling$" + | "addBackgroundImage$" > { const options$ = ( available$: Behavior>, @@ -158,7 +164,11 @@ function buildDeviceBehaviors( ), ), ), - backgroundEffects$: constant(backgroundEffectChoices()), + backgroundEffects$: scope.behavior( + addedBackgrounds.added$.pipe( + map((added) => backgroundEffectChoices(added ?? [])), + ), + ), backgroundEffectNotice$: scope.behavior( offered$.pipe( map((offered) => @@ -170,6 +180,22 @@ function buildDeviceBehaviors( backgroundEffectSettling$.pipe(distinctUntilChanged()), false, ), + // Kept and offered, not put on: that waits for the user to choose it. + addBackgroundImage$: scope.behavior( + offered$.pipe( + map((offered) => + offered + ? (file: File): void => { + addedBackgrounds + .add(file) + .catch((e) => + logger.warn("Could not keep that background", e), + ); + } + : undefined, + ), + ), + ), }; } @@ -337,7 +363,9 @@ export function createLobbyFooterViewModel( }; } -function backgroundEffectChoices(): BackgroundEffectGridItem[] { +function backgroundEffectChoices( + added: AddedBackground[], +): BackgroundEffectGridItem[] { return [ { id: serializeEffect({ kind: "none" }), kind: "none" }, { id: serializeEffect({ kind: "blur" }), kind: "blur" }, @@ -346,5 +374,10 @@ function backgroundEffectChoices(): BackgroundEffectGridItem[] { kind: "image" as const, imageUrl: background.imagePath, })), + ...added.map((background) => ({ + id: serializeEffect({ kind: "added", id: background.id }), + kind: "image" as const, + imageUrl: background.url, + })), ]; } diff --git a/src/components/MediaMuteAndSwitchButton.stories.tsx b/src/components/MediaMuteAndSwitchButton.stories.tsx index ba9163234..80b02d090 100644 --- a/src/components/MediaMuteAndSwitchButton.stories.tsx +++ b/src/components/MediaMuteAndSwitchButton.stories.tsx @@ -198,6 +198,7 @@ export const VideoUnmute: Story = { backgroundEffects: effects, selectedBackgroundEffect: "none", onSelectBackgroundEffect: fn(), + onAddBackgroundImage: fn(), }, }; @@ -241,6 +242,11 @@ export const BackgroundEffects: Story = { true, ); + const add = within(section).getByRole("menuitem", { name: "Add image" }); + await expect( + tiles[3].compareDocumentPosition(add) & Node.DOCUMENT_POSITION_FOLLOWING, + ).toBeTruthy(); + await userEvent.click(menu.getByRole("menuitemradio", { name: "Blur" })); await expect(args.onSelectBackgroundEffect).toHaveBeenCalledWith("blur"); await expect(menu.getByRole("menu")).toBeVisible(); diff --git a/src/components/MediaMuteAndSwitchButton.test.tsx b/src/components/MediaMuteAndSwitchButton.test.tsx index b851462fc..6613f45ea 100644 --- a/src/components/MediaMuteAndSwitchButton.test.tsx +++ b/src/components/MediaMuteAndSwitchButton.test.tsx @@ -367,6 +367,76 @@ describe("MediaMuteAndSwitchButton", () => { expect(onSelectBackgroundEffect).not.toHaveBeenCalled(); }); + test("added images are chosen like shipped ones", async () => { + const user = userEvent.setup(); + const onSelectBackgroundEffect = vi.fn(); + const { getByRole } = renderComponent( + , + ); + + await user.click(getByRole("button", { name: "Camera" })); + const section = screen.getByRole("group", { name: "Background effects" }); + await user.click( + within(section).getByRole("menuitemradio", { name: "Background 3" }), + ); + expect(onSelectBackgroundEffect).toHaveBeenCalledWith("added:mine"); + }); + + test("adds the file chosen from the add tile, and stays open", async () => { + const user = userEvent.setup(); + const onAddBackgroundImage = vi.fn(); + const pick = vi + .spyOn(HTMLInputElement.prototype, "click") + .mockImplementation(() => {}); + try { + const { getByRole, container } = renderComponent( + , + ); + + await user.click(getByRole("button", { name: "Camera" })); + await user.click(screen.getByRole("menuitem", { name: "Add image" })); + expect(pick).toHaveBeenCalled(); + // The picker takes the focus, which would otherwise close the menu. + await user.keyboard("{Escape}"); + expect(screen.getByRole("menu")).toBeInTheDocument(); + + const file = new File(["x"], "mine.png", { type: "image/png" }); + // What the picker hands back; the open menu blocks pointer events. + fireEvent.change( + container.querySelector("input[type=file]")!, + { target: { files: [file] } }, + ); + expect(onAddBackgroundImage).toHaveBeenCalledWith(file); + screen.getByRole("menuitemradio", { name: "None", checked: true }); + } finally { + pick.mockRestore(); + } + }); + test("offers the background effects in a phone's drawer", async () => { const userAgent = vi .spyOn(navigator, "userAgent", "get") diff --git a/src/components/MediaMuteAndSwitchButton.tsx b/src/components/MediaMuteAndSwitchButton.tsx index aa6d7afe6..b8630436a 100644 --- a/src/components/MediaMuteAndSwitchButton.tsx +++ b/src/components/MediaMuteAndSwitchButton.tsx @@ -8,6 +8,7 @@ Please see LICENSE in the repository root for full details. import { useCallback, useId, + useRef, useState, type CSSProperties, type FC, @@ -79,6 +80,8 @@ export interface MediaMuteAndSwitchButtonProps { backgroundEffectNotice?: string; /** Whether the first effect chosen is still being prepared. */ backgroundEffectSettling?: boolean; + /** Called with the file chosen from the add tile. Omit to leave it out. */ + onAddBackgroundImage?: (file: File) => void; /** * For any toggle and option this method will be called. * So toggles need to be implemented by listening here and setting the right toggle item to `enabled` @@ -113,6 +116,7 @@ export const MediaMuteAndSwitchButton: FC = ({ onSelectBackgroundEffect, backgroundEffectNotice, backgroundEffectSettling, + onAddBackgroundImage, onSelect, }) => { // Requested but not yet selected. Keyed by kind too, since Chrome uses @@ -122,11 +126,29 @@ export const MediaMuteAndSwitchButton: FC = ({ id: string; } | null>(null); const [menuOpen, setMenuOpen] = useState(false); + // The file picker takes the focus, which the menu reads as a click + // elsewhere; it is held open until the picker is done. + const choosingFile = useRef(false); + const fileInput = useRef(null); const onOpenChange = useCallback((open: boolean): void => { + if (!open && choosingFile.current) return; setMenuOpen(open); // Drop a request that never arrived. if (!open) setPlannedSelection(null); }, []); + const watchFileInput = useCallback( + (input: HTMLInputElement | null): (() => void) | undefined => { + fileInput.current = input; + if (input === null) return; + // Dismissing the picker fires cancel, which React doesn't type. + const done = (): void => { + choosingFile.current = false; + }; + input.addEventListener("cancel", done); + return (): void => input.removeEventListener("cancel", done); + }, + [], + ); const isBusy = busy ?? false; const { t } = useTranslation(); const devices = useMediaDevices(); @@ -326,6 +348,21 @@ export const MediaMuteAndSwitchButton: FC = ({ > {/* The mute button lives inside */} {button} + {onAddBackgroundImage !== undefined && ( + { + const file = e.target.files?.[0]; + // Cleared, so the same file can be chosen twice in a row. + e.target.value = ""; + choosingFile.current = false; + if (file) onAddBackgroundImage(file); + }} + /> + )} = ({ describedBy={ backgroundEffectNotice === undefined ? undefined : noticeId } + onAdd={ + onAddBackgroundImage === undefined + ? undefined + : (): void => { + choosingFile.current = true; + fileInput.current?.click(); + } + } + addLabel={t("action.add_background_image")} /> {/* In the list, so the menu grows no taller for it. */} {backgroundEffectNotice !== undefined && ( diff --git a/src/state/BackgroundEffects.test.ts b/src/state/BackgroundEffects.test.ts index 375a22384..b357a8058 100644 --- a/src/state/BackgroundEffects.test.ts +++ b/src/state/BackgroundEffects.test.ts @@ -15,6 +15,7 @@ import { type BackgroundEffectsOptions, type BackgroundEffectsState, } from "./BackgroundEffects"; +import { type AddedBackground } from "../tracks/backgroundImages"; import { shippedBackgrounds } from "../tracks/backgroundEffectCatalog"; import { flushPromises, testScope, withTestScheduler } from "../utils/test"; @@ -70,6 +71,7 @@ describe("the pipeline's state", () => { supported: true, effect$: behavior(effect, { n: "none", b: "blur" }), setEffect: vi.fn(), + added$: new BehaviorSubject([]), pipeline: fakePipeline().pipeline, transformer, }); @@ -95,6 +97,7 @@ describe("the pipeline's state", () => { describe("background effects", () => { let effect$: BehaviorSubject; + let added$: BehaviorSubject; let setEffect: (raw: string) => void; let fake: ReturnType; @@ -105,6 +108,7 @@ describe("background effects", () => { supported: true, effect$, setEffect, + added$, pipeline: fake.pipeline, transformer: { onFirstFrame: undefined }, ...options, @@ -119,6 +123,7 @@ describe("background effects", () => { beforeEach(() => { effect$ = new BehaviorSubject("none"); + added$ = new BehaviorSubject(undefined); setEffect = vi.fn((raw: string) => effect$.next(raw)); fake = fakePipeline(); }); @@ -180,4 +185,56 @@ describe("background effects", () => { expect(effect$.value).toBe("none"); expect(effects.state$.value.processor).toBeUndefined(); }); + + it("puts an added background on as its picture", async () => { + added$.next([{ id: "mine", url: "blob:mine" }]); + build(); + await choose("added:mine"); + expect(fake.switches).toEqual([ + { mode: "virtual-background", imagePath: "blob:mine" }, + ]); + }); + + it("switches between any two effects in place", async () => { + added$.next([{ id: "mine", url: "blob:mine" }]); + const shipped = shippedBackgrounds[0]; + const options: Record = { + none: { mode: "disabled" }, + blur: { mode: "background-blur", blurRadius: 15 }, + [`image:${shipped.id}`]: { + mode: "virtual-background", + imagePath: shipped.imagePath, + }, + "added:mine": { mode: "virtual-background", imagePath: "blob:mine" }, + }; + // Each kind to each other kind, and back. + const order = [ + "blur", + `image:${shipped.id}`, + "added:mine", + "blur", + "added:mine", + `image:${shipped.id}`, + "none", + "added:mine", + "none", + "blur", + ]; + const effects = build(); + await choose(order[0]); + const pipeline = effects.state$.value.processor; + for (const raw of order.slice(1)) await choose(raw); + + expect(effects.state$.value.processor).toBe(pipeline); + expect(fake.switches).toEqual(order.map((raw) => options[raw])); + }); + + it("forgets an added background the device no longer keeps", () => { + effect$.next("added:gone"); + build(); + // Not before the device has said what it keeps. + expect(effect$.value).toBe("added:gone"); + added$.next([]); + expect(effect$.value).toBe("none"); + }); }); diff --git a/src/state/BackgroundEffects.ts b/src/state/BackgroundEffects.ts index ecfd8b25a..ef40980e2 100644 --- a/src/state/BackgroundEffects.ts +++ b/src/state/BackgroundEffects.ts @@ -31,6 +31,7 @@ import { imagePathFor, parseEffect, } from "../tracks/backgroundEffectCatalog"; +import { type AddedBackground } from "../tracks/backgroundImages"; const blurRadius = 15; @@ -48,6 +49,8 @@ export interface BackgroundEffectsOptions { effect$: Behavior; /** Stores a choice, to forget one that can't be honoured. */ setEffect: (id: EffectId) => void; + /** The backgrounds the device keeps, undefined until it has said. */ + added$: Behavior; /** The background processor pipeline to be switched between effects. */ pipeline: BackgroundProcessorWrapper; /** Tells, once, that a frame carrying an effect has been drawn. */ @@ -66,6 +69,7 @@ export function createBackgroundEffects( supported, effect$, setEffect, + added$, pipeline, transformer, }: BackgroundEffectsOptions, @@ -78,10 +82,18 @@ export function createBackgroundEffects( distinctUntilChanged(), ); - choice$.pipe(scope.bind()).subscribe((effect) => { - if (effect.kind === "none") return; - if (!supported) setEffect("none"); - }); + combineLatest([choice$, added$]) + .pipe(scope.bind()) + .subscribe(([effect, added]) => { + if (effect.kind === "none") return; + if ( + !supported || + (effect.kind === "added" && + added !== undefined && + !added.some(({ id }) => id === effect.id)) + ) + setEffect("none"); + }); const drewAFrame$ = scope.behavior( new Observable((subscriber) => { @@ -115,9 +127,16 @@ export function createBackgroundEffects( const switchOptions$ = scope.behavior< SwitchBackgroundProcessorOptions | undefined >( - combineLatest([state$, choice$]).pipe( - map(([{ processor }, effect]) => - processor === undefined ? undefined : switchOptionsFor(effect), + combineLatest([state$, choice$, added$]).pipe( + map(([{ processor }, effect, added]) => + processor === undefined + ? undefined + : switchOptionsFor( + effect, + effect.kind === "added" + ? added?.find(({ id }) => id === effect.id)?.url + : undefined, + ), ), distinctUntilChanged(deepCompare), ), @@ -135,18 +154,24 @@ export function createBackgroundEffects( return { state$ }; } +/** `addedUrl` is the picture of the added background chosen, if still kept. */ function switchOptionsFor( effect: BackgroundEffect, + addedUrl: string | undefined, ): SwitchBackgroundProcessorOptions { + const withImage = ( + imagePath: string | undefined, + ): SwitchBackgroundProcessorOptions => + imagePath + ? { mode: "virtual-background", imagePath } + : { mode: "disabled" }; switch (effect.kind) { case "blur": return { mode: "background-blur", blurRadius }; - case "shipped": { - const imagePath = imagePathFor(effect.id); - return imagePath - ? { mode: "virtual-background", imagePath } - : { mode: "disabled" }; - } + case "shipped": + return withImage(imagePathFor(effect.id)); + case "added": + return withImage(addedUrl); default: return { mode: "disabled" }; } diff --git a/src/tracks/BackgroundEffectsContext.test.ts b/src/tracks/BackgroundEffectsContext.test.ts index 449bac4e5..8514293d8 100644 --- a/src/tracks/BackgroundEffectsContext.test.ts +++ b/src/tracks/BackgroundEffectsContext.test.ts @@ -19,6 +19,7 @@ import { type BackgroundEffectsState, } from "../state/BackgroundEffects"; import { backgroundEffect } from "../settings/settings"; +import { type AddedBackground } from "./backgroundImages"; import { flushPromises, testScope } from "../utils/test"; const sdkSupportMock = vi.hoisted(() => vi.fn(() => true)); @@ -46,6 +47,14 @@ vi.mock("@livekit/track-processors", () => ({ })); const platformMock = vi.hoisted(() => ({ platform: "desktop" })); vi.mock("../Platform", () => platformMock); +vi.mock("./backgroundImages", async () => { + const { BehaviorSubject } = await import("rxjs"); + return { + addedBackgrounds: { + added$: new BehaviorSubject(undefined), + }, + }; +}); vi.mock("./BackgroundEffectTransformer", () => ({ BackgroundEffectTransformer: vi.fn(), })); diff --git a/src/tracks/BackgroundEffectsContext.tsx b/src/tracks/BackgroundEffectsContext.tsx index d5a00d3bb..b36f65db7 100644 --- a/src/tracks/BackgroundEffectsContext.tsx +++ b/src/tracks/BackgroundEffectsContext.tsx @@ -13,6 +13,7 @@ import { createContext, type FC, type JSX, use } from "react"; import { backgroundEffect as backgroundEffectSetting } from "../settings/settings"; import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer"; +import { addedBackgrounds } from "./backgroundImages"; import { type Behavior } from "../state/Behavior"; import { type ObservableScope } from "../state/ObservableScope"; import { @@ -54,6 +55,7 @@ export function createAppBackgroundEffects( supported: supportsBackgroundProcessors(), effect$: backgroundEffectSetting.value$, setEffect: backgroundEffectSetting.setValue, + added$: addedBackgrounds.added$, pipeline: new BackgroundProcessorWrapper(transformer, "background-effect"), transformer, }); diff --git a/src/tracks/backgroundEffectCatalog.test.ts b/src/tracks/backgroundEffectCatalog.test.ts index 6b6b13e99..1f8f53d7a 100644 --- a/src/tracks/backgroundEffectCatalog.test.ts +++ b/src/tracks/backgroundEffectCatalog.test.ts @@ -22,6 +22,7 @@ describe("the chosen background effect", () => { { kind: "none" }, { kind: "blur" }, ...shippedBackgrounds.map(({ id }) => ({ kind: "shipped" as const, id })), + { kind: "added", id: "d3b07384" }, ]; for (const effect of effects) expect(parseEffect(serializeEffect(effect))).toEqual(effect); @@ -30,6 +31,7 @@ describe("the chosen background effect", () => { test("falls back to no effect when the stored form names nothing we ship", () => { expect(parseEffect("image:gone")).toEqual({ kind: "none" }); expect(parseEffect("")).toEqual({ kind: "none" }); + expect(parseEffect("added:")).toEqual({ kind: "none" }); }); test("reads a stored value that is not a string as no effect", () => { diff --git a/src/tracks/backgroundEffectCatalog.ts b/src/tracks/backgroundEffectCatalog.ts index 71c9e2d5d..e2039f4af 100644 --- a/src/tracks/backgroundEffectCatalog.ts +++ b/src/tracks/backgroundEffectCatalog.ts @@ -22,16 +22,18 @@ export const shippedBackgrounds: ShippedBackground[] = [ export type BackgroundEffect = | { kind: "none" } | { kind: "blur" } - | { kind: "shipped"; id: string }; + | { kind: "shipped"; id: string } + | { kind: "added"; id: string }; /** An effect as the setting stores it, and as the menu names its option. */ -export type EffectId = "none" | "blur" | `image:${string}`; +export type EffectId = "none" | "blur" | `image:${string}` | `added:${string}`; export const noEffect: BackgroundEffect = { kind: "none" }; /** - * Reads a stored effect; one we no longer ship reads as no effect. Stored by - * hand or by another build, the value need not even be a string. + * Reads a stored effect; one we no longer ship reads as no effect. Whether an + * added one is still kept only the device knows. Stored by hand or by another + * build, the value need not even be a string. */ export function parseEffect(raw: unknown): BackgroundEffect { if (typeof raw !== "string") return noEffect; @@ -41,6 +43,10 @@ export function parseEffect(raw: unknown): BackgroundEffect { if (shippedBackgrounds.some((b) => b.id === id)) return { kind: "shipped", id }; } + if (raw.startsWith("added:")) { + const id = raw.slice("added:".length); + if (id) return { kind: "added", id }; + } return noEffect; } @@ -50,6 +56,8 @@ export function serializeEffect(effect: BackgroundEffect): EffectId { return "blur"; case "shipped": return `image:${effect.id}`; + case "added": + return `added:${effect.id}`; default: return "none"; } diff --git a/src/tracks/backgroundImages.test.ts b/src/tracks/backgroundImages.test.ts new file mode 100644 index 000000000..252a879b6 --- /dev/null +++ b/src/tracks/backgroundImages.test.ts @@ -0,0 +1,95 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { IDBFactory } from "fake-indexeddb"; + +import { + AddedBackgrounds, + type BackgroundImageStorage, + IndexedDBImageStorage, + type KeptImage, +} from "./backgroundImages"; +import { flushPromises } from "../utils/test"; + +describe("AddedBackgrounds", () => { + beforeEach(() => { + let n = 0; + vi.spyOn(URL, "createObjectURL").mockImplementation(() => `blob:${++n}`); + }); + + it("offers what the device keeps, oldest first", async () => { + const image = new Blob(["x"], { type: "image/png" }); + const store = new AddedBackgrounds( + memoryStorage([ + { id: "later", image, addedAt: 2 }, + { id: "earlier", image, addedAt: 1 }, + ]), + ); + expect(store.added$.value).toBeUndefined(); + await flushPromises(); + expect(store.added$.value?.map(({ id }) => id)).toEqual([ + "earlier", + "later", + ]); + }); + + it("keeps an added image and offers it after the others", async () => { + const storage = memoryStorage(); + const store = new AddedBackgrounds(storage); + await flushPromises(); + const id = await store.add(new Blob(["x"], { type: "image/png" })); + expect(store.added$.value?.map((a) => a.id)).toEqual([id]); + expect((await storage.list()).map((k) => k.id)).toEqual([id]); + }); + + it("offers nothing where the browser keeps nothing", async () => { + const store = new AddedBackgrounds(null); + await flushPromises(); + expect(store.added$.value).toEqual([]); + await expect(store.add(new Blob(["x"]))).rejects.toThrow(); + }); + + it("offers none, rather than waiting, where the device can't be read", async () => { + const unreadable: BackgroundImageStorage = { + ...memoryStorage(), + list: async () => Promise.reject(new Error("storage cleared")), + }; + const store = new AddedBackgrounds(unreadable); + await flushPromises(); + expect(store.added$.value).toEqual([]); + }); +}); + +describe("IndexedDBImageStorage", () => { + it("keeps images across sessions", async () => { + const indexedDB = new IDBFactory(); + const image = new Blob(["x"], { type: "image/webp" }); + const first = new IndexedDBImageStorage(indexedDB); + await first.put({ id: "a", image, addedAt: 1 }); + await first.put({ id: "b", image, addedAt: 2 }); + + // A session of its own, as after a reload. + const second = new IndexedDBImageStorage(indexedDB); + expect( + (await second.list()).map(({ id, addedAt }) => [id, addedAt]), + ).toEqual([ + ["a", 1], + ["b", 2], + ]); + }); +}); + +function memoryStorage(kept: KeptImage[] = []): BackgroundImageStorage { + return { + list: async (): Promise => Promise.resolve([...kept]), + put: async (image): Promise => { + kept.push(image); + return Promise.resolve(); + }, + }; +} diff --git a/src/tracks/backgroundImages.ts b/src/tracks/backgroundImages.ts new file mode 100644 index 000000000..3465c3a24 --- /dev/null +++ b/src/tracks/backgroundImages.ts @@ -0,0 +1,126 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { BehaviorSubject } from "rxjs"; +import { logger } from "matrix-js-sdk/lib/logger"; + +import { type Behavior } from "../state/Behavior"; + +/** A background the user added, as the menu and the pipeline use it. */ +export interface AddedBackground { + id: string; + url: string; +} + +/** A background the user added, as it is kept on the device. */ +export interface KeptImage { + id: string; + image: Blob; + addedAt: number; +} + +/** Where added backgrounds are kept. */ +export interface BackgroundImageStorage { + list(): Promise; + put(image: KeptImage): Promise; +} + +/** The backgrounds this device keeps, which never leave it. */ +export class AddedBackgrounds { + private readonly subject$ = new BehaviorSubject< + AddedBackground[] | undefined + >(undefined); + /** Undefined until what the device keeps has been read. */ + public readonly added$: Behavior = + this.subject$; + + public constructor(private readonly storage: BackgroundImageStorage | null) { + this.read().catch((e) => { + logger.warn("Could not read added backgrounds", e); + this.subject$.next([]); + }); + } + + private async read(): Promise { + const kept = (await this.storage?.list()) ?? []; + kept.sort((a, b) => a.addedAt - b.addedAt); + this.subject$.next( + kept.map(({ id, image }) => ({ id, url: URL.createObjectURL(image) })), + ); + } + + /** Keeps a file as a background, and answers with its id. */ + public async add(file: Blob): Promise { + if (this.storage === null) + throw new Error("This browser keeps no backgrounds"); + const kept: KeptImage = { + id: crypto.randomUUID(), + image: file, + addedAt: Date.now(), + }; + await this.storage.put(kept); + this.subject$.next([ + ...(this.subject$.value ?? []), + { id: kept.id, url: URL.createObjectURL(file) }, + ]); + return kept.id; + } +} + +const DB_NAME = "element-call-background-images"; +const STORE = "backgrounds"; + +export class IndexedDBImageStorage implements BackgroundImageStorage { + private db?: Promise; + + public constructor(private readonly indexedDB: IDBFactory) {} + + private async open(): Promise { + return (this.db ??= new Promise((resolve, reject) => { + const request = this.indexedDB.open(DB_NAME, 1); + request.onupgradeneeded = (): void => { + request.result.createObjectStore(STORE, { keyPath: "id" }); + }; + request.onsuccess = (): void => resolve(request.result); + request.onerror = (): void => reject(request.error); + })); + } + + private async run( + mode: IDBTransactionMode, + body: (store: IDBObjectStore) => IDBRequest, + ): Promise { + const db = await this.open(); + return new Promise((resolve, reject) => { + const request = body(db.transaction(STORE, mode).objectStore(STORE)); + request.onsuccess = (): void => resolve(request.result); + request.onerror = (): void => reject(request.error); + }); + } + + public async list(): Promise { + return this.run("readonly", (s) => s.getAll()); + } + + public async put(image: KeptImage): Promise { + await this.run("readwrite", (s) => s.put(image)); + } +} + +function openStorage(): BackgroundImageStorage | null { + // Merely reading indexedDB throws where it is disabled. + try { + return globalThis.indexedDB + ? new IndexedDBImageStorage(globalThis.indexedDB) + : null; + } catch (e) { + logger.warn("Could not reach IndexedDB", e); + return null; + } +} + +export const addedBackgrounds = new AddedBackgrounds(openStorage()); From 373b1e0fc047148afb4565ebd1819ae7afa93760 Mon Sep 17 00:00:00 2001 From: fkwp Date: Thu, 24 Sep 2026 18:58:20 +0200 Subject: [PATCH 08/34] Guard what can be added as a background - Five of one's own at most, so the grid is whole: at the limit the add tile goes, and it comes back once one is removed. The store holds the limit too. - An added image is refused if it isn't an image, is animated, or can't be decoded; reduced to 1920px on its longest edge; and laid on an opaque ground, since a transparent pixel would be a hole in the background. - Animation is read from the file, as a still and an animated WebP share a type. Chromium has no selected track until the decoder's tracks are ready; without waiting for them an animated GIF got through. - A refusal says why, in place of the notice, and is scrolled into view. It can be dismissed, is gone once the menu closes, and the next shows even if it reads the same. What was in force stays. - End to end on Chromium and Firefox, from what the device then keeps: an animated GIF is refused with its message, a 4000x3000 image is kept at 1920x1440, and a half transparent one is kept opaque. All three fail with the preparation bypassed. Co-Authored-By: Claude Opus 5.5 (1M context) --- locales/en/app.json | 4 + playwright/background-effects.spec.ts | 132 +++++++++++++++++- src/components/CallFooter.stories.tsx | 1 + src/components/CallFooter.tsx | 23 +++ src/components/CallFooterViewModel.test.ts | 69 ++++++++- src/components/CallFooterViewModel.tsx | 29 ++-- .../MediaMuteAndSwitchButton.module.css | 4 + .../MediaMuteAndSwitchButton.stories.tsx | 36 +++++ .../MediaMuteAndSwitchButton.test.tsx | 46 +++++- src/components/MediaMuteAndSwitchButton.tsx | 49 +++++-- src/tracks/backgroundEffectCatalog.test.ts | 9 ++ src/tracks/backgroundImages.test.ts | 55 +++++++- src/tracks/backgroundImages.ts | 89 +++++++++++- 13 files changed, 517 insertions(+), 29 deletions(-) diff --git a/locales/en/app.json b/locales/en/app.json index 63a9b8b02..aa753edc9 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -99,6 +99,10 @@ }, "disconnected_banner": "Connectivity to the server has been lost.", "error": { + "background_animated": "Animated images cannot be used as a background", + "background_not_an_image": "That file is not a supported image", + "background_not_kept": "That background could not be saved", + "background_undecodable": "That image could not be opened", "call_is_not_supported": "Call is not supported", "call_not_found": "Call not found", "call_not_found_description": "<0>That link doesn't appear to belong to any existing call. Check that you have the right link, or <2>create a new one.", diff --git a/playwright/background-effects.spec.ts b/playwright/background-effects.spec.ts index 579955cde..96040ad4c 100644 --- a/playwright/background-effects.spec.ts +++ b/playwright/background-effects.spec.ts @@ -328,6 +328,128 @@ test.describe("a background of one's own", () => { }); }); +test.describe("what can be added", () => { + test("refuses an animated image", async ({ page }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall(page, "Refused user", "Animated refused"); + await addImage(page, ANIMATED_GIF, { + name: "moving.gif", + type: "image/gif", + }); + + await expect( + page.getByText("Animated images cannot be used as a background"), + ).toBeVisible(); + const section = page.getByRole("group", { name: "Background effects" }); + await expect( + section.getByRole("menuitemradio", { name: "Background 3" }), + ).toHaveCount(0); + await expect( + section.getByRole("menuitemradio", { name: "None" }), + ).toHaveAttribute("aria-checked", "true"); + expect(await keptImages(page)).toEqual([]); + }); + + test("reduces an oversized image", async ({ page }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall(page, "Large user", "Oversized reduced"); + await addImage(page, await pngImage(page, { width: 4000, height: 3000 })); + + await expect( + page + .getByRole("group", { name: "Background effects" }) + .getByRole("menuitemradio", { name: "Background 3" }), + ).toBeVisible(); + await expect(page.getByRole("alert")).toHaveCount(0); + const [kept] = await keptImages(page); + expect(kept).toMatchObject({ width: 1920, height: 1440 }); + }); + + test("lays every image on an opaque ground", async ({ page }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall(page, "Logo user", "Transparent made opaque"); + await addImage( + page, + await pngImage(page, { width: 320, height: 180, transparent: true }), + ); + + await expect( + page + .getByRole("group", { name: "Background effects" }) + .getByRole("menuitemradio", { name: "Background 3" }), + ).toBeVisible(); + const [kept] = await keptImages(page); + expect(kept.minAlpha).toBe(255); + }); +}); + +/** Two frames of one pixel, red then blue. */ +const ANIMATED_GIF = Buffer.from( + "R0lGODlhAQABAPEAAP8AAAAA/wAAAAAAACH/C05FVFNDQVBFMi4wAwEAAAAh+QQACgAAACwAAAAAAQABAAACAkQBACH5BAAKAAAALAAAAAABAAEAAAICTAEAOw==", + "base64", +); + +/** A red PNG of the given size; transparent leaves its right half empty. */ +async function pngImage( + page: Page, + size: { width: number; height: number; transparent?: boolean }, +): Promise { + const dataUrl = await page.evaluate(({ width, height, transparent }) => { + const canvas = document.createElement("canvas"); + canvas.width = width; + canvas.height = height; + const context = canvas.getContext("2d")!; + context.fillStyle = "rgb(255, 0, 0)"; + context.fillRect(0, 0, transparent ? width / 2 : width, height); + return canvas.toDataURL("image/png"); + }, size); + return Buffer.from(dataUrl.split(",")[1], "base64"); +} + +/** The images the device keeps, as it keeps them. */ +async function keptImages( + page: Page, +): Promise<{ width: number; height: number; minAlpha: number }[]> { + return page.evaluate(async () => { + const db = await new Promise((resolve, reject) => { + const request = indexedDB.open("element-call-background-images"); + request.onsuccess = (): void => resolve(request.result); + request.onerror = (): void => reject(request.error); + }); + if (!db.objectStoreNames.contains("backgrounds")) return []; + const kept = await new Promise<{ image: Blob }[]>((resolve, reject) => { + const request = db + .transaction("backgrounds") + .objectStore("backgrounds") + .getAll(); + request.onsuccess = (): void => resolve(request.result); + request.onerror = (): void => reject(request.error); + }); + db.close(); + return Promise.all( + kept.map(async ({ image }) => { + const bitmap = await createImageBitmap(image); + const canvas = new OffscreenCanvas(bitmap.width, bitmap.height); + const context = canvas.getContext("2d")!; + context.drawImage(bitmap, 0, 0); + const { data } = context.getImageData( + 0, + 0, + bitmap.width, + bitmap.height, + ); + let minAlpha = 255; + for (let i = 3; i < data.length; i += 4) + minAlpha = Math.min(minAlpha, data[i]); + return { width: bitmap.width, height: bitmap.height, minAlpha }; + }), + ); + }); +} + /** The bytes of the one image the device keeps. */ async function keptBytes(page: Page): Promise { const base64 = await page.evaluate( @@ -370,14 +492,16 @@ async function redImage(page: Page): Promise { } /** Adds an image from the camera menu's add tile, leaving the menu open. */ -async function addImage(page: Page, image: Buffer): Promise { +async function addImage( + page: Page, + image: Buffer, + { name, type } = { name: "mine.png", type: "image/png" }, +): Promise { if (!(await page.getByRole("menu").isVisible())) await page.getByRole("button", { name: "Camera", exact: true }).click(); const chooser = page.waitForEvent("filechooser"); await page.getByRole("menuitem", { name: "Add image" }).click(); - await ( - await chooser - ).setFiles({ name: "mine.png", mimeType: "image/png", buffer: image }); + await (await chooser).setFiles({ name, mimeType: type, buffer: image }); } test.describe("joining with a background chosen", () => { diff --git a/src/components/CallFooter.stories.tsx b/src/components/CallFooter.stories.tsx index 9e9e6168c..65397f824 100644 --- a/src/components/CallFooter.stories.tsx +++ b/src/components/CallFooter.stories.tsx @@ -137,6 +137,7 @@ export const Default: Story = { ], backgroundEffectNotice: undefined, backgroundEffectSettling: false, + backgroundImageRefusal: undefined, addBackgroundImage: fn(), hangup: fn(), buttonSize: "lg", diff --git a/src/components/CallFooter.tsx b/src/components/CallFooter.tsx index 652f67979..4b8cf30f7 100644 --- a/src/components/CallFooter.tsx +++ b/src/components/CallFooter.tsx @@ -33,6 +33,11 @@ import { useBehavior } from "../useBehavior"; import { type LayoutSwitchViewModel } from "../state/LayoutSwitchViewModel"; import { LayoutSwitch } from "../room/LayoutSwitch"; import { type BackgroundEffectGridItem } from "./BackgroundEffectGrid"; +import { type UnusableReason } from "../tracks/backgroundImages"; + +export interface BackgroundImageRefusal { + reason: UnusableReason | "not-kept"; +} export interface AudioOutputSwitcher { targetOutput: string; @@ -83,6 +88,8 @@ export interface FooterState { backgroundEffectNotice: "unavailable" | "slow" | undefined; /** Whether the first effect chosen is still being prepared. */ backgroundEffectSettling: boolean; + /** Why the last file offered couldn't be kept, if it couldn't. */ + backgroundImageRefusal: BackgroundImageRefusal | undefined; showFooter: boolean; /* This is needed for WindowMode = "flat" */ @@ -171,6 +178,21 @@ export const CallFooter: FC = ({ const backgroundEffectNotice = useBehavior(vm.backgroundEffectNotice$); const backgroundEffectSettling = useBehavior(vm.backgroundEffectSettling$); const addBackgroundImage = useBehavior(vm.addBackgroundImage$); + const refusal = useBehavior(vm.backgroundImageRefusal$); + const refusalMessage = useMemo(() => { + switch (refusal?.reason) { + case undefined: + return undefined; + case "not-an-image": + return { text: t("error.background_not_an_image") }; + case "animated": + return { text: t("error.background_animated") }; + case "undecodable": + return { text: t("error.background_undecodable") }; + case "not-kept": + return { text: t("error.background_not_kept") }; + } + }, [refusal, t]); const backgroundEffects = useBehavior(vm.backgroundEffects$); const buttonSize = useBehavior(vm.buttonSize$); const showLogo = useBehavior(vm.showLogo$); @@ -238,6 +260,7 @@ export const CallFooter: FC = ({ onSelectBackgroundEffect={selectBackgroundEffect} backgroundEffectSettling={backgroundEffectSettling} onAddBackgroundImage={addBackgroundImage} + backgroundImageRefusal={refusalMessage} backgroundEffectNotice={ backgroundEffectNotice === "unavailable" ? t("background_effects.unavailable") diff --git a/src/components/CallFooterViewModel.test.ts b/src/components/CallFooterViewModel.test.ts index 59376184e..706bc77f5 100644 --- a/src/components/CallFooterViewModel.test.ts +++ b/src/components/CallFooterViewModel.test.ts @@ -8,7 +8,12 @@ Please see LICENSE in the repository root for full details. import { afterEach, describe, expect, it, vi } from "vitest"; import { BehaviorSubject } from "rxjs"; -import { testScope, mockMuteStates, mockMediaDevices } from "../utils/test"; +import { + flushPromises, + testScope, + mockMuteStates, + mockMediaDevices, +} from "../utils/test"; import { constant } from "../state/Behavior"; import type { CallViewModel } from "../state/CallViewModel/CallViewModel"; import type { Layout } from "../state/layout-types"; @@ -21,9 +26,12 @@ import { import { HeaderStyle } from "../UrlParams"; import { backgroundEffect as backgroundEffectSetting } from "../settings/settings"; import { shippedBackgrounds } from "../tracks/backgroundEffectCatalog"; +import type * as BackgroundImages from "../tracks/backgroundImages"; import { type AddedBackground, addedBackgrounds, + maxAddedBackgrounds, + UnusableImage, } from "../tracks/backgroundImages"; const platformMock = vi.hoisted(() => vi.fn(() => "desktop")); @@ -41,6 +49,20 @@ vi.mock("@livekit/track-processors", () => ({ supportsModernBackgroundProcessors: (): boolean => modernRouteMock(), })); +const store = vi.hoisted(() => ({ + add: vi.fn<(file: Blob) => Promise>(), +})); +vi.mock("../tracks/backgroundImages", async (original) => { + const { BehaviorSubject } = await import("rxjs"); + return { + ...(await original()), + addedBackgrounds: { + added$: new BehaviorSubject([]), + add: store.add, + }, + }; +}); + const outputSelectionMock = vi.hoisted(() => vi.fn(() => true)); vi.mock("livekit-client", () => ({ supportsAudioOutputSelection: (): boolean => outputSelectionMock(), @@ -341,6 +363,51 @@ describe("createCallFooterViewModel", () => { expect(vm.backgroundEffectSettling$.value).toBe(false); }); + it("adding is unavailable at the limit", () => { + sdkSupportMock.mockReturnValue(true); + const added$ = addedBackgrounds.added$ as BehaviorSubject< + AddedBackground[] | undefined + >; + const vm = lobbyOn("desktop"); + expect(vm.addBackgroundImage$.value).toBeDefined(); + added$.next( + Array.from({ length: maxAddedBackgrounds }, (_, i) => ({ + id: `${i}`, + url: `blob:${i}`, + })), + ); + expect(vm.addBackgroundImage$.value).toBeUndefined(); + added$.next(added$.value!.slice(1)); + expect(vm.addBackgroundImage$.value).toBeDefined(); + added$.next([]); + }); + + it("says why each refused file was refused", async () => { + sdkSupportMock.mockReturnValue(true); + const vm = lobbyOn("desktop"); + store.add.mockRejectedValue(new UnusableImage("animated")); + vm.addBackgroundImage$.value!(new File(["x"], "a.gif")); + await flushPromises(); + const first = vm.backgroundImageRefusal$.value; + expect(first).toEqual({ reason: "animated" }); + + // The same again is a refusal of its own, so it is shown again. + vm.addBackgroundImage$.value!(new File(["x"], "a.gif")); + await flushPromises(); + expect(vm.backgroundImageRefusal$.value).toEqual({ reason: "animated" }); + expect(vm.backgroundImageRefusal$.value).not.toBe(first); + + store.add.mockRejectedValue(new Error("quota")); + vm.addBackgroundImage$.value!(new File(["x"], "b.png")); + await flushPromises(); + expect(vm.backgroundImageRefusal$.value).toEqual({ reason: "not-kept" }); + + store.add.mockResolvedValue("kept"); + vm.addBackgroundImage$.value!(new File(["x"], "c.png")); + await flushPromises(); + expect(vm.backgroundImageRefusal$.value).toBeUndefined(); + }); + it("availability is the same before and during a call", () => { for (const supported of [true, false]) { sdkSupportMock.mockReturnValue(supported); diff --git a/src/components/CallFooterViewModel.tsx b/src/components/CallFooterViewModel.tsx index 7dc4f412d..09e0f54f2 100644 --- a/src/components/CallFooterViewModel.tsx +++ b/src/components/CallFooterViewModel.tsx @@ -6,6 +6,7 @@ Please see LICENSE in the repository root for full details. */ import { + BehaviorSubject, combineLatest, distinctUntilChanged, map, @@ -27,6 +28,8 @@ import { import { type AddedBackground, addedBackgrounds, + maxAddedBackgrounds, + UnusableImage, } from "../tracks/backgroundImages"; import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; @@ -42,7 +45,7 @@ import { type MuteStates } from "../state/MuteStates"; import { createStaticViewModel, type ViewModel } from "../state/ViewModel"; import { HeaderStyle } from "../UrlParams"; import { platform } from "../Platform"; -import { type FooterSnapshot } from "./CallFooter"; +import { type BackgroundImageRefusal, type FooterSnapshot } from "./CallFooter"; import { type BackgroundEffectGridItem } from "./BackgroundEffectGrid"; /** @@ -102,6 +105,7 @@ function buildDeviceBehaviors( | "backgroundEffectNotice$" | "backgroundEffectSettling$" | "addBackgroundImage$" + | "backgroundImageRefusal$" > { const options$ = ( available$: Behavior>, @@ -123,6 +127,10 @@ function buildDeviceBehaviors( const offered$ = disableSwitcher$.pipe( map((switcherDisabled) => !switcherDisabled && supported), ); + // A new object for each refusal, so the same one twice is shown twice. + const refusal$ = new BehaviorSubject( + undefined, + ); return { audioOptions$: scope.behavior(options$(mediaDevices.audioInput.available$)), selectedAudio$: scope.behavior( @@ -182,20 +190,23 @@ function buildDeviceBehaviors( ), // Kept and offered, not put on: that waits for the user to choose it. addBackgroundImage$: scope.behavior( - offered$.pipe( - map((offered) => - offered + combineLatest([offered$, addedBackgrounds.added$]).pipe( + map(([offered, added]) => + offered && (added?.length ?? 0) < maxAddedBackgrounds ? (file: File): void => { - addedBackgrounds - .add(file) - .catch((e) => - logger.warn("Could not keep that background", e), - ); + refusal$.next(undefined); + addedBackgrounds.add(file).catch((e) => { + logger.warn("Could not keep that background", e); + refusal$.next({ + reason: e instanceof UnusableImage ? e.reason : "not-kept", + }); + }); } : undefined, ), ), ), + backgroundImageRefusal$: refusal$, }; } diff --git a/src/components/MediaMuteAndSwitchButton.module.css b/src/components/MediaMuteAndSwitchButton.module.css index 6e860db3c..10b3267f5 100644 --- a/src/components/MediaMuteAndSwitchButton.module.css +++ b/src/components/MediaMuteAndSwitchButton.module.css @@ -93,3 +93,7 @@ Please see LICENSE in the repository root for full details. flex-shrink: 0; color: var(--cpd-color-icon-secondary); } + +.refusal { + margin: var(--cpd-space-2x) var(--cpd-space-4x) 0; +} diff --git a/src/components/MediaMuteAndSwitchButton.stories.tsx b/src/components/MediaMuteAndSwitchButton.stories.tsx index 80b02d090..243068271 100644 --- a/src/components/MediaMuteAndSwitchButton.stories.tsx +++ b/src/components/MediaMuteAndSwitchButton.stories.tsx @@ -347,6 +347,42 @@ export const BackgroundEffectsSettling: Story = { }, }; +/** A file that can't be used: said where it was chosen, and nothing changes. */ +export const BackgroundImageRefused: Story = { + args: { + ...VideoUnmute.args, + selectedBackgroundEffect: "blur", + backgroundEffectNotice: + "Background effects run slowly on this platform, which may cause your video to stutter.", + backgroundImageRefusal: { text: "That file is not a supported image" }, + }, + decorators: [ + (Story): JSX.Element => ( + + + + ), + ], + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await userEvent.click(canvas.getByRole("button", { name: "Camera" })); + const menu = within(document.body); + const message = await menu.findByText("That file is not a supported image"); + const list = document.body.querySelector( + `.${styles.deviceList}`, + )!; + await expect(list.scrollHeight).toBeGreaterThan(list.clientHeight); + const scrollport = list.getBoundingClientRect(); + const shown = message.getBoundingClientRect(); + await expect(shown.bottom).toBeLessThanOrEqual(scrollport.bottom + 1); + await expect(shown.top).toBeGreaterThanOrEqual(scrollport.top - 1); + await expect(menu.queryByText(/run slowly/)).toBeNull(); + await expect( + menu.getByRole("menuitemradio", { name: "Blur" }), + ).toHaveAttribute("aria-checked", "true"); + }, +}; + /** In a short call the effects scroll into view with the list. */ export const BackgroundEffectsScrollWhenTheyDoNotFit: Story = { args: { diff --git a/src/components/MediaMuteAndSwitchButton.test.tsx b/src/components/MediaMuteAndSwitchButton.test.tsx index 6613f45ea..7f77f6dd8 100644 --- a/src/components/MediaMuteAndSwitchButton.test.tsx +++ b/src/components/MediaMuteAndSwitchButton.test.tsx @@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { afterEach, describe, expect, test, vi } from "vitest"; +import { afterEach, describe, expect, onTestFinished, test, vi } from "vitest"; import { axe } from "vitest-axe"; import { act, @@ -437,6 +437,50 @@ describe("MediaMuteAndSwitchButton", () => { } }); + test("a refusal can be dismissed and does not outlive the menu", async () => { + const user = userEvent.setup(); + // jsdom lays nothing out, so has nothing to scroll. + const scroll = Element.prototype.scrollIntoView; + Element.prototype.scrollIntoView = vi.fn(); + onTestFinished(() => { + Element.prototype.scrollIntoView = scroll; + }); + const menu = (refusal?: { text: string }): JSX.Element => + withProviders( + , + ); + const { rerender, getByRole } = render(menu()); + await user.click(getByRole("button", { name: "Camera" })); + + rerender(menu({ text: "That file is not a supported image" })); + expect( + screen.getByText("That file is not a supported image"), + ).toBeVisible(); + screen.getByRole("menuitemradio", { name: "Blur", checked: true }); + await user.click(screen.getByRole("button", { name: "Close" })); + expect(screen.queryByText("That file is not a supported image")).toBeNull(); + + // The next refusal is shown, even if it says the same. + rerender(menu({ text: "That file is not a supported image" })); + expect( + screen.getByText("That file is not a supported image"), + ).toBeVisible(); + + await user.keyboard("{Escape}"); + await user.click(getByRole("button", { name: "Camera" })); + expect(screen.queryByText("That file is not a supported image")).toBeNull(); + }); + test("offers the background effects in a phone's drawer", async () => { const userAgent = vi .spyOn(navigator, "userAgent", "get") diff --git a/src/components/MediaMuteAndSwitchButton.tsx b/src/components/MediaMuteAndSwitchButton.tsx index b8630436a..578346c39 100644 --- a/src/components/MediaMuteAndSwitchButton.tsx +++ b/src/components/MediaMuteAndSwitchButton.tsx @@ -16,6 +16,7 @@ import { type ReactElement, } from "react"; import { + Alert, Button, Menu, MenuItem, @@ -82,6 +83,8 @@ export interface MediaMuteAndSwitchButtonProps { backgroundEffectSettling?: boolean; /** Called with the file chosen from the add tile. Omit to leave it out. */ onAddBackgroundImage?: (file: File) => void; + /** Why the last file offered couldn't be used; a new object each time. */ + backgroundImageRefusal?: { text: string }; /** * For any toggle and option this method will be called. * So toggles need to be implemented by listening here and setting the right toggle item to `enabled` @@ -117,6 +120,7 @@ export const MediaMuteAndSwitchButton: FC = ({ backgroundEffectNotice, backgroundEffectSettling, onAddBackgroundImage, + backgroundImageRefusal, onSelect, }) => { // Requested but not yet selected. Keyed by kind too, since Chrome uses @@ -130,11 +134,24 @@ export const MediaMuteAndSwitchButton: FC = ({ // elsewhere; it is held open until the picker is done. const choosingFile = useRef(false); const fileInput = useRef(null); - const onOpenChange = useCallback((open: boolean): void => { - if (!open && choosingFile.current) return; - setMenuOpen(open); - // Drop a request that never arrived. - if (!open) setPlannedSelection(null); + // A refusal is about the attempt that caused it: shown until dismissed or + // until the menu closes, and again for the next one. + const [seenRefusal, setSeenRefusal] = useState(); + const refusal = + backgroundImageRefusal !== seenRefusal ? backgroundImageRefusal : undefined; + const onOpenChange = useCallback( + (open: boolean): void => { + if (!open && choosingFile.current) return; + setMenuOpen(open); + // Drop a request that never arrived. + if (!open) setPlannedSelection(null); + if (!open) setSeenRefusal(backgroundImageRefusal); + }, + [backgroundImageRefusal], + ); + // Scrolled to as it appears, as the list may be scrolled away from it. + const scrollIntoView = useCallback((element: HTMLElement | null): void => { + element?.scrollIntoView({ block: "nearest" }); }, []); const watchFileInput = useCallback( (input: HTMLInputElement | null): (() => void) | undefined => { @@ -442,12 +459,26 @@ export const MediaMuteAndSwitchButton: FC = ({ addLabel={t("action.add_background_image")} /> {/* In the list, so the menu grows no taller for it. */} - {backgroundEffectNotice !== undefined && ( -
- - {backgroundEffectNotice} + {refusal !== undefined && ( +
+ setSeenRefusal(refusal)} + />
)} + {refusal === undefined && + backgroundEffectNotice !== undefined && ( +
+ + {backgroundEffectNotice} +
+ )} )} diff --git a/src/tracks/backgroundEffectCatalog.test.ts b/src/tracks/backgroundEffectCatalog.test.ts index 1f8f53d7a..df587664e 100644 --- a/src/tracks/backgroundEffectCatalog.test.ts +++ b/src/tracks/backgroundEffectCatalog.test.ts @@ -15,6 +15,7 @@ import { serializeEffect, shippedBackgrounds, } from "./backgroundEffectCatalog"; +import { maxAddedBackgrounds } from "./backgroundImages"; describe("the chosen background effect", () => { test("round-trips through its stored form", () => { @@ -39,6 +40,14 @@ describe("the chosen background effect", () => { expect(parseEffect(raw)).toEqual({ kind: "none" }); }); + test("fills the grid at the limit and one short of it", () => { + // No effect, blur and the shipped ones, then the added ones. + const fixed = 2 + shippedBackgrounds.length; + expect((fixed + maxAddedBackgrounds) % 3).toBe(0); + // One short, the add tile takes the last place. + expect((fixed + maxAddedBackgrounds - 1 + 1) % 3).toBe(0); + }); + test("gives every shipped background an image to draw", () => { for (const { id } of shippedBackgrounds) expect(imagePathFor(id)).toBeTruthy(); diff --git a/src/tracks/backgroundImages.test.ts b/src/tracks/backgroundImages.test.ts index 252a879b6..6e52a7105 100644 --- a/src/tracks/backgroundImages.test.ts +++ b/src/tracks/backgroundImages.test.ts @@ -13,6 +13,9 @@ import { type BackgroundImageStorage, IndexedDBImageStorage, type KeptImage, + maxAddedBackgrounds, + prepareImage, + UnusableImage, } from "./backgroundImages"; import { flushPromises } from "../utils/test"; @@ -29,6 +32,7 @@ describe("AddedBackgrounds", () => { { id: "later", image, addedAt: 2 }, { id: "earlier", image, addedAt: 1 }, ]), + asGiven, ); expect(store.added$.value).toBeUndefined(); await flushPromises(); @@ -40,13 +44,43 @@ describe("AddedBackgrounds", () => { it("keeps an added image and offers it after the others", async () => { const storage = memoryStorage(); - const store = new AddedBackgrounds(storage); + const store = new AddedBackgrounds(storage, asGiven); await flushPromises(); const id = await store.add(new Blob(["x"], { type: "image/png" })); expect(store.added$.value?.map((a) => a.id)).toEqual([id]); expect((await storage.list()).map((k) => k.id)).toEqual([id]); }); + it("keeps the prepared image, not the file", async () => { + const storage = memoryStorage(); + const prepared = new Blob(["y"], { type: "image/webp" }); + const store = new AddedBackgrounds(storage, async () => + Promise.resolve(prepared), + ); + await store.add(new Blob(["x"], { type: "image/png" })); + expect((await storage.list())[0].image).toBe(prepared); + }); + + it("keeps nothing of a file it refuses", async () => { + const storage = memoryStorage(); + const store = new AddedBackgrounds(storage, async () => + Promise.reject(new UnusableImage("animated")), + ); + await flushPromises(); + await expect(store.add(new Blob(["x"]))).rejects.toThrow(UnusableImage); + expect(await storage.list()).toEqual([]); + expect(store.added$.value).toEqual([]); + }); + + it("keeps no more than the grid holds", async () => { + const storage = memoryStorage(); + const store = new AddedBackgrounds(storage, asGiven); + for (let i = 0; i < maxAddedBackgrounds; i++) + await store.add(new Blob([`${i}`])); + await expect(store.add(new Blob(["more"]))).rejects.toThrow(RangeError); + expect(await storage.list()).toHaveLength(maxAddedBackgrounds); + }); + it("offers nothing where the browser keeps nothing", async () => { const store = new AddedBackgrounds(null); await flushPromises(); @@ -65,6 +99,22 @@ describe("AddedBackgrounds", () => { }); }); +describe("prepareImage", () => { + // Refused before anything is decoded, which only a real browser could do. + it("refuses a file that isn't an image", async () => { + await expect( + prepareImage(new Blob(["x"], { type: "text/plain" })), + ).rejects.toEqual(new UnusableImage("not-an-image")); + }); + + it("refuses an animated type where the browser can't count frames", async () => { + expect(globalThis.ImageDecoder).toBeUndefined(); + await expect( + prepareImage(new Blob(["GIF89a"], { type: "image/gif" })), + ).rejects.toEqual(new UnusableImage("animated")); + }); +}); + describe("IndexedDBImageStorage", () => { it("keeps images across sessions", async () => { const indexedDB = new IDBFactory(); @@ -84,6 +134,9 @@ describe("IndexedDBImageStorage", () => { }); }); +// Decoding needs a real browser; the end-to-end checks cover it. +const asGiven = async (file: Blob): Promise => Promise.resolve(file); + function memoryStorage(kept: KeptImage[] = []): BackgroundImageStorage { return { list: async (): Promise => Promise.resolve([...kept]), diff --git a/src/tracks/backgroundImages.ts b/src/tracks/backgroundImages.ts index 3465c3a24..8f48abdf3 100644 --- a/src/tracks/backgroundImages.ts +++ b/src/tracks/backgroundImages.ts @@ -10,6 +10,21 @@ import { logger } from "matrix-js-sdk/lib/logger"; import { type Behavior } from "../state/Behavior"; +export const maxAddedBackgrounds = 5; + +/** The longest edge kept; the pipeline scales every image to the camera. */ +export const maxStoredEdge = 1920; + +/** Why a file can't be used as a background. */ +export type UnusableReason = "not-an-image" | "animated" | "undecodable"; + +export class UnusableImage extends Error { + public constructor(public readonly reason: UnusableReason) { + super(reason); + this.name = "UnusableImage"; + } +} + /** A background the user added, as the menu and the pipeline use it. */ export interface AddedBackground { id: string; @@ -38,7 +53,10 @@ export class AddedBackgrounds { public readonly added$: Behavior = this.subject$; - public constructor(private readonly storage: BackgroundImageStorage | null) { + public constructor( + private readonly storage: BackgroundImageStorage | null, + private readonly prepare: (file: Blob) => Promise = prepareImage, + ) { this.read().catch((e) => { logger.warn("Could not read added backgrounds", e); this.subject$.next([]); @@ -53,24 +71,87 @@ export class AddedBackgrounds { ); } - /** Keeps a file as a background, and answers with its id. */ + /** + * Keeps a file as a background, and answers with its id. Throws + * {@link UnusableImage} for a file that can't be used, and a RangeError + * once maxAddedBackgrounds are kept. + */ public async add(file: Blob): Promise { if (this.storage === null) throw new Error("This browser keeps no backgrounds"); + const image = await this.prepare(file); + if ((await this.storage.list()).length >= maxAddedBackgrounds) + throw new RangeError( + `${maxAddedBackgrounds} backgrounds are kept already`, + ); const kept: KeptImage = { id: crypto.randomUUID(), - image: file, + image, addedAt: Date.now(), }; await this.storage.put(kept); this.subject$.next([ ...(this.subject$.value ?? []), - { id: kept.id, url: URL.createObjectURL(file) }, + { id: kept.id, url: URL.createObjectURL(image) }, ]); return kept.id; } } +/** + * Refuses what can't be used, reduces what is larger than is kept, and lays + * every image on an opaque ground: a background has to cover what is behind + * it, so a transparent pixel would be a hole in it. + */ +export async function prepareImage(file: Blob): Promise { + if (!file.type.startsWith("image/")) throw new UnusableImage("not-an-image"); + if (await isAnimated(file)) throw new UnusableImage("animated"); + + let bitmap: ImageBitmap; + try { + bitmap = await createImageBitmap(file); + } catch (e) { + logger.debug("Could not decode the chosen file", e); + throw new UnusableImage("undecodable"); + } + try { + const scale = Math.min( + 1, + maxStoredEdge / Math.max(bitmap.width, bitmap.height), + ); + const canvas = new OffscreenCanvas( + Math.round(bitmap.width * scale), + Math.round(bitmap.height * scale), + ); + const context = canvas.getContext("2d"); + if (!context) throw new UnusableImage("undecodable"); + context.fillStyle = "black"; + context.fillRect(0, 0, canvas.width, canvas.height); + context.drawImage(bitmap, 0, 0, canvas.width, canvas.height); + return await canvas.convertToBlob({ type: "image/webp", quality: 0.9 }); + } finally { + bitmap.close(); + } +} + +// Read from the file, not its type: a still and an animated WebP share one. +async function isAnimated(file: Blob): Promise { + if (typeof ImageDecoder === "undefined") return /gif|apng/.test(file.type); + try { + const decoder = new ImageDecoder({ + data: await file.arrayBuffer(), + type: file.type, + }); + // Some browsers have no selected track until the tracks are ready. + await decoder.tracks.ready; + await decoder.completed; + return (decoder.tracks.selectedTrack?.frameCount ?? 1) > 1; + } catch (e) { + logger.debug("Could not read the frame count, judging by type", e); + return /gif|apng/.test(file.type); + } +} + const DB_NAME = "element-call-background-images"; const STORE = "backgrounds"; From a2cfc2af93c508c2aa0e54d628b4c322458e3ab9 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 6 Nov 2026 07:24:53 +0000 Subject: [PATCH 09/34] Update ghcr.io/element-hq/element-web:develop Docker digest to 49104e3 (#4415) * Update ghcr.io/element-hq/element-web:develop Docker digest to e063472 * Fix tests * Dismiss further toasts * Close even more toasts * Yet more toast fixes * Toast fixes all over --------- Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: Johannes Marbach --- docker-compose-playwright.yml | 4 +- playwright/fixtures/widget-user.ts | 29 +-------- playwright/widget/test-helpers.ts | 98 ++++++++++++++++++++++++------ 3 files changed, 84 insertions(+), 47 deletions(-) diff --git a/docker-compose-playwright.yml b/docker-compose-playwright.yml index e8deef680..7e58bcf64 100644 --- a/docker-compose-playwright.yml +++ b/docker-compose-playwright.yml @@ -19,7 +19,7 @@ services: - ./backend/playwright_homeserver-othersite.yaml:/data/cfg/homeserver.yaml:Z element-web: # Pin to a SHA so that upstream cannot break our tests. Renovate handles regular updates. - image: ghcr.io/element-hq/element-web:develop@sha256:a79801206c628a912d57fa6f355d5c5ab9c6c40c2d1b6208d4192678fb6e80c7 + image: ghcr.io/element-hq/element-web:develop@sha256:e063472b6bdf6d0bfa1d43dfa4ef7effee1efd2c9684daee665d1bedfe746a47 element-web-1: # Pin to a SHA so that upstream cannot break our tests. Renovate handles regular updates. - image: ghcr.io/element-hq/element-web:develop@sha256:a79801206c628a912d57fa6f355d5c5ab9c6c40c2d1b6208d4192678fb6e80c7 + image: ghcr.io/element-hq/element-web:develop@sha256:e063472b6bdf6d0bfa1d43dfa4ef7effee1efd2c9684daee665d1bedfe746a47 diff --git a/playwright/fixtures/widget-user.ts b/playwright/fixtures/widget-user.ts index a60776c99..d21f443df 100644 --- a/playwright/fixtures/widget-user.ts +++ b/playwright/fixtures/widget-user.ts @@ -109,15 +109,7 @@ export const widgetTest = test.extend({ await TestHelpers.dismissInviteUnknownUserModal(ewPage1); // Accept the invite - await TestHelpers.closeReleaseAnnouncement( - ewPage2, - "Introducing Sections", - ); - await TestHelpers.expandAllSections(ewPage2); - await expect( - TestHelpers.roomListItem(ewPage2, "Welcome Room"), - ).toBeVisible(); - await TestHelpers.roomListItem(ewPage2, "Welcome Room").click(); + await TestHelpers.openRoomInvite(ewPage2, "Welcome Room"); await ewPage2.getByRole("button", { name: "Accept" }).click(); await expect( ewPage2 @@ -125,17 +117,7 @@ export const widgetTest = test.extend({ .getByRole("heading", { name: "Welcome Room" }), ).toBeVisible(); } else if (callType === "dm") { - await TestHelpers.closeReleaseAnnouncement( - ewPage1, - "Introducing Sections", - ); - - await ewPage1 - .getByRole("navigation", { name: "Room list" }) - .getByRole("button", { name: "New", exact: true }) - .click(); - - await ewPage1.getByRole("menuitem", { name: "Start chat" }).click(); + await TestHelpers.chooseFromRoomListNewMenu(ewPage1, "Start chat"); await ewPage1.getByRole("textbox", { name: "Search" }).click(); await ewPage1.getByRole("textbox", { name: "Search" }).fill(whistlerMxId); await ewPage1.getByRole("button", { name: "Go" }).click(); @@ -157,12 +139,7 @@ export const widgetTest = test.extend({ ).toBeVisible(); // Accept the DM invite from brooks - await TestHelpers.closeReleaseAnnouncement( - ewPage2, - "Introducing Sections", - ); - await TestHelpers.expandAllSections(ewPage2); - await TestHelpers.roomListItem(ewPage2, brooksDisplayName).click(); + await TestHelpers.openRoomInvite(ewPage2, brooksDisplayName); await ewPage2.getByRole("button", { name: "Start chatting" }).click(); } diff --git a/playwright/widget/test-helpers.ts b/playwright/widget/test-helpers.ts index bfb20080b..8a821d511 100644 --- a/playwright/widget/test-helpers.ts +++ b/playwright/widget/test-helpers.ts @@ -157,25 +157,34 @@ export class TestHelpers { // Toast can be stacked, and only the top one can be dismiss, so just look at what is on top and // dismiss (if part of expected toats) public static async dismissStartupToasts(page: Page): Promise { - const expectedToasts = [ + const expectedToasts: { + title: string; + button: string; + confirm?: string; + }[] = [ { title: "Failed to load service worker", button: "OK" }, { title: "Back up your chats", button: "Not now" }, - { title: "Turn on key storage", button: "Dismiss" }, + { + title: "Turn on key storage", + button: "Dismiss", + confirm: "Yes, dismiss", + }, + { title: "Notifications", button: "Dismiss" }, { title: "Element does not support this browser", button: "Dismiss" }, ]; const toast = page.locator(".mx_Toast_toast"); + const toastTitle = toast.locator(".mx_Toast_title h2"); // eslint-disable-next-line no-constant-condition while (true) { try { await toast.waitFor({ state: "visible", timeout: 700 }); - const title = await toast.locator(".mx_Toast_title h2").textContent(); + const title = await toastTitle.textContent(); + if (!title) break; // Find the matching toast config - const toastConfig = expectedToasts.find((t) => - title?.includes(t.title), - ); + const toastConfig = expectedToasts.find((t) => title.includes(t.title)); if (toastConfig) { // Bounded so that a renamed button leaves the toast in place and the @@ -183,6 +192,16 @@ export class TestHelpers { await toast .getByRole("button", { name: toastConfig.button }) .click({ timeout: 5000 }); + if (toastConfig.confirm) { + const dialog = page.getByRole("dialog"); + await dialog + .getByRole("button", { name: toastConfig.confirm }) + .click({ timeout: 5000 }); + await expect(dialog).toBeHidden({ timeout: 5000 }); + } + // The same element shows the next toast a moment after the click, so + // reading it straight away acts on the dismissed one a second time + await expect(toastTitle).not.toHaveText(title, { timeout: 5000 }); } else { // Unknown toast. We don't want to act on unknown toasts break; @@ -194,6 +213,16 @@ export class TestHelpers { } } + /** + * Clears what Element Web stacks over the room list after sign-in. The + * release announcement is held back while any toast is up, so it is closed + * after the toasts, not before. + */ + public static async clearRoomListOverlays(page: Page): Promise { + await TestHelpers.dismissStartupToasts(page); + await TestHelpers.closeReleaseAnnouncement(page, "Introducing Sections"); + } + public static async closeReleaseAnnouncement( page: Page, name: string, @@ -223,16 +252,7 @@ export class TestHelpers { page: Page, andInvite: string[] = [], ): Promise { - await TestHelpers.closeReleaseAnnouncement(page, "Introducing Sections"); - - await page - .getByRole("navigation", { name: "Room list" }) - .getByRole("button", { name: "New", exact: true }) - .click(); - - await page.getByRole("menuitem", { name: "New Room" }).click({ - timeout: 5000, - }); + await TestHelpers.chooseFromRoomListNewMenu(page, "New Room"); await page.getByRole("textbox", { name: "Name" }).fill(name); await page.getByRole("button", { name: "Create room" }).click(); await expect(page.getByText("You created this room.")).toBeVisible({ @@ -259,6 +279,25 @@ export class TestHelpers { } } + /** + * Opens the room list's "New" menu and picks the given item. + * + * While the sections announcement is anchored to the "New" button, the + * button has no menu at all, so the overlays go first. Driven by keyboard, + * as a toast arriving afterwards sits over the button and swallows clicks. + */ + public static async chooseFromRoomListNewMenu( + page: Page, + itemName: string, + ): Promise { + await TestHelpers.clearRoomListOverlays(page); + await page + .getByRole("navigation", { name: "Room list" }) + .getByRole("button", { name: "New", exact: true }) + .press("Enter"); + await page.getByRole("menuitem", { name: itemName }).press("Enter"); + } + /** * Accepts a room invite using the room name. * Locatest the invite in the room list. @@ -268,9 +307,7 @@ export class TestHelpers { roomName: string, page: Page, ): Promise { - await TestHelpers.closeReleaseAnnouncement(page, "Introducing Sections"); - await TestHelpers.expandAllSections(page); - await TestHelpers.roomListItem(page, roomName).click({ timeout: 10000 }); + await TestHelpers.openRoomInvite(page, roomName); await page.getByRole("button", { name: "Accept" }).click({ timeout: 5000, }); @@ -376,6 +413,29 @@ export class TestHelpers { return page.locator(`[aria-label^="Open room ${roomName}"]`); } + /** + * Opens a pending invite from the room list by its room name. + * + * Element Web collapses the Invites section every time it appears, so it is + * expanded here once the invite has arrived. + */ + public static async openRoomInvite( + page: Page, + roomName: string, + ): Promise { + const invitesSection = page.getByRole("button", { + name: "Toggle Invites section", + }); + await expect(invitesSection).toBeVisible({ timeout: 10000 }); + // Toasts and the announcement sit over the top of the room list, where + // this header is + await TestHelpers.clearRoomListOverlays(page); + if ((await invitesSection.getAttribute("aria-expanded")) !== "true") { + await invitesSection.click(); + } + await TestHelpers.roomListItem(page, roomName).click({ timeout: 10000 }); + } + /** * Switches to a room in the room list by its name. * @param page - The EW page From fb4cbc72e261ea3d35b836269f3a15cf39af961f Mon Sep 17 00:00:00 2001 From: fkwp Date: Thu, 24 Sep 2026 19:07:43 +0200 Subject: [PATCH 10/34] Remove a background of one's own - A cross on the tile's top-right corner removes an added background, critical-coloured at rest and named by its tooltip, as design draws it. The one in force offers none, since taking it away would leave nothing chosen; the view model refuses it too. - The cross sits beside the item rather than in it, which would take the press first and choose the tile, and isn't a control of its own, so the menu keeps one keyboard model: Delete or Backspace on the tile does the same, and the tile says so with aria-keyshortcuts. - Removed from IndexedDB and its picture released. End to end on Chromium and Firefox, a background removed by Delete is still gone after a reload. Co-Authored-By: Claude Opus 5.5 (1M context) --- playwright/background-effects.spec.ts | 27 ++++ .../BackgroundEffectGrid.module.css | 39 ++++++ src/components/BackgroundEffectGrid.tsx | 73 +++++++++-- src/components/CallFooter.stories.tsx | 6 +- src/components/CallFooter.tsx | 4 + src/components/CallFooterViewModel.test.ts | 27 +++- src/components/CallFooterViewModel.tsx | 26 +++- .../MediaMuteAndSwitchButton.stories.tsx | 118 ++++++++++++++++++ src/components/MediaMuteAndSwitchButton.tsx | 5 + src/tracks/backgroundImages.test.ts | 26 +++- src/tracks/backgroundImages.ts | 13 ++ 11 files changed, 342 insertions(+), 22 deletions(-) diff --git a/playwright/background-effects.spec.ts b/playwright/background-effects.spec.ts index 96040ad4c..4d01c2d72 100644 --- a/playwright/background-effects.spec.ts +++ b/playwright/background-effects.spec.ts @@ -326,6 +326,33 @@ test.describe("a background of one's own", () => { await addImage(page, red); await expect(mine).toBeVisible(); }); + + test("a removed image stays removed", async ({ page }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall( + page, + "Removing user", + "Own background removed", + ); + await addImage(page, await redImage(page)); + const mine = page + .getByRole("group", { name: "Background effects" }) + .getByRole("menuitemradio", { name: "Background 3" }); + await expect(mine).toHaveAttribute("aria-keyshortcuts", "Delete"); + await mine.focus(); + await page.keyboard.press("Delete"); + await expect(mine).toHaveCount(0); + + await page.reload(); + await expect(page.getByTestId("lobby_joinCall")).toBeVisible(); + await page.getByRole("button", { name: "Camera", exact: true }).click(); + await expect( + page.getByRole("group", { name: "Background effects" }), + ).toBeVisible(); + await expect(mine).toHaveCount(0); + expect(await keptImages(page)).toEqual([]); + }); }); test.describe("what can be added", () => { diff --git a/src/components/BackgroundEffectGrid.module.css b/src/components/BackgroundEffectGrid.module.css index 472772c15..70c514a9e 100644 --- a/src/components/BackgroundEffectGrid.module.css +++ b/src/components/BackgroundEffectGrid.module.css @@ -90,3 +90,42 @@ Please see LICENSE in the repository root for full details. text-align: center; overflow-wrap: anywhere; } + +.removable { + position: relative; + display: flex; + min-inline-size: 0; +} + +.removable > .tile { + flex: 1; +} + +/* Clear of the target for choosing the tile. */ +.remove { + position: absolute; + inset-block-start: 0; + inset-inline-end: 0; + translate: 20% -20%; + display: none; + align-items: center; + justify-content: center; + inline-size: 28px; + block-size: 28px; + border-radius: 50%; + background: var(--cpd-color-bg-canvas-default); + color: var(--cpd-color-icon-critical-primary); + box-shadow: 0 1.2px 2.4px 0 rgb(0 0 0 / 15%); + cursor: pointer; +} + +.remove:hover { + background: var(--cpd-color-bg-critical-subtle-hovered); + box-shadow: 0 0 0 var(--cpd-border-width-1) + var(--cpd-color-border-critical-subtle); +} + +.removable:hover .remove, +.removable:focus-within .remove { + display: flex; +} diff --git a/src/components/BackgroundEffectGrid.tsx b/src/components/BackgroundEffectGrid.tsx index ef5944fb1..0c2803836 100644 --- a/src/components/BackgroundEffectGrid.tsx +++ b/src/components/BackgroundEffectGrid.tsx @@ -5,14 +5,15 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type FC, type ReactElement } from "react"; +import { type FC, type KeyboardEvent, type ReactElement } from "react"; import { useTranslation } from "react-i18next"; import { VisuallyHidden } from "@radix-ui/react-visually-hidden"; -import { InlineSpinner, MenuItem } from "@vector-im/compound-web"; +import { InlineSpinner, MenuItem, Tooltip } from "@vector-im/compound-web"; import { BlockIcon, BlurIcon, CheckCircleSolidIcon, + CloseIcon, PlusIcon, } from "@vector-im/compound-design-tokens/assets/web/icons"; @@ -28,6 +29,8 @@ export interface BackgroundEffectGridItem { kind: "none" | "blur" | "image"; /** For kind "image". */ imageUrl?: string; + /** Whether it is the user's to remove. */ + removable?: boolean; } interface Props { @@ -42,6 +45,9 @@ interface Props { /** Offers to add a background of one's own. Omit to leave that tile out. */ onAdd?: () => void; addLabel?: string; + /** Removes a removable one, never the one in force. Omit to offer none. */ + onRemove?: (id: string) => void; + removeLabel?: string; } /** @@ -57,6 +63,8 @@ export const BackgroundEffectGrid: FC = ({ describedBy, onAdd, addLabel, + onRemove, + removeLabel, }) => { const { t } = useTranslation(); const choose = (id: string): void => { @@ -72,7 +80,30 @@ export const BackgroundEffectGrid: FC = ({ : t("background_effects.numbered", { n: ++pictures }); const checked = effect.id === selected; const disabled = onSelect === undefined && effect.kind !== "none"; - return ( + const remove = + effect.removable && !checked && onRemove !== undefined + ? (): void => onRemove(effect.id) + : undefined; + const removeProps = + remove === undefined + ? {} + : { + "aria-keyshortcuts": "Delete", + onKeyDown: (e: KeyboardEvent): void => { + if (e.key !== "Delete" && e.key !== "Backspace") return; + e.preventDefault(); + remove(); + }, + }; + const content = ( + + ); + const tile = ( = ({ choose(effect.id); } } + {...removeProps} > - + {content} ); + if (remove === undefined) return tile; + // Beside the item, not in it: inside, the item takes the press first and + // the tile is chosen instead. + return ( +
+ {tile} + + {/* Not a control of its own, so the menu keeps one keyboard model: + the keyboard reaches the same action through Delete. */} + + + + +
+ ); }); // A command rather than a choice, so an item beside the radios. + const addContent = ( + <> + + + + {addLabel} + + ); const addTile = onAdd === undefined ? null : ( = ({ onAdd(); }} > - - - - {addLabel} + {addContent} ); diff --git a/src/components/CallFooter.stories.tsx b/src/components/CallFooter.stories.tsx index 65397f824..d7b1dcf90 100644 --- a/src/components/CallFooter.stories.tsx +++ b/src/components/CallFooter.stories.tsx @@ -127,18 +127,20 @@ export const Default: Story = { selectBackgroundEffect: fn(), backgroundEffect: "blur", backgroundEffects: [ - { id: "none", kind: "none" }, - { id: "blur", kind: "blur" }, + { id: "none", kind: "none", removable: false }, + { id: "blur", kind: "blur", removable: false }, ...shippedBackgrounds.map((background) => ({ id: `image:${background.id}`, kind: "image" as const, imageUrl: background.imagePath, + removable: false, })), ], backgroundEffectNotice: undefined, backgroundEffectSettling: false, backgroundImageRefusal: undefined, addBackgroundImage: fn(), + removeBackgroundEffect: fn(), hangup: fn(), buttonSize: "lg", showFooter: true, diff --git a/src/components/CallFooter.tsx b/src/components/CallFooter.tsx index 4b8cf30f7..fce982347 100644 --- a/src/components/CallFooter.tsx +++ b/src/components/CallFooter.tsx @@ -67,6 +67,8 @@ export interface FooterActions { selectBackgroundEffect: ((id: string) => void) | undefined; /** Undefined where no background can be added. */ addBackgroundImage: ((file: File) => void) | undefined; + /** Removes an added background by its option id; never the one in force. */ + removeBackgroundEffect: ((id: string) => void) | undefined; toggleScreenSharing: (() => void) | undefined; /** Also controls if the settings button is visible */ openSettings: (() => void) | undefined; @@ -178,6 +180,7 @@ export const CallFooter: FC = ({ const backgroundEffectNotice = useBehavior(vm.backgroundEffectNotice$); const backgroundEffectSettling = useBehavior(vm.backgroundEffectSettling$); const addBackgroundImage = useBehavior(vm.addBackgroundImage$); + const removeBackgroundEffect = useBehavior(vm.removeBackgroundEffect$); const refusal = useBehavior(vm.backgroundImageRefusal$); const refusalMessage = useMemo(() => { switch (refusal?.reason) { @@ -260,6 +263,7 @@ export const CallFooter: FC = ({ onSelectBackgroundEffect={selectBackgroundEffect} backgroundEffectSettling={backgroundEffectSettling} onAddBackgroundImage={addBackgroundImage} + onRemoveBackgroundEffect={removeBackgroundEffect} backgroundImageRefusal={refusalMessage} backgroundEffectNotice={ backgroundEffectNotice === "unavailable" diff --git a/src/components/CallFooterViewModel.test.ts b/src/components/CallFooterViewModel.test.ts index 706bc77f5..a1b174462 100644 --- a/src/components/CallFooterViewModel.test.ts +++ b/src/components/CallFooterViewModel.test.ts @@ -51,6 +51,7 @@ vi.mock("@livekit/track-processors", () => ({ const store = vi.hoisted(() => ({ add: vi.fn<(file: Blob) => Promise>(), + remove: vi.fn<(id: string) => Promise>(), })); vi.mock("../tracks/backgroundImages", async (original) => { const { BehaviorSubject } = await import("rxjs"); @@ -59,6 +60,7 @@ vi.mock("../tracks/backgroundImages", async (original) => { addedBackgrounds: { added$: new BehaviorSubject([]), add: store.add, + remove: store.remove, }, }; }); @@ -320,21 +322,27 @@ describe("createCallFooterViewModel", () => { expect(lobbyOn("desktop").backgroundEffectNotice$.value).toBeUndefined(); }); - it("offers every effect in order", () => { + it("offers every effect in order, the added ones removable", () => { sdkSupportMock.mockReturnValue(true); const added$ = addedBackgrounds.added$ as BehaviorSubject< AddedBackground[] | undefined >; added$.next([{ id: "mine", url: "blob:mine" }]); expect(lobbyOn("desktop").backgroundEffects$.value).toEqual([ - { id: "none", kind: "none" }, - { id: "blur", kind: "blur" }, + { id: "none", kind: "none", removable: false }, + { id: "blur", kind: "blur", removable: false }, ...shippedBackgrounds.map((background) => ({ id: `image:${background.id}`, kind: "image", imageUrl: background.imagePath, + removable: false, })), - { id: "added:mine", kind: "image", imageUrl: "blob:mine" }, + { + id: "added:mine", + kind: "image", + imageUrl: "blob:mine", + removable: true, + }, ]); added$.next([]); }); @@ -408,6 +416,17 @@ describe("createCallFooterViewModel", () => { expect(vm.backgroundImageRefusal$.value).toBeUndefined(); }); + it("never removes the background in force", () => { + sdkSupportMock.mockReturnValue(true); + store.remove.mockResolvedValue(); + const vm = lobbyOn("desktop"); + backgroundEffectSetting.setValue("added:worn"); + vm.removeBackgroundEffect$.value!("added:worn"); + expect(store.remove).not.toHaveBeenCalled(); + vm.removeBackgroundEffect$.value!("added:spare"); + expect(store.remove).toHaveBeenCalledWith("spare"); + }); + it("availability is the same before and during a call", () => { for (const supported of [true, false]) { sdkSupportMock.mockReturnValue(supported); diff --git a/src/components/CallFooterViewModel.tsx b/src/components/CallFooterViewModel.tsx index 09e0f54f2..8a22aa52e 100644 --- a/src/components/CallFooterViewModel.tsx +++ b/src/components/CallFooterViewModel.tsx @@ -106,6 +106,7 @@ function buildDeviceBehaviors( | "backgroundEffectSettling$" | "addBackgroundImage$" | "backgroundImageRefusal$" + | "removeBackgroundEffect$" > { const options$ = ( available$: Behavior>, @@ -207,6 +208,25 @@ function buildDeviceBehaviors( ), ), backgroundImageRefusal$: refusal$, + // Never the one in force: taking it away would leave nothing chosen. + removeBackgroundEffect$: scope.behavior( + offered$.pipe( + map((offered) => + offered + ? (optionId: string): void => { + const effect = parseEffect(optionId); + if (effect.kind !== "added") return; + if (backgroundEffectSetting.getValue() === optionId) return; + addedBackgrounds + .remove(effect.id) + .catch((e) => + logger.warn("Could not remove that background", e), + ); + } + : undefined, + ), + ), + ), }; } @@ -378,17 +398,19 @@ function backgroundEffectChoices( added: AddedBackground[], ): BackgroundEffectGridItem[] { return [ - { id: serializeEffect({ kind: "none" }), kind: "none" }, - { id: serializeEffect({ kind: "blur" }), kind: "blur" }, + { id: serializeEffect({ kind: "none" }), kind: "none", removable: false }, + { id: serializeEffect({ kind: "blur" }), kind: "blur", removable: false }, ...shippedBackgrounds.map((background) => ({ id: serializeEffect({ kind: "shipped", id: background.id }), kind: "image" as const, imageUrl: background.imagePath, + removable: false, })), ...added.map((background) => ({ id: serializeEffect({ kind: "added", id: background.id }), kind: "image" as const, imageUrl: background.url, + removable: true, })), ]; } diff --git a/src/components/MediaMuteAndSwitchButton.stories.tsx b/src/components/MediaMuteAndSwitchButton.stories.tsx index 243068271..38ace8aee 100644 --- a/src/components/MediaMuteAndSwitchButton.stories.tsx +++ b/src/components/MediaMuteAndSwitchButton.stories.tsx @@ -383,6 +383,124 @@ export const BackgroundImageRefused: Story = { }, }; +/** A flat picture, for backgrounds of one's own. */ +function swatch(colour: string): string { + return `data:image/svg+xml,${encodeURIComponent( + ``, + )}`; +} + +/** One's own backgrounds are removable, never the one in force. */ +export const AddedBackgroundsCanBeRemoved: Story = { + args: { + ...VideoUnmute.args, + backgroundEffects: [ + ...effects, + { + id: "added:one", + kind: "image", + imageUrl: swatch("#c2410c"), + removable: true, + }, + { + id: "added:two", + kind: "image", + imageUrl: swatch("#1d4ed8"), + removable: true, + }, + ], + selectedBackgroundEffect: "added:one", + onRemoveBackgroundEffect: fn(), + }, + play: async ({ args, canvasElement }) => { + const canvas = within(canvasElement); + await userEvent.click(canvas.getByRole("button", { name: "Camera" })); + const body = within(document.body); + + const inForce = await body.findByRole("menuitemradio", { + name: "Background 3", + }); + await expect(inForce).not.toHaveAttribute("aria-keyshortcuts"); + + const other = body.getByRole("menuitemradio", { name: "Background 4" }); + await expect(other).toHaveAttribute("aria-keyshortcuts", "Delete"); + other.focus(); + await userEvent.keyboard("{Delete}"); + await expect(args.onRemoveBackgroundEffect).toHaveBeenCalledWith( + "added:two", + ); + + const wrapper = other.parentElement!; + await userEvent.hover(wrapper); + const cross = wrapper.querySelector( + "[aria-hidden]:last-child", + )!; + const probe = document.createElement("span"); + probe.style.color = "var(--cpd-color-icon-critical-primary)"; + cross.append(probe); + const critical = getComputedStyle(probe).color; + probe.remove(); + await expect(getComputedStyle(cross).color).toBe(critical); + const tile = other.getBoundingClientRect(); + const box = cross.getBoundingClientRect(); + await expect(box.right).toBeGreaterThan(tile.right); + await expect(box.top).toBeLessThan(tile.top); + + await userEvent.hover(cross); + await waitFor(async () => + expect( + [...document.body.querySelectorAll("div")].some( + (d) => d.textContent === "Remove" && d.offsetParent !== null, + ), + ).toBe(true), + ); + }, +}; + +/** Removing by pointer while the selection really changes. */ +export const RemovingWithLiveSelection: Story = { + args: AddedBackgroundsCanBeRemoved.args, + render: function WithLiveSelection(args): JSX.Element { + const [selected, setSelected] = useState("none"); + const [offered, setOffered] = useState(args.backgroundEffects ?? []); + return ( + + setOffered((current) => current.filter((o) => o.id !== id)) + } + /> + ); + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await userEvent.click(canvas.getByRole("button", { name: "Camera" })); + const body = within(document.body); + + const tile = await body.findByRole("menuitemradio", { + name: "Background 4", + }); + await userEvent.hover(tile); + const cross = tile.parentElement!.querySelector( + "[aria-hidden]:last-child", + )!; + await expect(cross).toBeVisible(); + await userEvent.click(cross); + + await waitFor(async () => + expect( + body.queryByRole("menuitemradio", { name: "Background 4" }), + ).toBeNull(), + ); + await expect( + body.getByRole("menuitemradio", { name: "None" }), + ).toHaveAttribute("aria-checked", "true"); + }, +}; + /** In a short call the effects scroll into view with the list. */ export const BackgroundEffectsScrollWhenTheyDoNotFit: Story = { args: { diff --git a/src/components/MediaMuteAndSwitchButton.tsx b/src/components/MediaMuteAndSwitchButton.tsx index 578346c39..f4a125d09 100644 --- a/src/components/MediaMuteAndSwitchButton.tsx +++ b/src/components/MediaMuteAndSwitchButton.tsx @@ -83,6 +83,8 @@ export interface MediaMuteAndSwitchButtonProps { backgroundEffectSettling?: boolean; /** Called with the file chosen from the add tile. Omit to leave it out. */ onAddBackgroundImage?: (file: File) => void; + /** Removes a removable effect, never the one in force. */ + onRemoveBackgroundEffect?: (id: string) => void; /** Why the last file offered couldn't be used; a new object each time. */ backgroundImageRefusal?: { text: string }; /** @@ -120,6 +122,7 @@ export const MediaMuteAndSwitchButton: FC = ({ backgroundEffectNotice, backgroundEffectSettling, onAddBackgroundImage, + onRemoveBackgroundEffect, backgroundImageRefusal, onSelect, }) => { @@ -457,6 +460,8 @@ export const MediaMuteAndSwitchButton: FC = ({ } } addLabel={t("action.add_background_image")} + onRemove={onRemoveBackgroundEffect} + removeLabel={t("action.remove")} /> {/* In the list, so the menu grows no taller for it. */} {refusal !== undefined && ( diff --git a/src/tracks/backgroundImages.test.ts b/src/tracks/backgroundImages.test.ts index 6e52a7105..672d6b7e8 100644 --- a/src/tracks/backgroundImages.test.ts +++ b/src/tracks/backgroundImages.test.ts @@ -81,6 +81,23 @@ describe("AddedBackgrounds", () => { expect(await storage.list()).toHaveLength(maxAddedBackgrounds); }); + it("forgets a removed image, and lets its picture go", async () => { + const revoke = vi + .spyOn(URL, "revokeObjectURL") + .mockImplementation(() => {}); + const storage = memoryStorage(); + const store = new AddedBackgrounds(storage, asGiven); + await flushPromises(); + const first = await store.add(new Blob(["a"])); + const second = await store.add(new Blob(["b"])); + const url = store.added$.value![0].url; + + await store.remove(first); + expect(store.added$.value?.map((a) => a.id)).toEqual([second]); + expect((await storage.list()).map((k) => k.id)).toEqual([second]); + expect(revoke).toHaveBeenCalledWith(url); + }); + it("offers nothing where the browser keeps nothing", async () => { const store = new AddedBackgrounds(null); await flushPromises(); @@ -116,7 +133,7 @@ describe("prepareImage", () => { }); describe("IndexedDBImageStorage", () => { - it("keeps images across sessions", async () => { + it("keeps images across sessions until they are deleted", async () => { const indexedDB = new IDBFactory(); const image = new Blob(["x"], { type: "image/webp" }); const first = new IndexedDBImageStorage(indexedDB); @@ -131,6 +148,8 @@ describe("IndexedDBImageStorage", () => { ["a", 1], ["b", 2], ]); + await second.delete("a"); + expect((await first.list()).map(({ id }) => id)).toEqual(["b"]); }); }); @@ -144,5 +163,10 @@ function memoryStorage(kept: KeptImage[] = []): BackgroundImageStorage { kept.push(image); return Promise.resolve(); }, + delete: async (id): Promise => { + const index = kept.findIndex((k) => k.id === id); + if (index !== -1) kept.splice(index, 1); + return Promise.resolve(); + }, }; } diff --git a/src/tracks/backgroundImages.ts b/src/tracks/backgroundImages.ts index 8f48abdf3..f20e2f53b 100644 --- a/src/tracks/backgroundImages.ts +++ b/src/tracks/backgroundImages.ts @@ -42,6 +42,7 @@ export interface KeptImage { export interface BackgroundImageStorage { list(): Promise; put(image: KeptImage): Promise; + delete(id: string): Promise; } /** The backgrounds this device keeps, which never leave it. */ @@ -96,6 +97,14 @@ export class AddedBackgrounds { ]); return kept.id; } + + public async remove(id: string): Promise { + await this.storage?.delete(id); + const current = this.subject$.value ?? []; + const gone = current.find((a) => a.id === id); + if (gone) URL.revokeObjectURL(gone.url); + this.subject$.next(current.filter((a) => a.id !== id)); + } } /** @@ -190,6 +199,10 @@ export class IndexedDBImageStorage implements BackgroundImageStorage { public async put(image: KeptImage): Promise { await this.run("readwrite", (s) => s.put(image)); } + + public async delete(id: string): Promise { + await this.run("readwrite", (s) => s.delete(id)); + } } function openStorage(): BackgroundImageStorage | null { From 6b56c7e68367e5978a4343440b93364fef721a83 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 6 Nov 2026 09:35:24 +0000 Subject: [PATCH 11/34] Update dependency minimatch to v11 [SECURITY] --- pnpm-lock.yaml | 40 +++++++++++++++++++++++----------------- pnpm-workspace.yaml | 2 +- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2ced4cb90..32c4d95e0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,7 +8,7 @@ overrides: '@livekit/components-core>rxjs': ^7.8.1 '@livekit/track-processors>@mediapipe/tasks-vision': ^0.10.18 tar: ^7.5.11 - minimatch: ^10.2.3 + minimatch: ^11.0.0 glob: ^10.5.0 qs: ^6.15.2 js-yaml: ^4.2.0 @@ -1145,6 +1145,9 @@ packages: resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} engines: {node: '>=12'} + '@isaacs/nonrecursive-glob-match@1.0.3': + resolution: {integrity: sha512-SI7kyKHq4XXDRozw+r9GlRZkia1ZkFrpLW98r2IC2Bpr8YkFGE3fvjzmxCG0x78A++69HmDNrDvSMYBZmHGpKA==} + '@joshwooding/vite-plugin-react-docgen-typescript@0.7.0': resolution: {integrity: sha512-qvsTEwEFefhdirGOPnu9Wp6ChfIwy2dBCRuETU3uE+4cC+PFoxMSiiEhxk4lOluA34eARHA0OxqsEUYDqRMgeQ==} peerDependencies: @@ -3530,8 +3533,8 @@ packages: boolbase@1.0.0: resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==} - brace-expansion@5.0.9: - resolution: {integrity: sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==} + brace-expansion@5.0.12: + resolution: {integrity: sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ==} engines: {node: 20 || >=22} braces@3.0.3: @@ -4768,9 +4771,9 @@ packages: minimalistic-crypto-utils@1.0.1: resolution: {integrity: sha512-JIYlbt6g8i5jKfJ3xz7rF0LXmv2TkDxBLUkiBeZ7bAx4GnnNMr8xFpGnOxn6GhTEHx3SjRrZEoU+j04prX1ktg==} - minimatch@10.2.6: - resolution: {integrity: sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==} - engines: {node: 18 || 20 || >=22} + minimatch@11.0.4: + resolution: {integrity: sha512-/UJ1dEoVQTjPNDh6SGlUo+ueLMmoHTas4q+8yaAR/5t6nLAIiBDJtoYUh20IYleix7ehiQkgczb84PE3uzHkxg==} + engines: {node: 20 || >=22} minimist@1.2.8: resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} @@ -6902,7 +6905,7 @@ snapshots: ignore: 5.3.2 import-fresh: 3.3.1 js-yaml: 4.3.1 - minimatch: 10.2.6 + minimatch: 11.0.4 strip-json-comments: 3.1.1 transitivePeerDependencies: - supports-color @@ -6981,7 +6984,7 @@ snapshots: dependencies: '@humanwhocodes/object-schema': 2.0.3 debug: 4.4.3(supports-color@7.2.0) - minimatch: 10.2.6 + minimatch: 11.0.4 transitivePeerDependencies: - supports-color @@ -7117,6 +7120,8 @@ snapshots: wrap-ansi: 8.1.0 wrap-ansi-cjs: wrap-ansi@7.0.0 + '@isaacs/nonrecursive-glob-match@1.0.3': {} + '@joshwooding/vite-plugin-react-docgen-typescript@0.7.0(typescript@5.9.3)(vite@8.2.4(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.103.1)(terser@5.46.1)(yaml@2.9.0))': dependencies: glob: 10.5.0 @@ -8776,7 +8781,7 @@ snapshots: '@typescript-eslint/types': 8.69.0 '@typescript-eslint/visitor-keys': 8.69.0 debug: 4.4.3(supports-color@7.2.0) - minimatch: 10.2.6 + minimatch: 11.0.4 semver: 7.8.5 tinyglobby: 0.2.17 ts-api-utils: 2.5.0(typescript@5.9.3) @@ -8791,7 +8796,7 @@ snapshots: '@typescript-eslint/types': 8.72.0 '@typescript-eslint/visitor-keys': 8.72.0 debug: 4.4.3(supports-color@7.2.0) - minimatch: 10.2.6 + minimatch: 11.0.4 semver: 7.8.5 tinyglobby: 0.2.17 ts-api-utils: 2.5.0(typescript@5.9.3) @@ -9116,7 +9121,7 @@ snapshots: boolbase@1.0.0: {} - brace-expansion@5.0.9: + brace-expansion@5.0.12: dependencies: balanced-match: 4.0.4 @@ -9661,7 +9666,7 @@ snapshots: json-stable-stringify-without-jsonify: 1.0.1 levn: 0.4.1 lodash.merge: 4.6.2 - minimatch: 10.2.6 + minimatch: 11.0.4 natural-compare: 1.4.0 optionator: 0.9.4 strip-ansi: 6.0.1 @@ -9781,7 +9786,7 @@ snapshots: filelist@1.0.6: dependencies: - minimatch: 10.2.6 + minimatch: 11.0.4 fill-range@7.1.1: dependencies: @@ -9881,7 +9886,7 @@ snapshots: dependencies: foreground-child: 3.3.1 jackspeak: 3.4.3 - minimatch: 10.2.6 + minimatch: 11.0.4 minipass: 7.1.3 package-json-from-dist: 1.0.1 path-scurry: 1.11.1 @@ -10006,7 +10011,7 @@ snapshots: jiti: 2.7.0 jsonc-parser: 3.3.1 magic-string: 1.2.3 - minimatch: 10.2.6 + minimatch: 11.0.4 ora: 9.4.1 react: 19.2.8 react-i18next: 17.0.13(i18next@26.4.1(typescript@5.9.3))(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(typescript@5.9.3) @@ -10449,9 +10454,10 @@ snapshots: minimalistic-crypto-utils@1.0.1: {} - minimatch@10.2.6: + minimatch@11.0.4: dependencies: - brace-expansion: 5.0.9 + '@isaacs/nonrecursive-glob-match': 1.0.3 + brace-expansion: 5.0.12 minimist@1.2.8: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 9e4f0fd6f..72c15d304 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -27,7 +27,7 @@ overrides: # Security fixes: # - https://github.com/advisories/GHSA-7r86-cg39-jmmj # - https://github.com/advisories/GHSA-23c5-xmqv-rm74 - "minimatch": "^10.2.3" + "minimatch": "^11.0.0" # Security fix: https://github.com/element-hq/element-call/security/dependabot/109 "glob": "^10.5.0" # Security fixes: From ea977cecd93e0c17d9b970c4ff4424fa9683ff3c Mon Sep 17 00:00:00 2001 From: Johannes Marbach Date: Fri, 6 Nov 2026 11:31:01 +0100 Subject: [PATCH 12/34] Mark known release announcements as viewed --- backend/ew.test.config.json | 8 +++-- backend/ew.test.othersite.config.json | 8 +++-- docs/agents/testing.md | 2 ++ playwright/fixtures/widget-user.ts | 39 +----------------------- playwright/widget/test-helpers.ts | 44 ++++++--------------------- 5 files changed, 23 insertions(+), 78 deletions(-) diff --git a/backend/ew.test.config.json b/backend/ew.test.config.json index 52be51b8f..f8b3ee5c4 100644 --- a/backend/ew.test.config.json +++ b/backend/ew.test.config.json @@ -26,8 +26,7 @@ "features": { "feature_element_call_video_rooms": true, "feature_video_rooms": true, - "feature_group_calls": true, - "feature_release_announcement": false + "feature_group_calls": true }, "default_federate": true, "default_theme": "light", @@ -40,7 +39,10 @@ }, "setting_defaults": { "breadcrumbs": true, - "feature_group_calls": true + "feature_group_calls": true, + "releaseAnnouncementData": { + "room_list_section": true + } }, "jitsi": { "preferred_domain": "meet.element.io" diff --git a/backend/ew.test.othersite.config.json b/backend/ew.test.othersite.config.json index 2ad3df4f3..6b3cb3a9c 100644 --- a/backend/ew.test.othersite.config.json +++ b/backend/ew.test.othersite.config.json @@ -26,8 +26,7 @@ "features": { "feature_element_call_video_rooms": true, "feature_video_rooms": true, - "feature_group_calls": true, - "feature_release_announcement": false + "feature_group_calls": true }, "default_federate": true, "default_theme": "light", @@ -40,7 +39,10 @@ }, "setting_defaults": { "breadcrumbs": true, - "feature_group_calls": true + "feature_group_calls": true, + "releaseAnnouncementData": { + "room_list_section": true + } }, "jitsi": { "preferred_domain": "meet.element.io" diff --git a/docs/agents/testing.md b/docs/agents/testing.md index d1a325897..eb8792d67 100644 --- a/docs/agents/testing.md +++ b/docs/agents/testing.md @@ -48,6 +48,8 @@ Test what a user observes: the peer sees the change, it survives a reconnect, it right after a reload. Reuse `playwright/spa-helpers.ts`, `playwright/widget/test-helpers.ts`, `playwright/fixtures/`. +To mark new release announcements as viewed, add them in `backend/ew.test*.config.json`. + ```sh pnpm test # unit + storybook pnpm backend # Synapse + LiveKit, required for e2e diff --git a/playwright/fixtures/widget-user.ts b/playwright/fixtures/widget-user.ts index d21f443df..b21732594 100644 --- a/playwright/fixtures/widget-user.ts +++ b/playwright/fixtures/widget-user.ts @@ -29,47 +29,10 @@ export interface MyFixtures { addUser: (username: string, host: string) => Promise; } -// Minimal config.json for the local element-web instance -const CONFIG_JSON = { - default_server_config: { - "m.homeserver": { - base_url: "https://synapse.m.localhost", - server_name: "synapse.m.localhost", - }, - }, - - element_call: { - participant_limit: 8, - brand: "Element Call", - }, - - // The default language is set here for test consistency - setting_defaults: { - language: "en-GB", - feature_group_calls: true, - }, - - // the location tests want a map style url. - map_style_url: - "https://api.maptiler.com/maps/streets/style.json?key=fU3vlMsMn4Jb6dnEIFsx", - - features: { - // We don't want to go through the feature announcement during the e2e test - feature_release_announcement: false, - feature_element_call_video_rooms: true, - feature_video_rooms: true, - feature_group_calls: true, - }, -}; - export const widgetTest = test.extend({ // allow per-test override: `widgetTest.use({ callType: "dm" })` callType: ["room", { option: true }], - asWidget: async ({ browser, context, callType }, pUse) => { - await context.route(`http://localhost:8081/config.json*`, async (route) => { - await route.fulfill({ json: CONFIG_JSON }); - }); - + asWidget: async ({ browser, callType }, pUse) => { const brooksDisplayName = `brooks_${Date.now()}`; const whistlerDisplayName = `whistler_${Date.now()}`; diff --git a/playwright/widget/test-helpers.ts b/playwright/widget/test-helpers.ts index 8a821d511..4e255f869 100644 --- a/playwright/widget/test-helpers.ts +++ b/playwright/widget/test-helpers.ts @@ -153,9 +153,10 @@ export class TestHelpers { return { page, clientHandle, mxId }; } - // Dismisses any toasts that appear on startup, such as "Failed to load service worker" or "Back up your chats". - // Toast can be stacked, and only the top one can be dismiss, so just look at what is on top and - // dismiss (if part of expected toats) + // Dismisses the toasts a fresh device gets after sign-in. + // + // Toasts stack and only the top one can be dismissed, so look + // at what is on top each time. public static async dismissStartupToasts(page: Page): Promise { const expectedToasts: { title: string; @@ -163,6 +164,7 @@ export class TestHelpers { confirm?: string; }[] = [ { title: "Failed to load service worker", button: "OK" }, + { title: "Verify this device", button: "Later" }, { title: "Back up your chats", button: "Not now" }, { title: "Turn on key storage", @@ -213,30 +215,6 @@ export class TestHelpers { } } - /** - * Clears what Element Web stacks over the room list after sign-in. The - * release announcement is held back while any toast is up, so it is closed - * after the toasts, not before. - */ - public static async clearRoomListOverlays(page: Page): Promise { - await TestHelpers.dismissStartupToasts(page); - await TestHelpers.closeReleaseAnnouncement(page, "Introducing Sections"); - } - - public static async closeReleaseAnnouncement( - page: Page, - name: string, - ): Promise { - try { - await page - .getByRole("dialog", { name }) - .getByRole("button", { name: "OK" }) - .click({ timeout: 2000 }); - } catch { - // Announcement not shown; nothing to do - } - } - public static async expandAllSections(page: Page): Promise { try { await page @@ -282,15 +260,14 @@ export class TestHelpers { /** * Opens the room list's "New" menu and picks the given item. * - * While the sections announcement is anchored to the "New" button, the - * button has no menu at all, so the overlays go first. Driven by keyboard, - * as a toast arriving afterwards sits over the button and swallows clicks. + * Driven by keyboard, as a toast arriving late sits over the button and + * swallows clicks. */ public static async chooseFromRoomListNewMenu( page: Page, itemName: string, ): Promise { - await TestHelpers.clearRoomListOverlays(page); + await TestHelpers.dismissStartupToasts(page); await page .getByRole("navigation", { name: "Room list" }) .getByRole("button", { name: "New", exact: true }) @@ -427,9 +404,8 @@ export class TestHelpers { name: "Toggle Invites section", }); await expect(invitesSection).toBeVisible({ timeout: 10000 }); - // Toasts and the announcement sit over the top of the room list, where - // this header is - await TestHelpers.clearRoomListOverlays(page); + // Toasts sit over the top of the room list, where this header is. + await TestHelpers.dismissStartupToasts(page); if ((await invitesSection.getAttribute("aria-expanded")) !== "true") { await invitesSection.click(); } From 797a86815bca87150b2893fb853624c2a8037748 Mon Sep 17 00:00:00 2001 From: "Timo K." Date: Fri, 6 Nov 2026 14:34:14 +0100 Subject: [PATCH 13/34] Ship the component's assets as files beside the module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The component module was 18 MB, of which 15 MB was MediaPipe's runtime as a base64 data URL, with the segmentation model, every sound and the background pictures inlined the same way: a library build inlines every asset unless its import says `no-inline`. Every host downloaded all of it before the first call, whether or not a background effect was ever switched on, and a host with a Content Security Policy without `script-src data:` could not run effects at all, since MediaPipe injects its loader as a script tag. The imports now say `no-inline`, and the build renders each reference as `new URL("./assets/…", import.meta.url)`, which webpack 5, Vite and Rollup follow as an asset of the host's own; the `./` is spelled out because a bare `assets/…` is read as a package name. The module is 2.3 MB. The runtime is fetched when an effect is first switched on and cached on its own. MediaPipe's files come through the package's `exports` rather than a path into node_modules. MediaPipe's JavaScript (160 kB) stays in the module: LiveKit's track processors import it statically, so a dynamic import moves nothing. Co-Authored-By: Claude Fable 5.1 --- README.md | 7 +++ src/reactions/index.ts | 48 +++++++++---------- src/room/CallEventAudioRenderer.tsx | 24 +++++----- src/room/RingingAudioRenderer.tsx | 4 +- .../BackgroundEffectTransformer.test.ts | 18 +++++++ src/tracks/BackgroundEffectTransformer.ts | 29 ++++------- src/tracks/backgroundEffectCatalog.ts | 4 +- vite-component.config.ts | 25 ++++++++++ 8 files changed, 98 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index 17eae35d9..ebc26f936 100644 --- a/README.md +++ b/README.md @@ -268,6 +268,13 @@ widget's `theme` URL parameter: `light`, `dark`, `light-high-contrast` or `dark-high-contrast`. Both can change while a call is running without disturbing it. +The component's sounds, pictures and the MediaPipe runtime behind background +effects are files beside the module rather than data URLs inside it, each +referenced relative to the module (`new URL("./assets/…", import.meta.url)`), +so a host's bundler copies them as assets of its own — webpack 5, Vite and +Rollup all do — and the browser fetches the MediaPipe runtime, the largest of +them by far, only when an effect is first switched on. + A host must call and await `initializeElementCall(config)` once before rendering the component: it loads the `Intl` polyfills, applies the deployment-wide `config.json`-style configuration and sets up translations. diff --git a/src/reactions/index.ts b/src/reactions/index.ts index 26ca7f142..ffc14d43e 100644 --- a/src/reactions/index.ts +++ b/src/reactions/index.ts @@ -7,30 +7,30 @@ Please see LICENSE in the repository root for full details. import { type RelationType } from "matrix-js-sdk"; -import catSoundOgg from "../sound/reactions/cat.ogg?url"; -import catSoundMp3 from "../sound/reactions/cat.mp3?url"; -import clapSoundOgg from "../sound/reactions/clap.ogg?url"; -import clapSoundMp3 from "../sound/reactions/clap.mp3?url"; -import cricketsSoundOgg from "../sound/reactions/crickets.ogg?url"; -import cricketsSoundMp3 from "../sound/reactions/crickets.mp3?url"; -import dogSoundOgg from "../sound/reactions/dog.ogg?url"; -import dogSoundMp3 from "../sound/reactions/dog.mp3?url"; -import genericSoundOgg from "../sound/reactions/generic.ogg?url"; -import genericSoundMp3 from "../sound/reactions/generic.mp3?url"; -import lightbulbSoundOgg from "../sound/reactions/lightbulb.ogg?url"; -import lightbulbSoundMp3 from "../sound/reactions/lightbulb.mp3?url"; -import partySoundOgg from "../sound/reactions/party.ogg?url"; -import partySoundMp3 from "../sound/reactions/party.mp3?url"; -import deerSoundOgg from "../sound/reactions/deer.ogg?url"; -import deerSoundMp3 from "../sound/reactions/deer.mp3?url"; -import rockSoundOgg from "../sound/reactions/rock.ogg?url"; -import rockSoundMp3 from "../sound/reactions/rock.mp3?url"; -import waveSoundOgg from "../sound/reactions/wave.ogg?url"; -import waveSoundMp3 from "../sound/reactions/wave.mp3?url"; -import baduntssSoundOgg from "../sound/reactions/baduntss.ogg?url"; -import baduntssSoundMp3 from "../sound/reactions/baduntss.mp3?url"; -import bellSoundOgg from "../sound/reactions/bell.ogg?url"; -import bellSoundMp3 from "../sound/reactions/bell.mp3?url"; +import catSoundOgg from "../sound/reactions/cat.ogg?url&no-inline"; +import catSoundMp3 from "../sound/reactions/cat.mp3?url&no-inline"; +import clapSoundOgg from "../sound/reactions/clap.ogg?url&no-inline"; +import clapSoundMp3 from "../sound/reactions/clap.mp3?url&no-inline"; +import cricketsSoundOgg from "../sound/reactions/crickets.ogg?url&no-inline"; +import cricketsSoundMp3 from "../sound/reactions/crickets.mp3?url&no-inline"; +import dogSoundOgg from "../sound/reactions/dog.ogg?url&no-inline"; +import dogSoundMp3 from "../sound/reactions/dog.mp3?url&no-inline"; +import genericSoundOgg from "../sound/reactions/generic.ogg?url&no-inline"; +import genericSoundMp3 from "../sound/reactions/generic.mp3?url&no-inline"; +import lightbulbSoundOgg from "../sound/reactions/lightbulb.ogg?url&no-inline"; +import lightbulbSoundMp3 from "../sound/reactions/lightbulb.mp3?url&no-inline"; +import partySoundOgg from "../sound/reactions/party.ogg?url&no-inline"; +import partySoundMp3 from "../sound/reactions/party.mp3?url&no-inline"; +import deerSoundOgg from "../sound/reactions/deer.ogg?url&no-inline"; +import deerSoundMp3 from "../sound/reactions/deer.mp3?url&no-inline"; +import rockSoundOgg from "../sound/reactions/rock.ogg?url&no-inline"; +import rockSoundMp3 from "../sound/reactions/rock.mp3?url&no-inline"; +import waveSoundOgg from "../sound/reactions/wave.ogg?url&no-inline"; +import waveSoundMp3 from "../sound/reactions/wave.mp3?url&no-inline"; +import baduntssSoundOgg from "../sound/reactions/baduntss.ogg?url&no-inline"; +import baduntssSoundMp3 from "../sound/reactions/baduntss.mp3?url&no-inline"; +import bellSoundOgg from "../sound/reactions/bell.ogg?url&no-inline"; +import bellSoundMp3 from "../sound/reactions/bell.mp3?url&no-inline"; export const ElementCallReactionEventType = "io.element.call.reaction"; diff --git a/src/room/CallEventAudioRenderer.tsx b/src/room/CallEventAudioRenderer.tsx index d33f3b843..fa14bcd77 100644 --- a/src/room/CallEventAudioRenderer.tsx +++ b/src/room/CallEventAudioRenderer.tsx @@ -8,18 +8,18 @@ Please see LICENSE in the repository root for full details. import { type ReactNode, useEffect } from "react"; import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; -import joinCallSoundMp3 from "../sound/join_call.mp3"; -import joinCallSoundOgg from "../sound/join_call.ogg"; -import leftCallSoundMp3 from "../sound/left_call.mp3"; -import leftCallSoundOgg from "../sound/left_call.ogg"; -import handSoundOgg from "../sound/raise_hand.ogg"; -import handSoundMp3 from "../sound/raise_hand.mp3"; -import screenShareStartedOgg from "../sound/screen_share_started.ogg"; -import screenShareStartedMp3 from "../sound/screen_share_started.mp3"; -import declineMp3 from "../sound/call_declined.mp3?url"; -import declineOgg from "../sound/call_declined.ogg?url"; -import timeoutMp3 from "../sound/call_timeout.mp3?url"; -import timeoutOgg from "../sound/call_timeout.ogg?url"; +import joinCallSoundMp3 from "../sound/join_call.mp3?url&no-inline"; +import joinCallSoundOgg from "../sound/join_call.ogg?url&no-inline"; +import leftCallSoundMp3 from "../sound/left_call.mp3?url&no-inline"; +import leftCallSoundOgg from "../sound/left_call.ogg?url&no-inline"; +import handSoundOgg from "../sound/raise_hand.ogg?url&no-inline"; +import handSoundMp3 from "../sound/raise_hand.mp3?url&no-inline"; +import screenShareStartedOgg from "../sound/screen_share_started.ogg?url&no-inline"; +import screenShareStartedMp3 from "../sound/screen_share_started.mp3?url&no-inline"; +import declineMp3 from "../sound/call_declined.mp3?url&no-inline"; +import declineOgg from "../sound/call_declined.ogg?url&no-inline"; +import timeoutMp3 from "../sound/call_timeout.mp3?url&no-inline"; +import timeoutOgg from "../sound/call_timeout.ogg?url&no-inline"; import { useAudioContext } from "../useAudioContext"; import { prefetchSounds } from "../soundUtils"; import { useLatest } from "../useLatest"; diff --git a/src/room/RingingAudioRenderer.tsx b/src/room/RingingAudioRenderer.tsx index c0fe45d5b..f0ebd8386 100644 --- a/src/room/RingingAudioRenderer.tsx +++ b/src/room/RingingAudioRenderer.tsx @@ -12,8 +12,8 @@ import { type RingingMediaViewModel } from "../state/media/RingingMediaViewModel import { useBehavior } from "../useBehavior"; import { useInitial } from "../useInitial"; import { prefetchSounds } from "../soundUtils"; -import ringtoneMp3 from "../sound/ringtone.mp3?url"; -import ringtoneOgg from "../sound/ringtone.ogg?url"; +import ringtoneMp3 from "../sound/ringtone.mp3?url&no-inline"; +import ringtoneOgg from "../sound/ringtone.ogg?url&no-inline"; import { type UseAudioContext, useAudioContext } from "../useAudioContext"; import { useLatest } from "../useLatest"; diff --git a/src/tracks/BackgroundEffectTransformer.test.ts b/src/tracks/BackgroundEffectTransformer.test.ts index 400a50416..4e7ed8947 100644 --- a/src/tracks/BackgroundEffectTransformer.test.ts +++ b/src/tracks/BackgroundEffectTransformer.test.ts @@ -41,6 +41,24 @@ describe("BackgroundEffectTransformer", () => { expect(load).toHaveBeenCalledWith("/background.jpg"); }); + it("runs MediaPipe from its own bundle", async () => { + vi.spyOn(VideoTransformer.prototype, "init").mockResolvedValue(); + const create = vi + .spyOn(ImageSegmenter, "createFromOptions") + .mockResolvedValue({} as ImageSegmenter); + + await new BackgroundEffectTransformer({ backgroundDisabled: true }).init( + {} as VideoTransformerInitOptions, + ); + + const [fileset, options] = create.mock.calls[0]; + expect(fileset.wasmLoaderPath).toMatch(/vision_wasm_internal\.js/); + expect(fileset.wasmBinaryPath).toMatch(/vision_wasm_internal\.wasm/); + expect(options.baseOptions?.modelAssetPath).toMatch( + /selfie_segmenter\.tflite/, + ); + }); + it("reports the first frame carrying an effect, once", async () => { vi.spyOn(BackgroundTransformer.prototype, "transform").mockImplementation( async function (this: BackgroundTransformer) { diff --git a/src/tracks/BackgroundEffectTransformer.ts b/src/tracks/BackgroundEffectTransformer.ts index e3c0a9183..8d7221a6d 100644 --- a/src/tracks/BackgroundEffectTransformer.ts +++ b/src/tracks/BackgroundEffectTransformer.ts @@ -15,7 +15,13 @@ import { import { logger } from "matrix-js-sdk/lib/logger"; import { ImageSegmenter } from "@mediapipe/tasks-vision"; -import modelAssetPath from "../mediapipe/imageSegmenter/selfie_segmenter.tflite?url"; +// Only the SIMD build of MediaPipe's runtime: every browser Element Call +// supports has WebAssembly SIMD. As files rather than data URLs, so that a +// host's bundler serves them as assets of its own, and a browser fetches them +// only when an effect is first switched on. +import wasmLoaderPath from "@mediapipe/tasks-vision/vision_wasm_internal.js?url&no-inline"; +import wasmBinaryPath from "@mediapipe/tasks-vision/vision_wasm_internal.wasm?url&no-inline"; +import modelAssetPath from "../mediapipe/imageSegmenter/selfie_segmenter.tflite?url&no-inline"; interface WasmFileset { /** The path to the Wasm loader script. */ @@ -24,26 +30,7 @@ interface WasmFileset { wasmBinaryPath: string; } -// The MediaPipe package, by default, ships some alternative versions of the -// WASM files which avoid SIMD for compatibility with older browsers. But SIMD -// in WASM is actually fine by our support policy, so we include just the SIMD -// versions. -// It's really not ideal that we have to reference these internal files from -// MediaPipe and depend on node_modules having this specific structure. It's -// easy to see this breaking if our dependencies changed and MediaPipe were -// no longer hoisted, or if we switched to another dependency loader such as -// yarn PnP. -// https://github.com/google-ai-edge/mediapipe/issues/5961 -const wasmFileset: WasmFileset = { - wasmLoaderPath: new URL( - "../../node_modules/@mediapipe/tasks-vision/wasm/vision_wasm_internal.js", - import.meta.url, - ).href, - wasmBinaryPath: new URL( - "../../node_modules/@mediapipe/tasks-vision/wasm/vision_wasm_internal.wasm", - import.meta.url, - ).href, -}; +const wasmFileset: WasmFileset = { wasmLoaderPath, wasmBinaryPath }; /** * Track processor that applies effects such as blurring to a user's background. diff --git a/src/tracks/backgroundEffectCatalog.ts b/src/tracks/backgroundEffectCatalog.ts index 71c9e2d5d..9dd083ce1 100644 --- a/src/tracks/backgroundEffectCatalog.ts +++ b/src/tracks/backgroundEffectCatalog.ts @@ -5,8 +5,8 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import arc from "../graphics/background-arc.jpg?url"; -import glow from "../graphics/background-glow.jpg?url"; +import arc from "../graphics/background-arc.jpg?url&no-inline"; +import glow from "../graphics/background-glow.jpg?url&no-inline"; export interface ShippedBackground { /** Stored in the user's settings when chosen, so it must not change. */ diff --git a/vite-component.config.ts b/vite-component.config.ts index 63183c781..fc0c9ac4a 100644 --- a/vite-component.config.ts +++ b/vite-component.config.ts @@ -34,6 +34,22 @@ export default defineConfig(({ mode }) => { // whatever is in `public` — including the developer's own config.json, which // is not in the repository — into the output we would publish. publicDir: false, + // A library does not know where it will be served from, so every file it + // emits beside itself — the sounds, the pictures, MediaPipe's runtime — is + // referenced relative to the module's own URL. (A library build inlines + // each asset as a data URL unless its import says `no-inline`, so the + // imports do.) + base: "./", + experimental: { + // Spelled `./assets/…`, not `assets/…` as the relative base would have it: + // a host's bundler reads a bare name as a package to look up. + renderBuiltUrl: (filename, { hostType }) => + hostType === "js" + ? { + runtime: `new URL(${JSON.stringify(`./${filename}`)}, import.meta.url).href`, + } + : { relative: true }, + }, // A host's document is not ours to style: everything in the stylesheet is // confined to the element Element Call is mounted in css: { postcss: { plugins: [scopeStylesToRoot()] } }, @@ -64,6 +80,15 @@ export default defineConfig(({ mode }) => { cssFileName: "element-call", }, rollupOptions: { + output: { + // Hashed, as the app's are, so that a file a host serves as it is + // stays correct across versions of the package. Not the stylesheet, + // which the package exports by name (see `cssFileName`). + assetFileNames: ({ names }) => + names.some((name) => name.endsWith(".css")) + ? "[name][extname]" + : "assets/[name]-[hash][extname]", + }, // The host already has these, and a second copy of any of them does not // merely bloat the bundle: React would hold two sets of hooks, the // Matrix client would run two sync loops, and a second Compound would From 51fac690f4fd3f813f902c6b727bc76494189f06 Mon Sep 17 00:00:00 2001 From: "Timo K." Date: Fri, 6 Nov 2026 13:30:36 +0100 Subject: [PATCH 14/34] Hand the camera track to the footer view model --- src/components/CallFooter.stories.tsx | 1 + src/components/CallFooter.tsx | 3 ++ src/components/CallFooterViewModel.test.ts | 43 ++++++++++++++++++++++ src/components/CallFooterViewModel.tsx | 20 ++++++++++ src/room/LobbyView.tsx | 3 ++ src/state/CallViewModel/CallViewModel.ts | 3 ++ 6 files changed, 73 insertions(+) diff --git a/src/components/CallFooter.stories.tsx b/src/components/CallFooter.stories.tsx index d7b1dcf90..8d6cc2a77 100644 --- a/src/components/CallFooter.stories.tsx +++ b/src/components/CallFooter.stories.tsx @@ -141,6 +141,7 @@ export const Default: Story = { backgroundImageRefusal: undefined, addBackgroundImage: fn(), removeBackgroundEffect: fn(), + cameraTrack: null, hangup: fn(), buttonSize: "lg", showFooter: true, diff --git a/src/components/CallFooter.tsx b/src/components/CallFooter.tsx index fce982347..f09109357 100644 --- a/src/components/CallFooter.tsx +++ b/src/components/CallFooter.tsx @@ -34,6 +34,7 @@ import { type LayoutSwitchViewModel } from "../state/LayoutSwitchViewModel"; import { LayoutSwitch } from "../room/LayoutSwitch"; import { type BackgroundEffectGridItem } from "./BackgroundEffectGrid"; import { type UnusableReason } from "../tracks/backgroundImages"; +import { type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk"; export interface BackgroundImageRefusal { reason: UnusableReason | "not-kept"; @@ -92,6 +93,8 @@ export interface FooterState { backgroundEffectSettling: boolean; /** Why the last file offered couldn't be kept, if it couldn't. */ backgroundImageRefusal: BackgroundImageRefusal | undefined; + /** The camera to preview, with the effect in force; null while it is off. */ + cameraTrack: LocalVideoMediaTrack | null; showFooter: boolean; /* This is needed for WindowMode = "flat" */ diff --git a/src/components/CallFooterViewModel.test.ts b/src/components/CallFooterViewModel.test.ts index a1b174462..7036db800 100644 --- a/src/components/CallFooterViewModel.test.ts +++ b/src/components/CallFooterViewModel.test.ts @@ -7,12 +7,14 @@ Please see LICENSE in the repository root for full details. import { afterEach, describe, expect, it, vi } from "vitest"; import { BehaviorSubject } from "rxjs"; +import { type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk"; import { flushPromises, testScope, mockMuteStates, mockMediaDevices, + mockVideoTrack, } from "../utils/test"; import { constant } from "../state/Behavior"; import type { CallViewModel } from "../state/CallViewModel/CallViewModel"; @@ -78,6 +80,7 @@ vi.mock("livekit-client", () => ({ function buildMinimalCallViewModel(layout: Layout): CallViewModel { return { layout$: constant(layout), + camera$: constant(undefined), edgeToEdge$: constant(false), showHeader$: constant(false), hangup: (): void => {}, @@ -272,6 +275,7 @@ describe("createCallFooterViewModel", () => { mockMuteStates(), twoMicsAndOneCamMediaDevices, constant(false), + /* camera$ */ constant(null), /* openSettings */ undefined, /* hangup */ undefined, /* showLogo */ false, @@ -362,6 +366,7 @@ describe("createCallFooterViewModel", () => { mockMuteStates(), twoMicsAndOneCamMediaDevices, settling$, + /* camera$ */ constant(null), /* openSettings */ undefined, /* hangup */ undefined, /* showLogo */ false, @@ -427,6 +432,44 @@ describe("createCallFooterViewModel", () => { expect(store.remove).toHaveBeenCalledWith("spare"); }); + it("previews the lobby's camera while there is one", () => { + const track = mockVideoTrack({}); + const camera$ = new BehaviorSubject(track); + const vm = createLobbyFooterViewModel( + testScope(), + mockMuteStates(), + twoMicsAndOneCamMediaDevices, + constant(false), + camera$, + /* openSettings */ undefined, + /* hangup */ undefined, + /* showLogo */ false, + ); + expect(vm.cameraTrack$.value).toBe(track); + // The lobby releases the camera when it is turned off + camera$.next(null); + expect(vm.cameraTrack$.value).toBeNull(); + }); + + it("previews the call's camera only while it is not muted", () => { + const muted$ = new BehaviorSubject(false); + const track = mockVideoTrack({ muted$ }); + const vm = createCallFooterViewModel( + testScope(), + { ...buildMinimalCallViewModel(gridLayout), camera$: constant(track) }, + mockMuteStates(), + twoMicsAndOneCamMediaDevices, + constant(false), + /* reactionIdentifier */ undefined, + { showControls: true, header: HeaderStyle.Standard }, + ); + expect(vm.cameraTrack$.value).toBe(track); + muted$.next(true); + expect(vm.cameraTrack$.value).toBeNull(); + muted$.next(false); + expect(vm.cameraTrack$.value).toBe(track); + }); + it("availability is the same before and during a call", () => { for (const supported of [true, false]) { sdkSupportMock.mockReturnValue(supported); diff --git a/src/components/CallFooterViewModel.tsx b/src/components/CallFooterViewModel.tsx index 8a22aa52e..07f6dc0b1 100644 --- a/src/components/CallFooterViewModel.tsx +++ b/src/components/CallFooterViewModel.tsx @@ -11,9 +11,11 @@ import { distinctUntilChanged, map, type Observable, + of, switchMap, } from "rxjs"; import { supportsAudioOutputSelection } from "livekit-client"; +import { type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk"; import { supportsBackgroundProcessors, supportsModernBackgroundProcessors, @@ -88,6 +90,8 @@ function buildDeviceBehaviors( /** return empty arrays for audioOptions and videoOptions*/ disableSwitcher$: Behavior, backgroundEffectSettling$: Observable, + /** The camera as the pipeline has it, for the menu's preview. */ + camera$: Observable, ): Pick< ViewModel, | "audioOptions$" @@ -107,6 +111,7 @@ function buildDeviceBehaviors( | "addBackgroundImage$" | "backgroundImageRefusal$" | "removeBackgroundEffect$" + | "cameraTrack$" > { const options$ = ( available$: Behavior>, @@ -208,6 +213,17 @@ function buildDeviceBehaviors( ), ), backgroundImageRefusal$: refusal$, + // None while muted: the call mutes its track rather than releasing it. + cameraTrack$: scope.behavior( + camera$.pipe( + switchMap((camera) => + camera + ? camera.muted$.pipe(map((muted) => (muted ? null : camera))) + : of(null), + ), + ), + null, + ), // Never the one in force: taking it away would leave nothing chosen. removeBackgroundEffect$: scope.behavior( offered$.pipe( @@ -269,6 +285,7 @@ export function createCallFooterViewModel( mediaDevices, disableDeviceSwitcher$, backgroundEffectSettling$, + callModel.camera$, ), // candidat to move into the FooterViewModel showFooter$: callModel.showFooter$, @@ -333,6 +350,7 @@ export function createCallFooterViewModel( * @param mediaDevices - Available and selected input devices. * @param backgroundEffectSettling$ - Whether the first effect chosen is still * being prepared. + * @param camera$ - The lobby's camera, for the camera menu's preview. * @param openSettings - Callback to open the settings modal, or undefined. * @param hangup - Callback to leave/cancel, or undefined (hides the button). * @param showLogo - Whether to show the Element Call logo. @@ -342,6 +360,7 @@ export function createLobbyFooterViewModel( muteStates: MuteStates, mediaDevices: MediaDevices, backgroundEffectSettling$: Observable, + camera$: Behavior, openSettings: (() => void) | undefined, hangup: (() => void) | undefined, showLogo: boolean, @@ -390,6 +409,7 @@ export function createLobbyFooterViewModel( mediaDevices, constant(false), backgroundEffectSettling$, + camera$, ), }; } diff --git a/src/room/LobbyView.tsx b/src/room/LobbyView.tsx index 1a197b8d0..a72c04965 100644 --- a/src/room/LobbyView.tsx +++ b/src/room/LobbyView.tsx @@ -158,6 +158,7 @@ export const LobbyView: FC = ({ null, ); useEffect(() => { + if (mediaVm === null) return; const footerScope = new ObservableScope(); setFooterVm( createLobbyFooterViewModel( @@ -165,6 +166,7 @@ export const LobbyView: FC = ({ muteStates, devices, backgroundEffectsState$.pipe(map(({ settling }) => settling ?? false)), + mediaVm.videoTrack$, openSettings, hangup, // Logo and header are connected: only show the logo in SPA with header. @@ -179,6 +181,7 @@ export const LobbyView: FC = ({ backgroundEffectsState$, hangup, hideHeader, + mediaVm, muteStates, openSettings, ]); diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 9b1e853a3..ba83fad68 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -363,6 +363,8 @@ export interface CallViewModel { keyRotationSuppressed$: Behavior; /** The local member, once our membership has been seen in the room. */ localMember$: Behavior; + /** Our own camera track, while one is published. */ + camera$: Behavior; remoteMembers$: Behavior; /** The transports the client holds a connection to, for the developer panel. */ connectedTransports$: Behavior; @@ -1508,6 +1510,7 @@ export function createCallViewModel$( leave, toggleScreenSharing: toggleScreenSharing, sharingScreen$: sharingScreen$, + camera$, tapScreen: (): void => screenTap$.next(), tapControls: (): void => controlsTap$.next(), From f2f67019b41250e1f70b1c33cd08fdb8b3612cd9 Mon Sep 17 00:00:00 2001 From: "Timo K." Date: Fri, 6 Nov 2026 13:30:36 +0100 Subject: [PATCH 15/34] Show the self preview in the video mute button context menu --- playwright/background-effects.spec.ts | 60 +++++++ src/components/CallFooter.tsx | 3 + .../MediaMuteAndSwitchButton.module.css | 32 ++++ .../MediaMuteAndSwitchButton.stories.tsx | 159 +++++++++++++++++- src/components/MediaMuteAndSwitchButton.tsx | 60 ++++++- src/components/SelfPreview.module.css | 10 ++ src/components/SelfPreview.test.tsx | 47 ++++++ src/components/SelfPreview.tsx | 48 ++++++ 8 files changed, 412 insertions(+), 7 deletions(-) create mode 100644 src/components/SelfPreview.module.css create mode 100644 src/components/SelfPreview.test.tsx create mode 100644 src/components/SelfPreview.tsx diff --git a/playwright/background-effects.spec.ts b/playwright/background-effects.spec.ts index 4d01c2d72..019ccfd43 100644 --- a/playwright/background-effects.spec.ts +++ b/playwright/background-effects.spec.ts @@ -106,6 +106,66 @@ async function drawsPerSecond(page: Page): Promise { }); } +test.describe("the camera menu's preview", () => { + test("shows the camera with the effect in force", async ({ page }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall(page, "Preview user", "Preview in the menu"); + const camera = await cameraColour(page.locator("video").first()); + + await page.getByRole("button", { name: "Camera", exact: true }).click(); + const preview = page.getByRole("menu").locator("video"); + await expect(preview).toBeVisible(); + await expect + .poll(async () => distance(await averageColour(preview), camera), { + timeout: 20_000, + }) + .toBeLessThan(30); + + const tile = page.getByRole("menuitemradio", { name: "Background 1" }); + await tile.click(); + const picture = await averageColour(tile.locator("img")); + await expect + .poll(async () => distance(await averageColour(preview), picture), { + timeout: 60_000, + }) + .toBeLessThan(distance(camera, picture) / 2); + }); + + test("shows the call's own camera once joined", async ({ page }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall( + page, + "Call preview user", + "Preview in the call's menu", + true, + ); + await expect(page.getByTestId("incall_videomute")).toBeEnabled({ + timeout: 10_000, + }); + const camera = await cameraColour(page.locator("video").first()); + + await page.getByRole("button", { name: "Camera", exact: true }).click(); + const preview = page.getByRole("menu").locator("video"); + await expect(preview).toBeVisible(); + await expect + .poll(async () => distance(await averageColour(preview), camera), { + timeout: 20_000, + }) + .toBeLessThan(30); + + const tile = page.getByRole("menuitemradio", { name: "Background 1" }); + await tile.click(); + const picture = await averageColour(tile.locator("img")); + await expect + .poll(async () => distance(await averageColour(preview), picture), { + timeout: 60_000, + }) + .toBeLessThan(distance(camera, picture) / 2); + }); +}); + test.describe("background effects section", () => { test("section stays within the call area at every size", async ({ page }) => { test.setTimeout(120_000); diff --git a/src/components/CallFooter.tsx b/src/components/CallFooter.tsx index f09109357..4d13b4c6a 100644 --- a/src/components/CallFooter.tsx +++ b/src/components/CallFooter.tsx @@ -33,6 +33,7 @@ import { useBehavior } from "../useBehavior"; import { type LayoutSwitchViewModel } from "../state/LayoutSwitchViewModel"; import { LayoutSwitch } from "../room/LayoutSwitch"; import { type BackgroundEffectGridItem } from "./BackgroundEffectGrid"; +import { SelfPreview } from "./SelfPreview"; import { type UnusableReason } from "../tracks/backgroundImages"; import { type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk"; @@ -184,6 +185,7 @@ export const CallFooter: FC = ({ const backgroundEffectSettling = useBehavior(vm.backgroundEffectSettling$); const addBackgroundImage = useBehavior(vm.addBackgroundImage$); const removeBackgroundEffect = useBehavior(vm.removeBackgroundEffect$); + const cameraTrack = useBehavior(vm.cameraTrack$); const refusal = useBehavior(vm.backgroundImageRefusal$); const refusalMessage = useMemo(() => { switch (refusal?.reason) { @@ -267,6 +269,7 @@ export const CallFooter: FC = ({ backgroundEffectSettling={backgroundEffectSettling} onAddBackgroundImage={addBackgroundImage} onRemoveBackgroundEffect={removeBackgroundEffect} + selfPreview={} backgroundImageRefusal={refusalMessage} backgroundEffectNotice={ backgroundEffectNotice === "unavailable" diff --git a/src/components/MediaMuteAndSwitchButton.module.css b/src/components/MediaMuteAndSwitchButton.module.css index 10b3267f5..2e8c60f3f 100644 --- a/src/components/MediaMuteAndSwitchButton.module.css +++ b/src/components/MediaMuteAndSwitchButton.module.css @@ -97,3 +97,35 @@ Please see LICENSE in the repository root for full details. .refusal { margin: var(--cpd-space-2x) var(--cpd-space-4x) 0; } + +.selfPreview { + position: relative; + display: flex; + align-items: center; + justify-content: center; + flex-shrink: 0; + aspect-ratio: 16 / 9; + margin-block-end: var(--cpd-space-2x); + overflow: clip; + border-start-start-radius: var(--cpd-space-3x); + border-start-end-radius: var(--cpd-space-3x); + background: var(--cpd-color-bg-subtle-secondary); + color: var(--cpd-color-icon-secondary); +} + +.selfPreview > video { + position: absolute; + inset: 0; + inline-size: 100%; + block-size: 100%; + object-fit: cover; + /* Rounded itself too: a video can be drawn on a layer of its own, which the + frame's rounded clip doesn't always reach. */ + border-radius: inherit; +} + +/* Up to the menu's frame, over the padding at its top. */ +.selfPreviewPinned, +.deviceListLeadsWithPreview { + margin-block-start: calc(-1 * var(--cpd-space-5x)); +} diff --git a/src/components/MediaMuteAndSwitchButton.stories.tsx b/src/components/MediaMuteAndSwitchButton.stories.tsx index 38ace8aee..1d9447773 100644 --- a/src/components/MediaMuteAndSwitchButton.stories.tsx +++ b/src/components/MediaMuteAndSwitchButton.stories.tsx @@ -12,6 +12,7 @@ import type { Meta, StoryObj } from "@storybook/react-vite"; import { MediaMuteAndSwitchButton } from "./MediaMuteAndSwitchButton"; import styles from "./MediaMuteAndSwitchButton.module.css"; import { type BackgroundEffectGridItem } from "./BackgroundEffectGrid"; +import { SelfPreview } from "./SelfPreview"; import { shippedBackgrounds } from "../tracks/backgroundEffectCatalog"; import meterStyles from "./MicrophoneLevelMeter.module.css"; import { MediaDevicesContext } from "../MediaDevicesContext"; @@ -60,6 +61,50 @@ const WithAMicrophone: FC<{ children: ReactNode }> = ({ children }) => { return <>{children}; }; +/** A camera for the preview: a figure against a room, drawn to a stream. */ +const FakeCamera: FC = () => { + const [stream, setStream] = useState(); + useEffect(() => { + const canvas = document.createElement("canvas"); + canvas.width = 640; + canvas.height = 360; + const context = canvas.getContext("2d")!; + const draw = (): void => { + context.fillStyle = "#6b7f8e"; + context.fillRect(0, 0, 640, 360); + context.fillStyle = "#c9a27e"; + context.beginPath(); + context.arc(320, 150, 62, 0, 2 * Math.PI); + context.fill(); + context.fillStyle = "#34495e"; + context.beginPath(); + context.ellipse(320, 380, 150, 140, 0, Math.PI, 2 * Math.PI); + context.fill(); + }; + draw(); + // Redrawn, so the stream keeps delivering frames. + const timer = setInterval(draw, 200); + const captured = canvas.captureStream(10); + setStream(captured); + return (): void => { + clearInterval(timer); + captured.getTracks().forEach((track) => track.stop()); + }; + }, []); + return ( +