Compare commits

..
Author SHA1 Message Date
Timo K.andClaude Fable 5.1 3c8b381723 Fail the component build on what a host must not be handed
A library build inlines an asset as a data URL the moment its import
loses `no-inline`, and emits the standalone app's store worker the
moment a module the component reaches imports it; neither warns, and
the package grows by 11 MB without anyone seeing it in a diff. It had
been published at 18 MB for several releases before it was measured.

`scripts/check-component-bundle.mjs` now runs at the end of the
component build and fails it on a data URL over 16 kB in a module a
host bundles, an absolute `/assets/` reference, the store worker under
`assets/`, or a main module over 4 MB, each with a message saying what
to do. It passes only once both the assets and the store worker fixes
are in.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-11-06 14:44:22 +01:00
51 changed files with 414 additions and 3343 deletions
+5
View File
@@ -243,6 +243,11 @@ needs a certificate exception for `https://localhost:3001` as well (see the
note under [Backend](#backend)). It reads the same `public/config.json` as
`pnpm dev` if one exists, and runs with Element Call's defaults otherwise.
`pnpm build:component` ends with `scripts/check-component-bundle.mjs`, which
fails the build on what a host must not be handed: an asset inlined into the
module as a data URL, a file referenced by an absolute path, the standalone
app's client store worker, or a module over its size budget.
The call lays itself out for the size of the element it is mounted in, not the
window: a host that shrinks the container to a corner of its page gets the
picture-in-picture layout, just as a host that shrank the whole iframe used to.
+3 -5
View File
@@ -26,7 +26,8 @@
"features": {
"feature_element_call_video_rooms": true,
"feature_video_rooms": true,
"feature_group_calls": true
"feature_group_calls": true,
"feature_release_announcement": false
},
"default_federate": true,
"default_theme": "light",
@@ -39,10 +40,7 @@
},
"setting_defaults": {
"breadcrumbs": true,
"feature_group_calls": true,
"releaseAnnouncementData": {
"room_list_section": true
}
"feature_group_calls": true
},
"jitsi": {
"preferred_domain": "meet.element.io"
+3 -5
View File
@@ -26,7 +26,8 @@
"features": {
"feature_element_call_video_rooms": true,
"feature_video_rooms": true,
"feature_group_calls": true
"feature_group_calls": true,
"feature_release_announcement": false
},
"default_federate": true,
"default_theme": "light",
@@ -39,10 +40,7 @@
},
"setting_defaults": {
"breadcrumbs": true,
"feature_group_calls": true,
"releaseAnnouncementData": {
"room_list_section": true
}
"feature_group_calls": true
},
"jitsi": {
"preferred_domain": "meet.element.io"
+2 -2
View File
@@ -19,7 +19,7 @@ services:
- ./backend/playwright_homeserver-othersite.yaml:/data/cfg/homeserver.yaml:Z
element-web:
# Pin to a SHA so that upstream cannot break our tests. Renovate handles regular updates.
image: ghcr.io/element-hq/element-web:develop@sha256:e063472b6bdf6d0bfa1d43dfa4ef7effee1efd2c9684daee665d1bedfe746a47
image: ghcr.io/element-hq/element-web:develop@sha256:a79801206c628a912d57fa6f355d5c5ab9c6c40c2d1b6208d4192678fb6e80c7
element-web-1:
# Pin to a SHA so that upstream cannot break our tests. Renovate handles regular updates.
image: ghcr.io/element-hq/element-web:develop@sha256:e063472b6bdf6d0bfa1d43dfa4ef7effee1efd2c9684daee665d1bedfe746a47
image: ghcr.io/element-hq/element-web:develop@sha256:a79801206c628a912d57fa6f355d5c5ab9c6c40c2d1b6208d4192678fb6e80c7
-2
View File
@@ -48,8 +48,6 @@ Test what a user observes: the peer sees the change, it survives a reconnect, it
right after a reload. Reuse `playwright/spa-helpers.ts`,
`playwright/widget/test-helpers.ts`, `playwright/fixtures/`.
To mark new release announcements as viewed, add them in `backend/ew.test*.config.json`.
```sh
pnpm test # unit + storybook
pnpm backend # Synapse + LiveKit, required for e2e
-5
View File
@@ -3,7 +3,6 @@
"user_menu": "User menu"
},
"action": {
"add_background_image": "Add image",
"close": "Close",
"copy_link": "Copy link",
"edit": "Edit",
@@ -99,10 +98,6 @@
},
"disconnected_banner": "Connectivity to the server has been lost.",
"error": {
"background_animated": "Animated images cannot be used as a background",
"background_not_an_image": "That file is not a supported image",
"background_not_kept": "That background could not be saved",
"background_undecodable": "That image could not be opened",
"call_is_not_supported": "Call is not supported",
"call_not_found": "Call not found",
"call_not_found_description": "<0>That link doesn't appear to belong to any existing call. Check that you have the right link, or <2>create a new one</2>.</0>",
+1 -2
View File
@@ -22,7 +22,7 @@
"build:sdk:production": "pnpm build:sdk-target-based-on-call-view-model",
"build:sdk:development": "pnpm build:sdk-target-based-on-call-view-model --mode development",
"build:component": "pnpm build:component:js && pnpm build:component:types",
"build:component:js": "pnpm build:full --config vite-component.config.js",
"build:component:js": "pnpm build:full --config vite-component.config.js && node scripts/check-component-bundle.mjs",
"build:component:types": "tsc -p component/tsconfig.build.json",
"build:component:production": "pnpm build:component",
"build:component:development": "pnpm build:component:js --mode development && pnpm build:component:types",
@@ -97,7 +97,6 @@
"copy-to-clipboard": "^3.3.3",
"eslint-plugin-element-call": "link:eslint",
"eslint-plugin-storybook": "^10.3.6",
"fake-indexeddb": "^6.2.5",
"fetch-mock": "11.1.5",
"i18next": "^26.0.0",
"i18next-browser-languagedetector": "^8.0.0",
+23 -849
View File
@@ -5,15 +5,9 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { expect, test, type Browser, type Page } from "@playwright/test";
import { expect, test, type Locator, type Page } from "@playwright/test";
import { SpaHelpers } from "./spa-helpers.ts";
import {
averageColour,
cameraColour,
type Colour,
distance,
} from "./utils/colour.ts";
// Background effects need WebGL2, and headless Firefox on a CI runner has none,
// so it rightly offers none of them.
@@ -22,13 +16,15 @@ test.skip(
"Background effects need WebGL2, which headless Firefox on CI does not have",
);
type Colour = [number, number, number];
test.describe("background effects", () => {
test("pre-join preview shows the chosen effect", async ({ page }) => {
test.slow();
await page.goto("/");
await SpaHelpers.createCall(page, "Effect user", "Background effects");
const preview = page.locator("video").first();
const camera = await cameraColour(preview);
const camera = await averageColour(preview);
await page.getByRole("button", { name: "Camera", exact: true }).click();
const tile = page
@@ -106,847 +102,25 @@ async function drawsPerSecond(page: Page): Promise<number> {
});
}
test.describe("the camera menu's preview", () => {
test("shows the camera with the effect in force", async ({ page }) => {
test.setTimeout(120_000);
await page.goto("/");
await SpaHelpers.createCall(page, "Preview user", "Preview in the menu");
const camera = await cameraColour(page.locator("video").first());
await page.getByRole("button", { name: "Camera", exact: true }).click();
const preview = page.getByRole("menu").locator("video");
await expect(preview).toBeVisible();
await expect
.poll(async () => distance(await averageColour(preview), camera), {
timeout: 20_000,
})
.toBeLessThan(30);
const tile = page.getByRole("menuitemradio", { name: "Background 1" });
await tile.click();
const picture = await averageColour(tile.locator("img"));
await expect
.poll(async () => distance(await averageColour(preview), picture), {
timeout: 60_000,
})
.toBeLessThan(distance(camera, picture) / 2);
});
test("shows the call's own camera once joined", async ({ page }) => {
test.setTimeout(120_000);
await page.goto("/");
await SpaHelpers.createCall(
page,
"Call preview user",
"Preview in the call's menu",
true,
);
await expect(page.getByTestId("incall_videomute")).toBeEnabled({
timeout: 10_000,
});
const camera = await cameraColour(page.locator("video").first());
await page.getByRole("button", { name: "Camera", exact: true }).click();
const preview = page.getByRole("menu").locator("video");
await expect(preview).toBeVisible();
await expect
.poll(async () => distance(await averageColour(preview), camera), {
timeout: 20_000,
})
.toBeLessThan(30);
const tile = page.getByRole("menuitemradio", { name: "Background 1" });
await tile.click();
const picture = await averageColour(tile.locator("img"));
await expect
.poll(async () => distance(await averageColour(preview), picture), {
timeout: 60_000,
})
.toBeLessThan(distance(camera, picture) / 2);
});
});
test.describe("background effects section", () => {
test("section stays within the call area at every size", async ({ page }) => {
test.setTimeout(120_000);
await page.goto("/");
await SpaHelpers.createCall(page, "Sizes user", "Background effect sizes");
for (const size of [
{ width: 1280, height: 800 },
{ width: 800, height: 600 },
{ width: 600, height: 420 },
{ width: 480, height: 360 },
{ width: 360, height: 640 },
]) {
await page.setViewportSize(size);
await page.getByRole("button", { name: "Camera", exact: true }).click();
const menu = page.getByRole("menu");
await expect(menu).toBeVisible();
const box = (await menu.boundingBox())!;
const where = `at ${size.width}x${size.height}`;
expect(box.x, where).toBeGreaterThanOrEqual(0);
expect(box.y, where).toBeGreaterThanOrEqual(0);
expect(box.x + box.width, where).toBeLessThanOrEqual(size.width);
expect(box.y + box.height, where).toBeLessThanOrEqual(size.height);
const last = page
.getByRole("group", { name: "Background effects" })
.getByRole("menuitemradio")
.last();
await last.scrollIntoViewIfNeeded();
await expect(last, where).toBeInViewport({ ratio: 1 });
await last.click();
await expect(last, where).toHaveAttribute("aria-checked", "true");
// Back to no effect for the next size, and close.
await page
.getByRole("group", { name: "Background effects" })
.getByRole("menuitemradio", { name: "None" })
.click();
await page.keyboard.press("Escape");
await expect(menu).toBeHidden();
}
});
});
test.describe("a pipeline that fails to build", () => {
test("offers no effects for the session, and keeps the choice", async ({
page,
}) => {
test.setTimeout(120_000);
await page.goto("/");
await SpaHelpers.createCall(page, "Failing user", "Pipeline fails");
const preview = page.locator("video").first();
const camera = await cameraColour(preview);
await page.route("**/*.tflite*", async (route) => route.abort());
await page.getByRole("button", { name: "Camera", exact: true }).click();
const tile = page.getByRole("menuitemradio", { name: "Background 1" });
await tile.click();
await expect(
page.getByText("Background effects are not supported on this platform."),
).toBeVisible({ timeout: 30_000 });
await expect(tile).toBeDisabled();
await page.keyboard.press("Escape");
// The camera and microphone carry on without it.
expect(distance(await cameraColour(preview), camera)).toBeLessThan(30);
await expect(page.getByTestId("incall_mute")).toHaveAttribute(
"aria-checked",
"true",
);
await page.unroute("**/*.tflite*");
await page.reload();
await expect(page.getByTestId("lobby_joinCall")).toBeVisible();
await page.getByRole("button", { name: "Camera", exact: true }).click();
await expect(tile).toHaveAttribute("aria-checked", "true");
const picture = await averageColour(tile.locator("img"));
await expect
.poll(async () => distance(await averageColour(preview), picture), {
timeout: 60_000,
})
.toBeLessThan(distance(camera, picture) / 2);
});
});
test.describe("the chosen effect", () => {
test("chosen effect survives leaving and rejoining", async ({ page }) => {
test.setTimeout(120_000);
await page.goto("/");
await SpaHelpers.createCall(page, "Returning user", "Effect remembered");
const { camera, picture } = await chooseAndWear(page, "Background 1");
await page.getByTestId("lobby_joinCall").click();
await page.getByTestId("incall_leave").click();
await expect(page.getByRole("heading")).toContainText(
"your call has ended",
);
await page.goto("/");
await page.getByTestId("home_callName").fill("Effect remembered again");
await page.getByTestId("home_go").click();
await expect(page.getByTestId("lobby_joinCall")).toBeVisible();
await page.getByRole("button", { name: "Camera", exact: true }).click();
const tile = page
.getByRole("group", { name: "Background effects" })
.getByRole("menuitemradio", { name: "Background 1" });
await expect(tile).toHaveAttribute("aria-checked", "true");
const preview = page.locator("video").first();
await expect
.poll(async () => distance(await averageColour(preview), picture), {
timeout: 60_000,
})
.toBeLessThan(distance(camera, picture) / 2);
});
test("second session still shows the existing error", async ({ page }) => {
test.setTimeout(120_000);
await page.goto("/");
await SpaHelpers.createCall(page, "Two tabs", "Effect in two tabs");
await chooseAndWear(page, "Background 1");
const second = await page.context().newPage();
await second.goto("/");
await expect(
page.getByRole("heading", { name: "Opened in another tab" }),
).toBeVisible();
});
});
/** Chooses an effect in the lobby, and waits until the preview wears it. */
async function chooseAndWear(
page: Page,
name: string,
): Promise<{ camera: Colour; picture: Colour }> {
const preview = page.locator("video").first();
const camera = await cameraColour(preview);
await page.getByRole("button", { name: "Camera", exact: true }).click();
const tile = page.getByRole("menuitemradio", { name });
await tile.click();
const picture = await averageColour(tile.locator("img"));
await expect
.poll(async () => distance(await averageColour(preview), picture), {
timeout: 60_000,
})
.toBeLessThan(distance(camera, picture) / 2);
await page.keyboard.press("Escape");
return { camera, picture };
}
test.describe("a background of one's own", () => {
test("adds an image from the device", async ({ page }) => {
test.setTimeout(120_000);
await page.goto("/");
await SpaHelpers.createCall(page, "Adding user", "Own background");
const preview = page.locator("video").first();
const camera = await cameraColour(preview);
const red = await redImage(page);
await addImage(page, red);
const mine = page
.getByRole("group", { name: "Background effects" })
.getByRole("menuitemradio", { name: "Background 3" });
await expect(mine).toBeVisible();
await expect(mine).toHaveAttribute("aria-checked", "false");
await mine.click();
await expect
.poll(async () => distance(await averageColour(preview), RED), {
timeout: 60_000,
})
.toBeLessThan(distance(camera, RED) / 2);
});
test("adding an image makes no upload", async ({ page }) => {
test.setTimeout(120_000);
await page.goto("/");
await SpaHelpers.createCall(page, "Private user", "Own background kept");
const preview = page.locator("video").first();
const camera = await cameraColour(preview);
const red = await redImage(page);
const sent: { request: string; body: Buffer | null }[] = [];
page.on("request", (request) =>
sent.push({
request: `${request.method()} ${request.url()}`,
body: request.postDataBuffer(),
}),
);
await addImage(page, red);
await page
.getByRole("group", { name: "Background effects" })
.getByRole("menuitemradio", { name: "Background 3" })
.click();
// Kept, chosen and in force: anything that would send it has had its turn.
await expect
.poll(async () => distance(await averageColour(preview), RED), {
timeout: 60_000,
})
.toBeLessThan(distance(camera, RED) / 2);
// The file as given, and the copy kept, which is encoded anew.
const samples = [red, await keptBytes(page)].map((bytes) =>
bytes.subarray(bytes.length - 64),
);
const carriesIt = sent.filter(
({ request, body }) =>
request.includes("/_matrix/media/") ||
samples.some((sample) => body?.includes(sample)),
);
expect(carriesIt.map(({ request }) => request)).toEqual([]);
});
test("added image survives rejoining and is gone after storage is cleared", async ({
page,
}) => {
test.setTimeout(120_000);
await page.goto("/");
await SpaHelpers.createCall(page, "Keeping user", "Own background again");
const preview = page.locator("video").first();
const camera = await cameraColour(preview);
const red = await redImage(page);
await addImage(page, red);
const mine = page
.getByRole("group", { name: "Background effects" })
.getByRole("menuitemradio", { name: "Background 3" });
await mine.click();
await page.keyboard.press("Escape");
await page.reload();
await expect(page.getByTestId("lobby_joinCall")).toBeVisible();
await page.getByRole("button", { name: "Camera", exact: true }).click();
await expect(mine).toHaveAttribute("aria-checked", "true");
await page.keyboard.press("Escape");
await expect
.poll(async () => distance(await averageColour(preview), RED), {
timeout: 60_000,
})
.toBeLessThan(distance(camera, RED) / 2);
await page.evaluate(
async () =>
new Promise<void>((resolve, reject) => {
const request = indexedDB.deleteDatabase(
"element-call-background-images",
);
request.onsuccess = (): void => resolve();
request.onerror = (): void => reject(request.error);
request.onblocked = (): void => resolve();
}),
);
await page.reload();
await expect(page.getByTestId("lobby_joinCall")).toBeVisible();
await page.getByRole("button", { name: "Camera", exact: true }).click();
await expect(
page.getByRole("group", { name: "Background effects" }),
).toBeVisible();
await expect(mine).toHaveCount(0);
await page.keyboard.press("Escape");
await addImage(page, red);
await expect(mine).toBeVisible();
});
test("a removed image stays removed", async ({ page }) => {
test.setTimeout(120_000);
await page.goto("/");
await SpaHelpers.createCall(
page,
"Removing user",
"Own background removed",
);
await addImage(page, await redImage(page));
const mine = page
.getByRole("group", { name: "Background effects" })
.getByRole("menuitemradio", { name: "Background 3" });
await expect(mine).toHaveAttribute("aria-keyshortcuts", "Delete");
await mine.focus();
await page.keyboard.press("Delete");
await expect(mine).toHaveCount(0);
await page.reload();
await expect(page.getByTestId("lobby_joinCall")).toBeVisible();
await page.getByRole("button", { name: "Camera", exact: true }).click();
await expect(
page.getByRole("group", { name: "Background effects" }),
).toBeVisible();
await expect(mine).toHaveCount(0);
expect(await keptImages(page)).toEqual([]);
});
});
test.describe("what can be added", () => {
test("refuses an animated image", async ({ page }) => {
test.setTimeout(120_000);
await page.goto("/");
await SpaHelpers.createCall(page, "Refused user", "Animated refused");
await addImage(page, ANIMATED_GIF, {
name: "moving.gif",
type: "image/gif",
});
await expect(
page.getByText("Animated images cannot be used as a background"),
).toBeVisible();
const section = page.getByRole("group", { name: "Background effects" });
await expect(
section.getByRole("menuitemradio", { name: "Background 3" }),
).toHaveCount(0);
await expect(
section.getByRole("menuitemradio", { name: "None" }),
).toHaveAttribute("aria-checked", "true");
expect(await keptImages(page)).toEqual([]);
});
test("reduces an oversized image", async ({ page }) => {
test.setTimeout(120_000);
await page.goto("/");
await SpaHelpers.createCall(page, "Large user", "Oversized reduced");
await addImage(page, await pngImage(page, { width: 4000, height: 3000 }));
await expect(
page
.getByRole("group", { name: "Background effects" })
.getByRole("menuitemradio", { name: "Background 3" }),
).toBeVisible();
await expect(page.getByRole("alert")).toHaveCount(0);
const [kept] = await keptImages(page);
expect(kept).toMatchObject({ width: 1920, height: 1440 });
});
test("lays every image on an opaque ground", async ({ page }) => {
test.setTimeout(120_000);
await page.goto("/");
await SpaHelpers.createCall(page, "Logo user", "Transparent made opaque");
await addImage(
page,
await pngImage(page, { width: 320, height: 180, transparent: true }),
);
await expect(
page
.getByRole("group", { name: "Background effects" })
.getByRole("menuitemradio", { name: "Background 3" }),
).toBeVisible();
const [kept] = await keptImages(page);
expect(kept.minAlpha).toBe(255);
});
});
/** Two frames of one pixel, red then blue. */
const ANIMATED_GIF = Buffer.from(
"R0lGODlhAQABAPEAAP8AAAAA/wAAAAAAACH/C05FVFNDQVBFMi4wAwEAAAAh+QQACgAAACwAAAAAAQABAAACAkQBACH5BAAKAAAALAAAAAABAAEAAAICTAEAOw==",
"base64",
);
/** A red PNG of the given size; transparent leaves its right half empty. */
async function pngImage(
page: Page,
size: { width: number; height: number; transparent?: boolean },
): Promise<Buffer> {
const dataUrl = await page.evaluate(({ width, height, transparent }) => {
const canvas = document.createElement("canvas");
canvas.width = width;
canvas.height = height;
const context = canvas.getContext("2d")!;
context.fillStyle = "rgb(255, 0, 0)";
context.fillRect(0, 0, transparent ? width / 2 : width, height);
return canvas.toDataURL("image/png");
}, size);
return Buffer.from(dataUrl.split(",")[1], "base64");
}
/** The images the device keeps, as it keeps them. */
async function keptImages(
page: Page,
): Promise<{ width: number; height: number; minAlpha: number }[]> {
return page.evaluate(async () => {
const db = await new Promise<IDBDatabase>((resolve, reject) => {
const request = indexedDB.open("element-call-background-images");
request.onsuccess = (): void => resolve(request.result);
request.onerror = (): void => reject(request.error);
});
if (!db.objectStoreNames.contains("backgrounds")) return [];
const kept = await new Promise<{ image: Blob }[]>((resolve, reject) => {
const request = db
.transaction("backgrounds")
.objectStore("backgrounds")
.getAll();
request.onsuccess = (): void => resolve(request.result);
request.onerror = (): void => reject(request.error);
});
db.close();
return Promise.all(
kept.map(async ({ image }) => {
const bitmap = await createImageBitmap(image);
const canvas = new OffscreenCanvas(bitmap.width, bitmap.height);
const context = canvas.getContext("2d")!;
context.drawImage(bitmap, 0, 0);
const { data } = context.getImageData(
0,
0,
bitmap.width,
bitmap.height,
);
let minAlpha = 255;
for (let i = 3; i < data.length; i += 4)
minAlpha = Math.min(minAlpha, data[i]);
return { width: bitmap.width, height: bitmap.height, minAlpha };
}),
);
});
}
/** The bytes of the one image the device keeps. */
async function keptBytes(page: Page): Promise<Buffer> {
const base64 = await page.evaluate(
async () =>
new Promise<string>((resolve, reject) => {
const open = indexedDB.open("element-call-background-images");
open.onerror = (): void => reject(open.error);
open.onsuccess = (): void => {
const all = open.result
.transaction("backgrounds")
.objectStore("backgrounds")
.getAll();
all.onerror = (): void => reject(all.error);
all.onsuccess = (): void => {
const reader = new FileReader();
reader.onload = (): void =>
resolve((reader.result as string).split(",")[1]);
reader.readAsDataURL((all.result[0] as { image: Blob }).image);
};
};
}),
);
return Buffer.from(base64, "base64");
}
const RED: Colour = [255, 0, 0];
/** A solid red PNG, unlike the camera and the shipped pictures. */
async function redImage(page: Page): Promise<Buffer> {
const dataUrl = await page.evaluate(() => {
const canvas = document.createElement("canvas");
canvas.width = 320;
canvas.height = 180;
const context = canvas.getContext("2d")!;
context.fillStyle = "rgb(255, 0, 0)";
context.fillRect(0, 0, canvas.width, canvas.height);
return canvas.toDataURL("image/png");
});
return Buffer.from(dataUrl.split(",")[1], "base64");
}
/** Adds an image from the camera menu's add tile, leaving the menu open. */
async function addImage(
page: Page,
image: Buffer,
{ name, type } = { name: "mine.png", type: "image/png" },
): Promise<void> {
if (!(await page.getByRole("menu").isVisible()))
await page.getByRole("button", { name: "Camera", exact: true }).click();
const chooser = page.waitForEvent("filechooser");
await page.getByRole("menuitem", { name: "Add image" }).click();
await (await chooser).setFiles({ name, mimeType: type, buffer: image });
}
test.describe("joining with a background chosen", () => {
test("publishes no frame of the room", async ({ browser }) => {
// Two first builds of the pipeline, one of them held back.
test.setTimeout(180_000);
const hostContext = await browser.newContext();
const host = await hostContext.newPage();
await recordIncomingFrames(host);
await host.goto("/");
await SpaHelpers.createCall(host, "Host", "Join with a background", true);
await expect(host.getByTestId("incall_videomute")).toBeEnabled({
timeout: 10_000,
});
const inviteLink = await SpaHelpers.getCallInviteLink(host);
await host.keyboard.press("Escape");
const guestContext = await browser.newContext();
const guest = await guestContext.newPage();
await guest.goto(inviteLink);
await guest.getByTestId("joincall_displayName").fill("Guest");
await guest.getByTestId("joincall_joincall").click();
const preview = guest.locator("video").first();
await expect(preview).toBeVisible();
const camera = await cameraColour(preview);
await guest.getByRole("button", { name: "Camera", exact: true }).click();
const tile = guest.getByRole("menuitemradio", { name: "Background 1" });
await tile.click();
const picture = await averageColour(tile.locator("img"));
await expect
.poll(async () => distance(await averageColour(preview), picture), {
timeout: 60_000,
})
.toBeLessThan(distance(camera, picture) / 2);
// Were the pipeline built again for the call's track, it would take as
// long as on the slower route. Routing bypasses the cache, so it waits.
let heldBack = 0;
await guest.route("**/*.tflite*", async (route) => {
heldBack += 1;
await new Promise((resolve) => setTimeout(resolve, 15_000));
await route.continue();
});
await guest.keyboard.press("Escape");
await guest.getByTestId("lobby_joinCall").click();
const guestTile = host
.getByTestId("videoTile")
.filter({ has: host.getByTestId("name_tag").getByText("Guest") });
await expect(guestTile.locator("video")).toBeVisible({ timeout: 60_000 });
// Visible can come before the stream is attached.
let trackId: string | undefined;
await expect
.poll(async () => {
trackId = await guestTile
.locator("video")
.evaluate(
(video: HTMLVideoElement) =>
(video.srcObject as MediaStream | null)?.getVideoTracks()[0]?.id,
);
return trackId;
})
.toBeDefined();
await expectNoFrameOfTheRoom(host, trackId!, 0, camera, picture);
// The join adopts the lobby's pipeline, so the model is never fetched again.
expect(heldBack).toBe(0);
await hostContext.close();
await guestContext.close();
});
});
test.describe("the first build", () => {
test("survives the camera turned off and on while it runs", async ({
page,
}) => {
test.setTimeout(120_000);
const failures: string[] = [];
page.on("console", (m) => {
if (m.text().includes("Failed to attach video processor"))
failures.push(m.text());
});
await page.goto("/");
await SpaHelpers.createCall(page, "Toggling user", "Toggle while building");
const preview = page.locator("video").first();
const camera = await cameraColour(preview);
const held = await holdBackTheModel(page);
await page.getByRole("button", { name: "Camera", exact: true }).click();
const tile = page.getByRole("menuitemradio", { name: "Background 1" });
await tile.click();
const picture = await averageColour(tile.locator("img"));
await page.keyboard.press("Escape");
// New preview tracks, each attached while the first is still building.
for (let i = 0; i < 2; i++) {
await page.getByTestId("incall_videomute").click();
await page.getByTestId("incall_videomute").click();
}
await expect
.poll(async () => distance(await averageColour(preview), picture), {
timeout: 60_000,
})
.toBeLessThan(distance(camera, picture) / 2);
expect(failures).toEqual([]);
expect(held()).toBeGreaterThan(0);
});
});
test.describe("turning the camera on with a background chosen", () => {
test("publishes no frame of the room when the camera started off", async ({
browser,
}) => {
test.setTimeout(120_000);
const { host, guest, close } = await hostAndGuest(browser);
const camera = await cameraColour(guest.locator("video").first());
await guest.getByTestId("incall_videomute").click();
await guest.getByTestId("lobby_joinCall").click();
const picture = await chooseBackground(guest);
const held = await holdBackTheModel(guest);
await guest.getByTestId("incall_videomute").click();
const trackId = await incomingTrackId(host);
await expectNoFrameOfTheRoom(host, trackId, 0, camera, picture);
expect(held()).toBeGreaterThan(0);
await close();
});
test("publishes no frame of the room when the camera was turned off", async ({
browser,
}) => {
test.setTimeout(120_000);
const { host, guest, close } = await hostAndGuest(browser);
const camera = await cameraColour(guest.locator("video").first());
await guest.getByTestId("lobby_joinCall").click();
const trackId = await incomingTrackId(host);
await expect
.poll(async () => (await framesOf(host, trackId)).length, {
timeout: 20_000,
})
.toBeGreaterThan(0);
await guest.getByTestId("incall_videomute").click();
const picture = await chooseBackground(guest);
const held = await holdBackTheModel(guest);
const before = (await framesOf(host, trackId)).length;
await guest.getByTestId("incall_videomute").click();
await expectNoFrameOfTheRoom(host, trackId, before, camera, picture);
expect(held()).toBeGreaterThan(0);
await close();
});
});
/** A host in a call, recording what it receives, and a guest in its lobby. */
async function hostAndGuest(browser: Browser): Promise<{
host: Page;
guest: Page;
close: () => Promise<void>;
}> {
const hostContext = await browser.newContext();
const host = await hostContext.newPage();
await recordIncomingFrames(host);
await host.goto("/");
await SpaHelpers.createCall(host, "Host", "Camera with a background", true);
await expect(host.getByTestId("incall_videomute")).toBeEnabled({
timeout: 10_000,
});
const inviteLink = await SpaHelpers.getCallInviteLink(host);
await host.keyboard.press("Escape");
const guestContext = await browser.newContext();
const guest = await guestContext.newPage();
await guest.goto(inviteLink);
await guest.getByTestId("joincall_displayName").fill("Guest");
await guest.getByTestId("joincall_joincall").click();
await expect(guest.locator("video").first()).toBeVisible();
return {
host,
guest,
close: async (): Promise<void> => {
await hostContext.close();
await guestContext.close();
},
};
}
/** Chooses the first shipped picture in the call, and answers its colour. */
async function chooseBackground(guest: Page): Promise<Colour> {
// The lobby has a camera button of its own, so wait for the call's.
await expect(guest.getByTestId("incall_leave")).toBeVisible({
timeout: 30_000,
});
await expect(guest.getByTestId("incall_videomute")).toBeEnabled({
timeout: 10_000,
});
await guest.getByRole("button", { name: "Camera", exact: true }).click();
const tile = guest.getByRole("menuitemradio", { name: "Background 1" });
await tile.click();
await expect(tile).toHaveAttribute("aria-checked", "true");
const picture = await averageColour(tile.locator("img"));
await guest.keyboard.press("Escape");
return picture;
}
/** Makes the pipeline's first build as slow as on the slower route. */
/** Answers how often it held the model back, which a check should see. */
async function holdBackTheModel(page: Page): Promise<() => number> {
let held = 0;
// Routing bypasses the cache, so it waits.
await page.route("**/*.tflite*", async (route) => {
held += 1;
await new Promise((resolve) => setTimeout(resolve, 5_000));
await route.continue();
});
return () => held;
}
/** The id of the guest's camera track, as the host receives it. */
async function incomingTrackId(host: Page): Promise<string> {
const guestTile = host
.getByTestId("videoTile")
.filter({ has: host.getByTestId("name_tag").getByText("Guest") });
await expect(guestTile.locator("video")).toBeVisible({ timeout: 60_000 });
// Visible can come before the stream is attached.
let trackId: string | undefined;
await expect
.poll(async () => {
trackId = await guestTile
.locator("video")
.evaluate(
(video: HTMLVideoElement) =>
(video.srcObject as MediaStream | null)?.getVideoTracks()[0]?.id,
);
return trackId;
})
.toBeDefined();
return trackId!;
}
/** Waits for the picture to arrive, then finds no frame of the room since. */
async function expectNoFrameOfTheRoom(
host: Page,
trackId: string,
since: number,
camera: Colour,
picture: Colour,
): Promise<void> {
await expect
.poll(
async () => {
const frames = (await framesOf(host, trackId)).slice(since);
// A black frame, as a track starts, is nearer the picture than the
// camera, but shows neither.
return frames.some(
(f) =>
distance(f, picture) <
Math.min(distance(f, camera), distance(f, [0, 0, 0])),
);
},
{ timeout: 60_000 },
)
.toBe(true);
const frames = (await framesOf(host, trackId)).slice(since);
expect(
frames.filter((f) => distance(f, camera) < distance(f, picture)),
).toEqual([]);
}
/**
* Records the mean colour of every frame of every video track the page
* receives, from the moment it arrives: a tile may attach it later than its
* first frame.
*/
async function recordIncomingFrames(page: Page): Promise<void> {
await page.addInitScript(() => {
const frames: Record<string, [number, number, number][]> = {};
(window as unknown as { receivedFrames: typeof frames }).receivedFrames =
frames;
const record = (track: MediaStreamTrack): void => {
if (track.kind !== "video" || frames[track.id]) return;
const seen: [number, number, number][] = (frames[track.id] = []);
const video = document.createElement("video");
video.muted = true;
video.srcObject = new MediaStream([track]);
/** The mean colour of a video's current frame or an image, drawn small. */
async function averageColour(element: Locator): Promise<Colour> {
return element.evaluate(
async (source: HTMLVideoElement | HTMLImageElement) => {
if (source instanceof HTMLImageElement) await source.decode();
const canvas = document.createElement("canvas");
canvas.width = canvas.height = 16;
const context = canvas.getContext("2d", { willReadFrequently: true })!;
const step = (): void => {
context.drawImage(video, 0, 0, 16, 16);
const { data } = context.getImageData(0, 0, 16, 16);
const sum = [0, 0, 0];
for (let i = 0; i < data.length; i += 4)
for (let c = 0; c < 3; c++) sum[c] += data[i + c];
seen.push(sum.map((v) => v / (data.length / 4)) as Colour);
video.requestVideoFrameCallback(step);
};
video.requestVideoFrameCallback(step);
void video.play();
};
const Connection = window.RTCPeerConnection;
window.RTCPeerConnection = class extends Connection {
public constructor(...args: ConstructorParameters<typeof Connection>) {
super(...args);
this.addEventListener("track", (e) => record(e.track));
}
};
});
}
async function framesOf(page: Page, trackId: string): Promise<Colour[]> {
return page.evaluate(
(id) =>
(
window as unknown as {
receivedFrames: Record<string, [number, number, number][]>;
}
).receivedFrames[id] ?? [],
trackId,
canvas.width = canvas.height = 32;
const context = canvas.getContext("2d")!;
context.drawImage(source, 0, 0, 32, 32);
const { data } = context.getImageData(0, 0, 32, 32);
const sum = [0, 0, 0];
for (let i = 0; i < data.length; i += 4)
for (let c = 0; c < 3; c++) sum[c] += data[i + c];
const pixels = data.length / 4;
return sum.map((s) => s / pixels) as Colour;
},
);
}
function distance(a: Colour, b: Colour): number {
return Math.hypot(a[0] - b[0], a[1] - b[1], a[2] - b[2]);
}
@@ -1,85 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { expect, type Locator, type Page, test } from "@playwright/test";
import { createUserAndRoom, resizeContainer, startHarness } from "./harness.ts";
import { averageColour, cameraColour, expectWearing } from "../utils/colour.ts";
// Background effects need WebGL2, and headless Firefox on a CI runner has none,
// so it rightly offers none of them.
test.skip(
({ browserName }) => browserName === "firefox",
"Background effects need WebGL2, which headless Firefox on CI does not have",
);
/**
* Background effects with Element Call as a component in a host page: the
* pipeline's model and WebAssembly have to load from the component's bundle,
* and the portalled menu has to be sized against the call, not the window.
* Driven from the lobby, which builds the same menu and pipeline.
*/
// Sign-in, crypto setup and sync happen twice before anything shows.
test.describe.configure({ timeout: 180_000 });
test("puts a picture on the call's preview", async ({ page }) => {
const { username, roomId } = await createUserAndRoom("effects");
const panes = await startHarness(page, username, roomId);
const pane = panes.first();
await resizeContainer(pane.getByTestId("call-container"), {
width: 900,
height: 480,
});
await expect(pane.getByTestId("lobby_joinCall")).toBeVisible({
timeout: 60_000,
});
const preview = pane.locator("video").first();
const camera = await cameraColour(preview);
const tile = await openEffect(page, pane, "Background 1");
const picture = await averageColour(tile.locator("img"));
await tile.click();
await expect(tile).toHaveAttribute("aria-checked", "true");
await expectWearing(preview, picture, camera);
});
test("keeps every effect reachable in a small container", async ({ page }) => {
const { username, roomId } = await createUserAndRoom("effectsreach");
const panes = await startHarness(page, username, roomId);
const pane = panes.first();
const container = pane.getByTestId("call-container");
await resizeContainer(container, { width: 400, height: 360 });
await expect(pane.getByTestId("lobby_joinCall")).toBeVisible({
timeout: 60_000,
});
const last = await openEffect(page, pane, "Background 2");
const list = page.locator("[role='menu'] div[role='none']").first();
expect((await list.boundingBox())!.height).toBeLessThanOrEqual(
(await container.boundingBox())!.height,
);
await last.scrollIntoViewIfNeeded();
await expect(last).toBeInViewport({ ratio: 1 });
await last.click();
await expect(last).toHaveAttribute("aria-checked", "true");
});
/** Opens a component's camera menu and returns one of its effects. */
async function openEffect(
page: Page,
pane: Locator,
name: string,
): Promise<Locator> {
await pane
.getByRole("button", { name: "Camera", exact: true })
.click({ timeout: 60_000 });
// Portalled out of the pane, so found on the page.
return page
.getByRole("group", { name: "Background effects" })
.getByRole("menuitemradio", { name });
}
+64 -4
View File
@@ -29,10 +29,47 @@ export interface MyFixtures {
addUser: (username: string, host: string) => Promise<UserBaseFixture>;
}
// Minimal config.json for the local element-web instance
const CONFIG_JSON = {
default_server_config: {
"m.homeserver": {
base_url: "https://synapse.m.localhost",
server_name: "synapse.m.localhost",
},
},
element_call: {
participant_limit: 8,
brand: "Element Call",
},
// The default language is set here for test consistency
setting_defaults: {
language: "en-GB",
feature_group_calls: true,
},
// the location tests want a map style url.
map_style_url:
"https://api.maptiler.com/maps/streets/style.json?key=fU3vlMsMn4Jb6dnEIFsx",
features: {
// We don't want to go through the feature announcement during the e2e test
feature_release_announcement: false,
feature_element_call_video_rooms: true,
feature_video_rooms: true,
feature_group_calls: true,
},
};
export const widgetTest = test.extend<MyFixtures>({
// allow per-test override: `widgetTest.use({ callType: "dm" })`
callType: ["room", { option: true }],
asWidget: async ({ browser, callType }, pUse) => {
asWidget: async ({ browser, context, callType }, pUse) => {
await context.route(`http://localhost:8081/config.json*`, async (route) => {
await route.fulfill({ json: CONFIG_JSON });
});
const brooksDisplayName = `brooks_${Date.now()}`;
const whistlerDisplayName = `whistler_${Date.now()}`;
@@ -72,7 +109,15 @@ export const widgetTest = test.extend<MyFixtures>({
await TestHelpers.dismissInviteUnknownUserModal(ewPage1);
// Accept the invite
await TestHelpers.openRoomInvite(ewPage2, "Welcome Room");
await TestHelpers.closeReleaseAnnouncement(
ewPage2,
"Introducing Sections",
);
await TestHelpers.expandAllSections(ewPage2);
await expect(
TestHelpers.roomListItem(ewPage2, "Welcome Room"),
).toBeVisible();
await TestHelpers.roomListItem(ewPage2, "Welcome Room").click();
await ewPage2.getByRole("button", { name: "Accept" }).click();
await expect(
ewPage2
@@ -80,7 +125,17 @@ export const widgetTest = test.extend<MyFixtures>({
.getByRole("heading", { name: "Welcome Room" }),
).toBeVisible();
} else if (callType === "dm") {
await TestHelpers.chooseFromRoomListNewMenu(ewPage1, "Start chat");
await TestHelpers.closeReleaseAnnouncement(
ewPage1,
"Introducing Sections",
);
await ewPage1
.getByRole("navigation", { name: "Room list" })
.getByRole("button", { name: "New", exact: true })
.click();
await ewPage1.getByRole("menuitem", { name: "Start chat" }).click();
await ewPage1.getByRole("textbox", { name: "Search" }).click();
await ewPage1.getByRole("textbox", { name: "Search" }).fill(whistlerMxId);
await ewPage1.getByRole("button", { name: "Go" }).click();
@@ -102,7 +157,12 @@ export const widgetTest = test.extend<MyFixtures>({
).toBeVisible();
// Accept the DM invite from brooks
await TestHelpers.openRoomInvite(ewPage2, brooksDisplayName);
await TestHelpers.closeReleaseAnnouncement(
ewPage2,
"Introducing Sections",
);
await TestHelpers.expandAllSections(ewPage2);
await TestHelpers.roomListItem(ewPage2, brooksDisplayName).click();
await ewPage2.getByRole("button", { name: "Start chatting" }).click();
}
-55
View File
@@ -1,55 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { expect, type Locator } from "@playwright/test";
export type Colour = [number, number, number];
/** The camera's colour, once the preview is showing it rather than nothing. */
export async function cameraColour(preview: Locator): Promise<Colour> {
await expect
.poll(async () => Math.max(...(await averageColour(preview))))
.toBeGreaterThan(40);
return averageColour(preview);
}
/** The mean colour of a video's current frame or an image, drawn small. */
export async function averageColour(element: Locator): Promise<Colour> {
return element.evaluate(
async (source: HTMLVideoElement | HTMLImageElement) => {
if (source instanceof HTMLImageElement) await source.decode();
const canvas = document.createElement("canvas");
canvas.width = canvas.height = 32;
const context = canvas.getContext("2d")!;
context.drawImage(source, 0, 0, 32, 32);
const { data } = context.getImageData(0, 0, 32, 32);
const sum = [0, 0, 0];
for (let i = 0; i < data.length; i += 4)
for (let c = 0; c < 3; c++) sum[c] += data[i + c];
const pixels = data.length / 4;
return sum.map((s) => s / pixels) as Colour;
},
);
}
export function distance(a: Colour, b: Colour): number {
return Math.hypot(a[0] - b[0], a[1] - b[1], a[2] - b[2]);
}
/** Waits until a preview shows a picture rather than the camera. */
export async function expectWearing(
preview: Locator,
picture: Colour,
camera: Colour,
): Promise<void> {
// Long enough for the first effect of a session to be built.
await expect
.poll(async () => distance(await averageColour(preview), picture), {
timeout: 60_000,
})
.toBeLessThan(distance(camera, picture) / 2);
}
@@ -1,43 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { expect, test } from "@playwright/test";
import { widgetTest } from "../fixtures/widget-user.ts";
import { TestHelpers } from "./test-helpers.ts";
import { averageColour, cameraColour, expectWearing } from "../utils/colour.ts";
// Background effects need WebGL2, and headless Firefox on a CI runner has none,
// so it rightly offers none of them.
test.skip(
({ browserName }) => browserName === "firefox",
"Background effects need WebGL2, which headless Firefox on CI does not have",
);
widgetTest(
"puts a picture on the preview inside Element Web",
async ({ asWidget }) => {
test.slow();
const { brooks } = asWidget;
await TestHelpers.startCallInCurrentRoom(brooks.page, false);
const frame = brooks.page
.locator('iframe[title="Element Call"]')
.contentFrame();
await expect(frame.getByTestId("lobby_joinCall")).toBeVisible();
const preview = frame.locator("video").first();
const camera = await cameraColour(preview);
await frame.getByRole("button", { name: "Camera", exact: true }).click();
const tile = frame
.getByRole("group", { name: "Background effects" })
.getByRole("menuitemradio", { name: "Background 1" });
const picture = await averageColour(tile.locator("img"));
await tile.click();
await expect(tile).toHaveAttribute("aria-checked", "true");
await expectWearing(preview, picture, camera);
},
);
+36 -72
View File
@@ -153,40 +153,29 @@ export class TestHelpers {
return { page, clientHandle, mxId };
}
// Dismisses the toasts a fresh device gets after sign-in.
//
// Toasts stack and only the top one can be dismissed, so look
// at what is on top each time.
// Dismisses any toasts that appear on startup, such as "Failed to load service worker" or "Back up your chats".
// Toast can be stacked, and only the top one can be dismiss, so just look at what is on top and
// dismiss (if part of expected toats)
public static async dismissStartupToasts(page: Page): Promise<void> {
const expectedToasts: {
title: string;
button: string;
confirm?: string;
}[] = [
const expectedToasts = [
{ title: "Failed to load service worker", button: "OK" },
{ title: "Verify this device", button: "Later" },
{ title: "Back up your chats", button: "Not now" },
{
title: "Turn on key storage",
button: "Dismiss",
confirm: "Yes, dismiss",
},
{ title: "Notifications", button: "Dismiss" },
{ title: "Turn on key storage", button: "Dismiss" },
{ title: "Element does not support this browser", button: "Dismiss" },
];
const toast = page.locator(".mx_Toast_toast");
const toastTitle = toast.locator(".mx_Toast_title h2");
// eslint-disable-next-line no-constant-condition
while (true) {
try {
await toast.waitFor({ state: "visible", timeout: 700 });
const title = await toastTitle.textContent();
if (!title) break;
const title = await toast.locator(".mx_Toast_title h2").textContent();
// Find the matching toast config
const toastConfig = expectedToasts.find((t) => title.includes(t.title));
const toastConfig = expectedToasts.find((t) =>
title?.includes(t.title),
);
if (toastConfig) {
// Bounded so that a renamed button leaves the toast in place and the
@@ -194,16 +183,6 @@ export class TestHelpers {
await toast
.getByRole("button", { name: toastConfig.button })
.click({ timeout: 5000 });
if (toastConfig.confirm) {
const dialog = page.getByRole("dialog");
await dialog
.getByRole("button", { name: toastConfig.confirm })
.click({ timeout: 5000 });
await expect(dialog).toBeHidden({ timeout: 5000 });
}
// The same element shows the next toast a moment after the click, so
// reading it straight away acts on the dismissed one a second time
await expect(toastTitle).not.toHaveText(title, { timeout: 5000 });
} else {
// Unknown toast. We don't want to act on unknown toasts
break;
@@ -215,6 +194,20 @@ export class TestHelpers {
}
}
public static async closeReleaseAnnouncement(
page: Page,
name: string,
): Promise<void> {
try {
await page
.getByRole("dialog", { name })
.getByRole("button", { name: "OK" })
.click({ timeout: 2000 });
} catch {
// Announcement not shown; nothing to do
}
}
public static async expandAllSections(page: Page): Promise<void> {
try {
await page
@@ -230,7 +223,16 @@ export class TestHelpers {
page: Page,
andInvite: string[] = [],
): Promise<void> {
await TestHelpers.chooseFromRoomListNewMenu(page, "New Room");
await TestHelpers.closeReleaseAnnouncement(page, "Introducing Sections");
await page
.getByRole("navigation", { name: "Room list" })
.getByRole("button", { name: "New", exact: true })
.click();
await page.getByRole("menuitem", { name: "New Room" }).click({
timeout: 5000,
});
await page.getByRole("textbox", { name: "Name" }).fill(name);
await page.getByRole("button", { name: "Create room" }).click();
await expect(page.getByText("You created this room.")).toBeVisible({
@@ -257,24 +259,6 @@ export class TestHelpers {
}
}
/**
* Opens the room list's "New" menu and picks the given item.
*
* Driven by keyboard, as a toast arriving late sits over the button and
* swallows clicks.
*/
public static async chooseFromRoomListNewMenu(
page: Page,
itemName: string,
): Promise<void> {
await TestHelpers.dismissStartupToasts(page);
await page
.getByRole("navigation", { name: "Room list" })
.getByRole("button", { name: "New", exact: true })
.press("Enter");
await page.getByRole("menuitem", { name: itemName }).press("Enter");
}
/**
* Accepts a room invite using the room name.
* Locatest the invite in the room list.
@@ -284,7 +268,9 @@ export class TestHelpers {
roomName: string,
page: Page,
): Promise<void> {
await TestHelpers.openRoomInvite(page, roomName);
await TestHelpers.closeReleaseAnnouncement(page, "Introducing Sections");
await TestHelpers.expandAllSections(page);
await TestHelpers.roomListItem(page, roomName).click({ timeout: 10000 });
await page.getByRole("button", { name: "Accept" }).click({
timeout: 5000,
});
@@ -390,28 +376,6 @@ export class TestHelpers {
return page.locator(`[aria-label^="Open room ${roomName}"]`);
}
/**
* Opens a pending invite from the room list by its room name.
*
* Element Web collapses the Invites section every time it appears, so it is
* expanded here once the invite has arrived.
*/
public static async openRoomInvite(
page: Page,
roomName: string,
): Promise<void> {
const invitesSection = page.getByRole("button", {
name: "Toggle Invites section",
});
await expect(invitesSection).toBeVisible({ timeout: 10000 });
// Toasts sit over the top of the room list, where this header is.
await TestHelpers.dismissStartupToasts(page);
if ((await invitesSection.getAttribute("aria-expanded")) !== "true") {
await invitesSection.click();
}
await TestHelpers.roomListItem(page, roomName).click({ timeout: 10000 });
}
/**
* Switches to a room in the room list by its name.
* @param page - The EW page
+17 -32
View File
@@ -8,7 +8,7 @@ overrides:
'@livekit/components-core>rxjs': ^7.8.1
'@livekit/track-processors>@mediapipe/tasks-vision': ^0.10.18
tar: ^7.5.11
minimatch: ^11.0.0
minimatch: ^10.2.3
glob: ^10.5.0
qs: ^6.15.2
js-yaml: ^4.2.0
@@ -160,9 +160,6 @@ importers:
eslint-plugin-storybook:
specifier: ^10.3.6
version: 10.6.2(eslint@8.57.1(supports-color@7.2.0))(supports-color@7.2.0)(typescript@5.9.3)
fake-indexeddb:
specifier: ^6.2.5
version: 6.2.5
fetch-mock:
specifier: 11.1.5
version: 11.1.5(node-fetch@2.7.0)
@@ -1145,9 +1142,6 @@ packages:
resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==}
engines: {node: '>=12'}
'@isaacs/nonrecursive-glob-match@1.0.3':
resolution: {integrity: sha512-SI7kyKHq4XXDRozw+r9GlRZkia1ZkFrpLW98r2IC2Bpr8YkFGE3fvjzmxCG0x78A++69HmDNrDvSMYBZmHGpKA==}
'@joshwooding/vite-plugin-react-docgen-typescript@0.7.0':
resolution: {integrity: sha512-qvsTEwEFefhdirGOPnu9Wp6ChfIwy2dBCRuETU3uE+4cC+PFoxMSiiEhxk4lOluA34eARHA0OxqsEUYDqRMgeQ==}
peerDependencies:
@@ -3533,8 +3527,8 @@ packages:
boolbase@1.0.0:
resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==}
brace-expansion@5.0.12:
resolution: {integrity: sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ==}
brace-expansion@5.0.9:
resolution: {integrity: sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==}
engines: {node: 20 || >=22}
braces@3.0.3:
@@ -4054,10 +4048,6 @@ packages:
resolution: {integrity: sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==}
engines: {node: '>=12.0.0'}
fake-indexeddb@6.2.5:
resolution: {integrity: sha512-CGnyrvbhPlWYMngksqrSSUT1BAVP49dZocrHuK0SvtR0D5TMs5wP0o3j7jexDJW01KSadjBp1M/71o/KR3nD1w==}
engines: {node: '>=18'}
fast-deep-equal@3.1.3:
resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==}
@@ -4771,9 +4761,9 @@ packages:
minimalistic-crypto-utils@1.0.1:
resolution: {integrity: sha512-JIYlbt6g8i5jKfJ3xz7rF0LXmv2TkDxBLUkiBeZ7bAx4GnnNMr8xFpGnOxn6GhTEHx3SjRrZEoU+j04prX1ktg==}
minimatch@11.0.4:
resolution: {integrity: sha512-/UJ1dEoVQTjPNDh6SGlUo+ueLMmoHTas4q+8yaAR/5t6nLAIiBDJtoYUh20IYleix7ehiQkgczb84PE3uzHkxg==}
engines: {node: 20 || >=22}
minimatch@10.2.6:
resolution: {integrity: sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==}
engines: {node: 18 || 20 || >=22}
minimist@1.2.8:
resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==}
@@ -6905,7 +6895,7 @@ snapshots:
ignore: 5.3.2
import-fresh: 3.3.1
js-yaml: 4.3.1
minimatch: 11.0.4
minimatch: 10.2.6
strip-json-comments: 3.1.1
transitivePeerDependencies:
- supports-color
@@ -6984,7 +6974,7 @@ snapshots:
dependencies:
'@humanwhocodes/object-schema': 2.0.3
debug: 4.4.3(supports-color@7.2.0)
minimatch: 11.0.4
minimatch: 10.2.6
transitivePeerDependencies:
- supports-color
@@ -7120,8 +7110,6 @@ snapshots:
wrap-ansi: 8.1.0
wrap-ansi-cjs: wrap-ansi@7.0.0
'@isaacs/nonrecursive-glob-match@1.0.3': {}
'@joshwooding/vite-plugin-react-docgen-typescript@0.7.0(typescript@5.9.3)(vite@8.2.4(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.103.1)(terser@5.46.1)(yaml@2.9.0))':
dependencies:
glob: 10.5.0
@@ -8781,7 +8769,7 @@ snapshots:
'@typescript-eslint/types': 8.69.0
'@typescript-eslint/visitor-keys': 8.69.0
debug: 4.4.3(supports-color@7.2.0)
minimatch: 11.0.4
minimatch: 10.2.6
semver: 7.8.5
tinyglobby: 0.2.17
ts-api-utils: 2.5.0(typescript@5.9.3)
@@ -8796,7 +8784,7 @@ snapshots:
'@typescript-eslint/types': 8.72.0
'@typescript-eslint/visitor-keys': 8.72.0
debug: 4.4.3(supports-color@7.2.0)
minimatch: 11.0.4
minimatch: 10.2.6
semver: 7.8.5
tinyglobby: 0.2.17
ts-api-utils: 2.5.0(typescript@5.9.3)
@@ -9121,7 +9109,7 @@ snapshots:
boolbase@1.0.0: {}
brace-expansion@5.0.12:
brace-expansion@5.0.9:
dependencies:
balanced-match: 4.0.4
@@ -9666,7 +9654,7 @@ snapshots:
json-stable-stringify-without-jsonify: 1.0.1
levn: 0.4.1
lodash.merge: 4.6.2
minimatch: 11.0.4
minimatch: 10.2.6
natural-compare: 1.4.0
optionator: 0.9.4
strip-ansi: 6.0.1
@@ -9724,8 +9712,6 @@ snapshots:
expect-type@1.4.0: {}
fake-indexeddb@6.2.5: {}
fast-deep-equal@3.1.3: {}
fast-glob@3.3.3:
@@ -9786,7 +9772,7 @@ snapshots:
filelist@1.0.6:
dependencies:
minimatch: 11.0.4
minimatch: 10.2.6
fill-range@7.1.1:
dependencies:
@@ -9886,7 +9872,7 @@ snapshots:
dependencies:
foreground-child: 3.3.1
jackspeak: 3.4.3
minimatch: 11.0.4
minimatch: 10.2.6
minipass: 7.1.3
package-json-from-dist: 1.0.1
path-scurry: 1.11.1
@@ -10011,7 +9997,7 @@ snapshots:
jiti: 2.7.0
jsonc-parser: 3.3.1
magic-string: 1.2.3
minimatch: 11.0.4
minimatch: 10.2.6
ora: 9.4.1
react: 19.2.8
react-i18next: 17.0.13(i18next@26.4.1(typescript@5.9.3))(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(typescript@5.9.3)
@@ -10454,10 +10440,9 @@ snapshots:
minimalistic-crypto-utils@1.0.1: {}
minimatch@11.0.4:
minimatch@10.2.6:
dependencies:
'@isaacs/nonrecursive-glob-match': 1.0.3
brace-expansion: 5.0.12
brace-expansion: 5.0.9
minimist@1.2.8: {}
+1 -1
View File
@@ -27,7 +27,7 @@ overrides:
# Security fixes:
# - https://github.com/advisories/GHSA-7r86-cg39-jmmj
# - https://github.com/advisories/GHSA-23c5-xmqv-rm74
"minimatch": "^11.0.0"
"minimatch": "^10.2.3"
# Security fix: https://github.com/element-hq/element-call/security/dependabot/109
"glob": "^10.5.0"
# Security fixes:
+92
View File
@@ -0,0 +1,92 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
/**
* Checks the component build's output for what a host must not be handed.
*
* A library build inlines every asset it meets as a data URL unless the import
* says otherwise, and does so silently: one import without `no-inline` and the
* module grows by the asset, encoded a third larger, that every host then
* downloads and parses before the first call. MediaPipe's runtime alone is
* 11 MB. A reference to a file by an absolute path is just as silent and
* breaks at runtime instead, since a host serves the module from wherever it
* likes. And the standalone app's client store worker, which carries a copy of
* the Matrix SDK and its crypto, has no business in a package whose host
* supplies the client: the bundler emits it on loading its import, used or
* not (see src/utils/initClient.ts). This fails the build on each of those.
*/
import { readdir, readFile, stat } from "node:fs/promises";
import { join } from "node:path";
const dist = "component/dist";
/** The longest data URL a chunk may carry: enough for an icon, not a sound. */
const inlineLimit = 16 * 1024;
/**
* The module itself, in bytes. Well above where it sits with its assets as
* files, so that only an asset finding its way back in trips it.
*/
const moduleBudget = 4 * 1024 * 1024;
/**
* The modules a host bundles: the entry points and their chunks, at the top of
* the package. What sits under `assets/` is fetched as it is, by URL.
*/
async function modules() {
const names = await readdir(dist);
return names.filter((name) => name.endsWith(".js")).map((n) => join(dist, n));
}
const problems = [];
for (const name of await readdir(join(dist, "assets")))
if (name.includes("IndexedDBWorker"))
problems.push(
`${dist}/assets/${name}: the standalone app's client store worker is ` +
`in the package. Something the component reaches imports the worker; ` +
`keep it on the standalone app's side (see src/utils/initClient.ts ` +
`and the \`loadClient\` prop of ClientProvider).`,
);
for (const file of await modules()) {
const code = await readFile(file, "utf8");
for (const [, type, payload] of code.matchAll(
/data:([a-z0-9/.+-]*);base64,([A-Za-z0-9+/=]*)/g,
))
if (payload.length > inlineLimit)
problems.push(
`${file}: a ${type} asset of ${Math.round(payload.length / 1024)} kB ` +
`is inlined as a data URL. Import it with \`?url&no-inline\` so ` +
`that it is emitted as a file.`,
);
for (const [reference] of code.matchAll(/["'`]\/assets\/[^"'`]+["'`]/g))
problems.push(
`${file}: ${reference} is an absolute path, which only resolves ` +
`against an origin that serves the package at its root. The build ` +
`is meant to render relative references; check \`base\` in ` +
`vite-component.config.ts.`,
);
}
const moduleSize = (await stat(join(dist, "element-call.js"))).size;
if (moduleSize > moduleBudget)
problems.push(
`${dist}/element-call.js is ${Math.round(moduleSize / 1024 / 1024)} MB, ` +
`over its ${moduleBudget / 1024 / 1024} MB budget. Something large ` +
`has been bundled into the module that a host would rather fetch when ` +
`it is needed.`,
);
if (problems.length > 0) {
console.error(`The component build is not fit to publish:\n`);
for (const problem of problems) console.error(` ${problem}\n`);
process.exit(1);
}
@@ -225,51 +225,6 @@ describe("createLivekitMediaTrack", () => {
);
});
// LiveKit's media stream track is the processed one while a processor runs,
// and that one names no device; the camera behind it does
it.each([
{ deviceId: "cam-1", switches: false },
{ deviceId: "cam-2", switches: true },
])(
"leaves a processed camera alone unless its device changes: $deviceId",
async ({ deviceId, switches }) => {
vi.stubGlobal("MediaStreamTrack", class {});
onTestFinished(() => {
vi.unstubAllGlobals();
});
const camera = Object.assign(Object.create(LocalVideoTrack.prototype), {
source: Track.Source.Camera,
_mediaStreamTrack: {
readyState: "live",
label: "",
getSettings: () => ({ deviceId: "cam-1" }),
},
// What LiveKit shows as the track, with no device of its own
processor: {
processedTrack: { label: "", getSettings: () => ({}) },
},
}) as LocalVideoTrack;
const switchActiveDevice = vi.fn().mockResolvedValue(true);
const { participant, publication, room } = fakes({
track: camera as unknown as FakeTrack,
});
const track = createLocalLivekitMediaTrack(
scope,
Object.assign(participant, {
isLocal: true,
}) as unknown as LocalParticipant,
publication as unknown as TrackPublication,
Object.assign(room, { switchActiveDevice }),
vi.fn(),
);
await track.setDevice(deviceId);
expect(switchActiveDevice.mock.calls).toEqual(
switches ? [["videoinput", deviceId, false]] : [],
);
},
);
// LiveKit stops a camera that is turned off; the processor is built when it
// is turned on again
it.each([
@@ -199,13 +199,11 @@ export function createLocalLivekitMediaTrack(
throw new MatrixRTCError(`A ${base.source} track has no device`);
// LiveKit decides whether to restart by the constraints it keeps, which
// a browser that does not echo the device id back (Firefox) leaves
// without one; a track already on the device is left alone here. Asked
// of the camera track itself: with a processor on, LiveKit's media
// stream track is the processed one, which names no device
// without one; a track already on the device is left alone here
const current = publication.track;
if (
current instanceof LocalTrack &&
(await current.getDeviceId(false)) === deviceId
current.mediaStreamTrack.getSettings().deviceId === deviceId
)
return;
await room.switchActiveDevice(
@@ -90,42 +90,3 @@ Please see LICENSE in the repository root for full details.
text-align: center;
overflow-wrap: anywhere;
}
.removable {
position: relative;
display: flex;
min-inline-size: 0;
}
.removable > .tile {
flex: 1;
}
/* Clear of the target for choosing the tile. */
.remove {
position: absolute;
inset-block-start: 0;
inset-inline-end: 0;
translate: 20% -20%;
display: none;
align-items: center;
justify-content: center;
inline-size: 28px;
block-size: 28px;
border-radius: 50%;
background: var(--cpd-color-bg-canvas-default);
color: var(--cpd-color-icon-critical-primary);
box-shadow: 0 1.2px 2.4px 0 rgb(0 0 0 / 15%);
cursor: pointer;
}
.remove:hover {
background: var(--cpd-color-bg-critical-subtle-hovered);
box-shadow: 0 0 0 var(--cpd-border-width-1)
var(--cpd-color-border-critical-subtle);
}
.removable:hover .remove,
.removable:focus-within .remove {
display: flex;
}
+9 -82
View File
@@ -5,16 +5,14 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type FC, type KeyboardEvent, type ReactElement } from "react";
import { type FC, type ReactElement } from "react";
import { useTranslation } from "react-i18next";
import { VisuallyHidden } from "@radix-ui/react-visually-hidden";
import { InlineSpinner, MenuItem, Tooltip } from "@vector-im/compound-web";
import { InlineSpinner, MenuItem } from "@vector-im/compound-web";
import {
BlockIcon,
BlurIcon,
CheckCircleSolidIcon,
CloseIcon,
PlusIcon,
} from "@vector-im/compound-design-tokens/assets/web/icons";
import styles from "./BackgroundEffectGrid.module.css";
@@ -29,8 +27,6 @@ export interface BackgroundEffectGridItem {
kind: "none" | "blur" | "image";
/** For kind "image". */
imageUrl?: string;
/** Whether it is the user's to remove. */
removable?: boolean;
}
interface Props {
@@ -42,12 +38,6 @@ interface Props {
settling?: boolean;
/** Id of the text saying what the user should know before choosing. */
describedBy?: string;
/** Offers to add a background of one's own. Omit to leave that tile out. */
onAdd?: () => void;
addLabel?: string;
/** Removes a removable one, never the one in force. Omit to offer none. */
onRemove?: (id: string) => void;
removeLabel?: string;
}
/**
@@ -61,10 +51,6 @@ export const BackgroundEffectGrid: FC<Props> = ({
onSelect,
settling = false,
describedBy,
onAdd,
addLabel,
onRemove,
removeLabel,
}) => {
const { t } = useTranslation();
const choose = (id: string): void => {
@@ -80,30 +66,7 @@ export const BackgroundEffectGrid: FC<Props> = ({
: t("background_effects.numbered", { n: ++pictures });
const checked = effect.id === selected;
const disabled = onSelect === undefined && effect.kind !== "none";
const remove =
effect.removable && !checked && onRemove !== undefined
? (): void => onRemove(effect.id)
: undefined;
const removeProps =
remove === undefined
? {}
: {
"aria-keyshortcuts": "Delete",
onKeyDown: (e: KeyboardEvent): void => {
if (e.key !== "Delete" && e.key !== "Backspace") return;
e.preventDefault();
remove();
},
};
const content = (
<TileContent
effect={effect}
label={label}
checked={checked}
settling={checked && settling}
/>
);
const tile = (
return (
<MenuItem
key={effect.id}
role="menuitemradio"
@@ -125,57 +88,21 @@ export const BackgroundEffectGrid: FC<Props> = ({
choose(effect.id);
}
}
{...removeProps}
>
{content}
<TileContent
effect={effect}
label={label}
checked={checked}
settling={checked && settling}
/>
</MenuItem>
);
if (remove === undefined) return tile;
// Beside the item, not in it: inside, the item takes the press first and
// the tile is chosen instead.
return (
<div key={effect.id} className={styles.removable}>
{tile}
<Tooltip label={removeLabel ?? ""}>
{/* Not a control of its own, so the menu keeps one keyboard model:
the keyboard reaches the same action through Delete. */}
<span aria-hidden className={styles.remove} onClick={remove}>
<CloseIcon width={20} height={20} />
</span>
</Tooltip>
</div>
);
});
// A command rather than a choice, so an item beside the radios.
const addContent = (
<>
<span aria-hidden className={styles.swatch}>
<PlusIcon width={24} height={24} />
</span>
<VisuallyHidden>{addLabel}</VisuallyHidden>
</>
);
const addTile =
onAdd === undefined ? null : (
<MenuItem
label={null}
hideChevron
className={styles.tile}
onSelect={(e) => {
e.preventDefault();
onAdd();
}}
>
{addContent}
</MenuItem>
);
return (
// A group of its own, as the section's can't carry the description.
<div role="group" aria-describedby={describedBy} className={styles.grid}>
{tiles}
{addTile}
</div>
);
};
+2 -7
View File
@@ -127,21 +127,16 @@ export const Default: Story = {
selectBackgroundEffect: fn(),
backgroundEffect: "blur",
backgroundEffects: [
{ id: "none", kind: "none", removable: false },
{ id: "blur", kind: "blur", removable: false },
{ id: "none", kind: "none" },
{ id: "blur", kind: "blur" },
...shippedBackgrounds.map((background) => ({
id: `image:${background.id}`,
kind: "image" as const,
imageUrl: background.imagePath,
removable: false,
})),
],
backgroundEffectNotice: undefined,
backgroundEffectSettling: false,
backgroundImageRefusal: undefined,
addBackgroundImage: fn(),
removeBackgroundEffect: fn(),
cameraTrack: null,
hangup: fn(),
buttonSize: "lg",
showFooter: true,
-37
View File
@@ -33,13 +33,6 @@ import { useBehavior } from "../useBehavior";
import { type LayoutSwitchViewModel } from "../state/LayoutSwitchViewModel";
import { LayoutSwitch } from "../room/LayoutSwitch";
import { type BackgroundEffectGridItem } from "./BackgroundEffectGrid";
import { SelfPreview } from "./SelfPreview";
import { type UnusableReason } from "../tracks/backgroundImages";
import { type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk";
export interface BackgroundImageRefusal {
reason: UnusableReason | "not-kept";
}
export interface AudioOutputSwitcher {
targetOutput: string;
@@ -67,10 +60,6 @@ export interface FooterActions {
toggleVideo: (() => void) | undefined;
/** Undefined where background effects can't be chosen. */
selectBackgroundEffect: ((id: string) => void) | undefined;
/** Undefined where no background can be added. */
addBackgroundImage: ((file: File) => void) | undefined;
/** Removes an added background by its option id; never the one in force. */
removeBackgroundEffect: ((id: string) => void) | undefined;
toggleScreenSharing: (() => void) | undefined;
/** Also controls if the settings button is visible */
openSettings: (() => void) | undefined;
@@ -92,10 +81,6 @@ export interface FooterState {
backgroundEffectNotice: "unavailable" | "slow" | undefined;
/** Whether the first effect chosen is still being prepared. */
backgroundEffectSettling: boolean;
/** Why the last file offered couldn't be kept, if it couldn't. */
backgroundImageRefusal: BackgroundImageRefusal | undefined;
/** The camera to preview, with the effect in force; null while it is off. */
cameraTrack: LocalVideoMediaTrack | null;
showFooter: boolean;
/* This is needed for WindowMode = "flat" */
@@ -183,24 +168,6 @@ export const CallFooter: FC<FooterProps> = ({
const selectBackgroundEffect = useBehavior(vm.selectBackgroundEffect$);
const backgroundEffectNotice = useBehavior(vm.backgroundEffectNotice$);
const backgroundEffectSettling = useBehavior(vm.backgroundEffectSettling$);
const addBackgroundImage = useBehavior(vm.addBackgroundImage$);
const removeBackgroundEffect = useBehavior(vm.removeBackgroundEffect$);
const cameraTrack = useBehavior(vm.cameraTrack$);
const refusal = useBehavior(vm.backgroundImageRefusal$);
const refusalMessage = useMemo(() => {
switch (refusal?.reason) {
case undefined:
return undefined;
case "not-an-image":
return { text: t("error.background_not_an_image") };
case "animated":
return { text: t("error.background_animated") };
case "undecodable":
return { text: t("error.background_undecodable") };
case "not-kept":
return { text: t("error.background_not_kept") };
}
}, [refusal, t]);
const backgroundEffects = useBehavior(vm.backgroundEffects$);
const buttonSize = useBehavior(vm.buttonSize$);
const showLogo = useBehavior(vm.showLogo$);
@@ -267,10 +234,6 @@ export const CallFooter: FC<FooterProps> = ({
selectedBackgroundEffect={backgroundEffect}
onSelectBackgroundEffect={selectBackgroundEffect}
backgroundEffectSettling={backgroundEffectSettling}
onAddBackgroundImage={addBackgroundImage}
onRemoveBackgroundEffect={removeBackgroundEffect}
selfPreview={<SelfPreview track={cameraTrack} />}
backgroundImageRefusal={refusalMessage}
backgroundEffectNotice={
backgroundEffectNotice === "unavailable"
? t("background_effects.unavailable")
+14 -195
View File
@@ -7,17 +7,9 @@ Please see LICENSE in the repository root for full details.
import { afterEach, describe, expect, it, vi } from "vitest";
import { BehaviorSubject } from "rxjs";
import { type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk";
import {
flushPromises,
testScope,
mockMuteStates,
mockMediaDevices,
mockVideoTrack,
} from "../utils/test";
import { type Behavior, constant } from "../state/Behavior";
import { type BackgroundEffectsState } from "../state/BackgroundEffects";
import { testScope, mockMuteStates, mockMediaDevices } from "../utils/test";
import { constant } from "../state/Behavior";
import type { CallViewModel } from "../state/CallViewModel/CallViewModel";
import type { Layout } from "../state/layout-types";
import type { SpotlightTileViewModel } from "../state/TileViewModel";
@@ -29,13 +21,6 @@ import {
import { HeaderStyle } from "../UrlParams";
import { backgroundEffect as backgroundEffectSetting } from "../settings/settings";
import { shippedBackgrounds } from "../tracks/backgroundEffectCatalog";
import type * as BackgroundImages from "../tracks/backgroundImages";
import {
type AddedBackground,
addedBackgrounds,
maxAddedBackgrounds,
UnusableImage,
} from "../tracks/backgroundImages";
const platformMock = vi.hoisted(() => vi.fn(() => "desktop"));
vi.mock("../Platform", () => ({
@@ -52,22 +37,6 @@ vi.mock("@livekit/track-processors", () => ({
supportsModernBackgroundProcessors: (): boolean => modernRouteMock(),
}));
const store = vi.hoisted(() => ({
add: vi.fn<(file: Blob) => Promise<string>>(),
remove: vi.fn<(id: string) => Promise<void>>(),
}));
vi.mock("../tracks/backgroundImages", async (original) => {
const { BehaviorSubject } = await import("rxjs");
return {
...(await original<typeof BackgroundImages>()),
addedBackgrounds: {
added$: new BehaviorSubject<AddedBackground[] | undefined>([]),
add: store.add,
remove: store.remove,
},
};
});
const outputSelectionMock = vi.hoisted(() => vi.fn(() => true));
vi.mock("livekit-client", () => ({
supportsAudioOutputSelection: (): boolean => outputSelectionMock(),
@@ -81,7 +50,6 @@ vi.mock("livekit-client", () => ({
function buildMinimalCallViewModel(layout: Layout): CallViewModel {
return {
layout$: constant(layout),
camera$: constant(undefined),
edgeToEdge$: constant(false),
showHeader$: constant(false),
hangup: (): void => {},
@@ -113,16 +81,6 @@ const pipLayout: Layout = {
spotlight: {} as SpotlightTileViewModel,
};
const noPipeline = constant<BackgroundEffectsState>({
supported: false,
processor: undefined,
});
/** The provider's verdict, which the SDK's check stands for here. */
function pipelineAsTheSdkSays(): Behavior<BackgroundEffectsState> {
return constant({ supported: sdkSupportMock(), processor: undefined });
}
const twoMicsAndOneCamMediaDevices = mockMediaDevices({
audioInput: {
available$: constant(
@@ -154,7 +112,7 @@ describe("createCallFooterViewModel", () => {
buildMinimalCallViewModel(gridLayout),
mockMuteStates(),
twoMicsAndOneCamMediaDevices,
noPipeline,
constant(false),
/* reactionIdentifier */ undefined,
{ showControls: true, header: HeaderStyle.Standard },
);
@@ -197,7 +155,7 @@ describe("createCallFooterViewModel", () => {
select: vi.fn(),
},
}),
noPipeline,
constant(false),
/* reactionIdentifier */ undefined,
{ showControls: true, header: HeaderStyle.Standard },
);
@@ -216,7 +174,7 @@ describe("createCallFooterViewModel", () => {
buildMinimalCallViewModel(layout),
mockMuteStates(),
twoMicsAndOneCamMediaDevices,
noPipeline,
constant(false),
/* reactionIdentifier */ undefined,
{ showControls: true, header: HeaderStyle.Standard },
);
@@ -239,7 +197,7 @@ describe("createCallFooterViewModel", () => {
buildMinimalCallViewModel(gridLayout),
mockMuteStates(),
twoMicsAndOneCamMediaDevices,
noPipeline,
constant(false),
/* reactionIdentifier */ undefined,
{ showControls: true, header: HeaderStyle.Standard },
);
@@ -285,8 +243,7 @@ describe("createCallFooterViewModel", () => {
testScope(),
mockMuteStates(),
twoMicsAndOneCamMediaDevices,
pipelineAsTheSdkSays(),
/* camera$ */ constant(null),
constant(false),
/* openSettings */ undefined,
/* hangup */ undefined,
/* showLogo */ false,
@@ -306,7 +263,6 @@ describe("createCallFooterViewModel", () => {
it("offers nothing where the browser itself cannot run them", () => {
sdkSupportMock.mockReturnValue(false);
expect(lobbyOn("desktop").selectBackgroundEffect$.value).toBeUndefined();
expect(lobbyOn("desktop").addBackgroundImage$.value).toBeUndefined();
});
it("puts no effect in force where the browser cannot run them", () => {
@@ -337,29 +293,17 @@ describe("createCallFooterViewModel", () => {
expect(lobbyOn("desktop").backgroundEffectNotice$.value).toBeUndefined();
});
it("offers every effect in order, the added ones removable", () => {
it("offers every effect in order", () => {
sdkSupportMock.mockReturnValue(true);
const added$ = addedBackgrounds.added$ as BehaviorSubject<
AddedBackground[] | undefined
>;
added$.next([{ id: "mine", url: "blob:mine" }]);
expect(lobbyOn("desktop").backgroundEffects$.value).toEqual([
{ id: "none", kind: "none", removable: false },
{ id: "blur", kind: "blur", removable: false },
{ id: "none", kind: "none" },
{ id: "blur", kind: "blur" },
...shippedBackgrounds.map((background) => ({
id: `image:${background.id}`,
kind: "image",
imageUrl: background.imagePath,
removable: false,
})),
{
id: "added:mine",
kind: "image",
imageUrl: "blob:mine",
removable: true,
},
]);
added$.next([]);
});
it("says they are unavailable where they cannot be chosen", () => {
@@ -371,146 +315,21 @@ describe("createCallFooterViewModel", () => {
it("passes on the wait for the first effect", () => {
platformMock.mockReturnValue("desktop");
const pipeline$ = new BehaviorSubject<BackgroundEffectsState>({
supported: true,
processor: undefined,
wait: "settling",
});
const settling$ = new BehaviorSubject(true);
const vm = createLobbyFooterViewModel(
testScope(),
mockMuteStates(),
twoMicsAndOneCamMediaDevices,
pipeline$,
/* camera$ */ constant(null),
settling$,
/* openSettings */ undefined,
/* hangup */ undefined,
/* showLogo */ false,
);
expect(vm.backgroundEffectSettling$.value).toBe(true);
pipeline$.next({ ...pipeline$.value, wait: undefined });
settling$.next(false);
expect(vm.backgroundEffectSettling$.value).toBe(false);
});
it("offers nothing once the pipeline fails to build, keeping the choice", () => {
platformMock.mockReturnValue("desktop");
backgroundEffectSetting.setValue("blur");
const pipeline$ = new BehaviorSubject<BackgroundEffectsState>({
supported: true,
processor: undefined,
});
const vm = createLobbyFooterViewModel(
testScope(),
mockMuteStates(),
twoMicsAndOneCamMediaDevices,
pipeline$,
/* camera$ */ constant(null),
/* openSettings */ undefined,
/* hangup */ undefined,
/* showLogo */ false,
);
expect(vm.backgroundEffect$.value).toBe("blur");
pipeline$.next({ supported: false, processor: undefined });
expect(vm.selectBackgroundEffect$.value).toBeUndefined();
expect(vm.addBackgroundImage$.value).toBeUndefined();
expect(vm.backgroundEffect$.value).toBe("none");
expect(backgroundEffectSetting.getValue()).toBe("blur");
});
it("adding is unavailable at the limit", () => {
sdkSupportMock.mockReturnValue(true);
const added$ = addedBackgrounds.added$ as BehaviorSubject<
AddedBackground[] | undefined
>;
const vm = lobbyOn("desktop");
expect(vm.addBackgroundImage$.value).toBeDefined();
added$.next(
Array.from({ length: maxAddedBackgrounds }, (_, i) => ({
id: `${i}`,
url: `blob:${i}`,
})),
);
expect(vm.addBackgroundImage$.value).toBeUndefined();
added$.next(added$.value!.slice(1));
expect(vm.addBackgroundImage$.value).toBeDefined();
added$.next([]);
});
it("says why each refused file was refused", async () => {
sdkSupportMock.mockReturnValue(true);
const vm = lobbyOn("desktop");
store.add.mockRejectedValue(new UnusableImage("animated"));
vm.addBackgroundImage$.value!(new File(["x"], "a.gif"));
await flushPromises();
const first = vm.backgroundImageRefusal$.value;
expect(first).toEqual({ reason: "animated" });
// The same again is a refusal of its own, so it is shown again.
vm.addBackgroundImage$.value!(new File(["x"], "a.gif"));
await flushPromises();
expect(vm.backgroundImageRefusal$.value).toEqual({ reason: "animated" });
expect(vm.backgroundImageRefusal$.value).not.toBe(first);
store.add.mockRejectedValue(new Error("quota"));
vm.addBackgroundImage$.value!(new File(["x"], "b.png"));
await flushPromises();
expect(vm.backgroundImageRefusal$.value).toEqual({ reason: "not-kept" });
store.add.mockResolvedValue("kept");
vm.addBackgroundImage$.value!(new File(["x"], "c.png"));
await flushPromises();
expect(vm.backgroundImageRefusal$.value).toBeUndefined();
});
it("never removes the background in force", () => {
sdkSupportMock.mockReturnValue(true);
store.remove.mockResolvedValue();
const vm = lobbyOn("desktop");
backgroundEffectSetting.setValue("added:worn");
vm.removeBackgroundEffect$.value!("added:worn");
expect(store.remove).not.toHaveBeenCalled();
vm.removeBackgroundEffect$.value!("added:spare");
expect(store.remove).toHaveBeenCalledWith("spare");
});
it("previews the lobby's camera while there is one", () => {
const track = mockVideoTrack({});
const camera$ = new BehaviorSubject<LocalVideoMediaTrack | null>(track);
const vm = createLobbyFooterViewModel(
testScope(),
mockMuteStates(),
twoMicsAndOneCamMediaDevices,
noPipeline,
camera$,
/* openSettings */ undefined,
/* hangup */ undefined,
/* showLogo */ false,
);
expect(vm.cameraTrack$.value).toBe(track);
// The lobby releases the camera when it is turned off
camera$.next(null);
expect(vm.cameraTrack$.value).toBeNull();
});
it("previews the call's camera only while it is not muted", () => {
const muted$ = new BehaviorSubject(false);
const track = mockVideoTrack({ muted$ });
const vm = createCallFooterViewModel(
testScope(),
{ ...buildMinimalCallViewModel(gridLayout), camera$: constant(track) },
mockMuteStates(),
twoMicsAndOneCamMediaDevices,
noPipeline,
/* reactionIdentifier */ undefined,
{ showControls: true, header: HeaderStyle.Standard },
);
expect(vm.cameraTrack$.value).toBe(track);
muted$.next(true);
expect(vm.cameraTrack$.value).toBeNull();
muted$.next(false);
expect(vm.cameraTrack$.value).toBe(track);
});
it("availability is the same before and during a call", () => {
for (const supported of [true, false]) {
sdkSupportMock.mockReturnValue(supported);
@@ -520,7 +339,7 @@ describe("createCallFooterViewModel", () => {
buildMinimalCallViewModel(gridLayout),
mockMuteStates(),
twoMicsAndOneCamMediaDevices,
pipelineAsTheSdkSays(),
constant(false),
/* reactionIdentifier */ undefined,
{ showControls: true, header: HeaderStyle.Standard },
);
+24 -117
View File
@@ -6,33 +6,25 @@ Please see LICENSE in the repository root for full details.
*/
import {
BehaviorSubject,
combineLatest,
distinctUntilChanged,
map,
type Observable,
of,
switchMap,
} from "rxjs";
import { supportsAudioOutputSelection } from "livekit-client";
import { type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk";
import { supportsModernBackgroundProcessors } from "@livekit/track-processors";
import { logger } from "matrix-js-sdk/lib/logger";
import {
supportsBackgroundProcessors,
supportsModernBackgroundProcessors,
} from "@livekit/track-processors";
import {
parseEffect,
serializeEffect,
shippedBackgrounds,
} from "../tracks/backgroundEffectCatalog";
import {
type AddedBackground,
addedBackgrounds,
maxAddedBackgrounds,
UnusableImage,
} from "../tracks/backgroundImages";
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
import { type BackgroundEffectsState } from "../state/BackgroundEffects";
import { type MenuOptions } from "./MediaMuteAndSwitchButton";
import { type MediaDevices } from "../state/MediaDevices";
import {
@@ -45,7 +37,7 @@ import { type MuteStates } from "../state/MuteStates";
import { createStaticViewModel, type ViewModel } from "../state/ViewModel";
import { HeaderStyle } from "../UrlParams";
import { platform } from "../Platform";
import { type BackgroundImageRefusal, type FooterSnapshot } from "./CallFooter";
import { type FooterSnapshot } from "./CallFooter";
import { type BackgroundEffectGridItem } from "./BackgroundEffectGrid";
/**
@@ -87,9 +79,7 @@ function buildDeviceBehaviors(
mediaDevices: MediaDevices,
/** return empty arrays for audioOptions and videoOptions*/
disableSwitcher$: Behavior<boolean>,
backgroundEffectsState$: Observable<BackgroundEffectsState>,
/** The camera as the pipeline has it, for the menu's preview. */
camera$: Observable<LocalVideoMediaTrack | null | undefined>,
backgroundEffectSettling$: Observable<boolean>,
): Pick<
ViewModel<FooterSnapshot>,
| "audioOptions$"
@@ -106,10 +96,6 @@ function buildDeviceBehaviors(
| "backgroundEffects$"
| "backgroundEffectNotice$"
| "backgroundEffectSettling$"
| "addBackgroundImage$"
| "backgroundImageRefusal$"
| "removeBackgroundEffect$"
| "cameraTrack$"
> {
const options$ = (
available$: Behavior<Map<string, MenuOptions["label"]>>,
@@ -126,18 +112,10 @@ function buildDeviceBehaviors(
),
);
// Also false once the pipeline has failed to build this session.
const supported$ = backgroundEffectsState$.pipe(
map(({ supported }) => supported === true),
distinctUntilChanged(),
);
const supported = supportsBackgroundProcessors();
const slow = !supportsModernBackgroundProcessors();
const offered$ = combineLatest([disableSwitcher$, supported$]).pipe(
map(([switcherDisabled, supported]) => !switcherDisabled && supported),
);
// A new object for each refusal, so the same one twice is shown twice.
const refusal$ = new BehaviorSubject<BackgroundImageRefusal | undefined>(
undefined,
const offered$ = disableSwitcher$.pipe(
map((switcherDisabled) => !switcherDisabled && supported),
);
return {
audioOptions$: scope.behavior(options$(mediaDevices.audioInput.available$)),
@@ -164,10 +142,8 @@ function buildDeviceBehaviors(
),
selectVideoButtonOption$: constant(mediaDevices.videoInput.select),
backgroundEffect$: scope.behavior(
combineLatest([backgroundEffectSetting.value$, supported$]).pipe(
map(([raw, supported]) =>
supported ? serializeEffect(parseEffect(raw)) : "none",
),
backgroundEffectSetting.value$.pipe(
map((raw) => (supported ? serializeEffect(parseEffect(raw)) : "none")),
),
),
selectBackgroundEffect$: scope.behavior(
@@ -182,11 +158,7 @@ function buildDeviceBehaviors(
),
),
),
backgroundEffects$: scope.behavior(
addedBackgrounds.added$.pipe(
map((added) => backgroundEffectChoices(added ?? [])),
),
),
backgroundEffects$: constant(backgroundEffectChoices()),
backgroundEffectNotice$: scope.behavior(
offered$.pipe(
map((offered) =>
@@ -195,61 +167,9 @@ function buildDeviceBehaviors(
),
),
backgroundEffectSettling$: scope.behavior(
backgroundEffectsState$.pipe(
map(({ wait }) => wait !== undefined),
distinctUntilChanged(),
),
backgroundEffectSettling$.pipe(distinctUntilChanged()),
false,
),
// Kept and offered, not put on: that waits for the user to choose it.
addBackgroundImage$: scope.behavior(
combineLatest([offered$, addedBackgrounds.added$]).pipe(
map(([offered, added]) =>
offered && (added?.length ?? 0) < maxAddedBackgrounds
? (file: File): void => {
refusal$.next(undefined);
addedBackgrounds.add(file).catch((e) => {
logger.warn("Could not keep that background", e);
refusal$.next({
reason: e instanceof UnusableImage ? e.reason : "not-kept",
});
});
}
: undefined,
),
),
),
backgroundImageRefusal$: refusal$,
// None while muted: the call mutes its track rather than releasing it.
cameraTrack$: scope.behavior(
camera$.pipe(
switchMap((camera) =>
camera
? camera.muted$.pipe(map((muted) => (muted ? null : camera)))
: of(null),
),
),
null,
),
// Never the one in force: taking it away would leave nothing chosen.
removeBackgroundEffect$: scope.behavior(
offered$.pipe(
map((offered) =>
offered
? (optionId: string): void => {
const effect = parseEffect(optionId);
if (effect.kind !== "added") return;
if (backgroundEffectSetting.getValue() === optionId) return;
addedBackgrounds
.remove(effect.id)
.catch((e) =>
logger.warn("Could not remove that background", e),
);
}
: undefined,
),
),
),
};
}
@@ -260,8 +180,8 @@ function buildDeviceBehaviors(
* @param callModel - The root CallViewModel; provides layout, grid mode, reactions, etc.
* @param muteStates - Audio and video mute state + toggles.
* @param mediaDevices - Available and selected input devices.
* @param backgroundEffectsState$ - The background pipeline's state: whether
* effects run here, and the wait for the first.
* @param backgroundEffectSettling$ - Whether the first effect chosen is still
* being prepared.
* @param reactionIdentifier - The local user's reaction identifier string, or
* undefined when reactions are not supported (hides the reaction button).
* @param options - `showControls`: whether the call controls should be shown.
@@ -272,7 +192,7 @@ export function createCallFooterViewModel(
callModel: CallViewModel,
muteStates: MuteStates,
mediaDevices: MediaDevices,
backgroundEffectsState$: Observable<BackgroundEffectsState>,
backgroundEffectSettling$: Observable<boolean>,
reactionIdentifier: string | undefined,
options: { showControls: boolean; header: HeaderStyle },
): ViewModel<FooterSnapshot> {
@@ -291,8 +211,7 @@ export function createCallFooterViewModel(
scope,
mediaDevices,
disableDeviceSwitcher$,
backgroundEffectsState$,
callModel.camera$,
backgroundEffectSettling$,
),
// candidat to move into the FooterViewModel
showFooter$: callModel.showFooter$,
@@ -355,9 +274,8 @@ export function createCallFooterViewModel(
* @param scope - ObservableScope that bounds the lifetime of derived behaviors.
* @param muteStates - Audio and video mute state + toggles.
* @param mediaDevices - Available and selected input devices.
* @param backgroundEffectsState$ - The background pipeline's state: whether
* effects run here, and the wait for the first.
* @param camera$ - The lobby's camera, for the camera menu's preview.
* @param backgroundEffectSettling$ - Whether the first effect chosen is still
* being prepared.
* @param openSettings - Callback to open the settings modal, or undefined.
* @param hangup - Callback to leave/cancel, or undefined (hides the button).
* @param showLogo - Whether to show the Element Call logo.
@@ -366,8 +284,7 @@ export function createLobbyFooterViewModel(
scope: ObservableScope,
muteStates: MuteStates,
mediaDevices: MediaDevices,
backgroundEffectsState$: Observable<BackgroundEffectsState>,
camera$: Behavior<LocalVideoMediaTrack | null>,
backgroundEffectSettling$: Observable<boolean>,
openSettings: (() => void) | undefined,
hangup: (() => void) | undefined,
showLogo: boolean,
@@ -415,29 +332,19 @@ export function createLobbyFooterViewModel(
scope,
mediaDevices,
constant(false),
backgroundEffectsState$,
camera$,
backgroundEffectSettling$,
),
};
}
function backgroundEffectChoices(
added: AddedBackground[],
): BackgroundEffectGridItem[] {
function backgroundEffectChoices(): BackgroundEffectGridItem[] {
return [
{ id: serializeEffect({ kind: "none" }), kind: "none", removable: false },
{ id: serializeEffect({ kind: "blur" }), kind: "blur", removable: false },
{ id: serializeEffect({ kind: "none" }), kind: "none" },
{ id: serializeEffect({ kind: "blur" }), kind: "blur" },
...shippedBackgrounds.map((background) => ({
id: serializeEffect({ kind: "shipped", id: background.id }),
kind: "image" as const,
imageUrl: background.imagePath,
removable: false,
})),
...added.map((background) => ({
id: serializeEffect({ kind: "added", id: background.id }),
kind: "image" as const,
imageUrl: background.url,
removable: true,
})),
];
}
@@ -93,39 +93,3 @@ Please see LICENSE in the repository root for full details.
flex-shrink: 0;
color: var(--cpd-color-icon-secondary);
}
.refusal {
margin: var(--cpd-space-2x) var(--cpd-space-4x) 0;
}
.selfPreview {
position: relative;
display: flex;
align-items: center;
justify-content: center;
flex-shrink: 0;
aspect-ratio: 16 / 9;
margin-block-end: var(--cpd-space-2x);
overflow: clip;
border-start-start-radius: var(--cpd-space-3x);
border-start-end-radius: var(--cpd-space-3x);
background: var(--cpd-color-bg-subtle-secondary);
color: var(--cpd-color-icon-secondary);
}
.selfPreview > video {
position: absolute;
inset: 0;
inline-size: 100%;
block-size: 100%;
object-fit: cover;
/* Rounded itself too: a video can be drawn on a layer of its own, which the
frame's rounded clip doesn't always reach. */
border-radius: inherit;
}
/* Up to the menu's frame, over the padding at its top. */
.selfPreviewPinned,
.deviceListLeadsWithPreview {
margin-block-start: calc(-1 * var(--cpd-space-5x));
}
@@ -12,7 +12,6 @@ import type { Meta, StoryObj } from "@storybook/react-vite";
import { MediaMuteAndSwitchButton } from "./MediaMuteAndSwitchButton";
import styles from "./MediaMuteAndSwitchButton.module.css";
import { type BackgroundEffectGridItem } from "./BackgroundEffectGrid";
import { SelfPreview } from "./SelfPreview";
import { shippedBackgrounds } from "../tracks/backgroundEffectCatalog";
import meterStyles from "./MicrophoneLevelMeter.module.css";
import { MediaDevicesContext } from "../MediaDevicesContext";
@@ -61,50 +60,6 @@ const WithAMicrophone: FC<{ children: ReactNode }> = ({ children }) => {
return <>{children}</>;
};
/** A camera for the preview: a figure against a room, drawn to a stream. */
const FakeCamera: FC = () => {
const [stream, setStream] = useState<MediaStream>();
useEffect(() => {
const canvas = document.createElement("canvas");
canvas.width = 640;
canvas.height = 360;
const context = canvas.getContext("2d")!;
const draw = (): void => {
context.fillStyle = "#6b7f8e";
context.fillRect(0, 0, 640, 360);
context.fillStyle = "#c9a27e";
context.beginPath();
context.arc(320, 150, 62, 0, 2 * Math.PI);
context.fill();
context.fillStyle = "#34495e";
context.beginPath();
context.ellipse(320, 380, 150, 140, 0, Math.PI, 2 * Math.PI);
context.fill();
};
draw();
// Redrawn, so the stream keeps delivering frames.
const timer = setInterval(draw, 200);
const captured = canvas.captureStream(10);
setStream(captured);
return (): void => {
clearInterval(timer);
captured.getTracks().forEach((track) => track.stop());
};
}, []);
return (
<video
data-testid="preview-camera"
autoPlay
muted
playsInline
ref={(video): void => {
if (video && stream && video.srcObject !== stream)
video.srcObject = stream;
}}
/>
);
};
/** Supplies a call-sized root. Without one the list is bounded by the whole Storybook frame. */
const WithACallArea: FC<{ children: ReactNode; blockSize?: number }> = ({
children,
@@ -216,8 +171,8 @@ export const VideoMute: Story = {
{ label: { type: "name", name: "Camera 1" }, id: "1" },
{ label: { type: "name", name: "Camera 2" }, id: "2" },
],
selectedOption: "1",
selfPreview: <SelfPreview track={null} />,
},
};
@@ -243,9 +198,6 @@ export const VideoUnmute: Story = {
backgroundEffects: effects,
selectedBackgroundEffect: "none",
onSelectBackgroundEffect: fn(),
onAddBackgroundImage: fn(),
// As the footer always gives the camera menu one.
selfPreview: <FakeCamera />,
},
};
@@ -289,11 +241,6 @@ export const BackgroundEffects: Story = {
true,
);
const add = within(section).getByRole("menuitem", { name: "Add image" });
await expect(
tiles[3].compareDocumentPosition(add) & Node.DOCUMENT_POSITION_FOLLOWING,
).toBeTruthy();
await userEvent.click(menu.getByRole("menuitemradio", { name: "Blur" }));
await expect(args.onSelectBackgroundEffect).toHaveBeenCalledWith("blur");
await expect(menu.getByRole("menu")).toBeVisible();
@@ -391,270 +338,6 @@ export const BackgroundEffectsSettling: Story = {
checked: false,
}))
await expect(other).toHaveAttribute("aria-busy", "false");
await expect(menu.queryByTestId("preview-camera")).toBeNull();
const frames = document.body.querySelectorAll(`.${styles.selfPreview} svg`);
await expect(frames).toHaveLength(1);
},
};
/** A file that can't be used: said where it was chosen, and nothing changes. */
export const BackgroundImageRefused: Story = {
args: {
...VideoUnmute.args,
selectedBackgroundEffect: "blur",
backgroundEffectNotice:
"Background effects run slowly on this platform, which may cause your video to stutter.",
backgroundImageRefusal: { text: "That file is not a supported image" },
},
decorators: [
(Story): JSX.Element => (
<WithACallArea blockSize={300}>
<Story />
</WithACallArea>
),
],
play: async ({ canvasElement }) => {
const canvas = within(canvasElement);
await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
const menu = within(document.body);
const message = await menu.findByText("That file is not a supported image");
const list = document.body.querySelector<HTMLElement>(
`.${styles.deviceList}`,
)!;
await expect(list.scrollHeight).toBeGreaterThan(list.clientHeight);
const scrollport = list.getBoundingClientRect();
const shown = message.getBoundingClientRect();
await expect(shown.bottom).toBeLessThanOrEqual(scrollport.bottom + 1);
await expect(shown.top).toBeGreaterThanOrEqual(scrollport.top - 1);
await expect(menu.queryByText(/run slowly/)).toBeNull();
await expect(
menu.getByRole("menuitemradio", { name: "Blur" }),
).toHaveAttribute("aria-checked", "true");
},
};
/** A flat picture, for backgrounds of one's own. */
function swatch(colour: string): string {
return `data:image/svg+xml,${encodeURIComponent(
`<svg xmlns="http://www.w3.org/2000/svg" width="16" height="9"><rect width="16" height="9" fill="${colour}"/></svg>`,
)}`;
}
/** One's own backgrounds are removable, never the one in force. */
export const AddedBackgroundsCanBeRemoved: Story = {
args: {
...VideoUnmute.args,
backgroundEffects: [
...effects,
{
id: "added:one",
kind: "image",
imageUrl: swatch("#c2410c"),
removable: true,
},
{
id: "added:two",
kind: "image",
imageUrl: swatch("#1d4ed8"),
removable: true,
},
],
selectedBackgroundEffect: "added:one",
onRemoveBackgroundEffect: fn(),
},
play: async ({ args, canvasElement }) => {
const canvas = within(canvasElement);
await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
const body = within(document.body);
const inForce = await body.findByRole("menuitemradio", {
name: "Background 3",
});
await expect(inForce).not.toHaveAttribute("aria-keyshortcuts");
const other = body.getByRole("menuitemradio", { name: "Background 4" });
await expect(other).toHaveAttribute("aria-keyshortcuts", "Delete");
other.focus();
await userEvent.keyboard("{Delete}");
await expect(args.onRemoveBackgroundEffect).toHaveBeenCalledWith(
"added:two",
);
const wrapper = other.parentElement!;
await userEvent.hover(wrapper);
const cross = wrapper.querySelector<HTMLElement>(
"[aria-hidden]:last-child",
)!;
const probe = document.createElement("span");
probe.style.color = "var(--cpd-color-icon-critical-primary)";
cross.append(probe);
const critical = getComputedStyle(probe).color;
probe.remove();
await expect(getComputedStyle(cross).color).toBe(critical);
const tile = other.getBoundingClientRect();
const box = cross.getBoundingClientRect();
await expect(box.right).toBeGreaterThan(tile.right);
await expect(box.top).toBeLessThan(tile.top);
await userEvent.hover(cross);
await waitFor(async () =>
expect(
[...document.body.querySelectorAll("div")].some(
(d) => d.textContent === "Remove" && d.offsetParent !== null,
),
).toBe(true),
);
},
};
/** Removing by pointer while the selection really changes. */
export const RemovingWithLiveSelection: Story = {
args: AddedBackgroundsCanBeRemoved.args,
render: function WithLiveSelection(args): JSX.Element {
const [selected, setSelected] = useState("none");
const [offered, setOffered] = useState(args.backgroundEffects ?? []);
return (
<MediaMuteAndSwitchButton
{...args}
backgroundEffects={offered}
selectedBackgroundEffect={selected}
onSelectBackgroundEffect={setSelected}
onRemoveBackgroundEffect={(id): void =>
setOffered((current) => current.filter((o) => o.id !== id))
}
/>
);
},
play: async ({ canvasElement }) => {
const canvas = within(canvasElement);
await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
const body = within(document.body);
const tile = await body.findByRole("menuitemradio", {
name: "Background 4",
});
await userEvent.hover(tile);
const cross = tile.parentElement!.querySelector<HTMLElement>(
"[aria-hidden]:last-child",
)!;
await expect(cross).toBeVisible();
await userEvent.click(cross);
await waitFor(async () =>
expect(
body.queryByRole("menuitemradio", { name: "Background 4" }),
).toBeNull(),
);
await expect(
body.getByRole("menuitemradio", { name: "None" }),
).toHaveAttribute("aria-checked", "true");
},
};
/** Where the call leaves room, the camera holds the top while the options scroll. */
export const BackgroundEffectsWithPreview: Story = {
args: {
...VideoUnmute.args,
// Enough that the list fills its share, so the preview's cost counts.
options: Array.from({ length: 8 }, (_, i) => ({
label: { type: "name" as const, name: `Camera ${i + 1}` },
id: `${i + 1}`,
})),
},
decorators: [
(Story): JSX.Element => (
<WithACallArea blockSize={640}>
<Story />
</WithACallArea>
),
],
play: async ({ canvasElement }) => {
const canvas = within(canvasElement);
await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
const picture = await within(document.body).findByTestId("preview-camera");
const preview = picture.parentElement!;
const list = document.body.querySelector<HTMLElement>(
`.${styles.deviceList}`,
)!;
const frame = document.body
.querySelector("[role='menu']")!
.getBoundingClientRect();
const box = preview.getBoundingClientRect();
await expect(list.contains(preview)).toBe(false);
await expect(box.top - frame.top).toBeLessThanOrEqual(2);
await expect(frame.width - box.width).toBeLessThanOrEqual(2);
await expect(box.width / box.height).toBeCloseTo(16 / 9, 1);
await expect(box.bottom).toBeLessThanOrEqual(
list.getBoundingClientRect().top,
);
const listShare = Math.round(640 * 0.6);
// The list runs on through the menu's padding at its foot, which isn't
// part of its share.
const listEnd =
list.getBoundingClientRect().bottom -
Number.parseFloat(getComputedStyle(list).paddingBlockEnd);
await expect(listEnd - box.top).toBeLessThanOrEqual(listShare + 1);
await expect(list.scrollHeight).toBeGreaterThan(list.clientHeight);
list.scrollTop = list.scrollHeight;
await expect(preview.getBoundingClientRect().top).toBe(box.top);
},
};
/** With the camera off the preview says so, and effects can still be chosen. */
export const BackgroundEffectsPreviewWithTheCameraOff: Story = {
args: {
...VideoUnmute.args,
enabled: false,
selfPreview: <SelfPreview track={null} />,
},
play: async ({ args, canvasElement }) => {
const canvas = within(canvasElement);
await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
const menu = within(document.body);
await menu.findByRole("group", { name: "Background effects" });
const preview = document.body.querySelector(`.${styles.selfPreview}`)!;
await expect(preview.querySelector("video")).toBeNull();
await expect(preview.querySelector("svg")).not.toBeNull();
await userEvent.click(menu.getByRole("menuitemradio", { name: "Blur" }));
await expect(args.onSelectBackgroundEffect).toHaveBeenCalledWith("blur");
},
};
/** In a call too short to hold it, the camera leads the options and scrolls away. */
export const BackgroundEffectsPreviewScrollsWithTheList: Story = {
args: VideoUnmute.args,
decorators: [
(Story): JSX.Element => (
<WithACallArea blockSize={300}>
<Story />
</WithACallArea>
),
],
play: async ({ canvasElement }) => {
const canvas = within(canvasElement);
await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
const picture = await within(document.body).findByTestId("preview-camera");
const preview = picture.parentElement!;
const list = document.body.querySelector<HTMLElement>(
`.${styles.deviceList}`,
)!;
await expect(list.contains(preview)).toBe(true);
const cameras = within(document.body).getByRole("group", {
name: "Camera",
});
await expect(
preview.compareDocumentPosition(cameras) &
Node.DOCUMENT_POSITION_FOLLOWING,
).toBeTruthy();
list.scrollTop = list.scrollHeight;
await expect(preview.getBoundingClientRect().bottom).toBeLessThan(
list.getBoundingClientRect().top,
);
},
};
@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { afterEach, describe, expect, onTestFinished, test, vi } from "vitest";
import { afterEach, describe, expect, test, vi } from "vitest";
import { axe } from "vitest-axe";
import {
act,
@@ -367,120 +367,6 @@ describe("MediaMuteAndSwitchButton", () => {
expect(onSelectBackgroundEffect).not.toHaveBeenCalled();
});
test("added images are chosen like shipped ones", async () => {
const user = userEvent.setup();
const onSelectBackgroundEffect = vi.fn();
const { getByRole } = renderComponent(
<MediaMuteAndSwitchButton
iconsAndLabels="video"
enabled={true}
options={[{ label: { type: "name", name: "Camera 1" }, id: "cam1" }]}
selectedOption="cam1"
backgroundEffects={[
...effects,
{
id: "added:mine",
kind: "image",
imageUrl: "",
},
]}
selectedBackgroundEffect="none"
onSelectBackgroundEffect={onSelectBackgroundEffect}
/>,
);
await user.click(getByRole("button", { name: "Camera" }));
const section = screen.getByRole("group", { name: "Background effects" });
await user.click(
within(section).getByRole("menuitemradio", { name: "Background 3" }),
);
expect(onSelectBackgroundEffect).toHaveBeenCalledWith("added:mine");
});
test("adds the file chosen from the add tile, and stays open", async () => {
const user = userEvent.setup();
const onAddBackgroundImage = vi.fn();
const pick = vi
.spyOn(HTMLInputElement.prototype, "click")
.mockImplementation(() => {});
try {
const { getByRole, container } = renderComponent(
<MediaMuteAndSwitchButton
iconsAndLabels="video"
enabled={true}
options={[{ label: { type: "name", name: "Camera 1" }, id: "cam1" }]}
selectedOption="cam1"
backgroundEffects={effects}
selectedBackgroundEffect="none"
onSelectBackgroundEffect={vi.fn()}
onAddBackgroundImage={onAddBackgroundImage}
/>,
);
await user.click(getByRole("button", { name: "Camera" }));
await user.click(screen.getByRole("menuitem", { name: "Add image" }));
expect(pick).toHaveBeenCalled();
// The picker takes the focus, which would otherwise close the menu.
await user.keyboard("{Escape}");
expect(screen.getByRole("menu")).toBeInTheDocument();
const file = new File(["x"], "mine.png", { type: "image/png" });
// What the picker hands back; the open menu blocks pointer events.
fireEvent.change(
container.querySelector<HTMLInputElement>("input[type=file]")!,
{ target: { files: [file] } },
);
expect(onAddBackgroundImage).toHaveBeenCalledWith(file);
screen.getByRole("menuitemradio", { name: "None", checked: true });
} finally {
pick.mockRestore();
}
});
test("a refusal can be dismissed and does not outlive the menu", async () => {
const user = userEvent.setup();
// jsdom lays nothing out, so has nothing to scroll.
const scroll = Element.prototype.scrollIntoView;
Element.prototype.scrollIntoView = vi.fn();
onTestFinished(() => {
Element.prototype.scrollIntoView = scroll;
});
const menu = (refusal?: { text: string }): JSX.Element =>
withProviders(
<MediaMuteAndSwitchButton
iconsAndLabels="video"
enabled={true}
options={[{ label: { type: "name", name: "Camera 1" }, id: "cam1" }]}
selectedOption="cam1"
backgroundEffects={effects}
selectedBackgroundEffect="blur"
onSelectBackgroundEffect={vi.fn()}
onAddBackgroundImage={vi.fn()}
backgroundImageRefusal={refusal}
/>,
);
const { rerender, getByRole } = render(menu());
await user.click(getByRole("button", { name: "Camera" }));
rerender(menu({ text: "That file is not a supported image" }));
expect(
screen.getByText("That file is not a supported image"),
).toBeVisible();
screen.getByRole("menuitemradio", { name: "Blur", checked: true });
await user.click(screen.getByRole("button", { name: "Close" }));
expect(screen.queryByText("That file is not a supported image")).toBeNull();
// The next refusal is shown, even if it says the same.
rerender(menu({ text: "That file is not a supported image" }));
expect(
screen.getByText("That file is not a supported image"),
).toBeVisible();
await user.keyboard("{Escape}");
await user.click(getByRole("button", { name: "Camera" }));
expect(screen.queryByText("That file is not a supported image")).toBeNull();
});
test("offers the background effects in a phone's drawer", async () => {
const userAgent = vi
.spyOn(navigator, "userAgent", "get")
+14 -144
View File
@@ -8,18 +8,14 @@ Please see LICENSE in the repository root for full details.
import {
useCallback,
useId,
useRef,
useState,
type CSSProperties,
type FC,
useEffect,
type ReactElement,
type ReactNode,
} from "react";
import {
Alert,
Button,
InlineSpinner,
Menu,
MenuItem,
MenuScrollArea,
@@ -83,14 +79,6 @@ export interface MediaMuteAndSwitchButtonProps {
backgroundEffectNotice?: string;
/** Whether the first effect chosen is still being prepared. */
backgroundEffectSettling?: boolean;
/** Called with the file chosen from the add tile. Omit to leave it out. */
onAddBackgroundImage?: (file: File) => void;
/** Removes a removable effect, never the one in force. */
onRemoveBackgroundEffect?: (id: string) => void;
/** Why the last file offered couldn't be used; a new object each time. */
backgroundImageRefusal?: { text: string };
/** The user's own camera, shown at the top of the camera menu. */
selfPreview?: ReactNode;
/**
* For any toggle and option this method will be called.
* So toggles need to be implemented by listening here and setting the right toggle item to `enabled`
@@ -107,16 +95,9 @@ const LIST_SHARE_OF_CALL = 0.6;
/** Smallest device list height in px, so a short call still shows more than one device. */
const MIN_LIST_HEIGHT = 160;
/** The menu's width as its stylesheet sets it, to place the preview before
the menu is drawn. */
const MENU_WIDTH = 296;
/** Space kept between the menu and the call area's sides. */
const MENU_MARGIN = 16;
/** Below the preview, as its stylesheet draws it. */
const PREVIEW_GAP = 12;
export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
enabled,
busy,
@@ -132,10 +113,6 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
onSelectBackgroundEffect,
backgroundEffectNotice,
backgroundEffectSettling,
onAddBackgroundImage,
onRemoveBackgroundEffect,
backgroundImageRefusal,
selfPreview,
onSelect,
}) => {
// Requested but not yet selected. Keyed by kind too, since Chrome uses
@@ -145,42 +122,11 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
id: string;
} | null>(null);
const [menuOpen, setMenuOpen] = useState(false);
// The file picker takes the focus, which the menu reads as a click
// elsewhere; it is held open until the picker is done.
const choosingFile = useRef(false);
const fileInput = useRef<HTMLInputElement | null>(null);
// A refusal is about the attempt that caused it: shown until dismissed or
// until the menu closes, and again for the next one.
const [seenRefusal, setSeenRefusal] = useState<object>();
const refusal =
backgroundImageRefusal !== seenRefusal ? backgroundImageRefusal : undefined;
const onOpenChange = useCallback(
(open: boolean): void => {
if (!open && choosingFile.current) return;
setMenuOpen(open);
// Drop a request that never arrived.
if (!open) setPlannedSelection(null);
if (!open) setSeenRefusal(backgroundImageRefusal);
},
[backgroundImageRefusal],
);
// Scrolled to as it appears, as the list may be scrolled away from it.
const scrollIntoView = useCallback((element: HTMLElement | null): void => {
element?.scrollIntoView({ block: "nearest" });
const onOpenChange = useCallback((open: boolean): void => {
setMenuOpen(open);
// Drop a request that never arrived.
if (!open) setPlannedSelection(null);
}, []);
const watchFileInput = useCallback(
(input: HTMLInputElement | null): (() => void) | undefined => {
fileInput.current = input;
if (input === null) return;
// Dismissing the picker fires cancel, which React doesn't type.
const done = (): void => {
choosingFile.current = false;
};
input.addEventListener("cancel", done);
return (): void => input.removeEventListener("cancel", done);
},
[],
);
const isBusy = busy ?? false;
const { t } = useTranslation();
const devices = useMediaDevices();
@@ -214,53 +160,21 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
// Measured on the call area: CSS can't size the portalled menu against it.
const rootElement = useRootElement();
const [listShare, setListShare] = useState<number>();
const [previewBlock, setPreviewBlock] = useState(0);
const [listMaxHeight, setListMaxHeight] = useState<number>();
useEffect(() => {
if (!menuOpen) return;
// Followed, since a host can resize the call while the menu is open.
const subscription = observeElementSize$(rootElement)
.pipe(
map(({ width, height }) => ({
height: Math.round(height * LIST_SHARE_OF_CALL),
width: Math.min(MENU_WIDTH, Math.round(width - 2 * MENU_MARGIN)),
})),
distinctUntilChanged(
(a, b) => a.height === b.height && a.width === b.width,
map(({ height }) =>
Math.max(MIN_LIST_HEIGHT, Math.round(height * LIST_SHARE_OF_CALL)),
),
distinctUntilChanged(),
)
.subscribe(({ height, width }) => {
setListShare(height);
setPreviewBlock(Math.round((width * 9) / 16) + PREVIEW_GAP);
});
.subscribe(setListMaxHeight);
return (): void => subscription.unsubscribe();
}, [menuOpen, rootElement]);
// Paid for out of the list's share, so a preview makes the menu no taller.
const hasPreview =
iconsAndLabels === "video" &&
selfPreview !== undefined &&
listShare !== undefined;
const previewPinned =
hasPreview && listShare - previewBlock >= MIN_LIST_HEIGHT;
const listMaxHeight =
listShare === undefined
? undefined
: Math.max(
MIN_LIST_HEIGHT,
listShare - (previewPinned ? previewBlock : 0),
);
const preview = hasPreview && (
<div
aria-hidden
className={classNames(styles.selfPreview, {
[styles.selfPreviewPinned]: previewPinned,
})}
>
{backgroundEffectSettling ? <InlineSpinner size={32} /> : selfPreview}
</div>
);
useEffect(() => {
if (menuOpen) devices.requestDeviceNames(); // No-op after the first call
}, [menuOpen, devices]);
@@ -412,21 +326,6 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
>
{/* The mute button lives inside */}
{button}
{onAddBackgroundImage !== undefined && (
<input
ref={watchFileInput}
type="file"
accept="image/*"
hidden
onChange={(e) => {
const file = e.target.files?.[0];
// Cleared, so the same file can be chosen twice in a row.
e.target.value = "";
choosingFile.current = false;
if (file) onAddBackgroundImage(file);
}}
/>
)}
<Menu
className={styles.menu}
// Named for screen readers only: each section has its own heading.
@@ -451,12 +350,9 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
/>
}
>
{previewPinned && preview}
<MenuScrollArea
ref={trackFocusSource}
className={classNames(styles.deviceList, {
[styles.deviceListLeadsWithPreview]: hasPreview && !previewPinned,
})}
className={styles.deviceList}
data-testid="device-list"
style={
{
@@ -465,7 +361,6 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
} as CSSProperties
}
>
{hasPreview && !previewPinned && preview}
{audioMeter && speakerOptions && (
<MenuScrollSection title={t("settings.devices.speaker")}>
{deviceItems(
@@ -499,39 +394,14 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
describedBy={
backgroundEffectNotice === undefined ? undefined : noticeId
}
onAdd={
onAddBackgroundImage === undefined
? undefined
: (): void => {
choosingFile.current = true;
fileInput.current?.click();
}
}
addLabel={t("action.add_background_image")}
onRemove={onRemoveBackgroundEffect}
removeLabel={t("action.remove")}
/>
{/* In the list, so the menu grows no taller for it. */}
{refusal !== undefined && (
<div
ref={scrollIntoView}
role="none"
className={styles.refusal}
>
<Alert
type="critical"
title={refusal.text}
onClose={(): void => setSeenRefusal(refusal)}
/>
{backgroundEffectNotice !== undefined && (
<div id={noticeId} role="none" className={styles.notice}>
<InfoIcon width={20} height={20} />
<span>{backgroundEffectNotice}</span>
</div>
)}
{refusal === undefined &&
backgroundEffectNotice !== undefined && (
<div id={noticeId} role="none" className={styles.notice}>
<InfoIcon width={20} height={20} />
<span>{backgroundEffectNotice}</span>
</div>
)}
</MenuScrollSection>
)}
</MenuScrollArea>
-10
View File
@@ -1,10 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
.mirrored {
transform: scaleX(-1);
}
-47
View File
@@ -1,47 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { act, render } from "@testing-library/react";
import { describe, expect, it } from "vitest";
import { BehaviorSubject } from "rxjs";
import { SelfPreview } from "./SelfPreview";
import styles from "./SelfPreview.module.css";
import { mockVideoTrack } from "../utils/test";
describe("SelfPreview", () => {
it("detaches only its own element from a camera shown twice", () => {
const track = mockVideoTrack({});
const first = render(<SelfPreview track={track} />);
const second = render(<SelfPreview track={track} />);
const [a, b] = [first, second].map((r) =>
r.container.querySelector("video"),
);
expect(track.attach).toHaveBeenCalledWith(a);
expect(track.attach).toHaveBeenCalledWith(b);
first.unmount();
expect(track.detach).toHaveBeenCalledExactlyOnceWith(a);
});
it("mirrors the camera while it faces the user, and only then", () => {
const facingMode$ = new BehaviorSubject<"user" | "environment">("user");
const { container } = render(
<SelfPreview track={mockVideoTrack({ facingMode$ })} />,
);
const video = (): HTMLVideoElement => container.querySelector("video")!;
expect(video().classList.contains(styles.mirrored)).toBe(true);
act(() => facingMode$.next("environment"));
expect(video().classList.contains(styles.mirrored)).toBe(false);
});
it("says the camera is off while there is none", () => {
const { container } = render(<SelfPreview track={null} />);
expect(container.querySelector("video")).toBeNull();
expect(container.querySelector("svg")).not.toBeNull();
});
});
-48
View File
@@ -1,48 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type FC, useEffect, useRef } from "react";
import { constant, type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk";
import { VideoCallOffIcon } from "@vector-im/compound-design-tokens/assets/web/icons";
import styles from "./SelfPreview.module.css";
import { useBehavior } from "../useBehavior";
const noFacingMode = constant(undefined);
interface Props {
/** The camera as the pipeline has it, or none while the camera is off. */
track: LocalVideoMediaTrack | null;
}
/**
* The user's own camera with the effect in force: the pipeline's own track,
* shown a second time rather than captured again.
*/
export const SelfPreview: FC<Props> = ({ track }) => {
const videoRef = useRef<HTMLVideoElement | null>(null);
useEffect(() => {
const element = videoRef.current;
if (!track || !element) return;
track.attach(element);
// This element only: the pre-join preview may be showing the same camera
return (): void => {
track.detach(element);
};
}, [track]);
const facingMode = useBehavior(track?.facingMode$ ?? noFacingMode);
if (!track) return <VideoCallOffIcon width={32} height={32} />;
return (
<video
ref={videoRef}
muted
playsInline
className={facingMode === "user" ? styles.mirrored : undefined}
/>
);
};
-13
View File
@@ -95,19 +95,6 @@ describe("Config.initWith", () => {
expect(Config.get().ssla).toBe("https://example.invalid/ssla");
});
it("falls back to the defaults when fetching config.json rejects", async () => {
// A rejected promise is what a webview gives for a missing local file.
vi.spyOn(globalThis, "fetch").mockRejectedValue(
new TypeError("Load failed"),
);
vi.spyOn(logger, "info").mockImplementation(() => {});
Reflect.set(Config, "internalInstance", undefined);
await expect(Config.init()).resolves.toBeUndefined();
expect(Config.get().media_quality).toEqual(DEFAULT_CONFIG.media_quality);
});
it("replaces a config initialized earlier", () => {
Config.initWith({ ssla: "https://first.invalid/ssla" });
Config.initWith({ ssla: "https://second.invalid/ssla" });
+5 -12
View File
@@ -140,20 +140,13 @@ export function validateConfig(config: ConfigOptions): ConfigOptions {
}
async function downloadConfig(fetchTarget: string): Promise<ConfigOptions> {
let response: Response;
try {
response = await fetch(fetchTarget);
} catch (e) {
// The embedded package ships no config.json, and some webviews (Element X
// iOS) reject the fetch of a missing local file rather than answering 404.
logger.info(`No config at ${fetchTarget}, using defaults`, e);
// resolveConfig merges in the defaults.
return {};
}
const response = await fetch(fetchTarget);
if (isFailure(response)) {
// resolveConfig merges in the defaults.
return {};
// Lack of a config isn't an error, we should just use the defaults.
// Also treat a blank config as no config, assuming the status code is 0, because we don't get 404s from file:
// URIs so this is the only way we can not fail if the file doesn't exist when loading from a file:// URI.
return DEFAULT_CONFIG;
}
return response.json();
+6 -3
View File
@@ -75,6 +75,7 @@ import {
type RTCSlotOptions,
constant,
} from "@element-hq/matrixrtc-sdk";
import { map } from "rxjs";
import {
advancedCamera,
advancedScreenShare,
@@ -180,7 +181,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
props.muteStates,
mediaDevices,
capture,
backgroundEffectsState$.value,
backgroundEffectsState$.value.processor,
),
});
@@ -194,7 +195,9 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
...callViewModelOptionsFromParams(urlParams),
encryptionSystem: props.e2eeSystem,
capture,
backgroundEffectsState$,
videoProcessor$: scope.behavior(
backgroundEffectsState$.pipe(map((state) => state.processor)),
),
hostBridge,
autoLeaveWhenOthersLeft,
waitForCallPickup: waitForCallPickup && sendNotificationType === "ring",
@@ -238,7 +241,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
vm,
props.muteStates,
mediaDevices,
backgroundEffectsState$,
backgroundEffectsState$.pipe(map(({ settling }) => settling ?? false)),
`${props.client.getUserId()}:${props.client.getDeviceId()}`,
{ showControls: urlParams.showControls, header: urlParams.header },
);
+5 -5
View File
@@ -16,6 +16,7 @@ import {
type LocalMedia,
type LocalVideoMediaTrack,
} from "@element-hq/matrixrtc-sdk";
import { map } from "rxjs";
import inCallStyles from "./InCallView.module.css";
import styles from "./LobbyView.module.css";
@@ -133,7 +134,9 @@ export const LobbyView: FC<Props> = ({
muteStates,
devices,
captureSettings(),
backgroundEffectsState$,
mediaScope.behavior(
backgroundEffectsState$.pipe(map((state) => state.processor)),
),
),
);
return (): void => {
@@ -155,15 +158,13 @@ export const LobbyView: FC<Props> = ({
null,
);
useEffect(() => {
if (mediaVm === null) return;
const footerScope = new ObservableScope();
setFooterVm(
createLobbyFooterViewModel(
footerScope,
muteStates,
devices,
backgroundEffectsState$,
mediaVm.videoTrack$,
backgroundEffectsState$.pipe(map(({ settling }) => settling ?? false)),
openSettings,
hangup,
// Logo and header are connected: only show the logo in SPA with header.
@@ -178,7 +179,6 @@ export const LobbyView: FC<Props> = ({
backgroundEffectsState$,
hangup,
hideHeader,
mediaVm,
muteStates,
openSettings,
]);
+19 -109
View File
@@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details.
*/
import { beforeEach, describe, expect, it, vi } from "vitest";
import { BehaviorSubject, distinctUntilChanged, map, of } from "rxjs";
import { BehaviorSubject, distinctUntilChanged, map } from "rxjs";
import { type BackgroundProcessorWrapper } from "@livekit/track-processors";
import {
@@ -15,7 +15,6 @@ import {
type BackgroundEffectsOptions,
type BackgroundEffectsState,
} from "./BackgroundEffects";
import { type AddedBackground } from "../tracks/backgroundImages";
import { shippedBackgrounds } from "../tracks/backgroundEffectCatalog";
import { flushPromises, testScope, withTestScheduler } from "../utils/test";
@@ -47,37 +46,29 @@ function fakePipeline(): {
};
}
/** One letter per state: idle, preparing, waiting for a frame, attached. */
/** One letter per state: idle, waiting for a frame, attached. */
function letter(state: BackgroundEffectsState): string {
if (state.supported === false) return "x";
if (state.wait === "preparing") return "p";
if (state.processor === undefined) return "i";
return state.wait === "settling" ? "w" : "a";
return state.settling ? "w" : "a";
}
describe("the pipeline's state", () => {
function testState({
effect,
answer = "-y",
firstFrame = "",
expected,
}: {
effect: string;
/** When the trial build answers, from being asked. */
answer?: string;
firstFrame?: string;
expected: string;
}): void {
withTestScheduler(({ behavior, cold, schedule, expectObservable }) => {
withTestScheduler(({ behavior, schedule, expectObservable }) => {
const transformer = {
onFirstFrame: undefined as (() => void) | undefined,
};
const effects = createBackgroundEffects(testScope(), {
supported: true,
effect$: behavior(effect, { n: "none", b: "blur" }),
setEffect: vi.fn(),
added$: new BehaviorSubject<AddedBackground[] | undefined>([]),
canSegment: () => cold(answer, { y: true, n: false }),
pipeline: fakePipeline().pipeline,
transformer,
});
@@ -90,27 +81,19 @@ describe("the pipeline's state", () => {
it("defaults to no effect", () => testState({ effect: "n", expected: "i" }));
it("attaches only once it is known the pipeline builds", () =>
testState({ effect: "nb", answer: "--y", expected: "ip-w" }));
it("offers none where the pipeline fails to build", () =>
testState({ effect: "nb", answer: "-n", expected: "ipx" }));
it("waits from the first effect until a frame carries it", () =>
testState({
effect: " nb---n-b",
firstFrame: "---f",
expected: " ipwa",
effect: " nb-n-b",
firstFrame: " --f",
expected: " iwa",
}));
it("attaches on first use and stays attached", () =>
testState({ effect: "nb--n", firstFrame: "---f", expected: "ipwa" }));
testState({ effect: "nb-n", firstFrame: "--f", expected: "iwa" }));
});
describe("background effects", () => {
let effect$: BehaviorSubject<string>;
let added$: BehaviorSubject<AddedBackground[] | undefined>;
let setEffect: (raw: string) => void;
let fake: ReturnType<typeof fakePipeline>;
function build(
@@ -119,9 +102,6 @@ describe("background effects", () => {
return createBackgroundEffects(testScope(), {
supported: true,
effect$,
setEffect,
added$,
canSegment: () => of(true),
pipeline: fake.pipeline,
transformer: { onFirstFrame: undefined },
...options,
@@ -131,25 +111,14 @@ describe("background effects", () => {
effect$.next(raw);
await flushPromises();
};
const blur = async (on: boolean): Promise<void> =>
choose(on ? "blur" : "none");
beforeEach(() => {
effect$ = new BehaviorSubject("none");
added$ = new BehaviorSubject<AddedBackground[] | undefined>(undefined);
setEffect = vi.fn((raw: string) => effect$.next(raw));
fake = fakePipeline();
});
it("offers none for the session where the pipeline fails to build", async () => {
const effects = build({ canSegment: () => of(false) });
await choose("blur");
expect(effects.state$.value.supported).toBe(false);
expect(effects.state$.value.processor).toBeUndefined();
expect(effects.state$.value.wait).toBeUndefined();
// A browser update may fix it, so the next session asks again.
expect(effect$.value).toBe("blur");
});
it("puts a shipped background on as that picture", async () => {
build();
await choose(`image:${shippedBackgrounds[0].id}`);
@@ -163,10 +132,10 @@ describe("background effects", () => {
it("switches in place rather than reattaching", async () => {
const effects = build();
await choose("blur");
await blur(true);
const pipeline = effects.state$.value.processor;
await choose("none");
await choose("blur");
await blur(false);
await blur(true);
expect(effects.state$.value.processor).toBe(pipeline);
expect(fake.switches).toEqual([
@@ -179,18 +148,18 @@ describe("background effects", () => {
it("keeps switching after a switch fails", async () => {
vi.mocked(fake.pipeline.switchTo).mockRejectedValueOnce(new Error("lost"));
build();
await choose("blur");
await choose("none");
await blur(true);
await blur(false);
expect(fake.switches).toEqual([{ mode: "disabled" }]);
});
it("switches one at a time, skipping those overtaken", async () => {
const finished = fake.holdNext();
build();
await choose("blur");
await choose("none");
await choose("blur");
await choose("none");
await blur(true);
await blur(false);
await blur(true);
await blur(false);
expect(fake.switches).toHaveLength(1);
finished();
@@ -200,63 +169,4 @@ describe("background effects", () => {
{ mode: "disabled" },
]);
});
it("clears a remembered effect when it cannot be honoured", () => {
effect$.next(`image:${shippedBackgrounds[0].id}`);
const effects = build({ supported: false });
expect(effect$.value).toBe("none");
expect(effects.state$.value.processor).toBeUndefined();
});
it("puts an added background on as its picture", async () => {
added$.next([{ id: "mine", url: "blob:mine" }]);
build();
await choose("added:mine");
expect(fake.switches).toEqual([
{ mode: "virtual-background", imagePath: "blob:mine" },
]);
});
it("switches between any two effects in place", async () => {
added$.next([{ id: "mine", url: "blob:mine" }]);
const shipped = shippedBackgrounds[0];
const options: Record<string, unknown> = {
none: { mode: "disabled" },
blur: { mode: "background-blur", blurRadius: 15 },
[`image:${shipped.id}`]: {
mode: "virtual-background",
imagePath: shipped.imagePath,
},
"added:mine": { mode: "virtual-background", imagePath: "blob:mine" },
};
// Each kind to each other kind, and back.
const order = [
"blur",
`image:${shipped.id}`,
"added:mine",
"blur",
"added:mine",
`image:${shipped.id}`,
"none",
"added:mine",
"none",
"blur",
];
const effects = build();
await choose(order[0]);
const pipeline = effects.state$.value.processor;
for (const raw of order.slice(1)) await choose(raw);
expect(effects.state$.value.processor).toBe(pipeline);
expect(fake.switches).toEqual(order.map((raw) => options[raw]));
});
it("forgets an added background the device no longer keeps", () => {
effect$.next("added:gone");
build();
// Not before the device has said what it keeps.
expect(effect$.value).toBe("added:gone");
added$.next([]);
expect(effect$.value).toBe("none");
});
});
+21 -94
View File
@@ -8,15 +8,9 @@ Please see LICENSE in the repository root for full details.
import {
combineLatest,
distinctUntilChanged,
first,
from,
map,
Observable,
type ObservableInput,
of,
scan,
startWith,
switchMap,
} from "rxjs";
import {
type BackgroundProcessorWrapper,
@@ -33,23 +27,17 @@ import { type Behavior } from "./Behavior";
import { type ObservableScope } from "./ObservableScope";
import {
type BackgroundEffect,
type EffectId,
imagePathFor,
parseEffect,
} from "../tracks/backgroundEffectCatalog";
import { type AddedBackground } from "../tracks/backgroundImages";
const blurRadius = 15;
export interface BackgroundEffectsState {
supported: boolean | undefined;
processor: undefined | VideoProcessor;
/**
* The effect is not ready:
* preparing, asking whether the pipeline can be built.
* settling, once it is attached until a frame carrying the effect
*/
wait?: "preparing" | "settling";
/** From the first effect chosen until a frame carrying it is drawn. */
settling?: boolean;
}
export interface BackgroundEffectsOptions {
@@ -57,12 +45,6 @@ export interface BackgroundEffectsOptions {
supported: boolean;
/** The effect chosen, as the setting stores it. */
effect$: Behavior<string>;
/** Stores a choice, to forget one that can't be honoured. */
setEffect: (id: EffectId) => void;
/** The backgrounds the device keeps, undefined until it has said. */
added$: Behavior<AddedBackground[] | undefined>;
/** Builds a segmenter on trial, and answers whether it could. */
canSegment: () => ObservableInput<boolean>;
/** The background processor pipeline to be switched between effects. */
pipeline: BackgroundProcessorWrapper;
/** Tells, once, that a frame carrying an effect has been drawn. */
@@ -77,49 +59,14 @@ export interface BackgroundEffects {
/** Switches the pipeline as the choice changes, for as long as the scope lasts. */
export function createBackgroundEffects(
scope: ObservableScope,
{
supported,
effect$,
setEffect,
added$,
canSegment,
pipeline,
transformer,
}: BackgroundEffectsOptions,
{ supported, effect$, pipeline, transformer }: BackgroundEffectsOptions,
): BackgroundEffects {
// Wrapped once, so the call and the preview see one processor throughout.
const processor = fromLivekitProcessor(pipeline);
const choice$ = effect$.pipe(map(parseEffect));
const wanted$ = scope.behavior(
choice$.pipe(
map((effect) => effect.kind !== "none"),
distinctUntilChanged(),
),
);
combineLatest([choice$, added$])
.pipe(scope.bind())
.subscribe(([effect, added]) => {
if (effect.kind === "none") return;
if (
!supported ||
(effect.kind === "added" &&
added !== undefined &&
!added.some(({ id }) => id === effect.id))
)
setEffect("none");
});
// Asked once an effect is first wanted. A browser update may fix what
// fails, so the choice is kept for the next session rather than cleared.
const buildable$ = scope.behavior<boolean | undefined>(
supported
? wanted$.pipe(
first(Boolean),
switchMap(() => from(canSegment())),
startWith(undefined),
)
: of(undefined),
const wanted$ = choice$.pipe(
map((effect) => effect.kind !== "none"),
distinctUntilChanged(),
);
const drewAFrame$ = scope.behavior(
@@ -132,25 +79,18 @@ export function createBackgroundEffects(
}),
);
// In one step, so that nothing sees the pipeline neither preparing nor
// attached.
const state$ = scope.behavior(
combineLatest([wanted$, buildable$, drewAFrame$]).pipe(
scan<[boolean, boolean | undefined, boolean], BackgroundEffectsState>(
(previous, [wanted, buildable, drewAFrame]) => {
combineLatest([wanted$, drewAFrame$]).pipe(
scan<[boolean, boolean], BackgroundEffectsState>(
(previous, [wanted, drewAFrame]) => {
// Attached the first time an effect is wanted and never detached
// after, so someone who never turns one on pays for none of it.
const enable =
previous.processor !== undefined || (wanted && buildable === true);
const preparing = supported && wanted && buildable === undefined;
previous.processor !== undefined || (supported && wanted);
return {
supported: supported && buildable !== false,
supported,
processor: enable ? processor : undefined,
wait: preparing
? "preparing"
: enable && !drewAFrame
? "settling"
: undefined,
settling: enable && !drewAFrame,
};
},
{ supported, processor: undefined },
@@ -161,16 +101,9 @@ export function createBackgroundEffects(
const switchOptions$ = scope.behavior<
SwitchBackgroundProcessorOptions | undefined
>(
combineLatest([state$, choice$, added$]).pipe(
map(([{ processor }, effect, added]) =>
processor === undefined
? undefined
: switchOptionsFor(
effect,
effect.kind === "added"
? added?.find(({ id }) => id === effect.id)?.url
: undefined,
),
combineLatest([state$, choice$]).pipe(
map(([{ processor }, effect]) =>
processor === undefined ? undefined : switchOptionsFor(effect),
),
distinctUntilChanged(deepCompare),
),
@@ -188,24 +121,18 @@ export function createBackgroundEffects(
return { state$ };
}
/** `addedUrl` is the picture of the added background chosen, if still kept. */
function switchOptionsFor(
effect: BackgroundEffect,
addedUrl: string | undefined,
): SwitchBackgroundProcessorOptions {
const withImage = (
imagePath: string | undefined,
): SwitchBackgroundProcessorOptions =>
imagePath
? { mode: "virtual-background", imagePath }
: { mode: "disabled" };
switch (effect.kind) {
case "blur":
return { mode: "background-blur", blurRadius };
case "shipped":
return withImage(imagePathFor(effect.id));
case "added":
return withImage(addedUrl);
case "shipped": {
const imagePath = imagePathFor(effect.id);
return imagePath
? { mode: "virtual-background", imagePath }
: { mode: "disabled" };
}
default:
return { mode: "disabled" };
}
+6 -93
View File
@@ -70,11 +70,9 @@ import {
} from "./CallViewModel.ts";
import { type Handler, MuteState } from "../MuteStates.ts";
import {
type LocalVideoMediaTrack,
type RTCParticipation,
type VideoProcessor,
} from "@element-hq/matrixrtc-sdk";
import { type BackgroundEffectsState } from "../BackgroundEffects";
initializeWidget();
@@ -1694,113 +1692,28 @@ describe("a camera with a video processor", () => {
muteStates,
mockMediaDevices({}),
{},
{ supported: true, processor },
processor,
).find((request) => request.source === "camera"),
).toMatchObject({ processor });
});
it("leaves the camera out of the join while the processor is prepared", () => {
const muteStates = mockMuteStates();
vi.spyOn(muteStates.video.enabled$, "value", "get").mockReturnValue(true);
const sources = (wait?: "preparing"): string[] =>
initialPublishRequests(
muteStates,
mockMediaDevices({}),
{},
{
supported: true,
processor: undefined,
wait,
},
).map((request) => request.source);
expect(sources("preparing")).not.toContain("camera");
expect(sources()).toContain("camera");
});
it("waits to make a camera track until it is known the processor builds", async () => {
const setHandler = vi.spyOn(MuteState.prototype, "setHandler");
onTestFinished(() => setHandler.mockRestore());
const effects$ = new BehaviorSubject<BackgroundEffectsState>({
supported: true,
processor: undefined,
wait: "preparing",
});
let participation: RTCParticipation | undefined;
withCallViewModel(
{},
(_vm, _session, { rtcParticipation }) => {
participation = rtcParticipation;
},
{ backgroundEffectsState$: effects$ },
);
const camera = participation!.localMember$.value?.tracks$.value?.find(
(track) => track.source === "camera",
) as LocalVideoMediaTrack;
const turnedOn = cameraHandler(setHandler)(true);
await flushPromises();
expect(camera.setEnabled).not.toHaveBeenCalledWith(true);
effects$.next({ supported: true, processor });
await turnedOn;
expect(camera.setEnabled).toHaveBeenCalledWith(true);
expect(participation!.publish).not.toHaveBeenCalled();
});
it("turns the camera on only once its processor is on the track", async () => {
const setHandler = vi.spyOn(MuteState.prototype, "setHandler");
onTestFinished(() => setHandler.mockRestore());
const effects$ = new BehaviorSubject<BackgroundEffectsState>({
supported: true,
processor: undefined,
wait: "preparing",
});
let participation: RTCParticipation | undefined;
withCallViewModel(
{},
(_vm, _session, { rtcParticipation }) => {
participation = rtcParticipation;
},
{ backgroundEffectsState$: effects$ },
);
const camera = participation!.localMember$.value?.tracks$.value?.find(
(track) => track.source === "camera",
) as LocalVideoMediaTrack;
// Attaching takes its time, as the pipeline is built on the track
let attached!: () => void;
vi.mocked(camera.setProcessor).mockReturnValueOnce(
new Promise<void>((resolve) => {
attached = resolve;
}),
);
const turnedOn = cameraHandler(setHandler)(true);
effects$.next({ supported: true, processor });
await flushPromises();
expect(camera.setProcessor).toHaveBeenCalledWith(processor);
expect(camera.setEnabled).not.toHaveBeenCalledWith(true);
attached();
await turnedOn;
expect(camera.setEnabled).toHaveBeenCalledWith(true);
});
// The SDK builds the processor on the camera's restart, before the room
// sees a frame, so the track is unmuted rather than published anew
it("turns on a camera turned off before the processor was chosen", async () => {
const setHandler = vi.spyOn(MuteState.prototype, "setHandler");
onTestFinished(() => setHandler.mockRestore());
const effects$ = new BehaviorSubject<BackgroundEffectsState>({
supported: true,
processor: undefined,
});
const videoProcessor$ = new BehaviorSubject<VideoProcessor | undefined>(
undefined,
);
let participation: RTCParticipation | undefined;
withCallViewModel(
{},
(_vm, _session, { rtcParticipation }) => {
participation = rtcParticipation;
},
{ backgroundEffectsState$: effects$ },
{ videoProcessor$ },
);
effects$.next({ supported: true, processor });
videoProcessor$.next(processor);
await cameraHandler(setHandler)(true);
await flushPromises();
+8 -65
View File
@@ -37,7 +37,6 @@ import {
combineLatest,
distinctUntilChanged,
filter,
firstValueFrom,
fromEvent,
map,
merge,
@@ -78,7 +77,6 @@ import {
type ReactionInfo,
type ReactionOption,
} from "../../reactions";
import { type BackgroundEffectsState } from "../BackgroundEffects";
import { shallowEquals as arrayShallowEquals } from "../../utils/array";
import { type MediaDevices } from "../MediaDevices";
import { type MuteStates } from "../MuteStates";
@@ -172,16 +170,13 @@ export function initialPublishRequests(
muteStates: MuteStates,
mediaDevices: MediaDevices,
capture: CaptureSettings,
effects?: BackgroundEffectsState,
processor?: VideoProcessor,
): PublishRequest[] {
const requests: PublishRequest[] = [];
if (muteStates.audio.enabled$.value)
requests.push(publishRequest("microphone", mediaDevices, capture));
// Left out while the processor is prepared; the call publishes it after
if (muteStates.video.enabled$.value && effects?.wait !== "preparing")
requests.push(
publishRequest("camera", mediaDevices, capture, effects?.processor),
);
if (muteStates.video.enabled$.value)
requests.push(publishRequest("camera", mediaDevices, capture, processor));
return requests;
}
@@ -192,8 +187,8 @@ export interface CallViewModelOptions {
* defaults otherwise.
*/
capture?: CaptureSettings;
/** The background effects state (preparing, supported...) including the processor itself. */
backgroundEffectsState$?: Behavior<BackgroundEffectsState>;
/** Background blur and the like, applied to the camera as it changes. */
videoProcessor$?: Behavior<VideoProcessor | undefined>;
/**
* The application hosting Element Call, which can ask it to hang up and wants
* to know when the user joins or leaves. Defaults to no host.
@@ -368,8 +363,6 @@ export interface CallViewModel {
keyRotationSuppressed$: Behavior<boolean>;
/** The local member, once our membership has been seen in the room. */
localMember$: Behavior<LocalRTCMember | null>;
/** Our own camera track, while one is published. */
camera$: Behavior<LocalVideoMediaTrack | undefined>;
remoteMembers$: Behavior<RemoteRTCMember[]>;
/** The transports the client holds a connection to, for the developer panel. */
connectedTransports$: Behavior<TransportMetadata[]>;
@@ -495,21 +488,7 @@ export function createCallViewModel$(
sentCallNotification$ = of(null),
} = options;
const {
capture = {},
backgroundEffectsState$ = constant<BackgroundEffectsState>({
supported: false,
processor: undefined,
}),
} = options;
const videoProcessor$ = scope.behavior(
backgroundEffectsState$.pipe(map(({ processor }) => processor)),
);
// Resolves once it is known whether the processor builds
const processorKnown = async (): Promise<unknown> =>
firstValueFrom(
backgroundEffectsState$.pipe(filter(({ wait }) => wait !== "preparing")),
);
const { capture = {}, videoProcessor$ = constant(undefined) } = options;
const localTracks$ = scope.behavior<
(LocalAudioMediaTrack | LocalVideoMediaTrack)[]
>(
@@ -533,19 +512,6 @@ export function createCallViewModel$(
"screenShare",
) as Behavior<LocalVideoMediaTrack | undefined>;
const processorAttached = followProcessor(
scope,
camera$,
videoProcessor$,
logger,
);
// A camera turned on waits for its processor to be known, then to be on
// the track: LiveKit restarts the track with whatever processor it has then.
const cameraReady = async (): Promise<void> => {
await processorKnown();
await processorAttached();
};
const supply = (source: "microphone" | "camera"): TrackSupply => ({
publish: async () =>
rtcParticipation.publish(
@@ -556,7 +522,6 @@ export function createCallViewModel$(
source === "camera" ? videoProcessor$.value : undefined,
),
),
ready: source === "camera" ? cameraReady : undefined,
});
followMuteState(
scope,
@@ -583,29 +548,6 @@ export function createCallViewModel$(
);
followDevice(scope, camera$, mediaDevices.videoInput, "camera", logger);
// A camera wanted at the join was left out of it while the processor was
// prepared (see initialPublishRequests), and is published once that is known
if (
backgroundEffectsState$.value.wait === "preparing" &&
muteStates.video.enabled$.value
)
processorKnown()
.then(async () => {
if (camera$.value !== undefined || !muteStates.video.enabled$.value)
return;
await rtcParticipation.publish(
publishRequest(
"camera",
mediaDevices,
capture,
videoProcessor$.value,
),
);
})
.catch((e) => {
logger.error("Failed to publish the camera", e);
});
// The output device follows the selection onto the client the same way. A
// host that routes audio itself picks the output outside the browser, so the
// client is left alone
@@ -619,6 +561,8 @@ export function createCallViewModel$(
});
});
followProcessor(scope, camera$, videoProcessor$, logger);
const members$ = scope.behavior<(LocalRTCMember | RemoteRTCMember)[]>(
combineLatest(
[rtcParticipation.localMember$, rtcParticipation.remoteMembers$],
@@ -1564,7 +1508,6 @@ export function createCallViewModel$(
leave,
toggleScreenSharing: toggleScreenSharing,
sharingScreen$: sharingScreen$,
camera$,
tapScreen: (): void => screenTap$.next(),
tapControls: (): void => controlsTap$.next(),
+1 -2
View File
@@ -21,7 +21,6 @@ import { type MediaDevices } from "./MediaDevices";
import { MuteStates } from "./MuteStates";
import { nullHostBridge } from "../HostBridge";
import { flushPromises, mockMediaDevices, testScope } from "../utils/test";
import { type BackgroundEffectsState } from "./BackgroundEffects";
describe("LobbyMediaViewModel", () => {
it("opens what starts enabled in one capture and follows the devices", async () => {
@@ -136,7 +135,7 @@ function setup(
muteStates,
mediaDevices,
{},
constant<BackgroundEffectsState>({ supported: true, processor }),
constant(processor),
);
return { media, vm, muteStates, mediaDevices };
}
+4 -14
View File
@@ -13,6 +13,7 @@ import {
type ObservableScope,
type PublishRequest,
trackBySource$,
type VideoProcessor,
} from "@element-hq/matrixrtc-sdk";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import { combineLatest, debounceTime, map } from "rxjs";
@@ -24,7 +25,6 @@ import {
import { type MediaDevices } from "./MediaDevices";
import { type MuteStates } from "./MuteStates";
import { followDevice, followProcessor } from "./localMediaControls";
import { type BackgroundEffectsState } from "./BackgroundEffects";
export interface LobbyMediaViewModel {
/** The camera, once open, for the preview. */
@@ -42,13 +42,9 @@ export function createLobbyMediaViewModel(
muteStates: MuteStates,
mediaDevices: MediaDevices,
capture: CaptureSettings,
/** No camera is opened while its pipeline is prepared: it would show the room without the processor. */
backgroundEffectsState$: Behavior<BackgroundEffectsState>,
videoProcessor$: Behavior<VideoProcessor | undefined>,
): LobbyMediaViewModel {
const logger = rootLogger.getChild("[LobbyMedia]");
const videoProcessor$ = scope.behavior(
backgroundEffectsState$.pipe(map(({ processor }) => processor)),
);
const microphone$ = trackBySource$(
scope,
media.tracks$,
@@ -68,19 +64,13 @@ export function createLobbyMediaViewModel(
muteStates.video.enabled$,
microphone$,
camera$,
backgroundEffectsState$,
])
.pipe(debounceTime(0), scope.bind())
.subscribe(([audio, video, microphone, camera, { wait }]) => {
.subscribe(([audio, video, microphone, camera]) => {
const open: PublishRequest[] = [];
if (audio && microphone === undefined && !opening.has("microphone"))
open.push(publishRequest("microphone", mediaDevices, capture));
if (
video &&
camera === undefined &&
!opening.has("camera") &&
wait !== "preparing"
)
if (video && camera === undefined && !opening.has("camera"))
open.push(
publishRequest(
"camera",
+4 -12
View File
@@ -21,8 +21,6 @@ import { type Handler } from "./MuteStates";
/** How a mute switch gets a track where there is none. */
export interface TrackSupply {
publish: () => Promise<unknown>;
/** Awaited before a switch turned on acts: a camera waits for its processor to be known. */
ready?: () => Promise<unknown>;
}
/**
@@ -47,7 +45,6 @@ export function followMuteState(
let desired = muteState.enabled$.value;
muteState.setHandler(async (enabled) => {
desired = enabled;
if (enabled) await supply.ready?.();
const track = track$.value;
if (track !== undefined) {
desired = await track.setEnabled(enabled);
@@ -97,23 +94,18 @@ export function followDevice(
/**
* Attached once the camera track exists rather than carried in the publish
* request, so that a new track and a change of processor take the same path.
* Answers with a wait for the latest attachment to have gone through: a
* camera turned on before then would restart without its processor.
*/
export function followProcessor(
scope: ObservableScope,
camera$: Behavior<LocalVideoMediaTrack | undefined>,
videoProcessor$: Behavior<VideoProcessor | undefined>,
logger: Logger,
): () => Promise<void> {
let attached = Promise.resolve();
): void {
combineLatest([camera$, videoProcessor$])
.pipe(scope.bind())
.subscribe(([camera, processor]) => {
attached =
camera?.setProcessor(processor).catch((e) => {
logger.error("Failed to set the video processor", e);
}) ?? Promise.resolve();
camera?.setProcessor(processor).catch((e) => {
logger.error("Failed to set the video processor", e);
});
});
return async () => attached;
}
+2 -37
View File
@@ -13,19 +13,13 @@ import {
} from "@livekit/track-processors";
import { ImageSegmenter } from "@mediapipe/tasks-vision";
import {
BackgroundEffectTransformer,
canSegment,
} from "./BackgroundEffectTransformer";
import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer";
const frame = {} as VideoFrame;
const controller = {} as TransformStreamDefaultController<VideoFrame>;
describe("BackgroundEffectTransformer", () => {
afterEach(() => {
vi.restoreAllMocks();
vi.unstubAllGlobals();
});
afterEach(() => vi.restoreAllMocks());
it("loads a background chosen before it was built", async () => {
vi.spyOn(VideoTransformer.prototype, "init").mockResolvedValue();
@@ -84,33 +78,4 @@ describe("BackgroundEffectTransformer", () => {
expect(transformer.options.blurRadius).toBeUndefined();
expect(transformer.options.imagePath).toBeUndefined();
});
it("segments on the CPU where the GPU refuses", async () => {
vi.spyOn(VideoTransformer.prototype, "init").mockResolvedValue();
const build = vi
.spyOn(ImageSegmenter, "createFromOptions")
.mockRejectedValueOnce(new Error("blocklisted"))
.mockResolvedValue({} as ImageSegmenter);
await new BackgroundEffectTransformer({}).init(
{} as VideoTransformerInitOptions,
);
expect(build.mock.calls.map(([, o]) => o.baseOptions?.delegate)).toEqual([
"GPU",
"CPU",
]);
});
it("says whether a segmenter can be built at all", async () => {
vi.stubGlobal("OffscreenCanvas", class {});
const close = vi.fn();
const build = vi
.spyOn(ImageSegmenter, "createFromOptions")
.mockResolvedValue({ close } as unknown as ImageSegmenter);
expect(await canSegment()).toBe(true);
expect(close).toHaveBeenCalled();
build.mockRejectedValue(new Error("no model"));
expect(await canSegment()).toBe(false);
});
});
+11 -40
View File
@@ -11,7 +11,6 @@ import {
VideoTransformer,
type VideoTransformerInitOptions,
type BackgroundOptions,
type SegmenterOptions,
} from "@livekit/track-processors";
import { logger } from "matrix-js-sdk/lib/logger";
import { ImageSegmenter } from "@mediapipe/tasks-vision";
@@ -93,10 +92,17 @@ export class BackgroundEffectTransformer extends BackgroundTransformer {
inputElement: inputVideo,
});
this.imageSegmenter = await createSegmenter(
this.canvas,
this.options.segmenterOptions,
);
this.imageSegmenter = await ImageSegmenter.createFromOptions(wasmFileset, {
baseOptions: {
modelAssetPath,
delegate: "GPU",
...this.options.segmenterOptions,
},
canvas: this.canvas,
runningMode: "VIDEO",
outputCategoryMask: true,
outputConfidenceMasks: false,
});
// BackgroundTransformer's own init applies these, and this one replaces it.
if (this.options.imagePath) {
@@ -110,38 +116,3 @@ export class BackgroundEffectTransformer extends BackgroundTransformer {
this.gl?.setBackgroundDisabled(this.options.backgroundDisabled ?? false);
}
}
/**
* Whether a segmenter can be built here, asked once before any camera depends
* on one: a pipeline that fails to build takes the camera down with it.
*/
export async function canSegment(): Promise<boolean> {
try {
(await createSegmenter(new OffscreenCanvas(1, 1))).close();
return true;
} catch (e) {
logger.warn("Background effects cannot run here", e);
return false;
}
}
/** On the GPU, or on the CPU where the GPU refuses, as a blocklisted one does. */
async function createSegmenter(
canvas: HTMLCanvasElement | OffscreenCanvas | undefined,
segmenterOptions?: SegmenterOptions,
): Promise<ImageSegmenter> {
const build = async (delegate: "GPU" | "CPU"): Promise<ImageSegmenter> =>
ImageSegmenter.createFromOptions(wasmFileset, {
baseOptions: { modelAssetPath, delegate, ...segmenterOptions },
canvas,
runningMode: "VIDEO",
outputCategoryMask: true,
outputConfidenceMasks: false,
});
try {
return await build("GPU");
} catch (e) {
logger.warn("Segmenting on the CPU, as the GPU refused", e);
return build("CPU");
}
}
+2 -14
View File
@@ -19,10 +19,8 @@ import {
type BackgroundEffectsState,
} from "../state/BackgroundEffects";
import { backgroundEffect } from "../settings/settings";
import { type AddedBackground } from "./backgroundImages";
import { flushPromises, testScope } from "../utils/test";
const sdkSupportMock = vi.hoisted(() => vi.fn(() => true));
const pipelines = vi.hoisted(() => ({
built: 0,
destroyed: 0,
@@ -43,21 +41,12 @@ vi.mock("@livekit/track-processors", () => ({
return Promise.resolve();
});
}),
supportsBackgroundProcessors: (): boolean => sdkSupportMock(),
supportsBackgroundProcessors: (): boolean => true,
}));
const platformMock = vi.hoisted(() => ({ platform: "desktop" }));
vi.mock("../Platform", () => platformMock);
vi.mock("./backgroundImages", async () => {
const { BehaviorSubject } = await import("rxjs");
return {
addedBackgrounds: {
added$: new BehaviorSubject<AddedBackground[] | undefined>(undefined),
},
};
});
vi.mock("./BackgroundEffectTransformer", () => ({
BackgroundEffectTransformer: vi.fn(function () {}),
canSegment: async (): Promise<boolean> => Promise.resolve(true),
BackgroundEffectTransformer: vi.fn(),
}));
describe("BackgroundEffectsProvider", () => {
@@ -88,7 +77,6 @@ describe("BackgroundEffectsProvider", () => {
pipelines.destroyed = 0;
pipelines.switches = [];
platformMock.platform = "desktop";
sdkSupportMock.mockReturnValue(true);
backgroundEffect.setValue("none");
});
afterEach(() => backgroundEffect.setValue("none"));
+1 -8
View File
@@ -12,11 +12,7 @@ import {
import { createContext, type FC, type JSX, use } from "react";
import { backgroundEffect as backgroundEffectSetting } from "../settings/settings";
import {
BackgroundEffectTransformer,
canSegment,
} from "./BackgroundEffectTransformer";
import { addedBackgrounds } from "./backgroundImages";
import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer";
import { type Behavior } from "../state/Behavior";
import { type ObservableScope } from "../state/ObservableScope";
import {
@@ -57,9 +53,6 @@ export function createAppBackgroundEffects(
return createBackgroundEffects(scope, {
supported: supportsBackgroundProcessors(),
effect$: backgroundEffectSetting.value$,
setEffect: backgroundEffectSetting.setValue,
added$: addedBackgrounds.added$,
canSegment,
pipeline: new BackgroundProcessorWrapper(transformer, "background-effect"),
transformer,
});
@@ -15,7 +15,6 @@ import {
serializeEffect,
shippedBackgrounds,
} from "./backgroundEffectCatalog";
import { maxAddedBackgrounds } from "./backgroundImages";
describe("the chosen background effect", () => {
test("round-trips through its stored form", () => {
@@ -23,7 +22,6 @@ describe("the chosen background effect", () => {
{ kind: "none" },
{ kind: "blur" },
...shippedBackgrounds.map(({ id }) => ({ kind: "shipped" as const, id })),
{ kind: "added", id: "d3b07384" },
];
for (const effect of effects)
expect(parseEffect(serializeEffect(effect))).toEqual(effect);
@@ -32,7 +30,6 @@ describe("the chosen background effect", () => {
test("falls back to no effect when the stored form names nothing we ship", () => {
expect(parseEffect("image:gone")).toEqual({ kind: "none" });
expect(parseEffect("")).toEqual({ kind: "none" });
expect(parseEffect("added:")).toEqual({ kind: "none" });
});
test("reads a stored value that is not a string as no effect", () => {
@@ -40,14 +37,6 @@ describe("the chosen background effect", () => {
expect(parseEffect(raw)).toEqual({ kind: "none" });
});
test("fills the grid at the limit and one short of it", () => {
// No effect, blur and the shipped ones, then the added ones.
const fixed = 2 + shippedBackgrounds.length;
expect((fixed + maxAddedBackgrounds) % 3).toBe(0);
// One short, the add tile takes the last place.
expect((fixed + maxAddedBackgrounds - 1 + 1) % 3).toBe(0);
});
test("gives every shipped background an image to draw", () => {
for (const { id } of shippedBackgrounds)
expect(imagePathFor(id)).toBeTruthy();
+4 -12
View File
@@ -22,18 +22,16 @@ export const shippedBackgrounds: ShippedBackground[] = [
export type BackgroundEffect =
| { kind: "none" }
| { kind: "blur" }
| { kind: "shipped"; id: string }
| { kind: "added"; id: string };
| { kind: "shipped"; id: string };
/** An effect as the setting stores it, and as the menu names its option. */
export type EffectId = "none" | "blur" | `image:${string}` | `added:${string}`;
export type EffectId = "none" | "blur" | `image:${string}`;
export const noEffect: BackgroundEffect = { kind: "none" };
/**
* Reads a stored effect; one we no longer ship reads as no effect. Whether an
* added one is still kept only the device knows. Stored by hand or by another
* build, the value need not even be a string.
* Reads a stored effect; one we no longer ship reads as no effect. Stored by
* hand or by another build, the value need not even be a string.
*/
export function parseEffect(raw: unknown): BackgroundEffect {
if (typeof raw !== "string") return noEffect;
@@ -43,10 +41,6 @@ export function parseEffect(raw: unknown): BackgroundEffect {
if (shippedBackgrounds.some((b) => b.id === id))
return { kind: "shipped", id };
}
if (raw.startsWith("added:")) {
const id = raw.slice("added:".length);
if (id) return { kind: "added", id };
}
return noEffect;
}
@@ -56,8 +50,6 @@ export function serializeEffect(effect: BackgroundEffect): EffectId {
return "blur";
case "shipped":
return `image:${effect.id}`;
case "added":
return `added:${effect.id}`;
default:
return "none";
}
-172
View File
@@ -1,172 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { beforeEach, describe, expect, it, vi } from "vitest";
import { IDBFactory } from "fake-indexeddb";
import {
AddedBackgrounds,
type BackgroundImageStorage,
IndexedDBImageStorage,
type KeptImage,
maxAddedBackgrounds,
prepareImage,
UnusableImage,
} from "./backgroundImages";
import { flushPromises } from "../utils/test";
describe("AddedBackgrounds", () => {
beforeEach(() => {
let n = 0;
vi.spyOn(URL, "createObjectURL").mockImplementation(() => `blob:${++n}`);
});
it("offers what the device keeps, oldest first", async () => {
const image = new Blob(["x"], { type: "image/png" });
const store = new AddedBackgrounds(
memoryStorage([
{ id: "later", image, addedAt: 2 },
{ id: "earlier", image, addedAt: 1 },
]),
asGiven,
);
expect(store.added$.value).toBeUndefined();
await flushPromises();
expect(store.added$.value?.map(({ id }) => id)).toEqual([
"earlier",
"later",
]);
});
it("keeps an added image and offers it after the others", async () => {
const storage = memoryStorage();
const store = new AddedBackgrounds(storage, asGiven);
await flushPromises();
const id = await store.add(new Blob(["x"], { type: "image/png" }));
expect(store.added$.value?.map((a) => a.id)).toEqual([id]);
expect((await storage.list()).map((k) => k.id)).toEqual([id]);
});
it("keeps the prepared image, not the file", async () => {
const storage = memoryStorage();
const prepared = new Blob(["y"], { type: "image/webp" });
const store = new AddedBackgrounds(storage, async () =>
Promise.resolve(prepared),
);
await store.add(new Blob(["x"], { type: "image/png" }));
expect((await storage.list())[0].image).toBe(prepared);
});
it("keeps nothing of a file it refuses", async () => {
const storage = memoryStorage();
const store = new AddedBackgrounds(storage, async () =>
Promise.reject(new UnusableImage("animated")),
);
await flushPromises();
await expect(store.add(new Blob(["x"]))).rejects.toThrow(UnusableImage);
expect(await storage.list()).toEqual([]);
expect(store.added$.value).toEqual([]);
});
it("keeps no more than the grid holds", async () => {
const storage = memoryStorage();
const store = new AddedBackgrounds(storage, asGiven);
for (let i = 0; i < maxAddedBackgrounds; i++)
await store.add(new Blob([`${i}`]));
await expect(store.add(new Blob(["more"]))).rejects.toThrow(RangeError);
expect(await storage.list()).toHaveLength(maxAddedBackgrounds);
});
it("forgets a removed image, and lets its picture go", async () => {
const revoke = vi
.spyOn(URL, "revokeObjectURL")
.mockImplementation(() => {});
const storage = memoryStorage();
const store = new AddedBackgrounds(storage, asGiven);
await flushPromises();
const first = await store.add(new Blob(["a"]));
const second = await store.add(new Blob(["b"]));
const url = store.added$.value![0].url;
await store.remove(first);
expect(store.added$.value?.map((a) => a.id)).toEqual([second]);
expect((await storage.list()).map((k) => k.id)).toEqual([second]);
expect(revoke).toHaveBeenCalledWith(url);
});
it("offers nothing where the browser keeps nothing", async () => {
const store = new AddedBackgrounds(null);
await flushPromises();
expect(store.added$.value).toEqual([]);
await expect(store.add(new Blob(["x"]))).rejects.toThrow();
});
it("offers none, rather than waiting, where the device can't be read", async () => {
const unreadable: BackgroundImageStorage = {
...memoryStorage(),
list: async () => Promise.reject(new Error("storage cleared")),
};
const store = new AddedBackgrounds(unreadable);
await flushPromises();
expect(store.added$.value).toEqual([]);
});
});
describe("prepareImage", () => {
// Refused before anything is decoded, which only a real browser could do.
it("refuses a file that isn't an image", async () => {
await expect(
prepareImage(new Blob(["x"], { type: "text/plain" })),
).rejects.toEqual(new UnusableImage("not-an-image"));
});
it("refuses an animated type where the browser can't count frames", async () => {
expect(globalThis.ImageDecoder).toBeUndefined();
await expect(
prepareImage(new Blob(["GIF89a"], { type: "image/gif" })),
).rejects.toEqual(new UnusableImage("animated"));
});
});
describe("IndexedDBImageStorage", () => {
it("keeps images across sessions until they are deleted", async () => {
const indexedDB = new IDBFactory();
const image = new Blob(["x"], { type: "image/webp" });
const first = new IndexedDBImageStorage(indexedDB);
await first.put({ id: "a", image, addedAt: 1 });
await first.put({ id: "b", image, addedAt: 2 });
// A session of its own, as after a reload.
const second = new IndexedDBImageStorage(indexedDB);
expect(
(await second.list()).map(({ id, addedAt }) => [id, addedAt]),
).toEqual([
["a", 1],
["b", 2],
]);
await second.delete("a");
expect((await first.list()).map(({ id }) => id)).toEqual(["b"]);
});
});
// Decoding needs a real browser; the end-to-end checks cover it.
const asGiven = async (file: Blob): Promise<Blob> => Promise.resolve(file);
function memoryStorage(kept: KeptImage[] = []): BackgroundImageStorage {
return {
list: async (): Promise<KeptImage[]> => Promise.resolve([...kept]),
put: async (image): Promise<void> => {
kept.push(image);
return Promise.resolve();
},
delete: async (id): Promise<void> => {
const index = kept.findIndex((k) => k.id === id);
if (index !== -1) kept.splice(index, 1);
return Promise.resolve();
},
};
}
-220
View File
@@ -1,220 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { BehaviorSubject } from "rxjs";
import { logger } from "matrix-js-sdk/lib/logger";
import { type Behavior } from "../state/Behavior";
export const maxAddedBackgrounds = 5;
/** The longest edge kept; the pipeline scales every image to the camera. */
export const maxStoredEdge = 1920;
/** Why a file can't be used as a background. */
export type UnusableReason = "not-an-image" | "animated" | "undecodable";
export class UnusableImage extends Error {
public constructor(public readonly reason: UnusableReason) {
super(reason);
this.name = "UnusableImage";
}
}
/** A background the user added, as the menu and the pipeline use it. */
export interface AddedBackground {
id: string;
url: string;
}
/** A background the user added, as it is kept on the device. */
export interface KeptImage {
id: string;
image: Blob;
addedAt: number;
}
/** Where added backgrounds are kept. */
export interface BackgroundImageStorage {
list(): Promise<KeptImage[]>;
put(image: KeptImage): Promise<void>;
delete(id: string): Promise<void>;
}
/** The backgrounds this device keeps, which never leave it. */
export class AddedBackgrounds {
private readonly subject$ = new BehaviorSubject<
AddedBackground[] | undefined
>(undefined);
/** Undefined until what the device keeps has been read. */
public readonly added$: Behavior<AddedBackground[] | undefined> =
this.subject$;
public constructor(
private readonly storage: BackgroundImageStorage | null,
private readonly prepare: (file: Blob) => Promise<Blob> = prepareImage,
) {
this.read().catch((e) => {
logger.warn("Could not read added backgrounds", e);
this.subject$.next([]);
});
}
private async read(): Promise<void> {
const kept = (await this.storage?.list()) ?? [];
kept.sort((a, b) => a.addedAt - b.addedAt);
this.subject$.next(
kept.map(({ id, image }) => ({ id, url: URL.createObjectURL(image) })),
);
}
/**
* Keeps a file as a background, and answers with its id. Throws
* {@link UnusableImage} for a file that can't be used, and a RangeError
* once maxAddedBackgrounds are kept.
*/
public async add(file: Blob): Promise<string> {
if (this.storage === null)
throw new Error("This browser keeps no backgrounds");
const image = await this.prepare(file);
if ((await this.storage.list()).length >= maxAddedBackgrounds)
throw new RangeError(
`${maxAddedBackgrounds} backgrounds are kept already`,
);
const kept: KeptImage = {
id: crypto.randomUUID(),
image,
addedAt: Date.now(),
};
await this.storage.put(kept);
this.subject$.next([
...(this.subject$.value ?? []),
{ id: kept.id, url: URL.createObjectURL(image) },
]);
return kept.id;
}
public async remove(id: string): Promise<void> {
await this.storage?.delete(id);
const current = this.subject$.value ?? [];
const gone = current.find((a) => a.id === id);
if (gone) URL.revokeObjectURL(gone.url);
this.subject$.next(current.filter((a) => a.id !== id));
}
}
/**
* Refuses what can't be used, reduces what is larger than is kept, and lays
* every image on an opaque ground: a background has to cover what is behind
* it, so a transparent pixel would be a hole in it.
*/
export async function prepareImage(file: Blob): Promise<Blob> {
if (!file.type.startsWith("image/")) throw new UnusableImage("not-an-image");
if (await isAnimated(file)) throw new UnusableImage("animated");
let bitmap: ImageBitmap;
try {
bitmap = await createImageBitmap(file);
} catch (e) {
logger.debug("Could not decode the chosen file", e);
throw new UnusableImage("undecodable");
}
try {
const scale = Math.min(
1,
maxStoredEdge / Math.max(bitmap.width, bitmap.height),
);
const canvas = new OffscreenCanvas(
Math.round(bitmap.width * scale),
Math.round(bitmap.height * scale),
);
const context = canvas.getContext("2d");
if (!context) throw new UnusableImage("undecodable");
context.fillStyle = "black";
context.fillRect(0, 0, canvas.width, canvas.height);
context.drawImage(bitmap, 0, 0, canvas.width, canvas.height);
return await canvas.convertToBlob({ type: "image/webp", quality: 0.9 });
} finally {
bitmap.close();
}
}
// Read from the file, not its type: a still and an animated WebP share one.
async function isAnimated(file: Blob): Promise<boolean> {
if (typeof ImageDecoder === "undefined") return /gif|apng/.test(file.type);
try {
const decoder = new ImageDecoder({
data: await file.arrayBuffer(),
type: file.type,
});
// Some browsers have no selected track until the tracks are ready.
await decoder.tracks.ready;
await decoder.completed;
return (decoder.tracks.selectedTrack?.frameCount ?? 1) > 1;
} catch (e) {
logger.debug("Could not read the frame count, judging by type", e);
return /gif|apng/.test(file.type);
}
}
const DB_NAME = "element-call-background-images";
const STORE = "backgrounds";
export class IndexedDBImageStorage implements BackgroundImageStorage {
private db?: Promise<IDBDatabase>;
public constructor(private readonly indexedDB: IDBFactory) {}
private async open(): Promise<IDBDatabase> {
return (this.db ??= new Promise((resolve, reject) => {
const request = this.indexedDB.open(DB_NAME, 1);
request.onupgradeneeded = (): void => {
request.result.createObjectStore(STORE, { keyPath: "id" });
};
request.onsuccess = (): void => resolve(request.result);
request.onerror = (): void => reject(request.error);
}));
}
private async run<T>(
mode: IDBTransactionMode,
body: (store: IDBObjectStore) => IDBRequest<T>,
): Promise<T> {
const db = await this.open();
return new Promise((resolve, reject) => {
const request = body(db.transaction(STORE, mode).objectStore(STORE));
request.onsuccess = (): void => resolve(request.result);
request.onerror = (): void => reject(request.error);
});
}
public async list(): Promise<KeptImage[]> {
return this.run<KeptImage[]>("readonly", (s) => s.getAll());
}
public async put(image: KeptImage): Promise<void> {
await this.run("readwrite", (s) => s.put(image));
}
public async delete(id: string): Promise<void> {
await this.run("readwrite", (s) => s.delete(id));
}
}
function openStorage(): BackgroundImageStorage | null {
// Merely reading indexedDB throws where it is disabled.
try {
return globalThis.indexedDB
? new IndexedDBImageStorage(globalThis.indexedDB)
: null;
} catch (e) {
logger.warn("Could not reach IndexedDB", e);
return null;
}
}
export const addedBackgrounds = new AddedBackgrounds(openStorage());
+1 -1
View File
@@ -217,7 +217,7 @@ export function getBasicCallViewModelEnvironment(
vm,
muteStates,
mediaDevices,
constant({ supported: false, processor: undefined }),
constant(false),
"reactionId",
{ showControls: true, header: HeaderStyle.Standard },
);