mirror of
https://github.com/vector-im/element-call.git
synced 2026-10-16 23:45:50 +00:00
Compare commits
24
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a9fd753fbb | ||
|
|
24ca0eacc7 | ||
|
|
8637a1d736 | ||
|
|
bb11dda365 | ||
|
|
0a6780be24 | ||
|
|
c0035d95e4 | ||
|
|
a1a12f41ef | ||
|
|
deef868cd0 | ||
|
|
00402b54b5 | ||
|
|
db57962c50 | ||
|
|
5de48207d0 | ||
|
|
6c2b02a432 | ||
|
|
5ca9cedf30 | ||
|
|
de2b98251c | ||
|
|
32e7c407c3 | ||
|
|
e9365b85b5 | ||
|
|
cbb22a0094 | ||
|
|
ce41fee6ff | ||
|
|
e2b89a64b9 | ||
|
|
9823103ef7 | ||
|
|
98152f162d | ||
|
|
b1255f0424 | ||
|
|
65686ee9fc | ||
|
|
2faa6c3dac |
@@ -109,4 +109,12 @@ describe("scopeStylesToRoot", () => {
|
||||
".a .b_abc12:hover { border: 0 }",
|
||||
);
|
||||
});
|
||||
|
||||
it("leaves an excluded stylesheet as it is", async () => {
|
||||
const css = "body { margin: 0 } .menu { color: red }";
|
||||
const result = await postcss([
|
||||
scopeStylesToRoot({ exclude: [/host\.css$/] }),
|
||||
]).process(css, { from: "/app/host.css" });
|
||||
expect(result.css).toBe(css);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -60,12 +60,18 @@ const IS_ROOT = `:where(${ROOT_SELECTOR})`;
|
||||
* tokens, the component styles, the fonts — is not in this build at all: the
|
||||
* host supplies it, unscoped, along with its copy of Compound.
|
||||
*/
|
||||
export function scopeStylesToRoot(): Plugin {
|
||||
interface Options {
|
||||
/** Stylesheets left as they are: a host's own, which it doesn't scope. */
|
||||
exclude?: RegExp[];
|
||||
}
|
||||
|
||||
export function scopeStylesToRoot({ exclude = [] }: Options = {}): Plugin {
|
||||
return {
|
||||
postcssPlugin: "element-call-scope-styles-to-root",
|
||||
Once(root) {
|
||||
const isModule =
|
||||
root.source?.input.file?.endsWith(".module.css") ?? false;
|
||||
const file = root.source?.input.file ?? "";
|
||||
if (exclude.some((pattern) => pattern.test(file))) return;
|
||||
const isModule = file.endsWith(".module.css");
|
||||
root.walkRules((rule) => {
|
||||
if (isRelative(rule)) return;
|
||||
rule.selector = (isModule ? scopeBare : scopeAll).processSync(
|
||||
|
||||
+11
-12
@@ -72,9 +72,9 @@ import {
|
||||
useUrlParams,
|
||||
} from "../src/UrlParams";
|
||||
import { MediaDevicesContext } from "../src/MediaDevicesContext";
|
||||
import { MediaDevices } from "../src/state/MediaDevices";
|
||||
import { AppViewModel } from "../src/state/AppViewModel";
|
||||
import { ObservableScope } from "../src/state/ObservableScope";
|
||||
import { ProcessorProvider } from "../src/livekit/TrackProcessorContext";
|
||||
import { BackgroundEffectsProvider } from "../src/livekit/BackgroundEffectsContext";
|
||||
import { Config } from "../src/config/Config";
|
||||
import { type ConfigOptions } from "../src/config/ConfigOptions";
|
||||
import { i18n } from "../src/utils/i18n";
|
||||
@@ -192,14 +192,12 @@ export const ElementCall: FC<ElementCallProps> = ({
|
||||
// observers running for the rest of the page's life. Null until then, which
|
||||
// is one render.
|
||||
const { controlledAudioDevices, callIntent } = params;
|
||||
const [mediaDevices, setMediaDevices] = useState<MediaDevices | null>(null);
|
||||
const [vm, setVm] = useState<AppViewModel | null>(null);
|
||||
useEffect(() => {
|
||||
const scope = new ObservableScope();
|
||||
setMediaDevices(
|
||||
new MediaDevices(scope, { controlledAudioDevices, callIntent }),
|
||||
);
|
||||
setVm(new AppViewModel(scope, { controlledAudioDevices, callIntent }));
|
||||
return (): void => {
|
||||
setMediaDevices(null);
|
||||
setVm(null);
|
||||
scope.end();
|
||||
};
|
||||
}, [controlledAudioDevices, callIntent]);
|
||||
@@ -220,8 +218,7 @@ export const ElementCall: FC<ElementCallProps> = ({
|
||||
// is told that Element Call has loaded, as the widget tells its client once
|
||||
// its own initialisation is over. Once per mount, however often the pieces
|
||||
// are later swapped out.
|
||||
const ready =
|
||||
container !== null && rtcSession !== null && mediaDevices !== null;
|
||||
const ready = container !== null && rtcSession !== null && vm !== null;
|
||||
const announcedLoaded = useRef(false);
|
||||
useEffect(() => {
|
||||
if (!ready || announcedLoaded.current) return;
|
||||
@@ -251,8 +248,10 @@ export const ElementCall: FC<ElementCallProps> = ({
|
||||
<Decoration>
|
||||
<TooltipProvider>
|
||||
<ClientProvider client={client}>
|
||||
<MediaDevicesContext value={mediaDevices}>
|
||||
<ProcessorProvider>
|
||||
<MediaDevicesContext value={vm.mediaDevices}>
|
||||
<BackgroundEffectsProvider
|
||||
effects={vm.backgroundEffects}
|
||||
>
|
||||
<CallView
|
||||
client={client}
|
||||
rtcSession={rtcSession}
|
||||
@@ -261,7 +260,7 @@ export const ElementCall: FC<ElementCallProps> = ({
|
||||
preload={params.preload}
|
||||
skipLobby={params.skipLobby}
|
||||
/>
|
||||
</ProcessorProvider>
|
||||
</BackgroundEffectsProvider>
|
||||
</MediaDevicesContext>
|
||||
</ClientProvider>
|
||||
</TooltipProvider>
|
||||
|
||||
@@ -19,7 +19,7 @@ services:
|
||||
- ./backend/playwright_homeserver-othersite.yaml:/data/cfg/homeserver.yaml:Z
|
||||
element-web:
|
||||
# Pin to a SHA so that upstream cannot break our tests. Renovate handles regular updates.
|
||||
image: ghcr.io/element-hq/element-web:develop@sha256:39c94f0c29e7c5614ac7264a872bb3643320158b3bf1730fc4742ddf672c27d6
|
||||
image: ghcr.io/element-hq/element-web:develop@sha256:589c4909fd5844a9db7dd730f9796186eabbb981580cdf9bfa597186a2fb81f3
|
||||
element-web-1:
|
||||
# Pin to a SHA so that upstream cannot break our tests. Renovate handles regular updates.
|
||||
image: ghcr.io/element-hq/element-web:develop@sha256:39c94f0c29e7c5614ac7264a872bb3643320158b3bf1730fc4742ddf672c27d6
|
||||
image: ghcr.io/element-hq/element-web:develop@sha256:589c4909fd5844a9db7dd730f9796186eabbb981580cdf9bfa597186a2fb81f3
|
||||
|
||||
+13
-1
@@ -3,7 +3,7 @@
|
||||
"user_menu": "User menu"
|
||||
},
|
||||
"action": {
|
||||
"blur_background": "Blur background",
|
||||
"add_background_image": "Add image",
|
||||
"close": "Close",
|
||||
"copy_link": "Copy link",
|
||||
"edit": "Edit",
|
||||
@@ -23,6 +23,13 @@
|
||||
"upload_file": "Upload file"
|
||||
},
|
||||
"analytics_notice": "By participating in this beta, you consent to the collection of anonymous data, which we use to improve the product. You can find more information about which data we track in our <2>Privacy Policy</2> and our <6>Cookie Policy</6>.",
|
||||
"background_effects": {
|
||||
"blur": "Blur",
|
||||
"none": "None",
|
||||
"numbered": "Background {{n}}",
|
||||
"slow": "Background effects run slowly on this platform, which may cause your video to stutter.",
|
||||
"unavailable": "Background effects are not supported on this platform."
|
||||
},
|
||||
"call_ended_view": {
|
||||
"create_account_button": "Create account",
|
||||
"create_account_prompt": "<0>Why not finish by setting up a password to keep your account?</0><1>You'll be able to keep your name and set an avatar for use on future calls</1>",
|
||||
@@ -92,6 +99,10 @@
|
||||
},
|
||||
"disconnected_banner": "Connectivity to the server has been lost.",
|
||||
"error": {
|
||||
"background_animated": "Animated images cannot be used as a background",
|
||||
"background_not_an_image": "That file is not a supported image",
|
||||
"background_not_kept": "That background could not be saved",
|
||||
"background_undecodable": "That image could not be opened",
|
||||
"call_is_not_supported": "Call is not supported",
|
||||
"call_not_found": "Call not found",
|
||||
"call_not_found_description": "<0>That link doesn't appear to belong to any existing call. Check that you have the right link, or <2>create a new one</2>.</0>",
|
||||
@@ -212,6 +223,7 @@
|
||||
"auto_gain_control_label": "Automatic gain control",
|
||||
"background_blur_header": "Background",
|
||||
"background_blur_label": "Blur the background of the video",
|
||||
"background_effects_header": "Background effects",
|
||||
"bitrate_label": "Bitrate",
|
||||
"blur_not_supported_by_browser": "(Background blur is not supported by this device.)",
|
||||
"camera_header": "Camera quality",
|
||||
|
||||
+3
-2
@@ -83,8 +83,8 @@
|
||||
"@types/sdp-transform": "^2.4.5",
|
||||
"@typescript-eslint/utils": "^8.61.0",
|
||||
"@use-gesture/react": "^10.2.11",
|
||||
"@vector-im/compound-design-tokens": "^11.0.0",
|
||||
"@vector-im/compound-web": "^10.0.0",
|
||||
"@vector-im/compound-design-tokens": "^11.1.0",
|
||||
"@vector-im/compound-web": "^10.3.0",
|
||||
"@vitejs/plugin-react": "^6.0.2",
|
||||
"@vitest/browser-playwright": "^4.1.5",
|
||||
"@vitest/coverage-v8": "^4.0.18",
|
||||
@@ -94,6 +94,7 @@
|
||||
"copy-to-clipboard": "^3.3.3",
|
||||
"eslint-plugin-element-call": "link:eslint",
|
||||
"eslint-plugin-storybook": "^10.3.6",
|
||||
"fake-indexeddb": "^6.2.5",
|
||||
"fetch-mock": "11.1.5",
|
||||
"i18next": "^26.0.0",
|
||||
"i18next-browser-languagedetector": "^8.0.0",
|
||||
|
||||
@@ -0,0 +1,852 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { expect, test, type Browser, type Page } from "@playwright/test";
|
||||
|
||||
import { SpaHelpers } from "./spa-helpers.ts";
|
||||
import {
|
||||
averageColour,
|
||||
cameraColour,
|
||||
type Colour,
|
||||
distance,
|
||||
} from "./utils/colour.ts";
|
||||
|
||||
// Background effects need WebGL2, and headless Firefox on a CI runner has none,
|
||||
// so it rightly offers none of them.
|
||||
test.skip(
|
||||
({ browserName }) => browserName === "firefox",
|
||||
"Background effects need WebGL2, which headless Firefox on CI does not have",
|
||||
);
|
||||
|
||||
test.describe("background effects", () => {
|
||||
test("pre-join preview shows the chosen effect", async ({ page }) => {
|
||||
test.slow();
|
||||
await page.goto("/");
|
||||
await SpaHelpers.createCall(page, "Effect user", "Background effects");
|
||||
const preview = page.locator("video").first();
|
||||
const camera = await cameraColour(preview);
|
||||
|
||||
await page.getByRole("button", { name: "Camera", exact: true }).click();
|
||||
const tile = page
|
||||
.getByRole("group", { name: "Background effects" })
|
||||
.getByRole("menuitemradio", { name: "Background 1" });
|
||||
await tile.click();
|
||||
await expect(tile).toHaveAttribute("aria-checked", "true");
|
||||
|
||||
// The fake camera shows no person, so the whole frame is background: the
|
||||
// preview takes on the picture's colours and leaves the camera's.
|
||||
const picture = await averageColour(tile.locator("img"));
|
||||
// Long enough for the first effect of a session to be built.
|
||||
await expect
|
||||
.poll(async () => distance(await averageColour(preview), picture), {
|
||||
timeout: 60_000,
|
||||
})
|
||||
.toBeLessThan(distance(camera, picture) / 2);
|
||||
});
|
||||
});
|
||||
|
||||
test.describe("no effect", () => {
|
||||
test("draws nothing, even after blurring", async ({ page }) => {
|
||||
test.setTimeout(120_000);
|
||||
await countDraws(page);
|
||||
await page.goto("/");
|
||||
await SpaHelpers.createCall(page, "Idle user", "No effect after blur");
|
||||
|
||||
await page.getByRole("button", { name: "Camera", exact: true }).click();
|
||||
const effects = page.getByRole("group", { name: "Background effects" });
|
||||
await effects.getByRole("menuitemradio", { name: "Blur" }).click();
|
||||
// Long enough for the first effect of a session to be built.
|
||||
await expect
|
||||
.poll(async () => drawsPerSecond(page), { timeout: 60_000 })
|
||||
.toBeGreaterThan(0);
|
||||
const blurred = await drawsPerSecond(page);
|
||||
|
||||
await effects.getByRole("menuitemradio", { name: "None" }).click();
|
||||
// A frame already on its way may still be drawn.
|
||||
await expect.poll(async () => drawsPerSecond(page)).toBe(0);
|
||||
test.info().annotations.push({
|
||||
type: "WebGL draws per second",
|
||||
description: `blurred ${blurred}, none 0`,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
/** Counts every WebGL draw on the page, the segmenter's and the effect's. */
|
||||
async function countDraws(page: Page): Promise<void> {
|
||||
await page.addInitScript(() => {
|
||||
const counter = { draws: 0 };
|
||||
(window as unknown as { webglDraws: typeof counter }).webglDraws = counter;
|
||||
for (const name of ["drawArrays", "drawElements"] as const) {
|
||||
const draw = WebGL2RenderingContext.prototype[name] as (
|
||||
...args: unknown[]
|
||||
) => void;
|
||||
WebGL2RenderingContext.prototype[name] = function (
|
||||
this: WebGL2RenderingContext,
|
||||
...args: unknown[]
|
||||
): void {
|
||||
counter.draws += 1;
|
||||
draw.apply(this, args);
|
||||
} as never;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/** WebGL draws over the next second. */
|
||||
async function drawsPerSecond(page: Page): Promise<number> {
|
||||
return page.evaluate(async () => {
|
||||
const counter = (window as unknown as { webglDraws: { draws: number } })
|
||||
.webglDraws;
|
||||
const before = counter.draws;
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
return counter.draws - before;
|
||||
});
|
||||
}
|
||||
|
||||
test.describe("background effects section", () => {
|
||||
test("section stays within the call area at every size", async ({ page }) => {
|
||||
test.setTimeout(120_000);
|
||||
await page.goto("/");
|
||||
await SpaHelpers.createCall(page, "Sizes user", "Background effect sizes");
|
||||
|
||||
for (const size of [
|
||||
{ width: 1280, height: 800 },
|
||||
{ width: 800, height: 600 },
|
||||
{ width: 600, height: 420 },
|
||||
{ width: 480, height: 360 },
|
||||
{ width: 360, height: 640 },
|
||||
]) {
|
||||
await page.setViewportSize(size);
|
||||
await page.getByRole("button", { name: "Camera", exact: true }).click();
|
||||
const menu = page.getByRole("menu");
|
||||
await expect(menu).toBeVisible();
|
||||
|
||||
const box = (await menu.boundingBox())!;
|
||||
const where = `at ${size.width}x${size.height}`;
|
||||
expect(box.x, where).toBeGreaterThanOrEqual(0);
|
||||
expect(box.y, where).toBeGreaterThanOrEqual(0);
|
||||
expect(box.x + box.width, where).toBeLessThanOrEqual(size.width);
|
||||
expect(box.y + box.height, where).toBeLessThanOrEqual(size.height);
|
||||
|
||||
const last = page
|
||||
.getByRole("group", { name: "Background effects" })
|
||||
.getByRole("menuitemradio")
|
||||
.last();
|
||||
await last.scrollIntoViewIfNeeded();
|
||||
await expect(last, where).toBeInViewport({ ratio: 1 });
|
||||
await last.click();
|
||||
await expect(last, where).toHaveAttribute("aria-checked", "true");
|
||||
|
||||
// Back to no effect for the next size, and close.
|
||||
await page
|
||||
.getByRole("group", { name: "Background effects" })
|
||||
.getByRole("menuitemradio", { name: "None" })
|
||||
.click();
|
||||
await page.keyboard.press("Escape");
|
||||
await expect(menu).toBeHidden();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test.describe("the chosen effect", () => {
|
||||
test("chosen effect survives leaving and rejoining", async ({ page }) => {
|
||||
test.setTimeout(120_000);
|
||||
await page.goto("/");
|
||||
await SpaHelpers.createCall(page, "Returning user", "Effect remembered");
|
||||
const { camera, picture } = await chooseAndWear(page, "Background 1");
|
||||
await page.getByTestId("lobby_joinCall").click();
|
||||
await page.getByTestId("incall_leave").click();
|
||||
await expect(page.getByRole("heading")).toContainText(
|
||||
"your call has ended",
|
||||
);
|
||||
|
||||
await page.goto("/");
|
||||
await page.getByTestId("home_callName").fill("Effect remembered again");
|
||||
await page.getByTestId("home_go").click();
|
||||
await expect(page.getByTestId("lobby_joinCall")).toBeVisible();
|
||||
await page.getByRole("button", { name: "Camera", exact: true }).click();
|
||||
const tile = page
|
||||
.getByRole("group", { name: "Background effects" })
|
||||
.getByRole("menuitemradio", { name: "Background 1" });
|
||||
await expect(tile).toHaveAttribute("aria-checked", "true");
|
||||
const preview = page.locator("video").first();
|
||||
await expect
|
||||
.poll(async () => distance(await averageColour(preview), picture), {
|
||||
timeout: 60_000,
|
||||
})
|
||||
.toBeLessThan(distance(camera, picture) / 2);
|
||||
});
|
||||
|
||||
test("second session still shows the existing error", async ({ page }) => {
|
||||
test.setTimeout(120_000);
|
||||
await page.goto("/");
|
||||
await SpaHelpers.createCall(page, "Two tabs", "Effect in two tabs");
|
||||
await chooseAndWear(page, "Background 1");
|
||||
|
||||
const second = await page.context().newPage();
|
||||
await second.goto("/");
|
||||
await expect(
|
||||
page.getByRole("heading", { name: "Opened in another tab" }),
|
||||
).toBeVisible();
|
||||
});
|
||||
});
|
||||
|
||||
/** Chooses an effect in the lobby, and waits until the preview wears it. */
|
||||
async function chooseAndWear(
|
||||
page: Page,
|
||||
name: string,
|
||||
): Promise<{ camera: Colour; picture: Colour }> {
|
||||
const preview = page.locator("video").first();
|
||||
const camera = await cameraColour(preview);
|
||||
await page.getByRole("button", { name: "Camera", exact: true }).click();
|
||||
const tile = page.getByRole("menuitemradio", { name });
|
||||
await tile.click();
|
||||
const picture = await averageColour(tile.locator("img"));
|
||||
await expect
|
||||
.poll(async () => distance(await averageColour(preview), picture), {
|
||||
timeout: 60_000,
|
||||
})
|
||||
.toBeLessThan(distance(camera, picture) / 2);
|
||||
await page.keyboard.press("Escape");
|
||||
return { camera, picture };
|
||||
}
|
||||
|
||||
test.describe("a background of one's own", () => {
|
||||
test("adds an image from the device", async ({ page }) => {
|
||||
test.setTimeout(120_000);
|
||||
await page.goto("/");
|
||||
await SpaHelpers.createCall(page, "Adding user", "Own background");
|
||||
const preview = page.locator("video").first();
|
||||
const camera = await cameraColour(preview);
|
||||
const red = await redImage(page);
|
||||
|
||||
await addImage(page, red);
|
||||
const mine = page
|
||||
.getByRole("group", { name: "Background effects" })
|
||||
.getByRole("menuitemradio", { name: "Background 3" });
|
||||
await expect(mine).toBeVisible();
|
||||
await expect(mine).toHaveAttribute("aria-checked", "false");
|
||||
|
||||
await mine.click();
|
||||
await expect
|
||||
.poll(async () => distance(await averageColour(preview), RED), {
|
||||
timeout: 60_000,
|
||||
})
|
||||
.toBeLessThan(distance(camera, RED) / 2);
|
||||
});
|
||||
|
||||
test("adding an image makes no upload", async ({ page }) => {
|
||||
test.setTimeout(120_000);
|
||||
await page.goto("/");
|
||||
await SpaHelpers.createCall(page, "Private user", "Own background kept");
|
||||
const preview = page.locator("video").first();
|
||||
const camera = await cameraColour(preview);
|
||||
const red = await redImage(page);
|
||||
const sent: { request: string; body: Buffer | null }[] = [];
|
||||
page.on("request", (request) =>
|
||||
sent.push({
|
||||
request: `${request.method()} ${request.url()}`,
|
||||
body: request.postDataBuffer(),
|
||||
}),
|
||||
);
|
||||
|
||||
await addImage(page, red);
|
||||
await page
|
||||
.getByRole("group", { name: "Background effects" })
|
||||
.getByRole("menuitemradio", { name: "Background 3" })
|
||||
.click();
|
||||
// Kept, chosen and in force: anything that would send it has had its turn.
|
||||
await expect
|
||||
.poll(async () => distance(await averageColour(preview), RED), {
|
||||
timeout: 60_000,
|
||||
})
|
||||
.toBeLessThan(distance(camera, RED) / 2);
|
||||
// The file as given, and the copy kept, which is encoded anew.
|
||||
const samples = [red, await keptBytes(page)].map((bytes) =>
|
||||
bytes.subarray(bytes.length - 64),
|
||||
);
|
||||
const carriesIt = sent.filter(
|
||||
({ request, body }) =>
|
||||
request.includes("/_matrix/media/") ||
|
||||
samples.some((sample) => body?.includes(sample)),
|
||||
);
|
||||
expect(carriesIt.map(({ request }) => request)).toEqual([]);
|
||||
});
|
||||
|
||||
test("added image survives rejoining and is gone after storage is cleared", async ({
|
||||
page,
|
||||
}) => {
|
||||
test.setTimeout(120_000);
|
||||
await page.goto("/");
|
||||
await SpaHelpers.createCall(page, "Keeping user", "Own background again");
|
||||
const preview = page.locator("video").first();
|
||||
const camera = await cameraColour(preview);
|
||||
const red = await redImage(page);
|
||||
await addImage(page, red);
|
||||
const mine = page
|
||||
.getByRole("group", { name: "Background effects" })
|
||||
.getByRole("menuitemradio", { name: "Background 3" });
|
||||
await mine.click();
|
||||
await page.keyboard.press("Escape");
|
||||
|
||||
await page.reload();
|
||||
await expect(page.getByTestId("lobby_joinCall")).toBeVisible();
|
||||
await page.getByRole("button", { name: "Camera", exact: true }).click();
|
||||
await expect(mine).toHaveAttribute("aria-checked", "true");
|
||||
await page.keyboard.press("Escape");
|
||||
await expect
|
||||
.poll(async () => distance(await averageColour(preview), RED), {
|
||||
timeout: 60_000,
|
||||
})
|
||||
.toBeLessThan(distance(camera, RED) / 2);
|
||||
|
||||
await page.evaluate(
|
||||
async () =>
|
||||
new Promise<void>((resolve, reject) => {
|
||||
const request = indexedDB.deleteDatabase(
|
||||
"element-call-background-images",
|
||||
);
|
||||
request.onsuccess = (): void => resolve();
|
||||
request.onerror = (): void => reject(request.error);
|
||||
request.onblocked = (): void => resolve();
|
||||
}),
|
||||
);
|
||||
await page.reload();
|
||||
await expect(page.getByTestId("lobby_joinCall")).toBeVisible();
|
||||
await page.getByRole("button", { name: "Camera", exact: true }).click();
|
||||
await expect(
|
||||
page.getByRole("group", { name: "Background effects" }),
|
||||
).toBeVisible();
|
||||
await expect(mine).toHaveCount(0);
|
||||
await page.keyboard.press("Escape");
|
||||
|
||||
await addImage(page, red);
|
||||
await expect(mine).toBeVisible();
|
||||
});
|
||||
|
||||
test("a removed image stays removed", async ({ page }) => {
|
||||
test.setTimeout(120_000);
|
||||
await page.goto("/");
|
||||
await SpaHelpers.createCall(
|
||||
page,
|
||||
"Removing user",
|
||||
"Own background removed",
|
||||
);
|
||||
await addImage(page, await redImage(page));
|
||||
const mine = page
|
||||
.getByRole("group", { name: "Background effects" })
|
||||
.getByRole("menuitemradio", { name: "Background 3" });
|
||||
await expect(mine).toHaveAttribute("aria-keyshortcuts", "Delete");
|
||||
await mine.focus();
|
||||
await page.keyboard.press("Delete");
|
||||
await expect(mine).toHaveCount(0);
|
||||
|
||||
await page.reload();
|
||||
await expect(page.getByTestId("lobby_joinCall")).toBeVisible();
|
||||
await page.getByRole("button", { name: "Camera", exact: true }).click();
|
||||
await expect(
|
||||
page.getByRole("group", { name: "Background effects" }),
|
||||
).toBeVisible();
|
||||
await expect(mine).toHaveCount(0);
|
||||
expect(await keptImages(page)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
test.describe("what can be added", () => {
|
||||
test("refuses an animated image", async ({ page }) => {
|
||||
test.setTimeout(120_000);
|
||||
await page.goto("/");
|
||||
await SpaHelpers.createCall(page, "Refused user", "Animated refused");
|
||||
await addImage(page, ANIMATED_GIF, {
|
||||
name: "moving.gif",
|
||||
type: "image/gif",
|
||||
});
|
||||
|
||||
await expect(
|
||||
page.getByText("Animated images cannot be used as a background"),
|
||||
).toBeVisible();
|
||||
const section = page.getByRole("group", { name: "Background effects" });
|
||||
await expect(
|
||||
section.getByRole("menuitemradio", { name: "Background 3" }),
|
||||
).toHaveCount(0);
|
||||
await expect(
|
||||
section.getByRole("menuitemradio", { name: "None" }),
|
||||
).toHaveAttribute("aria-checked", "true");
|
||||
expect(await keptImages(page)).toEqual([]);
|
||||
});
|
||||
|
||||
test("reduces an oversized image", async ({ page }) => {
|
||||
test.setTimeout(120_000);
|
||||
await page.goto("/");
|
||||
await SpaHelpers.createCall(page, "Large user", "Oversized reduced");
|
||||
await addImage(page, await pngImage(page, { width: 4000, height: 3000 }));
|
||||
|
||||
await expect(
|
||||
page
|
||||
.getByRole("group", { name: "Background effects" })
|
||||
.getByRole("menuitemradio", { name: "Background 3" }),
|
||||
).toBeVisible();
|
||||
await expect(page.getByRole("alert")).toHaveCount(0);
|
||||
const [kept] = await keptImages(page);
|
||||
expect(kept).toMatchObject({ width: 1920, height: 1440 });
|
||||
});
|
||||
|
||||
test("lays every image on an opaque ground", async ({ page }) => {
|
||||
test.setTimeout(120_000);
|
||||
await page.goto("/");
|
||||
await SpaHelpers.createCall(page, "Logo user", "Transparent made opaque");
|
||||
await addImage(
|
||||
page,
|
||||
await pngImage(page, { width: 320, height: 180, transparent: true }),
|
||||
);
|
||||
|
||||
await expect(
|
||||
page
|
||||
.getByRole("group", { name: "Background effects" })
|
||||
.getByRole("menuitemradio", { name: "Background 3" }),
|
||||
).toBeVisible();
|
||||
const [kept] = await keptImages(page);
|
||||
expect(kept.minAlpha).toBe(255);
|
||||
});
|
||||
});
|
||||
|
||||
/** Two frames of one pixel, red then blue. */
|
||||
const ANIMATED_GIF = Buffer.from(
|
||||
"R0lGODlhAQABAPEAAP8AAAAA/wAAAAAAACH/C05FVFNDQVBFMi4wAwEAAAAh+QQACgAAACwAAAAAAQABAAACAkQBACH5BAAKAAAALAAAAAABAAEAAAICTAEAOw==",
|
||||
"base64",
|
||||
);
|
||||
|
||||
/** A red PNG of the given size; transparent leaves its right half empty. */
|
||||
async function pngImage(
|
||||
page: Page,
|
||||
size: { width: number; height: number; transparent?: boolean },
|
||||
): Promise<Buffer> {
|
||||
const dataUrl = await page.evaluate(({ width, height, transparent }) => {
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = width;
|
||||
canvas.height = height;
|
||||
const context = canvas.getContext("2d")!;
|
||||
context.fillStyle = "rgb(255, 0, 0)";
|
||||
context.fillRect(0, 0, transparent ? width / 2 : width, height);
|
||||
return canvas.toDataURL("image/png");
|
||||
}, size);
|
||||
return Buffer.from(dataUrl.split(",")[1], "base64");
|
||||
}
|
||||
|
||||
/** The images the device keeps, as it keeps them. */
|
||||
async function keptImages(
|
||||
page: Page,
|
||||
): Promise<{ width: number; height: number; minAlpha: number }[]> {
|
||||
return page.evaluate(async () => {
|
||||
const db = await new Promise<IDBDatabase>((resolve, reject) => {
|
||||
const request = indexedDB.open("element-call-background-images");
|
||||
request.onsuccess = (): void => resolve(request.result);
|
||||
request.onerror = (): void => reject(request.error);
|
||||
});
|
||||
if (!db.objectStoreNames.contains("backgrounds")) return [];
|
||||
const kept = await new Promise<{ image: Blob }[]>((resolve, reject) => {
|
||||
const request = db
|
||||
.transaction("backgrounds")
|
||||
.objectStore("backgrounds")
|
||||
.getAll();
|
||||
request.onsuccess = (): void => resolve(request.result);
|
||||
request.onerror = (): void => reject(request.error);
|
||||
});
|
||||
db.close();
|
||||
return Promise.all(
|
||||
kept.map(async ({ image }) => {
|
||||
const bitmap = await createImageBitmap(image);
|
||||
const canvas = new OffscreenCanvas(bitmap.width, bitmap.height);
|
||||
const context = canvas.getContext("2d")!;
|
||||
context.drawImage(bitmap, 0, 0);
|
||||
const { data } = context.getImageData(
|
||||
0,
|
||||
0,
|
||||
bitmap.width,
|
||||
bitmap.height,
|
||||
);
|
||||
let minAlpha = 255;
|
||||
for (let i = 3; i < data.length; i += 4)
|
||||
minAlpha = Math.min(minAlpha, data[i]);
|
||||
return { width: bitmap.width, height: bitmap.height, minAlpha };
|
||||
}),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/** The bytes of the one image the device keeps. */
|
||||
async function keptBytes(page: Page): Promise<Buffer> {
|
||||
const base64 = await page.evaluate(
|
||||
async () =>
|
||||
new Promise<string>((resolve, reject) => {
|
||||
const open = indexedDB.open("element-call-background-images");
|
||||
open.onerror = (): void => reject(open.error);
|
||||
open.onsuccess = (): void => {
|
||||
const all = open.result
|
||||
.transaction("backgrounds")
|
||||
.objectStore("backgrounds")
|
||||
.getAll();
|
||||
all.onerror = (): void => reject(all.error);
|
||||
all.onsuccess = (): void => {
|
||||
const reader = new FileReader();
|
||||
reader.onload = (): void =>
|
||||
resolve((reader.result as string).split(",")[1]);
|
||||
reader.readAsDataURL((all.result[0] as { image: Blob }).image);
|
||||
};
|
||||
};
|
||||
}),
|
||||
);
|
||||
return Buffer.from(base64, "base64");
|
||||
}
|
||||
|
||||
const RED: Colour = [255, 0, 0];
|
||||
|
||||
/** A solid red PNG, unlike the camera and the shipped pictures. */
|
||||
async function redImage(page: Page): Promise<Buffer> {
|
||||
const dataUrl = await page.evaluate(() => {
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = 320;
|
||||
canvas.height = 180;
|
||||
const context = canvas.getContext("2d")!;
|
||||
context.fillStyle = "rgb(255, 0, 0)";
|
||||
context.fillRect(0, 0, canvas.width, canvas.height);
|
||||
return canvas.toDataURL("image/png");
|
||||
});
|
||||
return Buffer.from(dataUrl.split(",")[1], "base64");
|
||||
}
|
||||
|
||||
/** Adds an image from the camera menu's add tile, leaving the menu open. */
|
||||
async function addImage(
|
||||
page: Page,
|
||||
image: Buffer,
|
||||
{ name, type } = { name: "mine.png", type: "image/png" },
|
||||
): Promise<void> {
|
||||
if (!(await page.getByRole("menu").isVisible()))
|
||||
await page.getByRole("button", { name: "Camera", exact: true }).click();
|
||||
const chooser = page.waitForEvent("filechooser");
|
||||
await page.getByRole("menuitem", { name: "Add image" }).click();
|
||||
await (await chooser).setFiles({ name, mimeType: type, buffer: image });
|
||||
}
|
||||
|
||||
test.describe("joining with a background chosen", () => {
|
||||
test("publishes no frame of the room", async ({ browser }) => {
|
||||
// Two first builds of the pipeline, one of them held back.
|
||||
test.setTimeout(180_000);
|
||||
const hostContext = await browser.newContext();
|
||||
const host = await hostContext.newPage();
|
||||
await recordIncomingFrames(host);
|
||||
await host.goto("/");
|
||||
await SpaHelpers.createCall(host, "Host", "Join with a background", true);
|
||||
await expect(host.getByTestId("incall_videomute")).toBeEnabled({
|
||||
timeout: 10_000,
|
||||
});
|
||||
const inviteLink = await SpaHelpers.getCallInviteLink(host);
|
||||
await host.keyboard.press("Escape");
|
||||
|
||||
const guestContext = await browser.newContext();
|
||||
const guest = await guestContext.newPage();
|
||||
await guest.goto(inviteLink);
|
||||
await guest.getByTestId("joincall_displayName").fill("Guest");
|
||||
await guest.getByTestId("joincall_joincall").click();
|
||||
const preview = guest.locator("video").first();
|
||||
await expect(preview).toBeVisible();
|
||||
const camera = await cameraColour(preview);
|
||||
await guest.getByRole("button", { name: "Camera", exact: true }).click();
|
||||
const tile = guest.getByRole("menuitemradio", { name: "Background 1" });
|
||||
await tile.click();
|
||||
const picture = await averageColour(tile.locator("img"));
|
||||
await expect
|
||||
.poll(async () => distance(await averageColour(preview), picture), {
|
||||
timeout: 60_000,
|
||||
})
|
||||
.toBeLessThan(distance(camera, picture) / 2);
|
||||
// A pipeline built again for the call's track now takes as long as it
|
||||
// does on the slower route. Routing bypasses the cache, so it waits.
|
||||
let heldBack = 0;
|
||||
await guest.route("**/*.tflite*", async (route) => {
|
||||
heldBack += 1;
|
||||
await new Promise((resolve) => setTimeout(resolve, 15_000));
|
||||
await route.continue();
|
||||
});
|
||||
await guest.keyboard.press("Escape");
|
||||
await guest.getByTestId("lobby_joinCall").click();
|
||||
|
||||
const guestTile = host
|
||||
.getByTestId("videoTile")
|
||||
.filter({ has: host.getByTestId("name_tag").getByText("Guest") });
|
||||
await expect(guestTile.locator("video")).toBeVisible({ timeout: 60_000 });
|
||||
// Visible can come before the stream is attached.
|
||||
let trackId: string | undefined;
|
||||
await expect
|
||||
.poll(async () => {
|
||||
trackId = await guestTile
|
||||
.locator("video")
|
||||
.evaluate(
|
||||
(video: HTMLVideoElement) =>
|
||||
(video.srcObject as MediaStream | null)?.getVideoTracks()[0]?.id,
|
||||
);
|
||||
return trackId;
|
||||
})
|
||||
.toBeDefined();
|
||||
await expectNoFrameOfTheRoom(host, trackId!, 0, camera, picture);
|
||||
// Otherwise nothing was waited for.
|
||||
expect(heldBack).toBeGreaterThan(0);
|
||||
|
||||
await hostContext.close();
|
||||
await guestContext.close();
|
||||
});
|
||||
});
|
||||
|
||||
test.describe("the first build", () => {
|
||||
test("survives the camera turned off and on while it runs", async ({
|
||||
page,
|
||||
}) => {
|
||||
test.setTimeout(120_000);
|
||||
const failures: string[] = [];
|
||||
page.on("console", (m) => {
|
||||
if (m.text().includes("Failed to attach video processor"))
|
||||
failures.push(m.text());
|
||||
});
|
||||
await page.goto("/");
|
||||
await SpaHelpers.createCall(page, "Toggling user", "Toggle while building");
|
||||
const preview = page.locator("video").first();
|
||||
const camera = await cameraColour(preview);
|
||||
|
||||
const held = await holdBackTheModel(page);
|
||||
await page.getByRole("button", { name: "Camera", exact: true }).click();
|
||||
const tile = page.getByRole("menuitemradio", { name: "Background 1" });
|
||||
await tile.click();
|
||||
const picture = await averageColour(tile.locator("img"));
|
||||
await page.keyboard.press("Escape");
|
||||
// New preview tracks, each attached while the first is still building.
|
||||
for (let i = 0; i < 2; i++) {
|
||||
await page.getByTestId("incall_videomute").click();
|
||||
await page.getByTestId("incall_videomute").click();
|
||||
}
|
||||
|
||||
await expect
|
||||
.poll(async () => distance(await averageColour(preview), picture), {
|
||||
timeout: 60_000,
|
||||
})
|
||||
.toBeLessThan(distance(camera, picture) / 2);
|
||||
expect(failures).toEqual([]);
|
||||
expect(held()).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
test.describe("turning the camera on with a background chosen", () => {
|
||||
test("publishes no frame of the room when the camera started off", async ({
|
||||
browser,
|
||||
}) => {
|
||||
test.setTimeout(120_000);
|
||||
const { host, guest, close } = await hostAndGuest(browser);
|
||||
const camera = await cameraColour(guest.locator("video").first());
|
||||
await guest.getByTestId("incall_videomute").click();
|
||||
await guest.getByTestId("lobby_joinCall").click();
|
||||
|
||||
const picture = await chooseBackground(guest);
|
||||
const held = await holdBackTheModel(guest);
|
||||
await guest.getByTestId("incall_videomute").click();
|
||||
|
||||
const trackId = await incomingTrackId(host);
|
||||
await expectNoFrameOfTheRoom(host, trackId, 0, camera, picture);
|
||||
expect(held()).toBeGreaterThan(0);
|
||||
await close();
|
||||
});
|
||||
|
||||
test("publishes no frame of the room when the camera was turned off", async ({
|
||||
browser,
|
||||
}) => {
|
||||
test.setTimeout(120_000);
|
||||
const { host, guest, close } = await hostAndGuest(browser);
|
||||
const camera = await cameraColour(guest.locator("video").first());
|
||||
await guest.getByTestId("lobby_joinCall").click();
|
||||
const trackId = await incomingTrackId(host);
|
||||
await expect
|
||||
.poll(async () => (await framesOf(host, trackId)).length, {
|
||||
timeout: 20_000,
|
||||
})
|
||||
.toBeGreaterThan(0);
|
||||
await guest.getByTestId("incall_videomute").click();
|
||||
|
||||
const picture = await chooseBackground(guest);
|
||||
const held = await holdBackTheModel(guest);
|
||||
const before = (await framesOf(host, trackId)).length;
|
||||
await guest.getByTestId("incall_videomute").click();
|
||||
|
||||
await expectNoFrameOfTheRoom(host, trackId, before, camera, picture);
|
||||
expect(held()).toBeGreaterThan(0);
|
||||
await close();
|
||||
});
|
||||
});
|
||||
|
||||
/** A host in a call, recording what it receives, and a guest in its lobby. */
|
||||
async function hostAndGuest(browser: Browser): Promise<{
|
||||
host: Page;
|
||||
guest: Page;
|
||||
close: () => Promise<void>;
|
||||
}> {
|
||||
const hostContext = await browser.newContext();
|
||||
const host = await hostContext.newPage();
|
||||
await recordIncomingFrames(host);
|
||||
await host.goto("/");
|
||||
await SpaHelpers.createCall(host, "Host", "Camera with a background", true);
|
||||
await expect(host.getByTestId("incall_videomute")).toBeEnabled({
|
||||
timeout: 10_000,
|
||||
});
|
||||
const inviteLink = await SpaHelpers.getCallInviteLink(host);
|
||||
await host.keyboard.press("Escape");
|
||||
|
||||
const guestContext = await browser.newContext();
|
||||
const guest = await guestContext.newPage();
|
||||
await guest.goto(inviteLink);
|
||||
await guest.getByTestId("joincall_displayName").fill("Guest");
|
||||
await guest.getByTestId("joincall_joincall").click();
|
||||
await expect(guest.locator("video").first()).toBeVisible();
|
||||
return {
|
||||
host,
|
||||
guest,
|
||||
close: async (): Promise<void> => {
|
||||
await hostContext.close();
|
||||
await guestContext.close();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Chooses the first shipped picture in the call, and answers its colour. */
|
||||
async function chooseBackground(guest: Page): Promise<Colour> {
|
||||
// The lobby has a camera button of its own, so wait for the call's.
|
||||
await expect(guest.getByTestId("incall_leave")).toBeVisible({
|
||||
timeout: 30_000,
|
||||
});
|
||||
await expect(guest.getByTestId("incall_videomute")).toBeEnabled({
|
||||
timeout: 10_000,
|
||||
});
|
||||
await guest.getByRole("button", { name: "Camera", exact: true }).click();
|
||||
const tile = guest.getByRole("menuitemradio", { name: "Background 1" });
|
||||
await tile.click();
|
||||
await expect(tile).toHaveAttribute("aria-checked", "true");
|
||||
const picture = await averageColour(tile.locator("img"));
|
||||
await guest.keyboard.press("Escape");
|
||||
return picture;
|
||||
}
|
||||
|
||||
/** Makes the pipeline's first build as slow as on the slower route. */
|
||||
/** Answers how often it held the model back, which a check should see. */
|
||||
async function holdBackTheModel(page: Page): Promise<() => number> {
|
||||
let held = 0;
|
||||
// Routing bypasses the cache, so it waits.
|
||||
await page.route("**/*.tflite*", async (route) => {
|
||||
held += 1;
|
||||
await new Promise((resolve) => setTimeout(resolve, 5_000));
|
||||
await route.continue();
|
||||
});
|
||||
return () => held;
|
||||
}
|
||||
|
||||
/** The id of the guest's camera track, as the host receives it. */
|
||||
async function incomingTrackId(host: Page): Promise<string> {
|
||||
const guestTile = host
|
||||
.getByTestId("videoTile")
|
||||
.filter({ has: host.getByTestId("name_tag").getByText("Guest") });
|
||||
await expect(guestTile.locator("video")).toBeVisible({ timeout: 60_000 });
|
||||
// Visible can come before the stream is attached.
|
||||
let trackId: string | undefined;
|
||||
await expect
|
||||
.poll(async () => {
|
||||
trackId = await guestTile
|
||||
.locator("video")
|
||||
.evaluate(
|
||||
(video: HTMLVideoElement) =>
|
||||
(video.srcObject as MediaStream | null)?.getVideoTracks()[0]?.id,
|
||||
);
|
||||
return trackId;
|
||||
})
|
||||
.toBeDefined();
|
||||
return trackId!;
|
||||
}
|
||||
|
||||
/** Waits for the picture to arrive, then finds no frame of the room since. */
|
||||
async function expectNoFrameOfTheRoom(
|
||||
host: Page,
|
||||
trackId: string,
|
||||
since: number,
|
||||
camera: Colour,
|
||||
picture: Colour,
|
||||
): Promise<void> {
|
||||
await expect
|
||||
.poll(
|
||||
async () => {
|
||||
const frames = (await framesOf(host, trackId)).slice(since);
|
||||
// A black frame, as a track starts, is nearer the picture than the
|
||||
// camera, but shows neither.
|
||||
return frames.some(
|
||||
(f) =>
|
||||
distance(f, picture) <
|
||||
Math.min(distance(f, camera), distance(f, [0, 0, 0])),
|
||||
);
|
||||
},
|
||||
{ timeout: 60_000 },
|
||||
)
|
||||
.toBe(true);
|
||||
const frames = (await framesOf(host, trackId)).slice(since);
|
||||
expect(
|
||||
frames.filter((f) => distance(f, camera) < distance(f, picture)),
|
||||
).toEqual([]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Records the mean colour of every frame of every video track the page
|
||||
* receives, from the moment it arrives: a tile may attach it later than its
|
||||
* first frame.
|
||||
*/
|
||||
async function recordIncomingFrames(page: Page): Promise<void> {
|
||||
await page.addInitScript(() => {
|
||||
const frames: Record<string, [number, number, number][]> = {};
|
||||
(window as unknown as { receivedFrames: typeof frames }).receivedFrames =
|
||||
frames;
|
||||
const record = (track: MediaStreamTrack): void => {
|
||||
if (track.kind !== "video" || frames[track.id]) return;
|
||||
const seen: [number, number, number][] = (frames[track.id] = []);
|
||||
const video = document.createElement("video");
|
||||
video.muted = true;
|
||||
video.srcObject = new MediaStream([track]);
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = canvas.height = 16;
|
||||
const context = canvas.getContext("2d", { willReadFrequently: true })!;
|
||||
const step = (): void => {
|
||||
context.drawImage(video, 0, 0, 16, 16);
|
||||
const { data } = context.getImageData(0, 0, 16, 16);
|
||||
const sum = [0, 0, 0];
|
||||
for (let i = 0; i < data.length; i += 4)
|
||||
for (let c = 0; c < 3; c++) sum[c] += data[i + c];
|
||||
seen.push(sum.map((v) => v / (data.length / 4)) as Colour);
|
||||
video.requestVideoFrameCallback(step);
|
||||
};
|
||||
video.requestVideoFrameCallback(step);
|
||||
void video.play();
|
||||
};
|
||||
const Connection = window.RTCPeerConnection;
|
||||
window.RTCPeerConnection = class extends Connection {
|
||||
public constructor(...args: ConstructorParameters<typeof Connection>) {
|
||||
super(...args);
|
||||
this.addEventListener("track", (e) => record(e.track));
|
||||
}
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function framesOf(page: Page, trackId: string): Promise<Colour[]> {
|
||||
return page.evaluate(
|
||||
(id) =>
|
||||
(
|
||||
window as unknown as {
|
||||
receivedFrames: Record<string, [number, number, number][]>;
|
||||
}
|
||||
).receivedFrames[id] ?? [],
|
||||
trackId,
|
||||
);
|
||||
}
|
||||
@@ -129,7 +129,10 @@ async function openDeviceList(page: Page, pane: Locator): Promise<Locator> {
|
||||
.getByRole("button", { name: "Microphone" })
|
||||
.click({ timeout: 60_000 });
|
||||
await expect(page.getByRole("menu")).toBeVisible();
|
||||
const list = page.locator("[role='menu'] div[role='none']").first();
|
||||
// The area of the menu holding the device groups, not the menu's own.
|
||||
const list = page.locator(
|
||||
"[role='menu'] div[role='none']:has(> [role='group'])",
|
||||
);
|
||||
await expect(list).toBeVisible();
|
||||
return list;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { expect, type Locator, type Page, test } from "@playwright/test";
|
||||
|
||||
import { createUserAndRoom, resizeContainer, startHarness } from "./harness.ts";
|
||||
import { averageColour, cameraColour, expectWearing } from "../utils/colour.ts";
|
||||
|
||||
// Background effects need WebGL2, and headless Firefox on a CI runner has none,
|
||||
// so it rightly offers none of them.
|
||||
test.skip(
|
||||
({ browserName }) => browserName === "firefox",
|
||||
"Background effects need WebGL2, which headless Firefox on CI does not have",
|
||||
);
|
||||
|
||||
/**
|
||||
* Background effects with Element Call as a component in a host page: the
|
||||
* pipeline's model and WebAssembly have to load from the component's bundle,
|
||||
* and the portalled menu has to be sized against the call, not the window.
|
||||
* Driven from the lobby, which builds the same menu and pipeline.
|
||||
*/
|
||||
|
||||
// Sign-in, crypto setup and sync happen twice before anything shows.
|
||||
test.describe.configure({ timeout: 180_000 });
|
||||
|
||||
test("puts a picture on the call's preview", async ({ page }) => {
|
||||
const { username, roomId } = await createUserAndRoom("effects");
|
||||
const panes = await startHarness(page, username, roomId);
|
||||
const pane = panes.first();
|
||||
await resizeContainer(pane.getByTestId("call-container"), {
|
||||
width: 900,
|
||||
height: 480,
|
||||
});
|
||||
await expect(pane.getByTestId("lobby_joinCall")).toBeVisible({
|
||||
timeout: 60_000,
|
||||
});
|
||||
const preview = pane.locator("video").first();
|
||||
const camera = await cameraColour(preview);
|
||||
|
||||
const tile = await openEffect(page, pane, "Background 1");
|
||||
const picture = await averageColour(tile.locator("img"));
|
||||
await tile.click();
|
||||
await expect(tile).toHaveAttribute("aria-checked", "true");
|
||||
await expectWearing(preview, picture, camera);
|
||||
});
|
||||
|
||||
test("keeps every effect reachable in a small container", async ({ page }) => {
|
||||
const { username, roomId } = await createUserAndRoom("effectsreach");
|
||||
const panes = await startHarness(page, username, roomId);
|
||||
const pane = panes.first();
|
||||
const container = pane.getByTestId("call-container");
|
||||
await resizeContainer(container, { width: 400, height: 360 });
|
||||
await expect(pane.getByTestId("lobby_joinCall")).toBeVisible({
|
||||
timeout: 60_000,
|
||||
});
|
||||
|
||||
const last = await openEffect(page, pane, "Background 2");
|
||||
const list = page.locator("[role='menu'] div[role='none']").first();
|
||||
expect((await list.boundingBox())!.height).toBeLessThanOrEqual(
|
||||
(await container.boundingBox())!.height,
|
||||
);
|
||||
await last.scrollIntoViewIfNeeded();
|
||||
await expect(last).toBeInViewport({ ratio: 1 });
|
||||
await last.click();
|
||||
await expect(last).toHaveAttribute("aria-checked", "true");
|
||||
});
|
||||
|
||||
/** Opens a component's camera menu and returns one of its effects. */
|
||||
async function openEffect(
|
||||
page: Page,
|
||||
pane: Locator,
|
||||
name: string,
|
||||
): Promise<Locator> {
|
||||
await pane
|
||||
.getByRole("button", { name: "Camera", exact: true })
|
||||
.click({ timeout: 60_000 });
|
||||
// Portalled out of the pane, so found on the page.
|
||||
return page
|
||||
.getByRole("group", { name: "Background effects" })
|
||||
.getByRole("menuitemradio", { name });
|
||||
}
|
||||
@@ -125,25 +125,19 @@ test("leaves the host's own page unstyled", async ({ page }) => {
|
||||
timeout: 60_000,
|
||||
});
|
||||
|
||||
// Element Call's stylesheet is written for a page of its own: normalize.css
|
||||
// gives `html` a line height, Compound gives `body` its font and feature
|
||||
// settings, and the design tokens live on `:root`. None of that may reach the
|
||||
// host's document — the harness sets none of these itself, so anything other
|
||||
// than the browser's defaults here came from us.
|
||||
// Compound's document styles reach the host, but it loads them itself.
|
||||
const host = await page.evaluate(() => {
|
||||
const html = getComputedStyle(document.documentElement);
|
||||
const body = getComputedStyle(document.body);
|
||||
const sub = document.body.appendChild(document.createElement("sub"));
|
||||
const verticalAlign = getComputedStyle(sub).verticalAlign;
|
||||
sub.remove();
|
||||
return {
|
||||
lineHeight: html.lineHeight,
|
||||
fontFeatureSettings: body.fontFeatureSettings,
|
||||
token: html.getPropertyValue("--cpd-color-text-primary"),
|
||||
verticalAlign,
|
||||
token: getComputedStyle(document.documentElement).getPropertyValue(
|
||||
"--font-scale",
|
||||
),
|
||||
};
|
||||
});
|
||||
expect(host).toEqual({
|
||||
lineHeight: "normal",
|
||||
fontFeatureSettings: "normal",
|
||||
token: "",
|
||||
});
|
||||
expect(host).toEqual({ verticalAlign: "sub", token: "" });
|
||||
|
||||
// While inside the container, the same rules do apply
|
||||
const root = panes.first().locator("[data-element-call-root]");
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { expect, type Locator } from "@playwright/test";
|
||||
|
||||
export type Colour = [number, number, number];
|
||||
|
||||
/** The camera's colour, once the preview is showing it rather than nothing. */
|
||||
export async function cameraColour(preview: Locator): Promise<Colour> {
|
||||
await expect
|
||||
.poll(async () => Math.max(...(await averageColour(preview))))
|
||||
.toBeGreaterThan(40);
|
||||
return averageColour(preview);
|
||||
}
|
||||
|
||||
/** The mean colour of a video's current frame or an image, drawn small. */
|
||||
export async function averageColour(element: Locator): Promise<Colour> {
|
||||
return element.evaluate(
|
||||
async (source: HTMLVideoElement | HTMLImageElement) => {
|
||||
if (source instanceof HTMLImageElement) await source.decode();
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = canvas.height = 32;
|
||||
const context = canvas.getContext("2d")!;
|
||||
context.drawImage(source, 0, 0, 32, 32);
|
||||
const { data } = context.getImageData(0, 0, 32, 32);
|
||||
const sum = [0, 0, 0];
|
||||
for (let i = 0; i < data.length; i += 4)
|
||||
for (let c = 0; c < 3; c++) sum[c] += data[i + c];
|
||||
const pixels = data.length / 4;
|
||||
return sum.map((s) => s / pixels) as Colour;
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
export function distance(a: Colour, b: Colour): number {
|
||||
return Math.hypot(a[0] - b[0], a[1] - b[1], a[2] - b[2]);
|
||||
}
|
||||
|
||||
/** Waits until a preview shows a picture rather than the camera. */
|
||||
export async function expectWearing(
|
||||
preview: Locator,
|
||||
picture: Colour,
|
||||
camera: Colour,
|
||||
): Promise<void> {
|
||||
// Long enough for the first effect of a session to be built.
|
||||
await expect
|
||||
.poll(async () => distance(await averageColour(preview), picture), {
|
||||
timeout: 60_000,
|
||||
})
|
||||
.toBeLessThan(distance(camera, picture) / 2);
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { expect, test } from "@playwright/test";
|
||||
|
||||
import { widgetTest } from "../fixtures/widget-user.ts";
|
||||
import { TestHelpers } from "./test-helpers.ts";
|
||||
import { averageColour, cameraColour, expectWearing } from "../utils/colour.ts";
|
||||
|
||||
// Background effects need WebGL2, and headless Firefox on a CI runner has none,
|
||||
// so it rightly offers none of them.
|
||||
test.skip(
|
||||
({ browserName }) => browserName === "firefox",
|
||||
"Background effects need WebGL2, which headless Firefox on CI does not have",
|
||||
);
|
||||
|
||||
widgetTest(
|
||||
"puts a picture on the preview inside Element Web",
|
||||
async ({ asWidget }) => {
|
||||
test.slow();
|
||||
const { brooks } = asWidget;
|
||||
await TestHelpers.startCallInCurrentRoom(brooks.page, false);
|
||||
const frame = brooks.page
|
||||
.locator('iframe[title="Element Call"]')
|
||||
.contentFrame();
|
||||
await expect(frame.getByTestId("lobby_joinCall")).toBeVisible();
|
||||
const preview = frame.locator("video").first();
|
||||
const camera = await cameraColour(preview);
|
||||
|
||||
await frame.getByRole("button", { name: "Camera", exact: true }).click();
|
||||
const tile = frame
|
||||
.getByRole("group", { name: "Background effects" })
|
||||
.getByRole("menuitemradio", { name: "Background 1" });
|
||||
const picture = await averageColour(tile.locator("img"));
|
||||
await tile.click();
|
||||
await expect(tile).toHaveAttribute("aria-checked", "true");
|
||||
await expectWearing(preview, picture, camera);
|
||||
},
|
||||
);
|
||||
Generated
+16
-7
@@ -131,11 +131,11 @@ importers:
|
||||
specifier: ^10.2.11
|
||||
version: 10.3.1(react@19.2.8)
|
||||
'@vector-im/compound-design-tokens':
|
||||
specifier: ^11.0.0
|
||||
specifier: ^11.1.0
|
||||
version: 11.1.0(@types/react@19.2.18)(react@19.2.8)
|
||||
'@vector-im/compound-web':
|
||||
specifier: ^10.0.0
|
||||
version: 10.1.0(@fontsource/inconsolata@5.3.0)(@fontsource/inter@5.3.0)(@types/react-dom@19.2.5(@types/react@19.2.18))(@types/react@19.2.18)(@vector-im/compound-design-tokens@11.1.0(@types/react@19.2.18)(react@19.2.8))(react-dom@19.2.8(react@19.2.8))(react@19.2.8)
|
||||
specifier: ^10.3.0
|
||||
version: 10.3.0(@fontsource/inconsolata@5.3.0)(@fontsource/inter@5.3.0)(@types/react-dom@19.2.5(@types/react@19.2.18))(@types/react@19.2.18)(@vector-im/compound-design-tokens@11.1.0(@types/react@19.2.18)(react@19.2.8))(react-dom@19.2.8(react@19.2.8))(react@19.2.8)
|
||||
'@vitejs/plugin-react':
|
||||
specifier: ^6.0.2
|
||||
version: 6.1.1(@rolldown/plugin-babel@0.2.3(@babel/core@7.29.7(supports-color@7.2.0))(@babel/runtime@8.0.0)(rolldown@1.2.7)(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.103.1)(terser@5.46.1)(yaml@2.9.0)))(babel-plugin-react-compiler@1.0.0)(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.103.1)(terser@5.46.1)(yaml@2.9.0))
|
||||
@@ -163,6 +163,9 @@ importers:
|
||||
eslint-plugin-storybook:
|
||||
specifier: ^10.3.6
|
||||
version: 10.6.0(eslint@8.57.1(supports-color@7.2.0))(supports-color@7.2.0)(typescript@5.9.3)
|
||||
fake-indexeddb:
|
||||
specifier: ^6.2.5
|
||||
version: 6.2.5
|
||||
fetch-mock:
|
||||
specifier: 11.1.5
|
||||
version: 11.1.5(node-fetch@2.7.0)
|
||||
@@ -3240,13 +3243,13 @@ packages:
|
||||
react:
|
||||
optional: true
|
||||
|
||||
'@vector-im/compound-web@10.1.0':
|
||||
resolution: {integrity: sha512-Lhw9Q+RZo1GtDYjF5PXWTD0zdfbrz/vnSVBYFHC8Hq+YeDule2QfGbTRo8IVHeV7jK6ZzsLc8zb6553eeIRLYg==}
|
||||
'@vector-im/compound-web@10.3.0':
|
||||
resolution: {integrity: sha512-tce5hsNpdvYXfB5nYQgTK7WjTDqjdxpV1BeGsw67gYAWSFKJr9p6X5gzE79o+Tp/q5uykLuO4ozTBdjLsmUWMw==}
|
||||
peerDependencies:
|
||||
'@fontsource/inconsolata': ^5
|
||||
'@fontsource/inter': ^5
|
||||
'@types/react': '*'
|
||||
'@vector-im/compound-design-tokens': '>=1.6.1 <11.0.0'
|
||||
'@vector-im/compound-design-tokens': '>=1.6.1 <12.0.0'
|
||||
react: ^18 || ^19.0.0
|
||||
peerDependenciesMeta:
|
||||
'@types/react':
|
||||
@@ -3998,6 +4001,10 @@ packages:
|
||||
resolution: {integrity: sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==}
|
||||
engines: {node: '>=12.0.0'}
|
||||
|
||||
fake-indexeddb@6.2.5:
|
||||
resolution: {integrity: sha512-CGnyrvbhPlWYMngksqrSSUT1BAVP49dZocrHuK0SvtR0D5TMs5wP0o3j7jexDJW01KSadjBp1M/71o/KR3nD1w==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
fast-deep-equal@3.1.3:
|
||||
resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==}
|
||||
|
||||
@@ -8672,7 +8679,7 @@ snapshots:
|
||||
'@types/react': 19.2.18
|
||||
react: 19.2.8
|
||||
|
||||
'@vector-im/compound-web@10.1.0(@fontsource/inconsolata@5.3.0)(@fontsource/inter@5.3.0)(@types/react-dom@19.2.5(@types/react@19.2.18))(@types/react@19.2.18)(@vector-im/compound-design-tokens@11.1.0(@types/react@19.2.18)(react@19.2.8))(react-dom@19.2.8(react@19.2.8))(react@19.2.8)':
|
||||
'@vector-im/compound-web@10.3.0(@fontsource/inconsolata@5.3.0)(@fontsource/inter@5.3.0)(@types/react-dom@19.2.5(@types/react@19.2.18))(@types/react@19.2.18)(@vector-im/compound-design-tokens@11.1.0(@types/react@19.2.18)(react@19.2.8))(react-dom@19.2.8(react@19.2.8))(react@19.2.8)':
|
||||
dependencies:
|
||||
'@floating-ui/react': 0.27.20(react-dom@19.2.8(react@19.2.8))(react@19.2.8)
|
||||
'@floating-ui/react-dom': 2.1.9(react-dom@19.2.8(react@19.2.8))(react@19.2.8)
|
||||
@@ -9548,6 +9555,8 @@ snapshots:
|
||||
|
||||
expect-type@1.4.0: {}
|
||||
|
||||
fake-indexeddb@6.2.5: {}
|
||||
|
||||
fast-deep-equal@3.1.3: {}
|
||||
|
||||
fast-glob@3.3.3:
|
||||
|
||||
+3
-3
@@ -35,7 +35,7 @@ import { ErrorPage, LoadingPage } from "./FullScreenView";
|
||||
import { Initializer } from "./initializer";
|
||||
import { type WidgetHelpers } from "./widget";
|
||||
import { useTheme } from "./useTheme";
|
||||
import { ProcessorProvider } from "./livekit/TrackProcessorContext";
|
||||
import { BackgroundEffectsProvider } from "./livekit/BackgroundEffectsContext";
|
||||
import { type AppViewModel } from "./state/AppViewModel";
|
||||
import { MediaDevicesContext } from "./MediaDevicesContext";
|
||||
import {
|
||||
@@ -148,7 +148,7 @@ export const App: FC<Props> = ({ vm, widget }) => {
|
||||
loaded && clientReady ? (
|
||||
<ClientProvider client={widgetClient}>
|
||||
<MediaDevicesContext value={vm.mediaDevices}>
|
||||
<ProcessorProvider>
|
||||
<BackgroundEffectsProvider effects={vm.backgroundEffects}>
|
||||
<Sentry.ErrorBoundary
|
||||
fallback={(error) => <ErrorPage error={error} />}
|
||||
>
|
||||
@@ -159,7 +159,7 @@ export const App: FC<Props> = ({ vm, widget }) => {
|
||||
<SentryRoute path="*" element={<RoomPage />} />
|
||||
</Routes>
|
||||
</Sentry.ErrorBoundary>
|
||||
</ProcessorProvider>
|
||||
</BackgroundEffectsProvider>
|
||||
</MediaDevicesContext>
|
||||
</ClientProvider>
|
||||
) : (
|
||||
|
||||
@@ -15,7 +15,7 @@ exports[`AppBar > renders 1`] = `
|
||||
tabindex="0"
|
||||
>
|
||||
<div
|
||||
class="_indicator-icon_147l5_17"
|
||||
class="_indicator-icon_j097f_17"
|
||||
style="--cpd-icon-button-size: 100%;"
|
||||
>
|
||||
<svg
|
||||
@@ -58,7 +58,7 @@ exports[`AppBar > renders with title and subtitle 1`] = `
|
||||
tabindex="0"
|
||||
>
|
||||
<div
|
||||
class="_indicator-icon_147l5_17"
|
||||
class="_indicator-icon_j097f_17"
|
||||
style="--cpd-icon-button-size: 100%;"
|
||||
>
|
||||
<svg
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
.grid {
|
||||
display: grid;
|
||||
/* A zero floor, so a long label wraps inside its tile rather than widening
|
||||
the grid past the menu. */
|
||||
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||
grid-auto-rows: 1fr;
|
||||
gap: var(--cpd-space-3x);
|
||||
padding: var(--cpd-space-2x) var(--cpd-space-4x) 0;
|
||||
}
|
||||
|
||||
/* Within the grid, to outweigh the menu item styles the tiles are built on. */
|
||||
.grid .tile {
|
||||
position: relative;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
gap: var(--cpd-space-1x);
|
||||
min-inline-size: 0;
|
||||
padding: var(--cpd-space-3x) var(--cpd-space-2x);
|
||||
/* Not hidden: that makes the tile a scroll container, which focus scrolls
|
||||
into view without its border, and so without the focus ring. */
|
||||
overflow: clip;
|
||||
border: var(--cpd-border-width-2) solid
|
||||
var(--cpd-color-border-interactive-secondary);
|
||||
border-radius: var(--cpd-space-2x);
|
||||
background: none;
|
||||
color: var(--cpd-color-icon-secondary);
|
||||
font: inherit;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
/* As the menu items highlight on hover. */
|
||||
.grid .tile[data-highlighted] {
|
||||
background: var(--cpd-color-bg-action-secondary-hovered);
|
||||
}
|
||||
|
||||
.grid .tile[aria-checked="true"] {
|
||||
border-color: var(--cpd-color-border-accent);
|
||||
}
|
||||
|
||||
.grid .tile[data-disabled] {
|
||||
cursor: default;
|
||||
opacity: 0.55;
|
||||
}
|
||||
|
||||
.tile[data-disabled] .picture {
|
||||
filter: grayscale(1);
|
||||
}
|
||||
|
||||
.swatch {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.picture {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
inline-size: 100%;
|
||||
block-size: 100%;
|
||||
object-fit: cover;
|
||||
}
|
||||
|
||||
/* Positioned, or the picture, positioned itself, would cover it. */
|
||||
.mark {
|
||||
position: relative;
|
||||
display: inline-flex;
|
||||
color: var(--cpd-color-icon-accent-primary);
|
||||
}
|
||||
|
||||
/* On a picture the mark needs a ground of its own to be seen. */
|
||||
.picture ~ .mark {
|
||||
border-radius: 50%;
|
||||
background: var(--cpd-color-bg-canvas-default);
|
||||
box-shadow: 0 0 0 var(--cpd-border-width-2) var(--cpd-color-bg-canvas-default);
|
||||
}
|
||||
|
||||
.label {
|
||||
max-inline-size: 100%;
|
||||
font: var(--cpd-font-body-sm-medium);
|
||||
color: var(--cpd-color-text-primary);
|
||||
text-align: center;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
.removable {
|
||||
position: relative;
|
||||
display: flex;
|
||||
min-inline-size: 0;
|
||||
}
|
||||
|
||||
.removable > .tile {
|
||||
flex: 1;
|
||||
}
|
||||
|
||||
/* Clear of the target for choosing the tile. */
|
||||
.remove {
|
||||
position: absolute;
|
||||
inset-block-start: 0;
|
||||
inset-inline-end: 0;
|
||||
translate: 20% -20%;
|
||||
display: none;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
inline-size: 28px;
|
||||
block-size: 28px;
|
||||
border-radius: 50%;
|
||||
background: var(--cpd-color-bg-canvas-default);
|
||||
color: var(--cpd-color-icon-critical-primary);
|
||||
box-shadow: 0 1.2px 2.4px 0 rgb(0 0 0 / 15%);
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.remove:hover {
|
||||
background: var(--cpd-color-bg-critical-subtle-hovered);
|
||||
box-shadow: 0 0 0 var(--cpd-border-width-1)
|
||||
var(--cpd-color-border-critical-subtle);
|
||||
}
|
||||
|
||||
.removable:hover .remove,
|
||||
.removable:focus-within .remove {
|
||||
display: flex;
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import {
|
||||
type FC,
|
||||
type KeyboardEvent,
|
||||
type ReactNode,
|
||||
type ReactElement,
|
||||
} from "react";
|
||||
import { VisuallyHidden } from "@radix-ui/react-visually-hidden";
|
||||
import { InlineSpinner, MenuItem, Tooltip } from "@vector-im/compound-web";
|
||||
import {
|
||||
BlockIcon,
|
||||
BlurIcon,
|
||||
CheckCircleSolidIcon,
|
||||
CloseIcon,
|
||||
PlusIcon,
|
||||
} from "@vector-im/compound-design-tokens/assets/web/icons";
|
||||
|
||||
import styles from "./BackgroundEffectGrid.module.css";
|
||||
|
||||
/** One choice in the camera menu's Background effects section. */
|
||||
export interface BackgroundEffectOption {
|
||||
id: string;
|
||||
/** The tile's accessible name; drawn only where the tile has no picture. */
|
||||
label: string;
|
||||
kind: "none" | "blur" | "image";
|
||||
/** For kind "image". */
|
||||
imageUrl?: string;
|
||||
/** Whether it is the user's to remove. */
|
||||
removable?: boolean;
|
||||
}
|
||||
|
||||
interface Props {
|
||||
/** Names the group, whose heading is hidden from assistive technology. */
|
||||
label: string;
|
||||
heading: ReactNode;
|
||||
effects: BackgroundEffectOption[];
|
||||
selected: string | undefined;
|
||||
/** Undefined where effects can't run: all but no effect are disabled. */
|
||||
onSelect: ((id: string) => void) | undefined;
|
||||
/** Shown on the selected tile until its effect is on screen. */
|
||||
settling?: boolean;
|
||||
/** Id of the text saying what the user should know before choosing. */
|
||||
describedBy?: string;
|
||||
/** Offers to add a background of one's own. Omit to leave that tile out. */
|
||||
onAdd?: () => void;
|
||||
addLabel?: string;
|
||||
/** Removes a removable one, never the one in force. Omit to offer none. */
|
||||
onRemove?: (id: string) => void;
|
||||
removeLabel?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* The background effects as a grid of tiles: one radio choice, walked by the
|
||||
* menu's arrow keys like the device rows.
|
||||
*/
|
||||
export const BackgroundEffectGrid: FC<Props> = ({
|
||||
label,
|
||||
heading,
|
||||
effects,
|
||||
selected,
|
||||
onSelect,
|
||||
settling = false,
|
||||
describedBy,
|
||||
onAdd,
|
||||
addLabel,
|
||||
onRemove,
|
||||
removeLabel,
|
||||
}) => {
|
||||
const choose = (id: string): void => {
|
||||
if (id !== selected) onSelect?.(id);
|
||||
};
|
||||
const tiles = effects.map((effect) => {
|
||||
const checked = effect.id === selected;
|
||||
const disabled = onSelect === undefined && effect.kind !== "none";
|
||||
const remove =
|
||||
effect.removable && !checked && onRemove !== undefined
|
||||
? (): void => onRemove(effect.id)
|
||||
: undefined;
|
||||
const removeProps =
|
||||
remove === undefined
|
||||
? {}
|
||||
: {
|
||||
"aria-keyshortcuts": "Delete",
|
||||
onKeyDown: (e: KeyboardEvent): void => {
|
||||
if (e.key !== "Delete" && e.key !== "Backspace") return;
|
||||
e.preventDefault();
|
||||
remove();
|
||||
},
|
||||
};
|
||||
const content = (
|
||||
<TileContent
|
||||
effect={effect}
|
||||
checked={checked}
|
||||
settling={checked && settling}
|
||||
/>
|
||||
);
|
||||
const tile = (
|
||||
<MenuItem
|
||||
key={effect.id}
|
||||
role="menuitemradio"
|
||||
aria-checked={checked}
|
||||
aria-busy={checked && settling}
|
||||
disabled={disabled}
|
||||
data-disabled={disabled ? "" : undefined}
|
||||
label={null}
|
||||
hideChevron
|
||||
className={styles.tile}
|
||||
// Without a handler it isn't one of the menu's items, so the keyboard
|
||||
// passes over it.
|
||||
onSelect={
|
||||
disabled
|
||||
? null
|
||||
: (e): void => {
|
||||
// Kept open, so the user sees what they chose take effect.
|
||||
e.preventDefault();
|
||||
choose(effect.id);
|
||||
}
|
||||
}
|
||||
{...removeProps}
|
||||
>
|
||||
{content}
|
||||
</MenuItem>
|
||||
);
|
||||
if (remove === undefined) return tile;
|
||||
// Beside the item, not in it: inside, the item takes the press first and
|
||||
// the tile is chosen instead.
|
||||
return (
|
||||
<div key={effect.id} className={styles.removable}>
|
||||
{tile}
|
||||
<Tooltip label={removeLabel ?? ""}>
|
||||
{/* Not a control of its own, so the menu keeps one keyboard model:
|
||||
the keyboard reaches the same action through Delete. */}
|
||||
<span aria-hidden className={styles.remove} onClick={remove}>
|
||||
<CloseIcon width={20} height={20} />
|
||||
</span>
|
||||
</Tooltip>
|
||||
</div>
|
||||
);
|
||||
});
|
||||
|
||||
// A command rather than a choice, so an item beside the radios.
|
||||
const addContent = (
|
||||
<>
|
||||
<span aria-hidden className={styles.swatch}>
|
||||
<PlusIcon width={24} height={24} />
|
||||
</span>
|
||||
<VisuallyHidden>{addLabel}</VisuallyHidden>
|
||||
</>
|
||||
);
|
||||
const addTile =
|
||||
onAdd === undefined ? null : (
|
||||
<MenuItem
|
||||
label={null}
|
||||
hideChevron
|
||||
className={styles.tile}
|
||||
onSelect={(e) => {
|
||||
e.preventDefault();
|
||||
onAdd();
|
||||
}}
|
||||
>
|
||||
{addContent}
|
||||
</MenuItem>
|
||||
);
|
||||
|
||||
return (
|
||||
<div role="group" aria-label={label} aria-describedby={describedBy}>
|
||||
{heading}
|
||||
<div role="none" className={styles.grid}>
|
||||
{tiles}
|
||||
{addTile}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
function TileContent({
|
||||
effect,
|
||||
checked,
|
||||
settling,
|
||||
}: {
|
||||
effect: BackgroundEffectOption;
|
||||
checked: boolean;
|
||||
settling: boolean;
|
||||
}): ReactElement {
|
||||
return (
|
||||
<>
|
||||
<span aria-hidden className={styles.swatch}>
|
||||
{effect.kind === "image" && (
|
||||
<img className={styles.picture} src={effect.imageUrl} alt="" />
|
||||
)}
|
||||
{settling ? (
|
||||
<span className={styles.mark}>
|
||||
<InlineSpinner size={20} />
|
||||
</span>
|
||||
) : checked ? (
|
||||
<CheckCircleSolidIcon
|
||||
className={styles.mark}
|
||||
width={20}
|
||||
height={20}
|
||||
/>
|
||||
) : effect.kind === "none" ? (
|
||||
<BlockIcon width={20} height={20} />
|
||||
) : effect.kind === "blur" ? (
|
||||
<BlurIcon width={20} height={20} />
|
||||
) : null}
|
||||
</span>
|
||||
{effect.kind === "image" ? (
|
||||
<VisuallyHidden>{effect.label}</VisuallyHidden>
|
||||
) : (
|
||||
<span className={styles.label}>{effect.label}</span>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -22,6 +22,7 @@ import { MediaDevices } from "../state/MediaDevices";
|
||||
import { globalScope } from "../state/ObservableScope";
|
||||
import { constant } from "../state/Behavior";
|
||||
import { type LayoutMode } from "../state/LayoutSwitchViewModel";
|
||||
import { shippedBackgrounds } from "../livekit/backgroundEffects";
|
||||
|
||||
// consts for tests
|
||||
const reactionIdentifier = "@user:example.com:DEVICE";
|
||||
@@ -123,8 +124,23 @@ export const Default: Story = {
|
||||
toggleAudio: fn(),
|
||||
toggleVideo: fn(),
|
||||
toggleScreenSharing: fn(),
|
||||
toggleBlur: fn(),
|
||||
videoBlurEnabled: true,
|
||||
selectBackgroundEffect: fn(),
|
||||
backgroundEffect: "blur",
|
||||
backgroundEffects: [
|
||||
{ id: "none", kind: "none", removable: false },
|
||||
{ id: "blur", kind: "blur", removable: false },
|
||||
...shippedBackgrounds.map((background) => ({
|
||||
id: `image:${background.id}`,
|
||||
kind: "image" as const,
|
||||
imageUrl: background.imagePath,
|
||||
removable: false,
|
||||
})),
|
||||
],
|
||||
backgroundEffectNotice: undefined,
|
||||
backgroundEffectSettling: false,
|
||||
backgroundImageRefusal: undefined,
|
||||
addBackgroundImage: fn(),
|
||||
removeBackgroundEffect: fn(),
|
||||
hangup: fn(),
|
||||
buttonSize: "lg",
|
||||
showFooter: true,
|
||||
|
||||
@@ -7,6 +7,7 @@ Please see LICENSE in the repository root for full details.
|
||||
|
||||
import { type FC, type JSX, type Ref, useMemo } from "react";
|
||||
import classNames from "classnames";
|
||||
import { useTranslation } from "react-i18next";
|
||||
|
||||
import LogoMark from "../icons/LogoMark.svg?react";
|
||||
import LogoType from "../icons/LogoType.svg?react";
|
||||
@@ -31,6 +32,22 @@ import { type ViewModel } from "../state/ViewModel";
|
||||
import { useBehavior } from "../useBehavior";
|
||||
import { type LayoutSwitchViewModel } from "../state/LayoutSwitchViewModel";
|
||||
import { LayoutSwitch } from "../room/LayoutSwitch";
|
||||
import { type BackgroundEffectOption } from "./BackgroundEffectGrid";
|
||||
import { type UnusableReason } from "../livekit/backgroundImages";
|
||||
|
||||
export interface BackgroundImageRefusal {
|
||||
reason: UnusableReason | "not-kept";
|
||||
}
|
||||
|
||||
/** A background effect on offer, which the view names. */
|
||||
export interface BackgroundEffectChoice {
|
||||
/** Its option id, as `backgroundEffect` gives the one in force. */
|
||||
id: string;
|
||||
kind: "none" | "blur" | "image";
|
||||
imageUrl?: string;
|
||||
/** One the user added, which they can take away again. */
|
||||
removable: boolean;
|
||||
}
|
||||
|
||||
export interface AudioOutputSwitcher {
|
||||
targetOutput: string;
|
||||
@@ -56,20 +73,35 @@ export interface FooterActions {
|
||||
toggleAudio: (() => void) | undefined;
|
||||
/** Also controls if the videoMute button is disabled */
|
||||
toggleVideo: (() => void) | undefined;
|
||||
toggleBlur: (() => void) | undefined;
|
||||
/** Undefined where background effects can't be chosen. */
|
||||
selectBackgroundEffect: ((id: string) => void) | undefined;
|
||||
/** Undefined where no background can be added. */
|
||||
addBackgroundImage: ((file: File) => void) | undefined;
|
||||
/** Removes an added background by its option id; never the one in force. */
|
||||
removeBackgroundEffect: ((id: string) => void) | undefined;
|
||||
toggleScreenSharing: (() => void) | undefined;
|
||||
/** Also controls if the settings button is visible */
|
||||
openSettings: (() => void) | undefined;
|
||||
/** Also controls if the hangup button is visible */
|
||||
hangup: (() => void) | undefined;
|
||||
}
|
||||
|
||||
// we do not use any ? optional properties so that the vm type is including all fields.
|
||||
export interface FooterState {
|
||||
audioEnabled: boolean;
|
||||
audioBusy: boolean;
|
||||
videoEnabled: boolean;
|
||||
videoBusy: boolean;
|
||||
videoBlurEnabled: boolean;
|
||||
/** The background effect in force, as its option id. */
|
||||
backgroundEffect: string;
|
||||
/** Every effect on offer, in the order they are shown. */
|
||||
backgroundEffects: BackgroundEffectChoice[];
|
||||
/** What the user is told about effects here, if anything. */
|
||||
backgroundEffectNotice: "unavailable" | "slow" | undefined;
|
||||
/** Whether the first effect chosen is still being prepared. */
|
||||
backgroundEffectSettling: boolean;
|
||||
/** Why the last file offered couldn't be kept, if it couldn't. */
|
||||
backgroundImageRefusal: BackgroundImageRefusal | undefined;
|
||||
showFooter: boolean;
|
||||
|
||||
/* This is needed for WindowMode = "flat" */
|
||||
@@ -117,12 +149,14 @@ export interface FooterProps {
|
||||
children?: JSX.Element | JSX.Element[] | false;
|
||||
vm: ViewModel<FooterSnapshot>;
|
||||
}
|
||||
|
||||
export const CallFooter: FC<FooterProps> = ({
|
||||
className,
|
||||
ref,
|
||||
children,
|
||||
vm,
|
||||
}) => {
|
||||
const { t } = useTranslation();
|
||||
const asOverlay = useBehavior(vm.asOverlay$);
|
||||
const showFooter = useBehavior(vm.showFooter$);
|
||||
const hideControls = useBehavior(vm.hideControls$);
|
||||
@@ -151,8 +185,30 @@ export const CallFooter: FC<FooterProps> = ({
|
||||
const selectedAudioOutput = useBehavior(vm.selectedAudioOutput$);
|
||||
const selectAudioOutputOption = useBehavior(vm.selectAudioOutputOption$);
|
||||
const selectVideoButtonOption = useBehavior(vm.selectVideoButtonOption$);
|
||||
const toggleBlur = useBehavior(vm.toggleBlur$);
|
||||
const videoBlurEnabled = useBehavior(vm.videoBlurEnabled$);
|
||||
const backgroundEffect = useBehavior(vm.backgroundEffect$);
|
||||
const selectBackgroundEffect = useBehavior(vm.selectBackgroundEffect$);
|
||||
const backgroundEffectNotice = useBehavior(vm.backgroundEffectNotice$);
|
||||
const backgroundEffectSettling = useBehavior(vm.backgroundEffectSettling$);
|
||||
const addBackgroundImage = useBehavior(vm.addBackgroundImage$);
|
||||
const removeBackgroundEffect = useBehavior(vm.removeBackgroundEffect$);
|
||||
const refusal = useBehavior(vm.backgroundImageRefusal$);
|
||||
const refusalMessage = useMemo(() => {
|
||||
switch (refusal?.reason) {
|
||||
case undefined:
|
||||
return undefined;
|
||||
case "not-an-image":
|
||||
return { text: t("error.background_not_an_image") };
|
||||
case "animated":
|
||||
return { text: t("error.background_animated") };
|
||||
case "undecodable":
|
||||
return { text: t("error.background_undecodable") };
|
||||
case "not-kept":
|
||||
return { text: t("error.background_not_kept") };
|
||||
}
|
||||
}, [refusal, t]);
|
||||
const backgroundEffects = useBackgroundEffectLabels(
|
||||
useBehavior(vm.backgroundEffects$),
|
||||
);
|
||||
const buttonSize = useBehavior(vm.buttonSize$);
|
||||
const showLogo = useBehavior(vm.showLogo$);
|
||||
|
||||
@@ -214,8 +270,20 @@ export const CallFooter: FC<FooterProps> = ({
|
||||
options={videoOptions}
|
||||
selectedOption={selectedVideo}
|
||||
onSelect={selectVideoButtonOption}
|
||||
videoBlurToggleClick={toggleBlur}
|
||||
videoBlurEnabled={videoBlurEnabled}
|
||||
backgroundEffects={backgroundEffects}
|
||||
selectedBackgroundEffect={backgroundEffect}
|
||||
onSelectBackgroundEffect={selectBackgroundEffect}
|
||||
backgroundEffectSettling={backgroundEffectSettling}
|
||||
onAddBackgroundImage={addBackgroundImage}
|
||||
onRemoveBackgroundEffect={removeBackgroundEffect}
|
||||
backgroundImageRefusal={refusalMessage}
|
||||
backgroundEffectNotice={
|
||||
backgroundEffectNotice === "unavailable"
|
||||
? t("background_effects.unavailable")
|
||||
: backgroundEffectNotice === "slow"
|
||||
? t("background_effects.slow")
|
||||
: undefined
|
||||
}
|
||||
/>,
|
||||
);
|
||||
} else {
|
||||
@@ -340,3 +408,21 @@ const TilesDebugInfo: FC<TilesDebugInfoProps> = ({ generation$ }) => {
|
||||
const generation = useBehavior(generation$);
|
||||
return `Tiles generation: ${generation}`;
|
||||
};
|
||||
|
||||
function useBackgroundEffectLabels(
|
||||
choices: BackgroundEffectChoice[],
|
||||
): BackgroundEffectOption[] {
|
||||
const { t } = useTranslation();
|
||||
return useMemo(() => {
|
||||
let pictures = 0;
|
||||
return choices.map((choice) => ({
|
||||
...choice,
|
||||
label:
|
||||
choice.kind === "none"
|
||||
? t("background_effects.none")
|
||||
: choice.kind === "blur"
|
||||
? t("background_effects.blur")
|
||||
: t("background_effects.numbered", { n: ++pictures }),
|
||||
}));
|
||||
}, [choices, t]);
|
||||
}
|
||||
|
||||
@@ -5,17 +5,34 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { BehaviorSubject } from "rxjs";
|
||||
|
||||
import { testScope, mockMuteStates, mockMediaDevices } from "../utils/test";
|
||||
import {
|
||||
flushPromises,
|
||||
testScope,
|
||||
mockMuteStates,
|
||||
mockMediaDevices,
|
||||
} from "../utils/test";
|
||||
import { constant } from "../state/Behavior";
|
||||
import type { CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import type { Alignment, Layout } from "../state/layout-types";
|
||||
import type { SpotlightTileViewModel } from "../state/TileViewModel";
|
||||
import type { DeviceLabel } from "../state/MediaDevices";
|
||||
import { createCallFooterViewModel } from "./CallFooterViewModel";
|
||||
import {
|
||||
createCallFooterViewModel,
|
||||
createLobbyFooterViewModel,
|
||||
} from "./CallFooterViewModel";
|
||||
import { HeaderStyle } from "../UrlParams";
|
||||
import { backgroundEffect as backgroundEffectSetting } from "../settings/settings";
|
||||
import { shippedBackgrounds } from "../livekit/backgroundEffects";
|
||||
import type * as BackgroundImages from "../livekit/backgroundImages";
|
||||
import {
|
||||
type AddedBackground,
|
||||
addedBackgrounds,
|
||||
maxAddedBackgrounds,
|
||||
UnusableImage,
|
||||
} from "../livekit/backgroundImages";
|
||||
|
||||
const platformMock = vi.hoisted(() => vi.fn(() => "desktop"));
|
||||
vi.mock("../Platform", () => ({
|
||||
@@ -24,12 +41,30 @@ vi.mock("../Platform", () => ({
|
||||
},
|
||||
}));
|
||||
|
||||
// Prevent supportsBackgroundProcessors from throwing in jsdom – it is not
|
||||
// exercised by these tests (only used in `videoToggles`, not `videoOptions`).
|
||||
// The SDK's own check needs WebGL and canvas APIs jsdom does not have.
|
||||
const sdkSupportMock = vi.hoisted(() => vi.fn(() => false));
|
||||
const modernRouteMock = vi.hoisted(() => vi.fn(() => true));
|
||||
vi.mock("@livekit/track-processors", () => ({
|
||||
supportsBackgroundProcessors: (): boolean => false,
|
||||
supportsBackgroundProcessors: (): boolean => sdkSupportMock(),
|
||||
supportsModernBackgroundProcessors: (): boolean => modernRouteMock(),
|
||||
}));
|
||||
|
||||
const store = vi.hoisted(() => ({
|
||||
add: vi.fn<(file: Blob) => Promise<string>>(),
|
||||
remove: vi.fn<(id: string) => Promise<void>>(),
|
||||
}));
|
||||
vi.mock("../livekit/backgroundImages", async (original) => {
|
||||
const { BehaviorSubject } = await import("rxjs");
|
||||
return {
|
||||
...(await original<typeof BackgroundImages>()),
|
||||
addedBackgrounds: {
|
||||
added$: new BehaviorSubject<AddedBackground[] | undefined>([]),
|
||||
add: store.add,
|
||||
remove: store.remove,
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
const outputSelectionMock = vi.hoisted(() => vi.fn(() => true));
|
||||
vi.mock("livekit-client", () => ({
|
||||
supportsAudioOutputSelection: (): boolean => outputSelectionMock(),
|
||||
@@ -109,6 +144,7 @@ describe("createCallFooterViewModel", () => {
|
||||
buildMinimalCallViewModel(gridLayout),
|
||||
mockMuteStates(),
|
||||
twoMicsAndOneCamMediaDevices,
|
||||
constant(false),
|
||||
/* reactionIdentifier */ undefined,
|
||||
{ showControls: true, header: HeaderStyle.Standard },
|
||||
);
|
||||
@@ -151,6 +187,7 @@ describe("createCallFooterViewModel", () => {
|
||||
select: vi.fn(),
|
||||
},
|
||||
}),
|
||||
constant(false),
|
||||
/* reactionIdentifier */ undefined,
|
||||
{ showControls: true, header: HeaderStyle.Standard },
|
||||
);
|
||||
@@ -169,6 +206,7 @@ describe("createCallFooterViewModel", () => {
|
||||
buildMinimalCallViewModel(layout),
|
||||
mockMuteStates(),
|
||||
twoMicsAndOneCamMediaDevices,
|
||||
constant(false),
|
||||
/* reactionIdentifier */ undefined,
|
||||
{ showControls: true, header: HeaderStyle.Standard },
|
||||
);
|
||||
@@ -191,6 +229,7 @@ describe("createCallFooterViewModel", () => {
|
||||
buildMinimalCallViewModel(gridLayout),
|
||||
mockMuteStates(),
|
||||
twoMicsAndOneCamMediaDevices,
|
||||
constant(false),
|
||||
/* reactionIdentifier */ undefined,
|
||||
{ showControls: true, header: HeaderStyle.Standard },
|
||||
);
|
||||
@@ -222,4 +261,196 @@ describe("createCallFooterViewModel", () => {
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("background effects", () => {
|
||||
afterEach(() => backgroundEffectSetting.setValue("none"));
|
||||
|
||||
// The lobby, because in a call the whole device switcher is already
|
||||
// withheld on a phone, whatever the verdict says.
|
||||
function lobbyOn(
|
||||
platform: string,
|
||||
): ReturnType<typeof createLobbyFooterViewModel> {
|
||||
platformMock.mockReturnValue(platform);
|
||||
return createLobbyFooterViewModel(
|
||||
testScope(),
|
||||
mockMuteStates(),
|
||||
twoMicsAndOneCamMediaDevices,
|
||||
constant(false),
|
||||
/* openSettings */ undefined,
|
||||
/* hangup */ undefined,
|
||||
/* showLogo */ false,
|
||||
);
|
||||
}
|
||||
|
||||
it("offers them on a phone whose browser can run them", () => {
|
||||
sdkSupportMock.mockReturnValue(true);
|
||||
expect(lobbyOn("ios").selectBackgroundEffect$.value).toBeDefined();
|
||||
});
|
||||
|
||||
it("offers them on a desktop whose browser can run them", () => {
|
||||
sdkSupportMock.mockReturnValue(true);
|
||||
expect(lobbyOn("desktop").selectBackgroundEffect$.value).toBeDefined();
|
||||
});
|
||||
|
||||
it("offers nothing where the browser itself cannot run them", () => {
|
||||
sdkSupportMock.mockReturnValue(false);
|
||||
expect(lobbyOn("desktop").selectBackgroundEffect$.value).toBeUndefined();
|
||||
expect(lobbyOn("desktop").addBackgroundImage$.value).toBeUndefined();
|
||||
});
|
||||
|
||||
it("puts no effect in force where the browser cannot run them", () => {
|
||||
backgroundEffectSetting.setValue("blur");
|
||||
sdkSupportMock.mockReturnValue(false);
|
||||
expect(lobbyOn("desktop").backgroundEffect$.value).toBe("none");
|
||||
sdkSupportMock.mockReturnValue(true);
|
||||
expect(lobbyOn("desktop").backgroundEffect$.value).toBe("blur");
|
||||
});
|
||||
|
||||
it("stores the effect chosen", () => {
|
||||
sdkSupportMock.mockReturnValue(true);
|
||||
const vm = lobbyOn("desktop");
|
||||
vm.selectBackgroundEffect$.value?.("image:arc");
|
||||
expect(vm.backgroundEffect$.value).toBe("image:arc");
|
||||
// What isn't an effect on offer is stored as none.
|
||||
vm.selectBackgroundEffect$.value?.("image:gone");
|
||||
expect(backgroundEffectSetting.getValue()).toBe("none");
|
||||
});
|
||||
|
||||
it("says they run slowly where only the slower route exists", () => {
|
||||
sdkSupportMock.mockReturnValue(true);
|
||||
modernRouteMock.mockReturnValue(false);
|
||||
const slow = lobbyOn("desktop");
|
||||
expect(slow.backgroundEffectNotice$.value).toBe("slow");
|
||||
expect(slow.selectBackgroundEffect$.value).toBeDefined();
|
||||
modernRouteMock.mockReturnValue(true);
|
||||
expect(lobbyOn("desktop").backgroundEffectNotice$.value).toBeUndefined();
|
||||
});
|
||||
|
||||
it("offers every effect in order, the added ones removable", () => {
|
||||
sdkSupportMock.mockReturnValue(true);
|
||||
const added$ = addedBackgrounds.added$ as BehaviorSubject<
|
||||
AddedBackground[] | undefined
|
||||
>;
|
||||
added$.next([{ id: "mine", url: "blob:mine" }]);
|
||||
expect(lobbyOn("desktop").backgroundEffects$.value).toEqual([
|
||||
{ id: "none", kind: "none", removable: false },
|
||||
{ id: "blur", kind: "blur", removable: false },
|
||||
...shippedBackgrounds.map((background) => ({
|
||||
id: `image:${background.id}`,
|
||||
kind: "image",
|
||||
imageUrl: background.imagePath,
|
||||
removable: false,
|
||||
})),
|
||||
{
|
||||
id: "added:mine",
|
||||
kind: "image",
|
||||
imageUrl: "blob:mine",
|
||||
removable: true,
|
||||
},
|
||||
]);
|
||||
added$.next([]);
|
||||
});
|
||||
|
||||
it("says they are unavailable where they cannot be chosen", () => {
|
||||
sdkSupportMock.mockReturnValue(false);
|
||||
expect(lobbyOn("desktop").backgroundEffectNotice$.value).toBe(
|
||||
"unavailable",
|
||||
);
|
||||
});
|
||||
|
||||
it("passes on the wait for the first effect", () => {
|
||||
platformMock.mockReturnValue("desktop");
|
||||
const settling$ = new BehaviorSubject(true);
|
||||
const vm = createLobbyFooterViewModel(
|
||||
testScope(),
|
||||
mockMuteStates(),
|
||||
twoMicsAndOneCamMediaDevices,
|
||||
settling$,
|
||||
/* openSettings */ undefined,
|
||||
/* hangup */ undefined,
|
||||
/* showLogo */ false,
|
||||
);
|
||||
expect(vm.backgroundEffectSettling$.value).toBe(true);
|
||||
settling$.next(false);
|
||||
expect(vm.backgroundEffectSettling$.value).toBe(false);
|
||||
});
|
||||
|
||||
it("adding is unavailable at the limit", () => {
|
||||
sdkSupportMock.mockReturnValue(true);
|
||||
const added$ = addedBackgrounds.added$ as BehaviorSubject<
|
||||
AddedBackground[] | undefined
|
||||
>;
|
||||
const vm = lobbyOn("desktop");
|
||||
expect(vm.addBackgroundImage$.value).toBeDefined();
|
||||
added$.next(
|
||||
Array.from({ length: maxAddedBackgrounds }, (_, i) => ({
|
||||
id: `${i}`,
|
||||
url: `blob:${i}`,
|
||||
})),
|
||||
);
|
||||
expect(vm.addBackgroundImage$.value).toBeUndefined();
|
||||
added$.next(added$.value!.slice(1));
|
||||
expect(vm.addBackgroundImage$.value).toBeDefined();
|
||||
added$.next([]);
|
||||
});
|
||||
|
||||
it("says why each refused file was refused", async () => {
|
||||
sdkSupportMock.mockReturnValue(true);
|
||||
const vm = lobbyOn("desktop");
|
||||
store.add.mockRejectedValue(new UnusableImage("animated"));
|
||||
vm.addBackgroundImage$.value!(new File(["x"], "a.gif"));
|
||||
await flushPromises();
|
||||
const first = vm.backgroundImageRefusal$.value;
|
||||
expect(first).toEqual({ reason: "animated" });
|
||||
|
||||
// The same again is a refusal of its own, so it is shown again.
|
||||
vm.addBackgroundImage$.value!(new File(["x"], "a.gif"));
|
||||
await flushPromises();
|
||||
expect(vm.backgroundImageRefusal$.value).toEqual({ reason: "animated" });
|
||||
expect(vm.backgroundImageRefusal$.value).not.toBe(first);
|
||||
|
||||
store.add.mockRejectedValue(new Error("quota"));
|
||||
vm.addBackgroundImage$.value!(new File(["x"], "b.png"));
|
||||
await flushPromises();
|
||||
expect(vm.backgroundImageRefusal$.value).toEqual({ reason: "not-kept" });
|
||||
|
||||
store.add.mockResolvedValue("kept");
|
||||
vm.addBackgroundImage$.value!(new File(["x"], "c.png"));
|
||||
await flushPromises();
|
||||
expect(vm.backgroundImageRefusal$.value).toBeUndefined();
|
||||
});
|
||||
|
||||
it("never removes the background in force", () => {
|
||||
sdkSupportMock.mockReturnValue(true);
|
||||
store.remove.mockResolvedValue();
|
||||
const vm = lobbyOn("desktop");
|
||||
backgroundEffectSetting.setValue("added:worn");
|
||||
vm.removeBackgroundEffect$.value!("added:worn");
|
||||
expect(store.remove).not.toHaveBeenCalled();
|
||||
vm.removeBackgroundEffect$.value!("added:spare");
|
||||
expect(store.remove).toHaveBeenCalledWith("spare");
|
||||
});
|
||||
|
||||
it("availability is the same before and during a call", () => {
|
||||
for (const supported of [true, false]) {
|
||||
sdkSupportMock.mockReturnValue(supported);
|
||||
const lobby = lobbyOn("desktop");
|
||||
const inCall = createCallFooterViewModel(
|
||||
testScope(),
|
||||
buildMinimalCallViewModel(gridLayout),
|
||||
mockMuteStates(),
|
||||
twoMicsAndOneCamMediaDevices,
|
||||
constant(false),
|
||||
/* reactionIdentifier */ undefined,
|
||||
{ showControls: true, header: HeaderStyle.Standard },
|
||||
);
|
||||
const offeredInLobby =
|
||||
lobby.selectBackgroundEffect$.value !== undefined;
|
||||
expect(inCall.selectBackgroundEffect$.value !== undefined).toBe(
|
||||
offeredInLobby,
|
||||
);
|
||||
expect(offeredInLobby).toBe(supported);
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -5,15 +5,38 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { combineLatest, map, type Observable, switchMap } from "rxjs";
|
||||
import { supportsBackgroundProcessors } from "@livekit/track-processors";
|
||||
import {
|
||||
BehaviorSubject,
|
||||
combineLatest,
|
||||
distinctUntilChanged,
|
||||
map,
|
||||
type Observable,
|
||||
switchMap,
|
||||
} from "rxjs";
|
||||
import { supportsAudioOutputSelection } from "livekit-client";
|
||||
import {
|
||||
supportsBackgroundProcessors,
|
||||
supportsModernBackgroundProcessors,
|
||||
} from "@livekit/track-processors";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
|
||||
import {
|
||||
parseEffect,
|
||||
serializeEffect,
|
||||
shippedBackgrounds,
|
||||
} from "../livekit/backgroundEffects";
|
||||
import {
|
||||
type AddedBackground,
|
||||
addedBackgrounds,
|
||||
maxAddedBackgrounds,
|
||||
UnusableImage,
|
||||
} from "../livekit/backgroundImages";
|
||||
|
||||
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import { type MenuOptions } from "./MediaMuteAndSwitchButton";
|
||||
import { type MediaDevices } from "../state/MediaDevices";
|
||||
import {
|
||||
backgroundBlur as backgroundBlurSettings,
|
||||
backgroundEffect as backgroundEffectSetting,
|
||||
debugTileLayout as debugTileLayoutSetting,
|
||||
} from "../settings/settings";
|
||||
import { type Behavior, constant } from "../state/Behavior";
|
||||
@@ -22,7 +45,11 @@ import { type MuteStates } from "../state/MuteStates";
|
||||
import { createStaticViewModel, type ViewModel } from "../state/ViewModel";
|
||||
import { HeaderStyle } from "../UrlParams";
|
||||
import { platform } from "../Platform";
|
||||
import { type FooterSnapshot } from "./CallFooter";
|
||||
import {
|
||||
type BackgroundEffectChoice,
|
||||
type BackgroundImageRefusal,
|
||||
type FooterSnapshot,
|
||||
} from "./CallFooter";
|
||||
|
||||
/**
|
||||
* Shared helper: maps MuteStates into the audio/video enabled + toggle behaviors
|
||||
@@ -56,13 +83,14 @@ function buildMuteBehaviors(
|
||||
|
||||
/**
|
||||
* Shared helper: maps MediaDevices into the audio/video device-list behaviors
|
||||
* needed by FooterSnapshot (options, selection, callbacks, blur toggle).
|
||||
* needed by FooterSnapshot (options, selection, callbacks, background effect).
|
||||
*/
|
||||
function buildDeviceBehaviors(
|
||||
scope: ObservableScope,
|
||||
mediaDevices: MediaDevices,
|
||||
/** return empty arrays for audioOptions and videoOptions*/
|
||||
disableSwitcher$: Behavior<boolean>,
|
||||
backgroundEffectSettling$: Observable<boolean>,
|
||||
): Pick<
|
||||
ViewModel<FooterSnapshot>,
|
||||
| "audioOptions$"
|
||||
@@ -74,8 +102,14 @@ function buildDeviceBehaviors(
|
||||
| "videoOptions$"
|
||||
| "selectedVideo$"
|
||||
| "selectVideoButtonOption$"
|
||||
| "toggleBlur$"
|
||||
| "videoBlurEnabled$"
|
||||
| "backgroundEffect$"
|
||||
| "selectBackgroundEffect$"
|
||||
| "backgroundEffects$"
|
||||
| "backgroundEffectNotice$"
|
||||
| "backgroundEffectSettling$"
|
||||
| "addBackgroundImage$"
|
||||
| "backgroundImageRefusal$"
|
||||
| "removeBackgroundEffect$"
|
||||
> {
|
||||
const options$ = (
|
||||
available$: Behavior<Map<string, MenuOptions["label"]>>,
|
||||
@@ -92,6 +126,15 @@ function buildDeviceBehaviors(
|
||||
),
|
||||
);
|
||||
|
||||
const supported = supportsBackgroundProcessors();
|
||||
const slow = !supportsModernBackgroundProcessors();
|
||||
const offered$ = disableSwitcher$.pipe(
|
||||
map((switcherDisabled) => !switcherDisabled && supported),
|
||||
);
|
||||
// A new object for each refusal, so the same one twice is shown twice.
|
||||
const refusal$ = new BehaviorSubject<BackgroundImageRefusal | undefined>(
|
||||
undefined,
|
||||
);
|
||||
return {
|
||||
audioOptions$: scope.behavior(options$(mediaDevices.audioInput.available$)),
|
||||
selectedAudio$: scope.behavior(
|
||||
@@ -116,18 +159,77 @@ function buildDeviceBehaviors(
|
||||
mediaDevices.videoInput.selected$.pipe(map((s) => s?.id)),
|
||||
),
|
||||
selectVideoButtonOption$: constant(mediaDevices.videoInput.select),
|
||||
toggleBlur$: scope.behavior(
|
||||
combineLatest([backgroundBlurSettings.value$, disableSwitcher$]).pipe(
|
||||
map(([current, switcherDisabled]) => {
|
||||
return !switcherDisabled && supportsBackgroundProcessors()
|
||||
? (): void => {
|
||||
backgroundBlurSettings.setValue(!current);
|
||||
backgroundEffect$: scope.behavior(
|
||||
backgroundEffectSetting.value$.pipe(
|
||||
map((raw) => (supported ? serializeEffect(parseEffect(raw)) : "none")),
|
||||
),
|
||||
),
|
||||
selectBackgroundEffect$: scope.behavior(
|
||||
offered$.pipe(
|
||||
map((offered) =>
|
||||
offered
|
||||
? (id: string): void =>
|
||||
backgroundEffectSetting.setValue(
|
||||
serializeEffect(parseEffect(id)),
|
||||
)
|
||||
: undefined,
|
||||
),
|
||||
),
|
||||
),
|
||||
backgroundEffects$: scope.behavior(
|
||||
addedBackgrounds.added$.pipe(
|
||||
map((added) => backgroundEffectChoices(added ?? [])),
|
||||
),
|
||||
),
|
||||
backgroundEffectNotice$: scope.behavior(
|
||||
offered$.pipe(
|
||||
map((offered) =>
|
||||
!offered ? "unavailable" : slow ? "slow" : undefined,
|
||||
),
|
||||
),
|
||||
),
|
||||
backgroundEffectSettling$: scope.behavior(
|
||||
backgroundEffectSettling$.pipe(distinctUntilChanged()),
|
||||
false,
|
||||
),
|
||||
// Kept and offered, not put on: that waits for the user to choose it.
|
||||
addBackgroundImage$: scope.behavior(
|
||||
combineLatest([offered$, addedBackgrounds.added$]).pipe(
|
||||
map(([offered, added]) =>
|
||||
offered && (added?.length ?? 0) < maxAddedBackgrounds
|
||||
? (file: File): void => {
|
||||
refusal$.next(undefined);
|
||||
addedBackgrounds.add(file).catch((e) => {
|
||||
logger.warn("Could not keep that background", e);
|
||||
refusal$.next({
|
||||
reason: e instanceof UnusableImage ? e.reason : "not-kept",
|
||||
});
|
||||
});
|
||||
}
|
||||
: undefined;
|
||||
}),
|
||||
: undefined,
|
||||
),
|
||||
),
|
||||
),
|
||||
backgroundImageRefusal$: refusal$,
|
||||
// Never the one in force: taking it away would leave nothing chosen.
|
||||
removeBackgroundEffect$: scope.behavior(
|
||||
offered$.pipe(
|
||||
map((offered) =>
|
||||
offered
|
||||
? (optionId: string): void => {
|
||||
const effect = parseEffect(optionId);
|
||||
if (effect.kind !== "added") return;
|
||||
if (backgroundEffectSetting.getValue() === optionId) return;
|
||||
addedBackgrounds
|
||||
.remove(effect.id)
|
||||
.catch((e) =>
|
||||
logger.warn("Could not remove that background", e),
|
||||
);
|
||||
}
|
||||
: undefined,
|
||||
),
|
||||
),
|
||||
),
|
||||
videoBlurEnabled$: backgroundBlurSettings.value$,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -138,6 +240,8 @@ function buildDeviceBehaviors(
|
||||
* @param callModel - The root CallViewModel; provides layout, grid mode, reactions, etc.
|
||||
* @param muteStates - Audio and video mute state + toggles.
|
||||
* @param mediaDevices - Available and selected input devices.
|
||||
* @param backgroundEffectSettling$ - Whether the first effect chosen is still
|
||||
* being prepared.
|
||||
* @param reactionIdentifier - The local user's reaction identifier string, or
|
||||
* undefined when reactions are not supported (hides the reaction button).
|
||||
* @param options - `showControls`: whether the call controls should be shown.
|
||||
@@ -148,6 +252,7 @@ export function createCallFooterViewModel(
|
||||
callModel: CallViewModel,
|
||||
muteStates: MuteStates,
|
||||
mediaDevices: MediaDevices,
|
||||
backgroundEffectSettling$: Observable<boolean>,
|
||||
reactionIdentifier: string | undefined,
|
||||
options: { showControls: boolean; header: HeaderStyle },
|
||||
): ViewModel<FooterSnapshot> {
|
||||
@@ -162,7 +267,12 @@ export function createCallFooterViewModel(
|
||||
);
|
||||
return {
|
||||
...buildMuteBehaviors(scope, muteStates),
|
||||
...buildDeviceBehaviors(scope, mediaDevices, disableDeviceSwitcher$),
|
||||
...buildDeviceBehaviors(
|
||||
scope,
|
||||
mediaDevices,
|
||||
disableDeviceSwitcher$,
|
||||
backgroundEffectSettling$,
|
||||
),
|
||||
// candidat to move into the FooterViewModel
|
||||
showFooter$: callModel.showFooter$,
|
||||
hideControls$: constant(!showControls),
|
||||
@@ -224,6 +334,8 @@ export function createCallFooterViewModel(
|
||||
* @param scope - ObservableScope that bounds the lifetime of derived behaviors.
|
||||
* @param muteStates - Audio and video mute state + toggles.
|
||||
* @param mediaDevices - Available and selected input devices.
|
||||
* @param backgroundEffectSettling$ - Whether the first effect chosen is still
|
||||
* being prepared.
|
||||
* @param openSettings - Callback to open the settings modal, or undefined.
|
||||
* @param hangup - Callback to leave/cancel, or undefined (hides the button).
|
||||
* @param showLogo - Whether to show the Element Call logo.
|
||||
@@ -232,6 +344,7 @@ export function createLobbyFooterViewModel(
|
||||
scope: ObservableScope,
|
||||
muteStates: MuteStates,
|
||||
mediaDevices: MediaDevices,
|
||||
backgroundEffectSettling$: Observable<boolean>,
|
||||
openSettings: (() => void) | undefined,
|
||||
hangup: (() => void) | undefined,
|
||||
showLogo: boolean,
|
||||
@@ -275,6 +388,32 @@ export function createLobbyFooterViewModel(
|
||||
selectVideoButtonOption: undefined,
|
||||
}),
|
||||
...buildMuteBehaviors(scope, muteStates),
|
||||
...buildDeviceBehaviors(scope, mediaDevices, constant(false)),
|
||||
...buildDeviceBehaviors(
|
||||
scope,
|
||||
mediaDevices,
|
||||
constant(false),
|
||||
backgroundEffectSettling$,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
function backgroundEffectChoices(
|
||||
added: AddedBackground[],
|
||||
): BackgroundEffectChoice[] {
|
||||
return [
|
||||
{ id: serializeEffect({ kind: "none" }), kind: "none", removable: false },
|
||||
{ id: serializeEffect({ kind: "blur" }), kind: "blur", removable: false },
|
||||
...shippedBackgrounds.map((background) => ({
|
||||
id: serializeEffect({ kind: "shipped", id: background.id }),
|
||||
kind: "image" as const,
|
||||
imageUrl: background.imagePath,
|
||||
removable: false,
|
||||
})),
|
||||
...added.map((background) => ({
|
||||
id: serializeEffect({ kind: "added", id: background.id }),
|
||||
kind: "image" as const,
|
||||
imageUrl: background.url,
|
||||
removable: true,
|
||||
})),
|
||||
];
|
||||
}
|
||||
|
||||
@@ -36,20 +36,19 @@ Please see LICENSE in the repository root for full details.
|
||||
}
|
||||
}
|
||||
|
||||
/* The width design sets: a long device name wraps. */
|
||||
.menu {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
inline-size: 296px;
|
||||
}
|
||||
|
||||
/* Only the device lists scroll, bounded by the measured call area: the menu is
|
||||
portalled out of the root, so CSS can't size it against the call. */
|
||||
/* Its share of the measured call area: the menu is portalled out of the root,
|
||||
so CSS can't size it against the call. */
|
||||
.deviceList {
|
||||
overflow-y: auto;
|
||||
min-block-size: 0;
|
||||
max-block-size: var(--device-list-max-height);
|
||||
/* Keeps a row reached by keyboard clear of the sticky heading and meter. */
|
||||
scroll-padding-block: var(--device-list-scroll-padding-start, 0)
|
||||
var(--device-list-scroll-padding-end, 0);
|
||||
/* The raised headings stay raised within the list, under the menu's frame. */
|
||||
isolation: isolate;
|
||||
/* Keeps a row reached by keyboard clear of the sticky heading. */
|
||||
scroll-padding-block-start: var(--device-list-scroll-padding-start, 0);
|
||||
}
|
||||
|
||||
/* Compound's heading rule heads each section; the design adds space below it. */
|
||||
@@ -62,23 +61,40 @@ Please see LICENSE in the repository root for full details.
|
||||
margin-block-start: var(--cpd-space-7x);
|
||||
}
|
||||
|
||||
/* Sticky within its own section. Opaque and inset a border width so it doesn't
|
||||
paint over the menu's outline. */
|
||||
/* Sticky within its own section: opaque, so rows scroll under it, and raised,
|
||||
or the positioned tiles scroll over it. */
|
||||
.sectionHeading {
|
||||
position: sticky;
|
||||
z-index: 1;
|
||||
inset-block-start: 0;
|
||||
background: var(--cpd-color-bg-canvas-default);
|
||||
margin-inline: var(--cpd-border-width-1);
|
||||
margin-block-start: var(--cpd-border-width-1);
|
||||
}
|
||||
|
||||
/* Sticky at the foot of the microphone section; opaque and inset as above. */
|
||||
/* Sticky at the foot of the microphone section; opaque as above. */
|
||||
.stickyMeter {
|
||||
position: sticky;
|
||||
inset-block-end: 0;
|
||||
background: var(--cpd-color-bg-canvas-default);
|
||||
margin-inline: var(--cpd-border-width-1);
|
||||
margin-block-end: var(--cpd-border-width-1);
|
||||
}
|
||||
|
||||
/* Fills the band between the meter and the frame, or rows show through it. */
|
||||
.stickyMeter::after {
|
||||
content: "";
|
||||
position: absolute;
|
||||
inset-inline: 0;
|
||||
inset-block-start: 100%;
|
||||
block-size: var(--cpd-menu-padding-block-end, 0px);
|
||||
background: var(--cpd-color-bg-canvas-default);
|
||||
}
|
||||
|
||||
/* Not Compound's fade: that stays at the edge while a bounce carries the meter.
|
||||
This one starts at the chevron, below the meter at rest. */
|
||||
.deviceList.deviceListWithMeter[data-more-below] {
|
||||
mask-image: linear-gradient(
|
||||
to bottom,
|
||||
#000 calc(100% - var(--cpd-space-6x)),
|
||||
transparent calc(100% - var(--cpd-space-3x))
|
||||
);
|
||||
}
|
||||
|
||||
/* Radix focuses whatever the pointer is over, so the browser's ring would
|
||||
@@ -92,3 +108,24 @@ Please see LICENSE in the repository root for full details.
|
||||
outline: var(--cpd-border-width-2) solid var(--cpd-color-border-focused);
|
||||
outline-offset: calc(-1 * var(--cpd-border-width-2));
|
||||
}
|
||||
|
||||
.notice {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
gap: var(--cpd-space-3x);
|
||||
margin: var(--cpd-space-2x) var(--cpd-space-4x) 0;
|
||||
padding: var(--cpd-space-3x);
|
||||
border-radius: var(--cpd-space-2x);
|
||||
background: var(--cpd-color-bg-subtle-secondary);
|
||||
color: var(--cpd-color-text-primary);
|
||||
font: var(--cpd-font-body-md-regular);
|
||||
}
|
||||
|
||||
.notice > svg {
|
||||
flex-shrink: 0;
|
||||
color: var(--cpd-color-icon-secondary);
|
||||
}
|
||||
|
||||
.refusal {
|
||||
margin: var(--cpd-space-2x) var(--cpd-space-4x) 0;
|
||||
}
|
||||
|
||||
@@ -11,6 +11,8 @@ import { useEffect, useState, type FC, type JSX, type ReactNode } from "react";
|
||||
import type { Meta, StoryObj } from "@storybook/react-vite";
|
||||
import { MediaMuteAndSwitchButton } from "./MediaMuteAndSwitchButton";
|
||||
import styles from "./MediaMuteAndSwitchButton.module.css";
|
||||
import { type BackgroundEffectOption } from "./BackgroundEffectGrid";
|
||||
import { shippedBackgrounds } from "../livekit/backgroundEffects";
|
||||
import meterStyles from "./MicrophoneLevelMeter.module.css";
|
||||
import { MediaDevicesContext } from "../MediaDevicesContext";
|
||||
import { RootElementProvider } from "../RootElementContext";
|
||||
@@ -59,14 +61,19 @@ const WithAMicrophone: FC<{ children: ReactNode }> = ({ children }) => {
|
||||
};
|
||||
|
||||
/** Supplies a call-sized root. Without one the list is bounded by the whole Storybook frame. */
|
||||
const WithACallArea: FC<{ children: ReactNode }> = ({ children }) => {
|
||||
const WithACallArea: FC<{ children: ReactNode; blockSize?: number }> = ({
|
||||
children,
|
||||
// A call's size: anything smaller makes a short device list scroll, and a
|
||||
// narrower one narrows the menu.
|
||||
blockSize = 720,
|
||||
}) => {
|
||||
const [callArea, setCallArea] = useState<HTMLElement | null>(null);
|
||||
return (
|
||||
<div
|
||||
ref={setCallArea}
|
||||
style={{
|
||||
// A call's size: anything smaller makes a short device list scroll.
|
||||
blockSize: 720,
|
||||
blockSize,
|
||||
inlineSize: 1024,
|
||||
display: "flex",
|
||||
alignItems: "flex-end",
|
||||
justifyContent: "center",
|
||||
@@ -127,8 +134,6 @@ export const AudioMute: Story = {
|
||||
{ label: { type: "name", name: "Microphone 1" }, id: "1" },
|
||||
{ label: { type: "name", name: "Microphone 2" }, id: "2" },
|
||||
],
|
||||
videoBlurEnabled: true,
|
||||
videoBlurToggleClick: fn(),
|
||||
selectedOption: "2",
|
||||
},
|
||||
play: async ({ args, canvasElement }) => {
|
||||
@@ -167,6 +172,17 @@ export const VideoMute: Story = {
|
||||
},
|
||||
};
|
||||
|
||||
const effects: BackgroundEffectOption[] = [
|
||||
{ id: "none", kind: "none", label: "None" },
|
||||
{ id: "blur", kind: "blur", label: "Blur" },
|
||||
...shippedBackgrounds.map((background, i) => ({
|
||||
id: `image:${background.id}`,
|
||||
kind: "image" as const,
|
||||
label: `Background ${i + 1}`,
|
||||
imageUrl: background.imagePath,
|
||||
})),
|
||||
];
|
||||
|
||||
export const VideoUnmute: Story = {
|
||||
args: {
|
||||
iconsAndLabels: "video",
|
||||
@@ -175,9 +191,372 @@ export const VideoUnmute: Story = {
|
||||
{ label: { type: "name", name: "Camera 1" }, id: "1" },
|
||||
{ label: { type: "name", name: "Camera 2" }, id: "2" },
|
||||
],
|
||||
videoBlurEnabled: true,
|
||||
videoBlurToggleClick: fn(),
|
||||
selectedOption: "2",
|
||||
backgroundEffects: effects,
|
||||
selectedBackgroundEffect: "none",
|
||||
onSelectBackgroundEffect: fn(),
|
||||
onAddBackgroundImage: fn(),
|
||||
},
|
||||
};
|
||||
|
||||
/** One choice among no effect, blur and the shipped images, under the cameras. */
|
||||
export const BackgroundEffects: Story = {
|
||||
args: VideoUnmute.args,
|
||||
play: async ({ args, canvasElement }) => {
|
||||
const canvas = within(canvasElement);
|
||||
await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
|
||||
const menu = within(document.body);
|
||||
const section = await menu.findByRole("group", {
|
||||
name: "Background effects",
|
||||
});
|
||||
const tiles = within(section).getAllByRole("menuitemradio");
|
||||
await expect(tiles.map((tile) => tile.textContent)).toEqual([
|
||||
"None",
|
||||
"Blur",
|
||||
"Background 1",
|
||||
"Background 2",
|
||||
]);
|
||||
await expect(
|
||||
within(section).getByRole("menuitemradio", { checked: true }),
|
||||
).toHaveTextContent("None");
|
||||
|
||||
const cameras = menu.getByRole("group", { name: "Camera" });
|
||||
await expect(
|
||||
cameras.compareDocumentPosition(section) &
|
||||
Node.DOCUMENT_POSITION_FOLLOWING,
|
||||
).toBeTruthy();
|
||||
const [first, second, third, fourth] = tiles.map((tile) =>
|
||||
tile.getBoundingClientRect(),
|
||||
);
|
||||
await expect(second.top).toBe(first.top);
|
||||
await expect(third.top).toBe(first.top);
|
||||
await expect(fourth.top).toBeGreaterThan(first.bottom);
|
||||
const frame = document.body
|
||||
.querySelector("[role='menu']")!
|
||||
.getBoundingClientRect();
|
||||
for (const tile of [first, second, third])
|
||||
await expect(tile.left >= frame.left && tile.right <= frame.right).toBe(
|
||||
true,
|
||||
);
|
||||
|
||||
const add = within(section).getByRole("menuitem", { name: "Add image" });
|
||||
await expect(
|
||||
tiles[3].compareDocumentPosition(add) & Node.DOCUMENT_POSITION_FOLLOWING,
|
||||
).toBeTruthy();
|
||||
|
||||
await userEvent.click(menu.getByRole("menuitemradio", { name: "Blur" }));
|
||||
await expect(args.onSelectBackgroundEffect).toHaveBeenCalledWith("blur");
|
||||
await expect(menu.getByRole("menu")).toBeVisible();
|
||||
},
|
||||
};
|
||||
|
||||
/** Where effects can't run: shown, disabled and explained, except no effect. */
|
||||
export const BackgroundEffectsUnavailable: Story = {
|
||||
args: {
|
||||
...VideoUnmute.args,
|
||||
onSelectBackgroundEffect: undefined,
|
||||
backgroundEffectNotice:
|
||||
"Background effects are not supported on this platform.",
|
||||
},
|
||||
play: async ({ canvasElement }) => {
|
||||
const canvas = within(canvasElement);
|
||||
await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
|
||||
const menu = within(document.body);
|
||||
const section = await menu.findByRole("group", {
|
||||
name: "Background effects",
|
||||
});
|
||||
await expect(section).toHaveAccessibleDescription(
|
||||
"Background effects are not supported on this platform.",
|
||||
);
|
||||
for (const tile of within(section).getAllByRole("menuitemradio"))
|
||||
if (tile.textContent === "None") await expect(tile).toBeEnabled();
|
||||
else await expect(tile).toBeDisabled();
|
||||
},
|
||||
};
|
||||
|
||||
/** Where only the slower route exists: offered and choosable, with the cost said. */
|
||||
export const BackgroundEffectsSlowInThisBrowser: Story = {
|
||||
args: {
|
||||
...VideoUnmute.args,
|
||||
backgroundEffectNotice:
|
||||
"Background effects run slowly on this platform, which may cause your video to stutter.",
|
||||
},
|
||||
play: async ({ args, canvasElement }) => {
|
||||
const canvas = within(canvasElement);
|
||||
await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
|
||||
const menu = within(document.body);
|
||||
const section = await menu.findByRole("group", {
|
||||
name: "Background effects",
|
||||
});
|
||||
await expect(section).toHaveAccessibleDescription(
|
||||
"Background effects run slowly on this platform, which may cause your video to stutter.",
|
||||
);
|
||||
for (const tile of within(section).getAllByRole("menuitemradio"))
|
||||
await expect(tile).not.toHaveAttribute("aria-disabled");
|
||||
await userEvent.click(
|
||||
within(section).getByRole("menuitemradio", { name: "Background 1" }),
|
||||
);
|
||||
await expect(args.onSelectBackgroundEffect).toHaveBeenCalledWith(
|
||||
"image:arc",
|
||||
);
|
||||
|
||||
const notice = document.body.querySelector(`.${styles.notice}`)!;
|
||||
const frame = document.body
|
||||
.querySelector("[role='menu']")!
|
||||
.getBoundingClientRect();
|
||||
await expect(frame.width).toBeLessThanOrEqual(296);
|
||||
await expect(notice.getBoundingClientRect().right).toBeLessThanOrEqual(
|
||||
frame.right,
|
||||
);
|
||||
},
|
||||
};
|
||||
|
||||
/** The first effect of a session shows its wait on its own tile. */
|
||||
export const BackgroundEffectsSettling: Story = {
|
||||
args: {
|
||||
...VideoUnmute.args,
|
||||
selectedBackgroundEffect: "blur",
|
||||
backgroundEffectSettling: true,
|
||||
},
|
||||
play: async ({ canvasElement }) => {
|
||||
const canvas = within(canvasElement);
|
||||
await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
|
||||
const menu = within(document.body);
|
||||
const chosen = await menu.findByRole("menuitemradio", {
|
||||
name: "Blur",
|
||||
checked: true,
|
||||
});
|
||||
await expect(chosen).toHaveAttribute("aria-busy", "true");
|
||||
const marks = chosen.querySelectorAll("svg");
|
||||
await expect(marks).toHaveLength(1);
|
||||
await expect(getComputedStyle(marks[0]).animationName).not.toBe("none");
|
||||
const section = menu.getByRole("group", { name: "Background effects" });
|
||||
for (const other of within(section).getAllByRole("menuitemradio", {
|
||||
checked: false,
|
||||
}))
|
||||
await expect(other).toHaveAttribute("aria-busy", "false");
|
||||
},
|
||||
};
|
||||
|
||||
/** A file that can't be used: said where it was chosen, and nothing changes. */
|
||||
export const BackgroundImageRefused: Story = {
|
||||
args: {
|
||||
...VideoUnmute.args,
|
||||
selectedBackgroundEffect: "blur",
|
||||
backgroundEffectNotice:
|
||||
"Background effects run slowly on this platform, which may cause your video to stutter.",
|
||||
backgroundImageRefusal: { text: "That file is not a supported image" },
|
||||
},
|
||||
decorators: [
|
||||
(Story): JSX.Element => (
|
||||
<WithACallArea blockSize={300}>
|
||||
<Story />
|
||||
</WithACallArea>
|
||||
),
|
||||
],
|
||||
play: async ({ canvasElement }) => {
|
||||
const canvas = within(canvasElement);
|
||||
await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
|
||||
const menu = within(document.body);
|
||||
const message = await menu.findByText("That file is not a supported image");
|
||||
const list = document.body.querySelector<HTMLElement>(
|
||||
`.${styles.deviceList}`,
|
||||
)!;
|
||||
await expect(list.scrollHeight).toBeGreaterThan(list.clientHeight);
|
||||
const scrollport = list.getBoundingClientRect();
|
||||
const shown = message.getBoundingClientRect();
|
||||
await expect(shown.bottom).toBeLessThanOrEqual(scrollport.bottom + 1);
|
||||
await expect(shown.top).toBeGreaterThanOrEqual(scrollport.top - 1);
|
||||
await expect(menu.queryByText(/run slowly/)).toBeNull();
|
||||
await expect(
|
||||
menu.getByRole("menuitemradio", { name: "Blur" }),
|
||||
).toHaveAttribute("aria-checked", "true");
|
||||
},
|
||||
};
|
||||
|
||||
/** A flat picture, for backgrounds of one's own. */
|
||||
function swatch(colour: string): string {
|
||||
return `data:image/svg+xml,${encodeURIComponent(
|
||||
`<svg xmlns="http://www.w3.org/2000/svg" width="16" height="9"><rect width="16" height="9" fill="${colour}"/></svg>`,
|
||||
)}`;
|
||||
}
|
||||
|
||||
/** One's own backgrounds are removable, never the one in force. */
|
||||
export const AddedBackgroundsCanBeRemoved: Story = {
|
||||
args: {
|
||||
...VideoUnmute.args,
|
||||
backgroundEffects: [
|
||||
...effects,
|
||||
{
|
||||
id: "added:one",
|
||||
label: "Background 3",
|
||||
kind: "image",
|
||||
imageUrl: swatch("#c2410c"),
|
||||
removable: true,
|
||||
},
|
||||
{
|
||||
id: "added:two",
|
||||
label: "Background 4",
|
||||
kind: "image",
|
||||
imageUrl: swatch("#1d4ed8"),
|
||||
removable: true,
|
||||
},
|
||||
],
|
||||
selectedBackgroundEffect: "added:one",
|
||||
onRemoveBackgroundEffect: fn(),
|
||||
},
|
||||
play: async ({ args, canvasElement }) => {
|
||||
const canvas = within(canvasElement);
|
||||
await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
|
||||
const body = within(document.body);
|
||||
|
||||
const inForce = await body.findByRole("menuitemradio", {
|
||||
name: "Background 3",
|
||||
});
|
||||
await expect(inForce).not.toHaveAttribute("aria-keyshortcuts");
|
||||
|
||||
const other = body.getByRole("menuitemradio", { name: "Background 4" });
|
||||
await expect(other).toHaveAttribute("aria-keyshortcuts", "Delete");
|
||||
other.focus();
|
||||
await userEvent.keyboard("{Delete}");
|
||||
await expect(args.onRemoveBackgroundEffect).toHaveBeenCalledWith(
|
||||
"added:two",
|
||||
);
|
||||
|
||||
const wrapper = other.parentElement!;
|
||||
await userEvent.hover(wrapper);
|
||||
const cross = wrapper.querySelector<HTMLElement>(
|
||||
"[aria-hidden]:last-child",
|
||||
)!;
|
||||
const probe = document.createElement("span");
|
||||
probe.style.color = "var(--cpd-color-icon-critical-primary)";
|
||||
cross.append(probe);
|
||||
const critical = getComputedStyle(probe).color;
|
||||
probe.remove();
|
||||
await expect(getComputedStyle(cross).color).toBe(critical);
|
||||
const tile = other.getBoundingClientRect();
|
||||
const box = cross.getBoundingClientRect();
|
||||
await expect(box.right).toBeGreaterThan(tile.right);
|
||||
await expect(box.top).toBeLessThan(tile.top);
|
||||
|
||||
await userEvent.hover(cross);
|
||||
await waitFor(async () =>
|
||||
expect(
|
||||
[...document.body.querySelectorAll("div")].some(
|
||||
(d) => d.textContent === "Remove" && d.offsetParent !== null,
|
||||
),
|
||||
).toBe(true),
|
||||
);
|
||||
},
|
||||
};
|
||||
|
||||
/** Removing by pointer while the selection really changes. */
|
||||
export const RemovingWithLiveSelection: Story = {
|
||||
args: AddedBackgroundsCanBeRemoved.args,
|
||||
render: function WithLiveSelection(args): JSX.Element {
|
||||
const [selected, setSelected] = useState("none");
|
||||
const [offered, setOffered] = useState(args.backgroundEffects ?? []);
|
||||
return (
|
||||
<MediaMuteAndSwitchButton
|
||||
{...args}
|
||||
backgroundEffects={offered}
|
||||
selectedBackgroundEffect={selected}
|
||||
onSelectBackgroundEffect={setSelected}
|
||||
onRemoveBackgroundEffect={(id): void =>
|
||||
setOffered((current) => current.filter((o) => o.id !== id))
|
||||
}
|
||||
/>
|
||||
);
|
||||
},
|
||||
play: async ({ canvasElement }) => {
|
||||
const canvas = within(canvasElement);
|
||||
await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
|
||||
const body = within(document.body);
|
||||
|
||||
const tile = await body.findByRole("menuitemradio", {
|
||||
name: "Background 4",
|
||||
});
|
||||
await userEvent.hover(tile);
|
||||
const cross = tile.parentElement!.querySelector<HTMLElement>(
|
||||
"[aria-hidden]:last-child",
|
||||
)!;
|
||||
await expect(cross).toBeVisible();
|
||||
await userEvent.click(cross);
|
||||
|
||||
await waitFor(async () =>
|
||||
expect(
|
||||
body.queryByRole("menuitemradio", { name: "Background 4" }),
|
||||
).toBeNull(),
|
||||
);
|
||||
await expect(
|
||||
body.getByRole("menuitemradio", { name: "None" }),
|
||||
).toHaveAttribute("aria-checked", "true");
|
||||
},
|
||||
};
|
||||
|
||||
/** In a short call the effects scroll into view with the list. */
|
||||
export const BackgroundEffectsScrollWhenTheyDoNotFit: Story = {
|
||||
args: {
|
||||
...VideoUnmute.args,
|
||||
// It sits with the effects, so it must fit the same share of the call.
|
||||
backgroundEffectNotice:
|
||||
"Background effects run slowly on this platform, which may cause your video to stutter.",
|
||||
},
|
||||
decorators: [
|
||||
(Story): JSX.Element => (
|
||||
<WithACallArea blockSize={300}>
|
||||
<Story />
|
||||
</WithACallArea>
|
||||
),
|
||||
],
|
||||
play: async ({ canvasElement }) => {
|
||||
const canvas = within(canvasElement);
|
||||
await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
|
||||
const menu = within(document.body);
|
||||
const last = await menu.findByRole("menuitemradio", {
|
||||
name: "Background 2",
|
||||
});
|
||||
const list = document.body.querySelector<HTMLElement>(
|
||||
`.${styles.deviceList}`,
|
||||
)!;
|
||||
await expect(list.scrollHeight).toBeGreaterThan(list.clientHeight);
|
||||
await expect(last.getBoundingClientRect().top).toBeGreaterThan(
|
||||
list.getBoundingClientRect().bottom,
|
||||
);
|
||||
|
||||
// Scrolled under the section's heading, the tiles pass beneath it.
|
||||
const heading = [
|
||||
...document.body.querySelectorAll<HTMLElement>(
|
||||
`.${styles.sectionHeading}`,
|
||||
),
|
||||
].find((h) => h.textContent === "Background effects")!;
|
||||
list.scrollTop = heading.offsetTop + 30;
|
||||
await waitFor(async () => {
|
||||
const box = heading.getBoundingClientRect();
|
||||
const drawn = document.elementFromPoint(
|
||||
box.left + box.width / 2,
|
||||
box.bottom - 2,
|
||||
);
|
||||
await expect(heading.contains(drawn)).toBe(true);
|
||||
});
|
||||
list.scrollTop = 0;
|
||||
|
||||
for (let i = 0; i < 10 && document.activeElement !== last; i++)
|
||||
await userEvent.keyboard("{ArrowDown}");
|
||||
await expect(document.activeElement).toBe(last);
|
||||
const scrollport = list.getBoundingClientRect();
|
||||
const reached = last.getBoundingClientRect();
|
||||
await expect(reached.bottom).toBeLessThanOrEqual(scrollport.bottom + 1);
|
||||
await expect(reached.top).toBeGreaterThanOrEqual(scrollport.top - 1);
|
||||
|
||||
const callArea = canvasElement
|
||||
.querySelector<HTMLElement>("[style*='block-size: 300px']")!
|
||||
.getBoundingClientRect();
|
||||
const frame = document.body
|
||||
.querySelector("[role='menu']")!
|
||||
.getBoundingClientRect();
|
||||
await expect(frame.top).toBeGreaterThanOrEqual(callArea.top);
|
||||
await expect(frame.bottom).toBeLessThanOrEqual(callArea.bottom);
|
||||
},
|
||||
};
|
||||
|
||||
@@ -256,6 +635,37 @@ export const SpeakerAndMicrophoneSections: Story = {
|
||||
},
|
||||
};
|
||||
|
||||
/** A long name wraps rather than widening the menu, which design sets. */
|
||||
export const LongDeviceNameWraps: Story = {
|
||||
args: {
|
||||
...SpeakerAndMicrophoneSections.args,
|
||||
outputOptions: [
|
||||
{
|
||||
label: {
|
||||
type: "name",
|
||||
name: "Logitech BRIO 4K Ultra HD Pro Business Webcam Speakers (046d:085e)",
|
||||
},
|
||||
id: "long",
|
||||
},
|
||||
{ label: { type: "name", name: "Headset" }, id: "spk2" },
|
||||
],
|
||||
selectedOutputOption: "long",
|
||||
},
|
||||
play: async ({ canvasElement }) => {
|
||||
const canvas = within(canvasElement);
|
||||
await userEvent.click(canvas.getByRole("button", { name: "Microphone" }));
|
||||
const body = within(document.body);
|
||||
const long = await body.findByRole("menuitemradio", { name: /Logitech/ });
|
||||
const short = await body.findByRole("menuitemradio", { name: "Headset" });
|
||||
|
||||
const menu = document.body.querySelector("[role='menu']")!;
|
||||
await expect(Math.round(menu.getBoundingClientRect().width)).toBe(296);
|
||||
await expect(long.getBoundingClientRect().height).toBeGreaterThan(
|
||||
short.getBoundingClientRect().height,
|
||||
);
|
||||
},
|
||||
};
|
||||
|
||||
export const OutputCannotBeChosen: Story = {
|
||||
args: {
|
||||
...Default.args,
|
||||
@@ -436,10 +846,6 @@ export const ManyDevices: Story = {
|
||||
.getBoundingClientRect();
|
||||
await expect(frame.top).toBeGreaterThanOrEqual(0);
|
||||
await expect(frame.bottom).toBeLessThanOrEqual(window.innerHeight + 1);
|
||||
|
||||
// The meter is the menu's one opaque part, so it must stay inside the frame.
|
||||
await expect(pinned.left).toBeGreaterThan(frame.left);
|
||||
await expect(pinned.right).toBeLessThan(frame.right);
|
||||
},
|
||||
};
|
||||
|
||||
@@ -607,13 +1013,6 @@ export const HeadingsStayWhileScrolling: Story = {
|
||||
await expect(heading.getBoundingClientRect().bottom).toBeGreaterThan(
|
||||
scrollport.top,
|
||||
);
|
||||
// Clear of the menu's frame.
|
||||
const frame = document.body
|
||||
.querySelector("[role='menu']")!
|
||||
.getBoundingClientRect();
|
||||
await expect(heading.getBoundingClientRect().left).toBeGreaterThan(
|
||||
frame.left,
|
||||
);
|
||||
},
|
||||
};
|
||||
|
||||
@@ -628,31 +1027,23 @@ function overlapping(element: Element, overlays: Element[]): number {
|
||||
}, 0);
|
||||
}
|
||||
|
||||
/** The blur toggle gets the keyboard ring too: it is the menu's child, not the list's. */
|
||||
export const FocusRingCoversTheBlurToggle: Story = {
|
||||
args: {
|
||||
...VideoUnmute.args,
|
||||
iconsAndLabels: "video",
|
||||
videoBlurEnabled: false,
|
||||
videoBlurToggleClick: fn(),
|
||||
},
|
||||
/** The effect tiles get the keyboard ring the device rows get. */
|
||||
export const FocusRingCoversTheEffectTiles: Story = {
|
||||
args: VideoUnmute.args,
|
||||
play: async ({ canvasElement }) => {
|
||||
const canvas = within(canvasElement);
|
||||
await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
|
||||
const toggle = await within(document.body).findByRole("menuitemcheckbox", {
|
||||
name: /Blur background/,
|
||||
const tile = await within(document.body).findByRole("menuitemradio", {
|
||||
name: "Blur",
|
||||
});
|
||||
|
||||
// Arrow down past the cameras to the toggle.
|
||||
for (let i = 0; i < 6 && document.activeElement !== toggle; i++)
|
||||
for (let i = 0; i < 6 && document.activeElement !== tile; i++)
|
||||
await userEvent.keyboard("{ArrowDown}");
|
||||
await expect(document.activeElement).toBe(toggle);
|
||||
// The same ring the device rows get.
|
||||
await expect(outlineWidth(toggle)).toBeGreaterThan(0);
|
||||
await expect(document.activeElement).toBe(tile);
|
||||
await expect(outlineWidth(tile)).toBeGreaterThan(0);
|
||||
|
||||
// And the pointer takes it away, with the toggle still focused.
|
||||
await userEvent.hover(toggle);
|
||||
await expect(document.activeElement).toBe(toggle);
|
||||
await expect(outlineWidth(toggle)).toBe(0);
|
||||
await userEvent.hover(tile);
|
||||
await expect(document.activeElement).toBe(tile);
|
||||
await expect(outlineWidth(tile)).toBe(0);
|
||||
},
|
||||
};
|
||||
|
||||
@@ -5,10 +5,11 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { afterEach, describe, expect, test, vi } from "vitest";
|
||||
import { afterEach, describe, expect, onTestFinished, test, vi } from "vitest";
|
||||
import { axe } from "vitest-axe";
|
||||
import {
|
||||
act,
|
||||
fireEvent,
|
||||
render,
|
||||
screen,
|
||||
within,
|
||||
@@ -22,10 +23,18 @@ import {
|
||||
MediaMuteAndSwitchButton,
|
||||
type MenuOptions,
|
||||
} from "./MediaMuteAndSwitchButton";
|
||||
import { type BackgroundEffectOption } from "./BackgroundEffectGrid";
|
||||
import { MediaDevicesContext } from "../MediaDevicesContext";
|
||||
import { type MediaDevices } from "../state/MediaDevices";
|
||||
import { restoreAudioCapture, stubAudioCapture } from "../utils/test";
|
||||
|
||||
const effects: BackgroundEffectOption[] = [
|
||||
{ id: "none", kind: "none", label: "None" },
|
||||
{ id: "blur", kind: "blur", label: "Blur" },
|
||||
{ id: "image:arc", kind: "image", label: "Background 1", imageUrl: "" },
|
||||
{ id: "image:glow", kind: "image", label: "Background 2", imageUrl: "" },
|
||||
];
|
||||
|
||||
interface RenderOptions {
|
||||
requestDeviceNames: () => void;
|
||||
}
|
||||
@@ -305,30 +314,203 @@ describe("MediaMuteAndSwitchButton", () => {
|
||||
expect(mic2ItemAfter.querySelector(".rotate")).toBeNull();
|
||||
});
|
||||
|
||||
test("renders menu with toggle control and calls toggle callback", async () => {
|
||||
test("background effects are one keyboard-operable labelled choice", async () => {
|
||||
const user = userEvent.setup();
|
||||
const onSelect = vi.fn();
|
||||
const onVideoBlurToggle = vi.fn();
|
||||
const onSelectBackgroundEffect = vi.fn();
|
||||
const { getByRole } = renderComponent(
|
||||
<MediaMuteAndSwitchButton
|
||||
iconsAndLabels="video"
|
||||
enabled={true}
|
||||
videoBlurToggleClick={onVideoBlurToggle}
|
||||
onSelect={onSelect}
|
||||
options={[{ label: { type: "name", name: "Camera 1" }, id: "cam1" }]}
|
||||
selectedOption="cam1"
|
||||
onSelect={vi.fn()}
|
||||
backgroundEffects={effects}
|
||||
selectedBackgroundEffect="none"
|
||||
onSelectBackgroundEffect={onSelectBackgroundEffect}
|
||||
/>,
|
||||
);
|
||||
|
||||
await user.click(getByRole("button", { name: "Camera" }));
|
||||
|
||||
const toggle = screen.getByRole("menuitemcheckbox", {
|
||||
name: "Blur background",
|
||||
const section = screen.getByRole("group", { name: "Background effects" });
|
||||
within(section).getByRole("menuitemradio", { name: "None", checked: true });
|
||||
within(section).getByRole("menuitemradio", {
|
||||
name: "Background 1",
|
||||
checked: false,
|
||||
});
|
||||
expect(toggle).toBeInTheDocument();
|
||||
expect(toggle).toHaveAttribute("aria-checked", "false");
|
||||
|
||||
await user.click(toggle);
|
||||
const blur = within(section).getByRole("menuitemradio", { name: "Blur" });
|
||||
for (let i = 0; i < 6 && document.activeElement !== blur; i++)
|
||||
await user.keyboard("{ArrowDown}");
|
||||
expect(document.activeElement).toBe(blur);
|
||||
await user.keyboard("{Enter}");
|
||||
expect(onSelectBackgroundEffect).toHaveBeenCalledWith("blur");
|
||||
expect(screen.getByRole("menu")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
expect(onVideoBlurToggle).toHaveBeenCalled();
|
||||
test("does not choose the effect already in force again", async () => {
|
||||
const user = userEvent.setup();
|
||||
const onSelectBackgroundEffect = vi.fn();
|
||||
const { getByRole } = renderComponent(
|
||||
<MediaMuteAndSwitchButton
|
||||
iconsAndLabels="video"
|
||||
enabled={true}
|
||||
options={[{ label: { type: "name", name: "Camera 1" }, id: "cam1" }]}
|
||||
selectedOption="cam1"
|
||||
backgroundEffects={effects}
|
||||
selectedBackgroundEffect="blur"
|
||||
onSelectBackgroundEffect={onSelectBackgroundEffect}
|
||||
/>,
|
||||
);
|
||||
|
||||
await user.click(getByRole("button", { name: "Camera" }));
|
||||
await user.click(screen.getByRole("menuitemradio", { name: "Blur" }));
|
||||
expect(onSelectBackgroundEffect).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("added images are chosen like shipped ones", async () => {
|
||||
const user = userEvent.setup();
|
||||
const onSelectBackgroundEffect = vi.fn();
|
||||
const { getByRole } = renderComponent(
|
||||
<MediaMuteAndSwitchButton
|
||||
iconsAndLabels="video"
|
||||
enabled={true}
|
||||
options={[{ label: { type: "name", name: "Camera 1" }, id: "cam1" }]}
|
||||
selectedOption="cam1"
|
||||
backgroundEffects={[
|
||||
...effects,
|
||||
{
|
||||
id: "added:mine",
|
||||
kind: "image",
|
||||
label: "Background 3",
|
||||
imageUrl: "",
|
||||
},
|
||||
]}
|
||||
selectedBackgroundEffect="none"
|
||||
onSelectBackgroundEffect={onSelectBackgroundEffect}
|
||||
/>,
|
||||
);
|
||||
|
||||
await user.click(getByRole("button", { name: "Camera" }));
|
||||
const section = screen.getByRole("group", { name: "Background effects" });
|
||||
await user.click(
|
||||
within(section).getByRole("menuitemradio", { name: "Background 3" }),
|
||||
);
|
||||
expect(onSelectBackgroundEffect).toHaveBeenCalledWith("added:mine");
|
||||
});
|
||||
|
||||
test("adds the file chosen from the add tile, and stays open", async () => {
|
||||
const user = userEvent.setup();
|
||||
const onAddBackgroundImage = vi.fn();
|
||||
const pick = vi
|
||||
.spyOn(HTMLInputElement.prototype, "click")
|
||||
.mockImplementation(() => {});
|
||||
try {
|
||||
const { getByRole, container } = renderComponent(
|
||||
<MediaMuteAndSwitchButton
|
||||
iconsAndLabels="video"
|
||||
enabled={true}
|
||||
options={[{ label: { type: "name", name: "Camera 1" }, id: "cam1" }]}
|
||||
selectedOption="cam1"
|
||||
backgroundEffects={effects}
|
||||
selectedBackgroundEffect="none"
|
||||
onSelectBackgroundEffect={vi.fn()}
|
||||
onAddBackgroundImage={onAddBackgroundImage}
|
||||
/>,
|
||||
);
|
||||
|
||||
await user.click(getByRole("button", { name: "Camera" }));
|
||||
await user.click(screen.getByRole("menuitem", { name: "Add image" }));
|
||||
expect(pick).toHaveBeenCalled();
|
||||
// The picker takes the focus, which would otherwise close the menu.
|
||||
await user.keyboard("{Escape}");
|
||||
expect(screen.getByRole("menu")).toBeInTheDocument();
|
||||
|
||||
const file = new File(["x"], "mine.png", { type: "image/png" });
|
||||
// What the picker hands back; the open menu blocks pointer events.
|
||||
fireEvent.change(
|
||||
container.querySelector<HTMLInputElement>("input[type=file]")!,
|
||||
{ target: { files: [file] } },
|
||||
);
|
||||
expect(onAddBackgroundImage).toHaveBeenCalledWith(file);
|
||||
screen.getByRole("menuitemradio", { name: "None", checked: true });
|
||||
} finally {
|
||||
pick.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
test("a refusal can be dismissed and does not outlive the menu", async () => {
|
||||
const user = userEvent.setup();
|
||||
// jsdom lays nothing out, so has nothing to scroll.
|
||||
const scroll = Element.prototype.scrollIntoView;
|
||||
Element.prototype.scrollIntoView = vi.fn();
|
||||
onTestFinished(() => {
|
||||
Element.prototype.scrollIntoView = scroll;
|
||||
});
|
||||
const menu = (refusal?: { text: string }): JSX.Element =>
|
||||
withProviders(
|
||||
<MediaMuteAndSwitchButton
|
||||
iconsAndLabels="video"
|
||||
enabled={true}
|
||||
options={[{ label: { type: "name", name: "Camera 1" }, id: "cam1" }]}
|
||||
selectedOption="cam1"
|
||||
backgroundEffects={effects}
|
||||
selectedBackgroundEffect="blur"
|
||||
onSelectBackgroundEffect={vi.fn()}
|
||||
onAddBackgroundImage={vi.fn()}
|
||||
backgroundImageRefusal={refusal}
|
||||
/>,
|
||||
);
|
||||
const { rerender, getByRole } = render(menu());
|
||||
await user.click(getByRole("button", { name: "Camera" }));
|
||||
|
||||
rerender(menu({ text: "That file is not a supported image" }));
|
||||
expect(
|
||||
screen.getByText("That file is not a supported image"),
|
||||
).toBeVisible();
|
||||
screen.getByRole("menuitemradio", { name: "Blur", checked: true });
|
||||
await user.click(screen.getByRole("button", { name: "Close" }));
|
||||
expect(screen.queryByText("That file is not a supported image")).toBeNull();
|
||||
|
||||
// The next refusal is shown, even if it says the same.
|
||||
rerender(menu({ text: "That file is not a supported image" }));
|
||||
expect(
|
||||
screen.getByText("That file is not a supported image"),
|
||||
).toBeVisible();
|
||||
|
||||
await user.keyboard("{Escape}");
|
||||
await user.click(getByRole("button", { name: "Camera" }));
|
||||
expect(screen.queryByText("That file is not a supported image")).toBeNull();
|
||||
});
|
||||
|
||||
test("offers the background effects in a phone's drawer", async () => {
|
||||
const userAgent = vi
|
||||
.spyOn(navigator, "userAgent", "get")
|
||||
.mockReturnValue("Mozilla/5.0 (Linux; Android 14)");
|
||||
try {
|
||||
const user = userEvent.setup();
|
||||
const onSelectBackgroundEffect = vi.fn();
|
||||
const { getByRole } = renderComponent(
|
||||
<MediaMuteAndSwitchButton
|
||||
iconsAndLabels="video"
|
||||
enabled={true}
|
||||
options={[{ label: { type: "name", name: "Camera 1" }, id: "cam1" }]}
|
||||
selectedOption="cam1"
|
||||
backgroundEffects={effects}
|
||||
selectedBackgroundEffect="none"
|
||||
onSelectBackgroundEffect={onSelectBackgroundEffect}
|
||||
/>,
|
||||
);
|
||||
|
||||
await user.click(getByRole("button", { name: "Camera" }));
|
||||
const section = screen.getByRole("group", { name: "Background effects" });
|
||||
// A click alone: the drawer's drag handling reads layout jsdom lacks.
|
||||
fireEvent.click(
|
||||
within(section).getByRole("menuitemradio", { name: "Background 2" }),
|
||||
);
|
||||
expect(onSelectBackgroundEffect).toHaveBeenCalledWith("image:glow");
|
||||
} finally {
|
||||
userAgent.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
test("marks the selected menu item as checked", async () => {
|
||||
@@ -552,7 +734,9 @@ describe("MediaMuteAndSwitchButton", () => {
|
||||
]}
|
||||
selectedOption="cam1"
|
||||
onSelect={vi.fn()}
|
||||
videoBlurToggleClick={vi.fn()}
|
||||
backgroundEffects={effects}
|
||||
selectedBackgroundEffect="none"
|
||||
onSelectBackgroundEffect={vi.fn()}
|
||||
/>,
|
||||
);
|
||||
|
||||
@@ -560,9 +744,10 @@ describe("MediaMuteAndSwitchButton", () => {
|
||||
|
||||
screen.getByRole("menuitemradio", { name: "Camera 1", checked: true });
|
||||
screen.getByRole("menuitemradio", { name: "Camera 2", checked: false });
|
||||
expect(
|
||||
screen.getByRole("menuitemcheckbox", { name: "Blur background" }),
|
||||
).toBeInTheDocument();
|
||||
within(screen.getByRole("group", { name: "Background effects" })).getByRole(
|
||||
"menuitemradio",
|
||||
{ name: "Blur" },
|
||||
);
|
||||
});
|
||||
|
||||
test("marks focus as keyboard-driven only when the keyboard moved it", async () => {
|
||||
|
||||
@@ -7,6 +7,8 @@ Please see LICENSE in the repository root for full details.
|
||||
|
||||
import {
|
||||
useCallback,
|
||||
useId,
|
||||
useRef,
|
||||
useState,
|
||||
type CSSProperties,
|
||||
type FC,
|
||||
@@ -14,16 +16,18 @@ import {
|
||||
type ReactElement,
|
||||
} from "react";
|
||||
import {
|
||||
Alert,
|
||||
Button,
|
||||
Menu,
|
||||
MenuItem,
|
||||
MenuScrollArea,
|
||||
MenuTitle,
|
||||
RadioInput,
|
||||
ToggleMenuItem,
|
||||
} from "@vector-im/compound-web";
|
||||
import {
|
||||
ChevronUpIcon,
|
||||
ChevronDownIcon,
|
||||
InfoIcon,
|
||||
SpinnerIcon,
|
||||
} from "@vector-im/compound-design-tokens/assets/web/icons";
|
||||
import classNames from "classnames";
|
||||
@@ -40,6 +44,10 @@ import { useMediaDevices } from "../MediaDevicesContext";
|
||||
import { useRootElement } from "../RootElementContext";
|
||||
import { observeElementSize$ } from "../utils/elementSize";
|
||||
import { LiveMicrophoneLevelMeter } from "./MicrophoneLevelMeter";
|
||||
import {
|
||||
BackgroundEffectGrid,
|
||||
type BackgroundEffectOption,
|
||||
} from "./BackgroundEffectGrid";
|
||||
|
||||
export interface MenuOptions {
|
||||
label: DeviceLabel | AudioOutputDeviceLabel;
|
||||
@@ -64,8 +72,21 @@ export interface MediaMuteAndSwitchButtonProps {
|
||||
selectedOutputOption?: string;
|
||||
/** Picks an output device. Undefined disables the speaker section. */
|
||||
onSelectOutput?: (id: string) => void;
|
||||
videoBlurToggleClick?: () => void;
|
||||
videoBlurEnabled?: boolean;
|
||||
/** Camera menu only. None or an empty list leaves the section out. */
|
||||
backgroundEffects?: BackgroundEffectOption[];
|
||||
selectedBackgroundEffect?: string;
|
||||
/** Undefined where effects can't run: all but no effect are disabled. */
|
||||
onSelectBackgroundEffect?: (id: string) => void;
|
||||
/** What the user should know before choosing an effect, if anything. */
|
||||
backgroundEffectNotice?: string;
|
||||
/** Whether the first effect chosen is still being prepared. */
|
||||
backgroundEffectSettling?: boolean;
|
||||
/** Called with the file chosen from the add tile. Omit to leave it out. */
|
||||
onAddBackgroundImage?: (file: File) => void;
|
||||
/** Removes a removable effect, never the one in force. */
|
||||
onRemoveBackgroundEffect?: (id: string) => void;
|
||||
/** Why the last file offered couldn't be used; a new object each time. */
|
||||
backgroundImageRefusal?: { text: string };
|
||||
/**
|
||||
* For any toggle and option this method will be called.
|
||||
* So toggles need to be implemented by listening here and setting the right toggle item to `enabled`
|
||||
@@ -73,8 +94,6 @@ export interface MediaMuteAndSwitchButtonProps {
|
||||
onSelect?: (id: string) => void;
|
||||
}
|
||||
|
||||
const BLUR_ID = "blur";
|
||||
|
||||
/** Id of the placeholder "Default" row, shown when the platform lists no outputs. */
|
||||
const DEFAULT_OUTPUT_ID = "default";
|
||||
|
||||
@@ -84,6 +103,9 @@ const LIST_SHARE_OF_CALL = 0.6;
|
||||
/** Smallest device list height in px, so a short call still shows more than one device. */
|
||||
const MIN_LIST_HEIGHT = 160;
|
||||
|
||||
/** Space kept between the menu and the call area's sides. */
|
||||
const MENU_MARGIN = 16;
|
||||
|
||||
export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
|
||||
enabled,
|
||||
busy,
|
||||
@@ -94,8 +116,14 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
|
||||
outputOptions,
|
||||
selectedOutputOption,
|
||||
onSelectOutput,
|
||||
videoBlurEnabled,
|
||||
videoBlurToggleClick,
|
||||
backgroundEffects,
|
||||
selectedBackgroundEffect,
|
||||
onSelectBackgroundEffect,
|
||||
backgroundEffectNotice,
|
||||
backgroundEffectSettling,
|
||||
onAddBackgroundImage,
|
||||
onRemoveBackgroundEffect,
|
||||
backgroundImageRefusal,
|
||||
onSelect,
|
||||
}) => {
|
||||
// Requested but not yet selected. Keyed by kind too, since Chrome uses
|
||||
@@ -105,14 +133,46 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
|
||||
id: string;
|
||||
} | null>(null);
|
||||
const [menuOpen, setMenuOpen] = useState(false);
|
||||
const onOpenChange = useCallback((open: boolean): void => {
|
||||
setMenuOpen(open);
|
||||
// Drop a request that never arrived.
|
||||
if (!open) setPlannedSelection(null);
|
||||
// The file picker takes the focus, which the menu reads as a click
|
||||
// elsewhere; it is held open until the picker is done.
|
||||
const choosingFile = useRef(false);
|
||||
const fileInput = useRef<HTMLInputElement | null>(null);
|
||||
// A refusal is about the attempt that caused it: shown until dismissed or
|
||||
// until the menu closes, and again for the next one.
|
||||
const [seenRefusal, setSeenRefusal] = useState<object>();
|
||||
const refusal =
|
||||
backgroundImageRefusal !== seenRefusal ? backgroundImageRefusal : undefined;
|
||||
const onOpenChange = useCallback(
|
||||
(open: boolean): void => {
|
||||
if (!open && choosingFile.current) return;
|
||||
setMenuOpen(open);
|
||||
// Drop a request that never arrived.
|
||||
if (!open) setPlannedSelection(null);
|
||||
if (!open) setSeenRefusal(backgroundImageRefusal);
|
||||
},
|
||||
[backgroundImageRefusal],
|
||||
);
|
||||
// Scrolled to as it appears, as the list may be scrolled away from it.
|
||||
const scrollIntoView = useCallback((element: HTMLElement | null): void => {
|
||||
element?.scrollIntoView({ block: "nearest" });
|
||||
}, []);
|
||||
const watchFileInput = useCallback(
|
||||
(input: HTMLInputElement | null): (() => void) | undefined => {
|
||||
fileInput.current = input;
|
||||
if (input === null) return;
|
||||
// Dismissing the picker fires cancel, which React doesn't type.
|
||||
const done = (): void => {
|
||||
choosingFile.current = false;
|
||||
};
|
||||
input.addEventListener("cancel", done);
|
||||
return (): void => input.removeEventListener("cancel", done);
|
||||
},
|
||||
[],
|
||||
);
|
||||
const isBusy = busy ?? false;
|
||||
const { t } = useTranslation();
|
||||
const devices = useMediaDevices();
|
||||
const noticeId = useId();
|
||||
|
||||
/**
|
||||
* Records on the menu whether the keyboard or the pointer moved focus, for
|
||||
@@ -185,16 +245,8 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
|
||||
/>
|
||||
);
|
||||
|
||||
const toggles =
|
||||
iconsAndLabels === "video" && videoBlurToggleClick !== undefined
|
||||
? [
|
||||
{
|
||||
label: t("action.blur_background"),
|
||||
enabled: videoBlurEnabled ?? false,
|
||||
id: BLUR_ID,
|
||||
},
|
||||
]
|
||||
: [];
|
||||
const showEffects =
|
||||
iconsAndLabels === "video" && (backgroundEffects?.length ?? 0) > 0;
|
||||
|
||||
let optionsButtonLabel: string;
|
||||
let menuTitle: string;
|
||||
@@ -318,6 +370,21 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
|
||||
>
|
||||
{/* The mute button lives inside */}
|
||||
{button}
|
||||
{onAddBackgroundImage !== undefined && (
|
||||
<input
|
||||
ref={watchFileInput}
|
||||
type="file"
|
||||
accept="image/*"
|
||||
hidden
|
||||
onChange={(e) => {
|
||||
const file = e.target.files?.[0];
|
||||
// Cleared, so the same file can be chosen twice in a row.
|
||||
e.target.value = "";
|
||||
choosingFile.current = false;
|
||||
if (file) onAddBackgroundImage(file);
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
<Menu
|
||||
className={styles.menu}
|
||||
// Named for screen readers only: each section has its own heading.
|
||||
@@ -326,6 +393,8 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
|
||||
open={menuOpen}
|
||||
onOpenChange={onOpenChange}
|
||||
side="top"
|
||||
collisionBoundary={rootElement}
|
||||
collisionPadding={MENU_MARGIN}
|
||||
trigger={
|
||||
<Button
|
||||
iconOnly
|
||||
@@ -340,17 +409,21 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
|
||||
/>
|
||||
}
|
||||
>
|
||||
<div
|
||||
<MenuScrollArea
|
||||
ref={trackFocusSource}
|
||||
// Keeps the items the menu's own children for assistive tech.
|
||||
role="none"
|
||||
className={styles.deviceList}
|
||||
className={classNames(styles.deviceList, {
|
||||
[styles.deviceListWithMeter]: iconsAndLabels === "audio",
|
||||
})}
|
||||
style={
|
||||
{
|
||||
"--device-list-max-height":
|
||||
listMaxHeight === undefined ? undefined : `${listMaxHeight}px`,
|
||||
"--device-list-scroll-padding-end":
|
||||
meterHeight === undefined ? undefined : `${meterHeight}px`,
|
||||
// A row reached by keyboard stays above the meter and the band its
|
||||
// ground fills.
|
||||
scrollPaddingBlockEnd:
|
||||
meterHeight === undefined
|
||||
? undefined
|
||||
: `calc(${meterHeight}px + var(--cpd-menu-padding-block-end, 0px))`,
|
||||
"--device-list-scroll-padding-start":
|
||||
headingHeight === undefined ? undefined : `${headingHeight}px`,
|
||||
} as CSSProperties
|
||||
@@ -398,19 +471,53 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{toggles.length > 0 && <hr />}
|
||||
{toggles.map((toggle) => (
|
||||
<ToggleMenuItem
|
||||
label={toggle.label}
|
||||
onSelect={(e) => {
|
||||
videoBlurToggleClick?.();
|
||||
e.preventDefault();
|
||||
}}
|
||||
checked={toggle.enabled}
|
||||
key={toggle.id}
|
||||
/>
|
||||
))}
|
||||
{showEffects && (
|
||||
<BackgroundEffectGrid
|
||||
label={t("settings.background_effects_header")}
|
||||
heading={
|
||||
<div aria-hidden className={styles.sectionHeading}>
|
||||
<MenuTitle title={t("settings.background_effects_header")} />
|
||||
</div>
|
||||
}
|
||||
effects={backgroundEffects ?? []}
|
||||
selected={selectedBackgroundEffect}
|
||||
onSelect={onSelectBackgroundEffect}
|
||||
settling={backgroundEffectSettling}
|
||||
describedBy={
|
||||
backgroundEffectNotice === undefined ? undefined : noticeId
|
||||
}
|
||||
onAdd={
|
||||
onAddBackgroundImage === undefined
|
||||
? undefined
|
||||
: (): void => {
|
||||
choosingFile.current = true;
|
||||
fileInput.current?.click();
|
||||
}
|
||||
}
|
||||
addLabel={t("action.add_background_image")}
|
||||
onRemove={onRemoveBackgroundEffect}
|
||||
removeLabel={t("action.remove")}
|
||||
/>
|
||||
)}
|
||||
{/* In the list, so the menu grows no taller for it. */}
|
||||
{showEffects && refusal !== undefined && (
|
||||
<div ref={scrollIntoView} role="none" className={styles.refusal}>
|
||||
<Alert
|
||||
type="critical"
|
||||
title={refusal.text}
|
||||
onClose={(): void => setSeenRefusal(refusal)}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
{showEffects &&
|
||||
refusal === undefined &&
|
||||
backgroundEffectNotice !== undefined && (
|
||||
<div id={noticeId} role="none" className={styles.notice}>
|
||||
<InfoIcon width={20} height={20} />
|
||||
<span>{backgroundEffectNotice}</span>
|
||||
</div>
|
||||
)}
|
||||
</MenuScrollArea>
|
||||
</Menu>
|
||||
</div>
|
||||
);
|
||||
|
||||
@@ -16,10 +16,10 @@ Please see LICENSE in the repository root for full details.
|
||||
align-items: center;
|
||||
gap: var(--cpd-space-3x);
|
||||
padding-block: var(--cpd-space-2x);
|
||||
/* Centres the icon on the radio column: less the row's inset border and half
|
||||
the icon's extra width. Checked in the MeterAlignsWithTheDeviceRows story. */
|
||||
/* Centres the icon on the radio column: less half the icon's extra width.
|
||||
Checked in the MeterAlignsWithTheDeviceRows story. */
|
||||
padding-inline-start: calc(
|
||||
var(--cpd-space-4x) - var(--cpd-border-width-1) -
|
||||
var(--cpd-space-4x) -
|
||||
(var(--meter-icon-size) - var(--device-control-size)) / 2
|
||||
);
|
||||
padding-inline-end: calc(var(--cpd-space-4x) * 2 + var(--cpd-space-2x));
|
||||
|
||||
@@ -9,7 +9,7 @@ exports[`MediaMuteAndSwitchButton > renders 1`] = `
|
||||
aria-busy="false"
|
||||
aria-checked="false"
|
||||
aria-disabled="true"
|
||||
aria-labelledby="_r_0_"
|
||||
aria-labelledby="_r_1_"
|
||||
class="_button_1nw83_8 _has-icon_1nw83_60 _icon-only_1nw83_53"
|
||||
data-kind="primary"
|
||||
data-size="lg"
|
||||
@@ -39,7 +39,7 @@ exports[`MediaMuteAndSwitchButton > renders 1`] = `
|
||||
data-kind="tertiary"
|
||||
data-size="lg"
|
||||
data-state="closed"
|
||||
id="radix-_r_5_"
|
||||
id="radix-_r_6_"
|
||||
role="button"
|
||||
tabindex="0"
|
||||
type="button"
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 27 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 22 KiB |
@@ -0,0 +1,81 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
BackgroundTransformer,
|
||||
VideoTransformer,
|
||||
type VideoTransformerInitOptions,
|
||||
} from "@livekit/track-processors";
|
||||
import { ImageSegmenter } from "@mediapipe/tasks-vision";
|
||||
|
||||
import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer";
|
||||
|
||||
const frame = {} as VideoFrame;
|
||||
const controller = {} as TransformStreamDefaultController<VideoFrame>;
|
||||
|
||||
describe("BackgroundEffectTransformer", () => {
|
||||
afterEach(() => vi.restoreAllMocks());
|
||||
|
||||
it("loads a background chosen before it was built", async () => {
|
||||
vi.spyOn(VideoTransformer.prototype, "init").mockResolvedValue();
|
||||
vi.spyOn(ImageSegmenter, "createFromOptions").mockResolvedValue(
|
||||
{} as ImageSegmenter,
|
||||
);
|
||||
const load = vi
|
||||
.spyOn(BackgroundTransformer.prototype, "loadAndSetBackground")
|
||||
.mockResolvedValue();
|
||||
const transformer = new BackgroundEffectTransformer({
|
||||
backgroundDisabled: true,
|
||||
});
|
||||
// Chosen before the camera is processed, as the first effect always is:
|
||||
// loaded then, there is nothing yet to draw it with.
|
||||
await transformer.update({ imagePath: "/background.jpg" });
|
||||
load.mockClear();
|
||||
|
||||
await transformer.init({} as VideoTransformerInitOptions);
|
||||
expect(load).toHaveBeenCalledWith("/background.jpg");
|
||||
});
|
||||
|
||||
it("reports the first frame carrying an effect, once", async () => {
|
||||
vi.spyOn(BackgroundTransformer.prototype, "transform").mockImplementation(
|
||||
async function (this: BackgroundTransformer) {
|
||||
this.isFirstFrame = false;
|
||||
return Promise.resolve();
|
||||
},
|
||||
);
|
||||
const transformer = new BackgroundEffectTransformer({});
|
||||
transformer.onFirstFrame = vi.fn();
|
||||
|
||||
await transformer.transform(frame, controller);
|
||||
await transformer.transform(frame, controller);
|
||||
expect(transformer.onFirstFrame).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not report a frame passed through untouched", async () => {
|
||||
vi.spyOn(BackgroundTransformer.prototype, "transform").mockResolvedValue();
|
||||
const transformer = new BackgroundEffectTransformer({
|
||||
backgroundDisabled: true,
|
||||
});
|
||||
transformer.onFirstFrame = vi.fn();
|
||||
|
||||
await transformer.transform(frame, controller);
|
||||
expect(transformer.onFirstFrame).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("passes frames on untouched once disabled, even after blurring", async () => {
|
||||
const transformer = new BackgroundEffectTransformer({});
|
||||
await transformer.update({ blurRadius: 15, backgroundDisabled: false });
|
||||
await transformer.update({
|
||||
imagePath: undefined,
|
||||
backgroundDisabled: true,
|
||||
});
|
||||
// With neither, the library passes a frame on without segmenting it.
|
||||
expect(transformer.options.blurRadius).toBeUndefined();
|
||||
expect(transformer.options.imagePath).toBeUndefined();
|
||||
});
|
||||
});
|
||||
+36
-1
@@ -1,5 +1,6 @@
|
||||
/*
|
||||
Copyright 2024-2025 New Vector Ltd.
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
@@ -9,7 +10,9 @@ import {
|
||||
BackgroundTransformer,
|
||||
VideoTransformer,
|
||||
type VideoTransformerInitOptions,
|
||||
type BackgroundOptions,
|
||||
} from "@livekit/track-processors";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
import { ImageSegmenter } from "@mediapipe/tasks-vision";
|
||||
|
||||
import modelAssetPath from "../mediapipe/imageSegmenter/selfie_segmenter.tflite?url";
|
||||
@@ -49,7 +52,32 @@ const wasmFileset: WasmFileset = {
|
||||
* loads the segmentation models from our own bundle rather than as an external
|
||||
* resource fetched from the public internet.
|
||||
*/
|
||||
export class BlurBackgroundTransformer extends BackgroundTransformer {
|
||||
export class BackgroundEffectTransformer extends BackgroundTransformer {
|
||||
/** Called once, when the first frame carrying an effect has been drawn. */
|
||||
public onFirstFrame: (() => void) | undefined;
|
||||
|
||||
public override async transform(
|
||||
frame: VideoFrame,
|
||||
controller: TransformStreamDefaultController<VideoFrame>,
|
||||
): Promise<void> {
|
||||
const priming = this.isFirstFrame;
|
||||
await super.transform(frame, controller);
|
||||
// Cleared only once a frame has been through the segmenter; a frame
|
||||
// passed through untouched leaves it set.
|
||||
if (priming && !this.isFirstFrame) this.onFirstFrame?.();
|
||||
}
|
||||
|
||||
/**
|
||||
* As the library's, except that disabling also drops the blur radius: kept,
|
||||
* it has every frame segmented and the result thrown away, where with
|
||||
* neither a radius nor a picture frames are passed on untouched.
|
||||
*/
|
||||
public override async update(opts: BackgroundOptions): Promise<void> {
|
||||
await super.update(
|
||||
opts.backgroundDisabled ? { ...opts, blurRadius: undefined } : opts,
|
||||
);
|
||||
}
|
||||
|
||||
public async init({
|
||||
outputCanvas,
|
||||
inputElement: inputVideo,
|
||||
@@ -73,8 +101,15 @@ export class BlurBackgroundTransformer extends BackgroundTransformer {
|
||||
outputConfidenceMasks: false,
|
||||
});
|
||||
|
||||
// BackgroundTransformer's own init applies these, and this one replaces it.
|
||||
if (this.options.imagePath) {
|
||||
await this.loadAndSetBackground(this.options.imagePath).catch((e) =>
|
||||
logger.warn("Failed to load the background image", e),
|
||||
);
|
||||
}
|
||||
if (this.options.blurRadius) {
|
||||
this.gl?.setBlurRadius(this.options.blurRadius);
|
||||
}
|
||||
this.gl?.setBackgroundDisabled(this.options.backgroundDisabled ?? false);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,275 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { act, createElement, type FC } from "react";
|
||||
import { render } from "@testing-library/react";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { type LocalVideoTrack } from "livekit-client";
|
||||
import {
|
||||
type BackgroundOptions,
|
||||
type ProcessorWrapper,
|
||||
} from "@livekit/track-processors";
|
||||
|
||||
import {
|
||||
applyProcessor,
|
||||
BackgroundEffectsProvider,
|
||||
type BackgroundEffectsState,
|
||||
syncBackgroundEffects,
|
||||
useBackgroundEffects,
|
||||
useSyncBackgroundEffects,
|
||||
createAppBackgroundEffects,
|
||||
} from "./BackgroundEffectsContext";
|
||||
import { type BackgroundEffects } from "../state/BackgroundEffects";
|
||||
import { backgroundEffect } from "../settings/settings";
|
||||
import { type AddedBackground } from "./backgroundImages";
|
||||
import { constant } from "../state/Behavior";
|
||||
import { flushPromises, testScope } from "../utils/test";
|
||||
|
||||
const sdkSupportMock = vi.hoisted(() => vi.fn(() => true));
|
||||
const pipelines = vi.hoisted(() => ({
|
||||
built: 0,
|
||||
destroyed: 0,
|
||||
switches: [] as unknown[],
|
||||
}));
|
||||
|
||||
// The pipeline itself needs WebGL and MediaPipe; what these tests are about is
|
||||
// what the provider asks of it.
|
||||
vi.mock("@livekit/track-processors", () => ({
|
||||
BackgroundProcessorWrapper: vi.fn(function (this: Record<string, unknown>) {
|
||||
pipelines.built += 1;
|
||||
this.switchTo = vi.fn(async (options: unknown) => {
|
||||
pipelines.switches.push(options);
|
||||
return Promise.resolve();
|
||||
});
|
||||
this.destroy = vi.fn(async () => {
|
||||
pipelines.destroyed += 1;
|
||||
return Promise.resolve();
|
||||
});
|
||||
}),
|
||||
supportsBackgroundProcessors: (): boolean => sdkSupportMock(),
|
||||
}));
|
||||
const platformMock = vi.hoisted(() => ({ platform: "desktop" }));
|
||||
vi.mock("../Platform", () => platformMock);
|
||||
vi.mock("./backgroundImages", async () => {
|
||||
const { BehaviorSubject } = await import("rxjs");
|
||||
return {
|
||||
addedBackgrounds: {
|
||||
added$: new BehaviorSubject<AddedBackground[] | undefined>(undefined),
|
||||
},
|
||||
};
|
||||
});
|
||||
vi.mock("./BackgroundEffectTransformer", () => ({
|
||||
BackgroundEffectTransformer: vi.fn(),
|
||||
}));
|
||||
|
||||
const processor = {} as ProcessorWrapper<BackgroundOptions>;
|
||||
|
||||
function mockTrack(
|
||||
readyState: MediaStreamTrackState,
|
||||
current?: ProcessorWrapper<BackgroundOptions>,
|
||||
): LocalVideoTrack {
|
||||
return {
|
||||
mediaStreamTrack: { readyState },
|
||||
getProcessor: vi.fn().mockReturnValue(current),
|
||||
setProcessor: vi.fn().mockResolvedValue(undefined),
|
||||
stopProcessor: vi.fn().mockResolvedValue(undefined),
|
||||
} as unknown as LocalVideoTrack;
|
||||
}
|
||||
|
||||
describe("applyProcessor", () => {
|
||||
it("attaches the processor to a live track", () => {
|
||||
const track = mockTrack("live");
|
||||
applyProcessor(track, processor);
|
||||
expect(track.setProcessor).toHaveBeenCalledWith(processor);
|
||||
});
|
||||
|
||||
it("does not attach the processor to an ended track", () => {
|
||||
const track = mockTrack("ended");
|
||||
applyProcessor(track, processor);
|
||||
expect(track.setProcessor).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not surface a rejected setProcessor", async () => {
|
||||
const track = mockTrack("live");
|
||||
vi.mocked(track.setProcessor).mockRejectedValue(
|
||||
new TypeError("Input track cannot be ended"),
|
||||
);
|
||||
const unhandled = vi.fn();
|
||||
process.on("unhandledRejection", unhandled);
|
||||
applyProcessor(track, processor);
|
||||
await flushPromises();
|
||||
process.off("unhandledRejection", unhandled);
|
||||
expect(unhandled).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("stops the processor when none is wanted", () => {
|
||||
const track = mockTrack("live", processor);
|
||||
applyProcessor(track, undefined);
|
||||
expect(track.stopProcessor).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not surface a rejected stopProcessor", async () => {
|
||||
const track = mockTrack("live", processor);
|
||||
vi.mocked(track.stopProcessor).mockRejectedValue(new Error("nope"));
|
||||
const unhandled = vi.fn();
|
||||
process.on("unhandledRejection", unhandled);
|
||||
applyProcessor(track, undefined);
|
||||
await flushPromises();
|
||||
process.off("unhandledRejection", unhandled);
|
||||
expect(unhandled).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("syncBackgroundEffects", () => {
|
||||
it("applies the processor to the current track", () => {
|
||||
const track = mockTrack("live");
|
||||
syncBackgroundEffects(
|
||||
testScope(),
|
||||
constant(track),
|
||||
constant({ supported: true, processor }),
|
||||
);
|
||||
expect(track.setProcessor).toHaveBeenCalledWith(processor);
|
||||
});
|
||||
});
|
||||
|
||||
describe("BackgroundEffectsProvider", () => {
|
||||
/** A camera track that holds whatever processor it is given. */
|
||||
function cameraTrack(): LocalVideoTrack {
|
||||
let current: unknown;
|
||||
return {
|
||||
mediaStreamTrack: { readyState: "live" },
|
||||
getProcessor: vi.fn(() => current),
|
||||
setProcessor: vi.fn(async (next: unknown) => {
|
||||
current = next;
|
||||
return Promise.resolve();
|
||||
}),
|
||||
stopProcessor: vi.fn(async () => {
|
||||
current = undefined;
|
||||
return Promise.resolve();
|
||||
}),
|
||||
} as unknown as LocalVideoTrack;
|
||||
}
|
||||
|
||||
let seen: BackgroundEffectsState[];
|
||||
const Surface: FC<{ track: LocalVideoTrack | null }> = ({ track }) => {
|
||||
seen.push(useBackgroundEffects());
|
||||
useSyncBackgroundEffects(track);
|
||||
return null;
|
||||
};
|
||||
// One component for every render, so a rerender updates the tree rather than
|
||||
// mounting a second provider with a pipeline of its own.
|
||||
let effects: BackgroundEffects | undefined;
|
||||
const Surfaces: FC<{ tracks: (LocalVideoTrack | null)[] }> = ({ tracks }) =>
|
||||
createElement(BackgroundEffectsProvider, {
|
||||
// Made on first render, once a test has set up the platform.
|
||||
effects: (effects ??= createAppBackgroundEffects(testScope())),
|
||||
children: createElement(
|
||||
"div",
|
||||
null,
|
||||
...tracks.map((track, key) => createElement(Surface, { key, track })),
|
||||
),
|
||||
});
|
||||
const surfaces = (
|
||||
...tracks: (LocalVideoTrack | null)[]
|
||||
): ReturnType<typeof createElement> => createElement(Surfaces, { tracks });
|
||||
const blur = async (on: boolean): Promise<void> => {
|
||||
await act(async () => {
|
||||
backgroundEffect.setValue(on ? "blur" : "none");
|
||||
await flushPromises();
|
||||
});
|
||||
};
|
||||
const latest = (): BackgroundEffectsState => seen[seen.length - 1];
|
||||
|
||||
beforeEach(() => {
|
||||
seen = [];
|
||||
effects = undefined;
|
||||
pipelines.built = 0;
|
||||
pipelines.destroyed = 0;
|
||||
pipelines.switches = [];
|
||||
platformMock.platform = "desktop";
|
||||
sdkSupportMock.mockReturnValue(true);
|
||||
backgroundEffect.setValue("none");
|
||||
});
|
||||
afterEach(() => backgroundEffect.setValue("none"));
|
||||
|
||||
// A pipeline primes itself when it is built and when it is destroyed, and a
|
||||
// primed pipeline lets one frame through untouched: so the frame is spent
|
||||
// once, as long as it is built once and never taken off the camera.
|
||||
it("spends one priming frame on first use and none afterwards", async () => {
|
||||
const track = cameraTrack();
|
||||
render(surfaces(track));
|
||||
for (const on of [true, false, true, false, true]) await blur(on);
|
||||
|
||||
expect(pipelines.built).toBe(1);
|
||||
expect(pipelines.destroyed).toBe(0);
|
||||
expect(track.setProcessor).toHaveBeenCalledTimes(1);
|
||||
expect(track.stopProcessor).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("camera on with an effect chosen publishes nothing unprocessed", async () => {
|
||||
const view = render(surfaces(cameraTrack()));
|
||||
await blur(true);
|
||||
const pipeline = latest().processor;
|
||||
|
||||
// Camera off, then on again: a new track, given the pipeline as it
|
||||
// arrives, and the pipeline neither rebuilt nor switched again.
|
||||
await act(async () => {
|
||||
view.rerender(surfaces(null));
|
||||
await flushPromises();
|
||||
});
|
||||
const again = cameraTrack();
|
||||
await act(async () => {
|
||||
view.rerender(surfaces(again));
|
||||
await flushPromises();
|
||||
});
|
||||
expect(again.setProcessor).toHaveBeenCalledTimes(1);
|
||||
expect(again.setProcessor).toHaveBeenCalledWith(pipeline);
|
||||
expect(pipelines.built).toBe(1);
|
||||
expect(pipelines.switches).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("runs on a phone whose browser can run it", async () => {
|
||||
platformMock.platform = "ios";
|
||||
const track = cameraTrack();
|
||||
render(surfaces(track));
|
||||
await blur(true);
|
||||
|
||||
expect(latest().supported).toBe(true);
|
||||
expect(track.setProcessor).toHaveBeenCalledWith(latest().processor);
|
||||
});
|
||||
|
||||
it("preview and call share one pipeline", async () => {
|
||||
const preview = cameraTrack();
|
||||
const call = cameraTrack();
|
||||
render(surfaces(preview, call));
|
||||
await blur(true);
|
||||
|
||||
const pipeline = latest().processor;
|
||||
expect(preview.setProcessor).toHaveBeenCalledWith(pipeline);
|
||||
expect(call.setProcessor).toHaveBeenCalledWith(pipeline);
|
||||
expect(pipelines.built).toBe(1);
|
||||
});
|
||||
|
||||
it("applies an effect chosen while the camera is off", async () => {
|
||||
const view = render(surfaces(null));
|
||||
await blur(true);
|
||||
// Already switched to blur, with nothing yet to attach it to.
|
||||
expect(pipelines.switches).toEqual([
|
||||
{ mode: "background-blur", blurRadius: 15 },
|
||||
]);
|
||||
|
||||
const track = cameraTrack();
|
||||
await act(async () => {
|
||||
view.rerender(surfaces(track));
|
||||
await flushPromises();
|
||||
});
|
||||
expect(track.setProcessor).toHaveBeenCalledTimes(1);
|
||||
expect(track.setProcessor).toHaveBeenCalledWith(latest().processor);
|
||||
expect(pipelines.built).toBe(1);
|
||||
expect(pipelines.switches).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,143 @@
|
||||
/*
|
||||
Copyright 2024-2025 New Vector Ltd.
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import {
|
||||
type ProcessorWrapper,
|
||||
type BackgroundOptions,
|
||||
supportsBackgroundProcessors,
|
||||
} from "@livekit/track-processors";
|
||||
import { createContext, type FC, type JSX, use, useEffect } from "react";
|
||||
import { type LocalVideoTrack } from "livekit-client";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
import { combineLatest } from "rxjs";
|
||||
|
||||
import { backgroundEffect as backgroundEffectSetting } from "../settings/settings";
|
||||
import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer";
|
||||
import { OneStepPipeline } from "./OneStepPipeline";
|
||||
import { addedBackgrounds } from "./backgroundImages";
|
||||
import { type Behavior } from "../state/Behavior";
|
||||
import { type ObservableScope } from "../state/ObservableScope";
|
||||
import {
|
||||
type BackgroundEffects,
|
||||
createBackgroundEffects,
|
||||
} from "../state/BackgroundEffects";
|
||||
import { useBehavior } from "../useBehavior";
|
||||
|
||||
//TODO-MULTI-SFU: This is not yet fully there.
|
||||
// it is a combination of exposing observable and react hooks.
|
||||
// preferably we should not make this a context anymore and instead just a vm?
|
||||
|
||||
export type BackgroundEffectsState = {
|
||||
supported: boolean | undefined;
|
||||
processor: undefined | ProcessorWrapper<BackgroundOptions>;
|
||||
/** From the first effect chosen until a frame carrying it is drawn. */
|
||||
settling?: boolean;
|
||||
};
|
||||
|
||||
const BackgroundEffectsContext = createContext<BackgroundEffects | undefined>(
|
||||
undefined,
|
||||
);
|
||||
|
||||
export function useBackgroundEffects(): BackgroundEffectsState {
|
||||
return useBehavior(useBackgroundEffectsState$());
|
||||
}
|
||||
|
||||
export function useBackgroundEffectsState$(): Behavior<BackgroundEffectsState> {
|
||||
const effects = use(BackgroundEffectsContext);
|
||||
if (effects === undefined)
|
||||
throw new Error(
|
||||
"useBackgroundEffects must be used within a BackgroundEffectsProvider",
|
||||
);
|
||||
return effects.state$;
|
||||
}
|
||||
|
||||
/**
|
||||
* Attaches or detaches the processor so that the track matches the desired
|
||||
* state, without throwing.
|
||||
*/
|
||||
export function applyProcessor(
|
||||
videoTrack: LocalVideoTrack,
|
||||
processor: ProcessorWrapper<BackgroundOptions> | undefined,
|
||||
): void {
|
||||
if (processor && !videoTrack.getProcessor()) {
|
||||
// A MediaStreamTrackProcessor cannot be constructed on an ended track
|
||||
// (e.g. the camera was stopped while the processor was being applied),
|
||||
// and setProcessor rejects with a TypeError. The track is going away
|
||||
// anyway, so there is nothing to attach to.
|
||||
if (videoTrack.mediaStreamTrack.readyState === "ended") {
|
||||
logger.debug("Not attaching video processor to an ended track");
|
||||
return;
|
||||
}
|
||||
const track = videoTrack.mediaStreamTrack;
|
||||
videoTrack.setProcessor(processor).catch((e) => {
|
||||
if (track.readyState === "ended")
|
||||
logger.debug("Video processor not attached: the track ended first");
|
||||
else logger.warn("Failed to attach video processor", e);
|
||||
});
|
||||
}
|
||||
if (!processor && videoTrack.getProcessor()) {
|
||||
videoTrack.stopProcessor().catch((e) => {
|
||||
logger.warn("Failed to stop video processor", e);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates your video tracks to always use the given processor.
|
||||
*/
|
||||
export const syncBackgroundEffects = (
|
||||
scope: ObservableScope,
|
||||
videoTrack$: Behavior<LocalVideoTrack | null>,
|
||||
backgroundEffectsState$: Behavior<BackgroundEffectsState>,
|
||||
): void => {
|
||||
combineLatest([videoTrack$, backgroundEffectsState$])
|
||||
.pipe(scope.bind())
|
||||
.subscribe(([videoTrack, backgroundEffectsState]) => {
|
||||
if (!backgroundEffectsState) return;
|
||||
if (!videoTrack) return;
|
||||
applyProcessor(videoTrack, backgroundEffectsState.processor);
|
||||
});
|
||||
};
|
||||
|
||||
export const useSyncBackgroundEffects = (
|
||||
videoTrack: LocalVideoTrack | null,
|
||||
): void => {
|
||||
const { processor } = useBackgroundEffects();
|
||||
useEffect(() => {
|
||||
if (!videoTrack) return;
|
||||
applyProcessor(videoTrack, processor);
|
||||
}, [processor, videoTrack]);
|
||||
};
|
||||
|
||||
/** The app's one pipeline, shared by every camera track it opens. */
|
||||
export function createAppBackgroundEffects(
|
||||
scope: ObservableScope,
|
||||
): BackgroundEffects {
|
||||
const transformer = new BackgroundEffectTransformer({
|
||||
backgroundDisabled: true,
|
||||
});
|
||||
return createBackgroundEffects(scope, {
|
||||
supported: supportsBackgroundProcessors(),
|
||||
effect$: backgroundEffectSetting.value$,
|
||||
setEffect: backgroundEffectSetting.setValue,
|
||||
added$: addedBackgrounds.added$,
|
||||
pipeline: new OneStepPipeline(transformer, "background-effect"),
|
||||
transformer,
|
||||
});
|
||||
}
|
||||
|
||||
interface Props {
|
||||
effects: BackgroundEffects;
|
||||
children: JSX.Element;
|
||||
}
|
||||
|
||||
export const BackgroundEffectsProvider: FC<Props> = ({ effects, children }) => (
|
||||
<BackgroundEffectsContext value={effects}>
|
||||
{children}
|
||||
</BackgroundEffectsContext>
|
||||
);
|
||||
@@ -0,0 +1,80 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { ProcessorWrapper } from "@livekit/track-processors";
|
||||
|
||||
import { OneStepPipeline } from "./OneStepPipeline";
|
||||
import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer";
|
||||
import { flushPromises } from "../utils/test";
|
||||
|
||||
const opts = {} as Parameters<OneStepPipeline["init"]>[0];
|
||||
|
||||
describe("OneStepPipeline", () => {
|
||||
afterEach(() => vi.restoreAllMocks());
|
||||
|
||||
it("stops and builds again only once a build has finished", async () => {
|
||||
let finishBuild!: () => void;
|
||||
const init = vi
|
||||
.spyOn(ProcessorWrapper.prototype, "init")
|
||||
.mockImplementationOnce(
|
||||
async () => new Promise((resolve) => (finishBuild = resolve)),
|
||||
)
|
||||
.mockResolvedValue();
|
||||
const destroy = vi
|
||||
.spyOn(ProcessorWrapper.prototype, "destroy")
|
||||
.mockResolvedValue();
|
||||
const pipeline = new OneStepPipeline(
|
||||
new BackgroundEffectTransformer({}),
|
||||
"test",
|
||||
);
|
||||
|
||||
const first = pipeline.init(opts);
|
||||
const stopped = pipeline.destroy();
|
||||
const second = pipeline.init(opts);
|
||||
await flushPromises();
|
||||
expect(init).toHaveBeenCalledTimes(1);
|
||||
expect(destroy).not.toHaveBeenCalled();
|
||||
|
||||
finishBuild();
|
||||
await Promise.all([first, stopped, second]);
|
||||
expect(init).toHaveBeenCalledTimes(2);
|
||||
expect(destroy.mock.invocationCallOrder[0]).toBeLessThan(
|
||||
init.mock.invocationCallOrder[1],
|
||||
);
|
||||
});
|
||||
|
||||
it("restarts in one step, without waiting on itself", async () => {
|
||||
const init = vi
|
||||
.spyOn(ProcessorWrapper.prototype, "init")
|
||||
.mockResolvedValue();
|
||||
const destroy = vi
|
||||
.spyOn(ProcessorWrapper.prototype, "destroy")
|
||||
.mockResolvedValue();
|
||||
const pipeline = new OneStepPipeline(
|
||||
new BackgroundEffectTransformer({}),
|
||||
"test",
|
||||
);
|
||||
|
||||
await pipeline.restart(opts);
|
||||
expect(destroy).toHaveBeenCalledWith({ willProcessorRestart: true });
|
||||
expect(init).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("carries on after a step that failed", async () => {
|
||||
vi.spyOn(ProcessorWrapper.prototype, "init")
|
||||
.mockRejectedValueOnce(new Error("no GPU"))
|
||||
.mockResolvedValue();
|
||||
const pipeline = new OneStepPipeline(
|
||||
new BackgroundEffectTransformer({}),
|
||||
"test",
|
||||
);
|
||||
|
||||
await expect(pipeline.init(opts)).rejects.toThrow("no GPU");
|
||||
await expect(pipeline.init(opts)).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,43 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { BackgroundProcessorWrapper } from "@livekit/track-processors";
|
||||
|
||||
type InitOptions = Parameters<BackgroundProcessorWrapper["init"]>[0];
|
||||
type DestroyOptions = Parameters<BackgroundProcessorWrapper["destroy"]>[0];
|
||||
|
||||
/**
|
||||
* A background pipeline that is built, rebuilt and destroyed one step at a
|
||||
* time. The camera tracks sharing it attach and stop it independently, and
|
||||
* the SDK's wrapper holds a single track's streams: two builds at once
|
||||
* overwrite each other's, and a destroy during a build does nothing.
|
||||
*/
|
||||
export class OneStepPipeline extends BackgroundProcessorWrapper {
|
||||
private steps: Promise<void> = Promise.resolve();
|
||||
|
||||
public override async init(opts: InitOptions): Promise<void> {
|
||||
return this.inTurn(async () => super.init(opts));
|
||||
}
|
||||
|
||||
// The SDK's restart calls init and destroy, which would wait on itself.
|
||||
public override async restart(opts: InitOptions): Promise<void> {
|
||||
return this.inTurn(async () => {
|
||||
await super.destroy({ willProcessorRestart: true });
|
||||
await super.init(opts);
|
||||
});
|
||||
}
|
||||
|
||||
public override async destroy(options?: DestroyOptions): Promise<void> {
|
||||
return this.inTurn(async () => super.destroy(options));
|
||||
}
|
||||
|
||||
private async inTurn(step: () => Promise<void>): Promise<void> {
|
||||
const turn = this.steps.then(step);
|
||||
this.steps = turn.catch(() => {});
|
||||
return turn;
|
||||
}
|
||||
}
|
||||
@@ -1,87 +0,0 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { type LocalVideoTrack } from "livekit-client";
|
||||
import {
|
||||
type BackgroundOptions,
|
||||
type ProcessorWrapper,
|
||||
} from "@livekit/track-processors";
|
||||
|
||||
import { applyProcessor, trackProcessorSync } from "./TrackProcessorContext";
|
||||
import { constant } from "../state/Behavior";
|
||||
import { flushPromises, testScope } from "../utils/test";
|
||||
|
||||
const processor = {} as ProcessorWrapper<BackgroundOptions>;
|
||||
|
||||
function mockTrack(
|
||||
readyState: MediaStreamTrackState,
|
||||
current?: ProcessorWrapper<BackgroundOptions>,
|
||||
): LocalVideoTrack {
|
||||
return {
|
||||
mediaStreamTrack: { readyState },
|
||||
getProcessor: vi.fn().mockReturnValue(current),
|
||||
setProcessor: vi.fn().mockResolvedValue(undefined),
|
||||
stopProcessor: vi.fn().mockResolvedValue(undefined),
|
||||
} as unknown as LocalVideoTrack;
|
||||
}
|
||||
|
||||
describe("applyProcessor", () => {
|
||||
it("attaches the processor to a live track", () => {
|
||||
const track = mockTrack("live");
|
||||
applyProcessor(track, processor);
|
||||
expect(track.setProcessor).toHaveBeenCalledWith(processor);
|
||||
});
|
||||
|
||||
it("does not attach the processor to an ended track", () => {
|
||||
const track = mockTrack("ended");
|
||||
applyProcessor(track, processor);
|
||||
expect(track.setProcessor).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not surface a rejected setProcessor", async () => {
|
||||
const track = mockTrack("live");
|
||||
vi.mocked(track.setProcessor).mockRejectedValue(
|
||||
new TypeError("Input track cannot be ended"),
|
||||
);
|
||||
const unhandled = vi.fn();
|
||||
process.on("unhandledRejection", unhandled);
|
||||
applyProcessor(track, processor);
|
||||
await flushPromises();
|
||||
process.off("unhandledRejection", unhandled);
|
||||
expect(unhandled).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("stops the processor when none is wanted", () => {
|
||||
const track = mockTrack("live", processor);
|
||||
applyProcessor(track, undefined);
|
||||
expect(track.stopProcessor).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not surface a rejected stopProcessor", async () => {
|
||||
const track = mockTrack("live", processor);
|
||||
vi.mocked(track.stopProcessor).mockRejectedValue(new Error("nope"));
|
||||
const unhandled = vi.fn();
|
||||
process.on("unhandledRejection", unhandled);
|
||||
applyProcessor(track, undefined);
|
||||
await flushPromises();
|
||||
process.off("unhandledRejection", unhandled);
|
||||
expect(unhandled).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("trackProcessorSync", () => {
|
||||
it("applies the processor to the current track", () => {
|
||||
const track = mockTrack("live");
|
||||
trackProcessorSync(
|
||||
testScope(),
|
||||
constant(track),
|
||||
constant({ supported: true, processor }),
|
||||
);
|
||||
expect(track.setProcessor).toHaveBeenCalledWith(processor);
|
||||
});
|
||||
});
|
||||
@@ -1,145 +0,0 @@
|
||||
/*
|
||||
Copyright 2024-2025 New Vector Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import {
|
||||
ProcessorWrapper,
|
||||
supportsBackgroundProcessors as supportsBackgroundProcessorsLivekitSdk,
|
||||
type BackgroundOptions,
|
||||
} from "@livekit/track-processors";
|
||||
import {
|
||||
createContext,
|
||||
type FC,
|
||||
type JSX,
|
||||
use,
|
||||
useEffect,
|
||||
useMemo,
|
||||
} from "react";
|
||||
import { type LocalVideoTrack } from "livekit-client";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
import { combineLatest } from "rxjs";
|
||||
|
||||
import {
|
||||
backgroundBlur as backgroundBlurSettings,
|
||||
useSetting,
|
||||
} from "../settings/settings";
|
||||
import { BlurBackgroundTransformer } from "./BlurBackgroundTransformer";
|
||||
import { type Behavior } from "../state/Behavior";
|
||||
import { type ObservableScope } from "../state/ObservableScope";
|
||||
import { platform } from "../Platform";
|
||||
import { useValueBehavior } from "../useValueBehavior";
|
||||
|
||||
//TODO-MULTI-SFU: This is not yet fully there.
|
||||
// it is a combination of exposing observable and react hooks.
|
||||
// preferably we should not make this a context anymore and instead just a vm?
|
||||
|
||||
export type ProcessorState = {
|
||||
supported: boolean | undefined;
|
||||
processor: undefined | ProcessorWrapper<BackgroundOptions>;
|
||||
};
|
||||
|
||||
const ProcessorContext = createContext<ProcessorState | undefined>(undefined);
|
||||
|
||||
export function useTrackProcessor(): ProcessorState {
|
||||
const state = use(ProcessorContext);
|
||||
if (state === undefined)
|
||||
throw new Error(
|
||||
"useTrackProcessor must be used within a ProcessorProvider",
|
||||
);
|
||||
return state;
|
||||
}
|
||||
|
||||
export function useTrackProcessorState$(): Behavior<ProcessorState> {
|
||||
return useValueBehavior(useTrackProcessor());
|
||||
}
|
||||
|
||||
/**
|
||||
* Attaches or detaches the processor so that the track matches the desired
|
||||
* state, without throwing.
|
||||
*/
|
||||
export function applyProcessor(
|
||||
videoTrack: LocalVideoTrack,
|
||||
processor: ProcessorWrapper<BackgroundOptions> | undefined,
|
||||
): void {
|
||||
if (processor && !videoTrack.getProcessor()) {
|
||||
// A MediaStreamTrackProcessor cannot be constructed on an ended track
|
||||
// (e.g. the camera was stopped while the processor was being applied),
|
||||
// and setProcessor rejects with a TypeError. The track is going away
|
||||
// anyway, so there is nothing to attach to.
|
||||
if (videoTrack.mediaStreamTrack.readyState === "ended") {
|
||||
logger.debug("Not attaching video processor to an ended track");
|
||||
return;
|
||||
}
|
||||
videoTrack.setProcessor(processor).catch((e) => {
|
||||
logger.warn("Failed to attach video processor", e);
|
||||
});
|
||||
}
|
||||
if (!processor && videoTrack.getProcessor()) {
|
||||
videoTrack.stopProcessor().catch((e) => {
|
||||
logger.warn("Failed to stop video processor", e);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates your video tracks to always use the given processor.
|
||||
*/
|
||||
export const trackProcessorSync = (
|
||||
scope: ObservableScope,
|
||||
videoTrack$: Behavior<LocalVideoTrack | null>,
|
||||
processor$: Behavior<ProcessorState>,
|
||||
): void => {
|
||||
combineLatest([videoTrack$, processor$])
|
||||
.pipe(scope.bind())
|
||||
.subscribe(([videoTrack, processorState]) => {
|
||||
if (!processorState) return;
|
||||
if (!videoTrack) return;
|
||||
applyProcessor(videoTrack, processorState.processor);
|
||||
});
|
||||
};
|
||||
|
||||
export const useTrackProcessorSync = (
|
||||
videoTrack: LocalVideoTrack | null,
|
||||
): void => {
|
||||
const { processor } = useTrackProcessor();
|
||||
useEffect(() => {
|
||||
if (!videoTrack) return;
|
||||
applyProcessor(videoTrack, processor);
|
||||
}, [processor, videoTrack]);
|
||||
};
|
||||
|
||||
interface Props {
|
||||
children: JSX.Element;
|
||||
}
|
||||
|
||||
function supportsBackgroundProcessors(): boolean {
|
||||
return supportsBackgroundProcessorsLivekitSdk() && platform === "desktop";
|
||||
}
|
||||
|
||||
export const ProcessorProvider: FC<Props> = ({ children }) => {
|
||||
// The setting the user wants to have
|
||||
const [blurActivated] = useSetting(backgroundBlurSettings);
|
||||
const supported = useMemo(() => supportsBackgroundProcessors(), []);
|
||||
const blur = useMemo(
|
||||
() =>
|
||||
new ProcessorWrapper(
|
||||
new BlurBackgroundTransformer({ blurRadius: 15 }),
|
||||
"background-blur",
|
||||
),
|
||||
[],
|
||||
);
|
||||
|
||||
// This is the actual state exposed through the context
|
||||
const processorState = useMemo(
|
||||
() => ({
|
||||
supported,
|
||||
processor: supported && blurActivated ? blur : undefined,
|
||||
}),
|
||||
[supported, blurActivated, blur],
|
||||
);
|
||||
|
||||
return <ProcessorContext value={processorState}>{children}</ProcessorContext>;
|
||||
};
|
||||
@@ -0,0 +1,64 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { describe, expect, test } from "vitest";
|
||||
|
||||
import {
|
||||
type BackgroundEffect,
|
||||
imagePathFor,
|
||||
parseEffect,
|
||||
serializeEffect,
|
||||
shippedBackgrounds,
|
||||
} from "./backgroundEffects";
|
||||
import { maxAddedBackgrounds } from "./backgroundImages";
|
||||
|
||||
describe("the chosen background effect", () => {
|
||||
test("round-trips through its stored form", () => {
|
||||
const effects: BackgroundEffect[] = [
|
||||
{ kind: "none" },
|
||||
{ kind: "blur" },
|
||||
...shippedBackgrounds.map(({ id }) => ({ kind: "shipped" as const, id })),
|
||||
{ kind: "added", id: "d3b07384" },
|
||||
];
|
||||
for (const effect of effects)
|
||||
expect(parseEffect(serializeEffect(effect))).toEqual(effect);
|
||||
});
|
||||
|
||||
test("falls back to no effect when the stored form names nothing we ship", () => {
|
||||
expect(parseEffect("image:gone")).toEqual({ kind: "none" });
|
||||
expect(parseEffect("")).toEqual({ kind: "none" });
|
||||
expect(parseEffect("added:")).toEqual({ kind: "none" });
|
||||
});
|
||||
|
||||
test("reads a stored value that is not a string as no effect", () => {
|
||||
for (const raw of [null, 1, {}, ["blur"]])
|
||||
expect(parseEffect(raw)).toEqual({ kind: "none" });
|
||||
});
|
||||
|
||||
test("fills the grid at the limit and one short of it", () => {
|
||||
// No effect, blur and the shipped ones, then the added ones.
|
||||
const fixed = 2 + shippedBackgrounds.length;
|
||||
expect((fixed + maxAddedBackgrounds) % 3).toBe(0);
|
||||
// One short, the add tile takes the last place.
|
||||
expect((fixed + maxAddedBackgrounds - 1 + 1) % 3).toBe(0);
|
||||
});
|
||||
|
||||
test("gives every shipped background an image to draw", () => {
|
||||
for (const { id } of shippedBackgrounds)
|
||||
expect(imagePathFor(id)).toBeTruthy();
|
||||
});
|
||||
|
||||
// A background has to cover what is behind it: JPEG has no alpha channel.
|
||||
test("ships only opaque backgrounds", async () => {
|
||||
for (const { imagePath } of shippedBackgrounds) {
|
||||
const name = imagePath.split("/").pop()!.split("?")[0];
|
||||
const bytes = await readFile(`src/graphics/${name}`);
|
||||
expect([bytes[0], bytes[1], bytes[2]]).toEqual([0xff, 0xd8, 0xff]);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,70 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import arc from "../graphics/background-arc.jpg?url";
|
||||
import glow from "../graphics/background-glow.jpg?url";
|
||||
|
||||
export const blurRadius = 15;
|
||||
|
||||
export interface ShippedBackground {
|
||||
/** Stored in the user's settings when chosen, so it must not change. */
|
||||
id: string;
|
||||
imagePath: string;
|
||||
}
|
||||
|
||||
export const shippedBackgrounds: ShippedBackground[] = [
|
||||
{ id: "arc", imagePath: arc },
|
||||
{ id: "glow", imagePath: glow },
|
||||
];
|
||||
|
||||
export type BackgroundEffect =
|
||||
| { kind: "none" }
|
||||
| { kind: "blur" }
|
||||
| { kind: "shipped"; id: string }
|
||||
| { kind: "added"; id: string };
|
||||
|
||||
/** An effect as the setting stores it, and as the menu names its option. */
|
||||
export type EffectId = "none" | "blur" | `image:${string}` | `added:${string}`;
|
||||
|
||||
export const noEffect: BackgroundEffect = { kind: "none" };
|
||||
|
||||
/**
|
||||
* Reads a stored effect; one we no longer ship reads as no effect. Whether an
|
||||
* added one is still kept only the device knows. Stored by hand or by another
|
||||
* build, the value need not even be a string.
|
||||
*/
|
||||
export function parseEffect(raw: unknown): BackgroundEffect {
|
||||
if (typeof raw !== "string") return noEffect;
|
||||
if (raw === "blur") return { kind: "blur" };
|
||||
if (raw.startsWith("image:")) {
|
||||
const id = raw.slice("image:".length);
|
||||
if (shippedBackgrounds.some((b) => b.id === id))
|
||||
return { kind: "shipped", id };
|
||||
}
|
||||
if (raw.startsWith("added:")) {
|
||||
const id = raw.slice("added:".length);
|
||||
if (id) return { kind: "added", id };
|
||||
}
|
||||
return noEffect;
|
||||
}
|
||||
|
||||
export function serializeEffect(effect: BackgroundEffect): EffectId {
|
||||
switch (effect.kind) {
|
||||
case "blur":
|
||||
return "blur";
|
||||
case "shipped":
|
||||
return `image:${effect.id}`;
|
||||
case "added":
|
||||
return `added:${effect.id}`;
|
||||
default:
|
||||
return "none";
|
||||
}
|
||||
}
|
||||
|
||||
export function imagePathFor(id: string): string | undefined {
|
||||
return shippedBackgrounds.find((b) => b.id === id)?.imagePath;
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { IDBFactory } from "fake-indexeddb";
|
||||
|
||||
import {
|
||||
AddedBackgrounds,
|
||||
type BackgroundImageStorage,
|
||||
IndexedDBImageStorage,
|
||||
type KeptImage,
|
||||
maxAddedBackgrounds,
|
||||
prepareImage,
|
||||
UnusableImage,
|
||||
} from "./backgroundImages";
|
||||
import { flushPromises } from "../utils/test";
|
||||
|
||||
describe("AddedBackgrounds", () => {
|
||||
beforeEach(() => {
|
||||
let n = 0;
|
||||
vi.spyOn(URL, "createObjectURL").mockImplementation(() => `blob:${++n}`);
|
||||
});
|
||||
|
||||
it("offers what the device keeps, oldest first", async () => {
|
||||
const image = new Blob(["x"], { type: "image/png" });
|
||||
const store = new AddedBackgrounds(
|
||||
memoryStorage([
|
||||
{ id: "later", image, addedAt: 2 },
|
||||
{ id: "earlier", image, addedAt: 1 },
|
||||
]),
|
||||
asGiven,
|
||||
);
|
||||
expect(store.added$.value).toBeUndefined();
|
||||
await flushPromises();
|
||||
expect(store.added$.value?.map(({ id }) => id)).toEqual([
|
||||
"earlier",
|
||||
"later",
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps an added image and offers it after the others", async () => {
|
||||
const storage = memoryStorage();
|
||||
const store = new AddedBackgrounds(storage, asGiven);
|
||||
await flushPromises();
|
||||
const id = await store.add(new Blob(["x"], { type: "image/png" }));
|
||||
expect(store.added$.value?.map((a) => a.id)).toEqual([id]);
|
||||
expect((await storage.list()).map((k) => k.id)).toEqual([id]);
|
||||
});
|
||||
|
||||
it("keeps the prepared image, not the file", async () => {
|
||||
const storage = memoryStorage();
|
||||
const prepared = new Blob(["y"], { type: "image/webp" });
|
||||
const store = new AddedBackgrounds(storage, async () =>
|
||||
Promise.resolve(prepared),
|
||||
);
|
||||
await store.add(new Blob(["x"], { type: "image/png" }));
|
||||
expect((await storage.list())[0].image).toBe(prepared);
|
||||
});
|
||||
|
||||
it("keeps nothing of a file it refuses", async () => {
|
||||
const storage = memoryStorage();
|
||||
const store = new AddedBackgrounds(storage, async () =>
|
||||
Promise.reject(new UnusableImage("animated")),
|
||||
);
|
||||
await flushPromises();
|
||||
await expect(store.add(new Blob(["x"]))).rejects.toThrow(UnusableImage);
|
||||
expect(await storage.list()).toEqual([]);
|
||||
expect(store.added$.value).toEqual([]);
|
||||
});
|
||||
|
||||
it("keeps no more than the grid holds", async () => {
|
||||
const storage = memoryStorage();
|
||||
const store = new AddedBackgrounds(storage, asGiven);
|
||||
for (let i = 0; i < maxAddedBackgrounds; i++)
|
||||
await store.add(new Blob([`${i}`]));
|
||||
await expect(store.add(new Blob(["more"]))).rejects.toThrow(RangeError);
|
||||
expect(await storage.list()).toHaveLength(maxAddedBackgrounds);
|
||||
});
|
||||
|
||||
it("forgets a removed image, and lets its picture go", async () => {
|
||||
const revoke = vi
|
||||
.spyOn(URL, "revokeObjectURL")
|
||||
.mockImplementation(() => {});
|
||||
const storage = memoryStorage();
|
||||
const store = new AddedBackgrounds(storage, asGiven);
|
||||
await flushPromises();
|
||||
const first = await store.add(new Blob(["a"]));
|
||||
const second = await store.add(new Blob(["b"]));
|
||||
const url = store.added$.value![0].url;
|
||||
|
||||
await store.remove(first);
|
||||
expect(store.added$.value?.map((a) => a.id)).toEqual([second]);
|
||||
expect((await storage.list()).map((k) => k.id)).toEqual([second]);
|
||||
expect(revoke).toHaveBeenCalledWith(url);
|
||||
});
|
||||
|
||||
it("offers nothing where the browser keeps nothing", async () => {
|
||||
const store = new AddedBackgrounds(null);
|
||||
await flushPromises();
|
||||
expect(store.added$.value).toEqual([]);
|
||||
await expect(store.add(new Blob(["x"]))).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("offers none, rather than waiting, where the device can't be read", async () => {
|
||||
const unreadable: BackgroundImageStorage = {
|
||||
...memoryStorage(),
|
||||
list: async () => Promise.reject(new Error("storage cleared")),
|
||||
};
|
||||
const store = new AddedBackgrounds(unreadable);
|
||||
await flushPromises();
|
||||
expect(store.added$.value).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("prepareImage", () => {
|
||||
// Refused before anything is decoded, which only a real browser could do.
|
||||
it("refuses a file that isn't an image", async () => {
|
||||
await expect(
|
||||
prepareImage(new Blob(["x"], { type: "text/plain" })),
|
||||
).rejects.toEqual(new UnusableImage("not-an-image"));
|
||||
});
|
||||
|
||||
it("refuses an animated type where the browser can't count frames", async () => {
|
||||
expect(globalThis.ImageDecoder).toBeUndefined();
|
||||
await expect(
|
||||
prepareImage(new Blob(["GIF89a"], { type: "image/gif" })),
|
||||
).rejects.toEqual(new UnusableImage("animated"));
|
||||
});
|
||||
});
|
||||
|
||||
describe("IndexedDBImageStorage", () => {
|
||||
it("keeps images across sessions until they are deleted", async () => {
|
||||
const indexedDB = new IDBFactory();
|
||||
const image = new Blob(["x"], { type: "image/webp" });
|
||||
const first = new IndexedDBImageStorage(indexedDB);
|
||||
await first.put({ id: "a", image, addedAt: 1 });
|
||||
await first.put({ id: "b", image, addedAt: 2 });
|
||||
|
||||
// A session of its own, as after a reload.
|
||||
const second = new IndexedDBImageStorage(indexedDB);
|
||||
expect(
|
||||
(await second.list()).map(({ id, addedAt }) => [id, addedAt]),
|
||||
).toEqual([
|
||||
["a", 1],
|
||||
["b", 2],
|
||||
]);
|
||||
await second.delete("a");
|
||||
expect((await first.list()).map(({ id }) => id)).toEqual(["b"]);
|
||||
});
|
||||
});
|
||||
|
||||
// Decoding needs a real browser; the end-to-end checks cover it.
|
||||
const asGiven = async (file: Blob): Promise<Blob> => Promise.resolve(file);
|
||||
|
||||
function memoryStorage(kept: KeptImage[] = []): BackgroundImageStorage {
|
||||
return {
|
||||
list: async (): Promise<KeptImage[]> => Promise.resolve([...kept]),
|
||||
put: async (image): Promise<void> => {
|
||||
kept.push(image);
|
||||
return Promise.resolve();
|
||||
},
|
||||
delete: async (id): Promise<void> => {
|
||||
const index = kept.findIndex((k) => k.id === id);
|
||||
if (index !== -1) kept.splice(index, 1);
|
||||
return Promise.resolve();
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { BehaviorSubject } from "rxjs";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
|
||||
import { type Behavior } from "../state/Behavior";
|
||||
|
||||
export const maxAddedBackgrounds = 5;
|
||||
|
||||
/** The longest edge kept; the pipeline scales every image to the camera. */
|
||||
export const maxStoredEdge = 1920;
|
||||
|
||||
/** Why a file can't be used as a background. */
|
||||
export type UnusableReason = "not-an-image" | "animated" | "undecodable";
|
||||
|
||||
export class UnusableImage extends Error {
|
||||
public constructor(public readonly reason: UnusableReason) {
|
||||
super(reason);
|
||||
this.name = "UnusableImage";
|
||||
}
|
||||
}
|
||||
|
||||
/** A background the user added, as the menu and the pipeline use it. */
|
||||
export interface AddedBackground {
|
||||
id: string;
|
||||
url: string;
|
||||
}
|
||||
|
||||
/** A background the user added, as it is kept on the device. */
|
||||
export interface KeptImage {
|
||||
id: string;
|
||||
image: Blob;
|
||||
addedAt: number;
|
||||
}
|
||||
|
||||
/** Where added backgrounds are kept. */
|
||||
export interface BackgroundImageStorage {
|
||||
list(): Promise<KeptImage[]>;
|
||||
put(image: KeptImage): Promise<void>;
|
||||
delete(id: string): Promise<void>;
|
||||
}
|
||||
|
||||
/** The backgrounds this device keeps, which never leave it. */
|
||||
export class AddedBackgrounds {
|
||||
private readonly subject$ = new BehaviorSubject<
|
||||
AddedBackground[] | undefined
|
||||
>(undefined);
|
||||
/** Undefined until what the device keeps has been read. */
|
||||
public readonly added$: Behavior<AddedBackground[] | undefined> =
|
||||
this.subject$;
|
||||
|
||||
public constructor(
|
||||
private readonly storage: BackgroundImageStorage | null,
|
||||
private readonly prepare: (file: Blob) => Promise<Blob> = prepareImage,
|
||||
) {
|
||||
this.read().catch((e) => {
|
||||
logger.warn("Could not read added backgrounds", e);
|
||||
this.subject$.next([]);
|
||||
});
|
||||
}
|
||||
|
||||
private async read(): Promise<void> {
|
||||
const kept = (await this.storage?.list()) ?? [];
|
||||
kept.sort((a, b) => a.addedAt - b.addedAt);
|
||||
this.subject$.next(
|
||||
kept.map(({ id, image }) => ({ id, url: URL.createObjectURL(image) })),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Keeps a file as a background, and answers with its id. Throws
|
||||
* {@link UnusableImage} for a file that can't be used, and a RangeError
|
||||
* once maxAddedBackgrounds are kept.
|
||||
*/
|
||||
public async add(file: Blob): Promise<string> {
|
||||
if (this.storage === null)
|
||||
throw new Error("This browser keeps no backgrounds");
|
||||
const image = await this.prepare(file);
|
||||
if ((await this.storage.list()).length >= maxAddedBackgrounds)
|
||||
throw new RangeError(
|
||||
`${maxAddedBackgrounds} backgrounds are kept already`,
|
||||
);
|
||||
const kept: KeptImage = {
|
||||
id: crypto.randomUUID(),
|
||||
image,
|
||||
addedAt: Date.now(),
|
||||
};
|
||||
await this.storage.put(kept);
|
||||
this.subject$.next([
|
||||
...(this.subject$.value ?? []),
|
||||
{ id: kept.id, url: URL.createObjectURL(image) },
|
||||
]);
|
||||
return kept.id;
|
||||
}
|
||||
|
||||
public async remove(id: string): Promise<void> {
|
||||
await this.storage?.delete(id);
|
||||
const current = this.subject$.value ?? [];
|
||||
const gone = current.find((a) => a.id === id);
|
||||
if (gone) URL.revokeObjectURL(gone.url);
|
||||
this.subject$.next(current.filter((a) => a.id !== id));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Refuses what can't be used, reduces what is larger than is kept, and lays
|
||||
* every image on an opaque ground: a background has to cover what is behind
|
||||
* it, so a transparent pixel would be a hole in it.
|
||||
*/
|
||||
export async function prepareImage(file: Blob): Promise<Blob> {
|
||||
if (!file.type.startsWith("image/")) throw new UnusableImage("not-an-image");
|
||||
if (await isAnimated(file)) throw new UnusableImage("animated");
|
||||
|
||||
let bitmap: ImageBitmap;
|
||||
try {
|
||||
bitmap = await createImageBitmap(file);
|
||||
} catch (e) {
|
||||
logger.debug("Could not decode the chosen file", e);
|
||||
throw new UnusableImage("undecodable");
|
||||
}
|
||||
try {
|
||||
const scale = Math.min(
|
||||
1,
|
||||
maxStoredEdge / Math.max(bitmap.width, bitmap.height),
|
||||
);
|
||||
const canvas = new OffscreenCanvas(
|
||||
Math.round(bitmap.width * scale),
|
||||
Math.round(bitmap.height * scale),
|
||||
);
|
||||
const context = canvas.getContext("2d");
|
||||
if (!context) throw new UnusableImage("undecodable");
|
||||
context.fillStyle = "black";
|
||||
context.fillRect(0, 0, canvas.width, canvas.height);
|
||||
context.drawImage(bitmap, 0, 0, canvas.width, canvas.height);
|
||||
return await canvas.convertToBlob({ type: "image/webp", quality: 0.9 });
|
||||
} finally {
|
||||
bitmap.close();
|
||||
}
|
||||
}
|
||||
|
||||
// Read from the file, not its type: a still and an animated WebP share one.
|
||||
async function isAnimated(file: Blob): Promise<boolean> {
|
||||
if (typeof ImageDecoder === "undefined") return /gif|apng/.test(file.type);
|
||||
try {
|
||||
const decoder = new ImageDecoder({
|
||||
data: await file.arrayBuffer(),
|
||||
type: file.type,
|
||||
});
|
||||
// Some browsers have no selected track until the tracks are ready.
|
||||
await decoder.tracks.ready;
|
||||
await decoder.completed;
|
||||
return (decoder.tracks.selectedTrack?.frameCount ?? 1) > 1;
|
||||
} catch (e) {
|
||||
logger.debug("Could not read the frame count, judging by type", e);
|
||||
return /gif|apng/.test(file.type);
|
||||
}
|
||||
}
|
||||
|
||||
const DB_NAME = "element-call-background-images";
|
||||
const STORE = "backgrounds";
|
||||
|
||||
export class IndexedDBImageStorage implements BackgroundImageStorage {
|
||||
private db?: Promise<IDBDatabase>;
|
||||
|
||||
public constructor(private readonly indexedDB: IDBFactory) {}
|
||||
|
||||
private async open(): Promise<IDBDatabase> {
|
||||
return (this.db ??= new Promise((resolve, reject) => {
|
||||
const request = this.indexedDB.open(DB_NAME, 1);
|
||||
request.onupgradeneeded = (): void => {
|
||||
request.result.createObjectStore(STORE, { keyPath: "id" });
|
||||
};
|
||||
request.onsuccess = (): void => resolve(request.result);
|
||||
request.onerror = (): void => reject(request.error);
|
||||
}));
|
||||
}
|
||||
|
||||
private async run<T>(
|
||||
mode: IDBTransactionMode,
|
||||
body: (store: IDBObjectStore) => IDBRequest<T>,
|
||||
): Promise<T> {
|
||||
const db = await this.open();
|
||||
return new Promise((resolve, reject) => {
|
||||
const request = body(db.transaction(STORE, mode).objectStore(STORE));
|
||||
request.onsuccess = (): void => resolve(request.result);
|
||||
request.onerror = (): void => reject(request.error);
|
||||
});
|
||||
}
|
||||
|
||||
public async list(): Promise<KeptImage[]> {
|
||||
return this.run<KeptImage[]>("readonly", (s) => s.getAll());
|
||||
}
|
||||
|
||||
public async put(image: KeptImage): Promise<void> {
|
||||
await this.run("readwrite", (s) => s.put(image));
|
||||
}
|
||||
|
||||
public async delete(id: string): Promise<void> {
|
||||
await this.run("readwrite", (s) => s.delete(id));
|
||||
}
|
||||
}
|
||||
|
||||
function openStorage(): BackgroundImageStorage | null {
|
||||
// Merely reading indexedDB throws where it is disabled.
|
||||
try {
|
||||
return globalThis.indexedDB
|
||||
? new IndexedDBImageStorage(globalThis.indexedDB)
|
||||
: null;
|
||||
} catch (e) {
|
||||
logger.warn("Could not reach IndexedDB", e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export const addedBackgrounds = new AddedBackgrounds(openStorage());
|
||||
@@ -50,6 +50,7 @@ import {
|
||||
mockMediaDevices,
|
||||
mockRtcMembership,
|
||||
MockRTCSession,
|
||||
testScope,
|
||||
} from "../utils/test";
|
||||
import { CallView } from "./CallView";
|
||||
import { GroupCallErrorBoundary } from "./GroupCallErrorBoundary";
|
||||
@@ -61,7 +62,10 @@ import {
|
||||
} from "../HostBridge";
|
||||
import { type JoinCallData } from "../widget";
|
||||
import { MatrixRTCTransportMissingError } from "../utils/errors";
|
||||
import { ProcessorProvider } from "../livekit/TrackProcessorContext";
|
||||
import {
|
||||
BackgroundEffectsProvider,
|
||||
createAppBackgroundEffects,
|
||||
} from "../livekit/BackgroundEffectsContext";
|
||||
import { MediaDevicesContext } from "../MediaDevicesContext";
|
||||
import { constant } from "../state/Behavior";
|
||||
|
||||
@@ -193,7 +197,9 @@ function createCallView(
|
||||
<HostBridgeProvider value={hostBridge}>
|
||||
<TooltipProvider>
|
||||
<MediaDevicesContext value={mockMediaDevices({})}>
|
||||
<ProcessorProvider>
|
||||
<BackgroundEffectsProvider
|
||||
effects={createAppBackgroundEffects(testScope())}
|
||||
>
|
||||
{options.withErrorBoundary ? (
|
||||
<GroupCallErrorBoundary recoveryActionHandler={vi.fn()}>
|
||||
{callView}
|
||||
@@ -201,7 +207,7 @@ function createCallView(
|
||||
) : (
|
||||
callView
|
||||
)}
|
||||
</ProcessorProvider>
|
||||
</BackgroundEffectsProvider>
|
||||
</MediaDevicesContext>
|
||||
</TooltipProvider>
|
||||
</HostBridgeProvider>
|
||||
|
||||
@@ -31,6 +31,7 @@ import {
|
||||
mockRemoteParticipant,
|
||||
mockRtcMembership,
|
||||
type MockRTCSession,
|
||||
testScope,
|
||||
} from "../utils/test";
|
||||
import { E2eeType } from "../e2ee/e2eeType";
|
||||
import {
|
||||
@@ -49,7 +50,10 @@ import { MediaDevicesContext } from "../MediaDevicesContext";
|
||||
import { type MediaDevices as ECMediaDevices } from "../state/MediaDevices";
|
||||
import { AppBar } from "../AppBar";
|
||||
import { type MatrixInfo } from "./VideoPreview";
|
||||
import { ProcessorProvider } from "../livekit/TrackProcessorContext";
|
||||
import {
|
||||
BackgroundEffectsProvider,
|
||||
createAppBackgroundEffects,
|
||||
} from "../livekit/BackgroundEffectsContext";
|
||||
import { initializeWidget } from "../widget";
|
||||
import { RootElementProvider } from "../RootElementContext";
|
||||
|
||||
@@ -242,7 +246,9 @@ describe("ActiveCall", () => {
|
||||
const { findByTestId } = render(
|
||||
<BrowserRouter>
|
||||
<MediaDevicesContext value={mediaDevices}>
|
||||
<ProcessorProvider>
|
||||
<BackgroundEffectsProvider
|
||||
effects={createAppBackgroundEffects(testScope())}
|
||||
>
|
||||
<TooltipProvider>
|
||||
<RoomContext value={mockLivekitRoom({ localParticipant })}>
|
||||
<ActiveCall
|
||||
@@ -257,7 +263,7 @@ describe("ActiveCall", () => {
|
||||
/>
|
||||
</RoomContext>
|
||||
</TooltipProvider>
|
||||
</ProcessorProvider>
|
||||
</BackgroundEffectsProvider>
|
||||
</MediaDevicesContext>
|
||||
</BrowserRouter>,
|
||||
);
|
||||
@@ -292,7 +298,9 @@ describe("ActiveCall", () => {
|
||||
<BrowserRouter>
|
||||
<RootElementProvider value={root}>
|
||||
<MediaDevicesContext value={mediaDevices}>
|
||||
<ProcessorProvider>
|
||||
<BackgroundEffectsProvider
|
||||
effects={createAppBackgroundEffects(testScope())}
|
||||
>
|
||||
<TooltipProvider>
|
||||
<RoomContext value={mockLivekitRoom({ localParticipant })}>
|
||||
<ActiveCall
|
||||
@@ -307,7 +315,7 @@ describe("ActiveCall", () => {
|
||||
/>
|
||||
</RoomContext>
|
||||
</TooltipProvider>
|
||||
</ProcessorProvider>
|
||||
</BackgroundEffectsProvider>
|
||||
</MediaDevicesContext>
|
||||
</RootElementProvider>
|
||||
</BrowserRouter>,
|
||||
|
||||
@@ -22,6 +22,7 @@ import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
|
||||
import classNames from "classnames";
|
||||
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { map } from "rxjs";
|
||||
|
||||
import { Header, LeftNav, RightNav, RoomHeaderInfo } from "../Header";
|
||||
import { HeaderStyle, useUrlParams } from "../UrlParams";
|
||||
@@ -79,7 +80,7 @@ import { useValueBehavior } from "../useValueBehavior.ts";
|
||||
import { constant } from "../state/Behavior.ts";
|
||||
import { Toast } from "../Toast.tsx";
|
||||
import overlayStyles from "../Overlay.module.css";
|
||||
import { useTrackProcessorState$ } from "../livekit/TrackProcessorContext.tsx";
|
||||
import { useBackgroundEffectsState$ } from "../livekit/BackgroundEffectsContext.tsx";
|
||||
import { type Layout } from "../state/layout-types.ts";
|
||||
import { ObservableScope } from "../state/ObservableScope.ts";
|
||||
import { CallFooter, type FooterSnapshot } from "../components/CallFooter.tsx";
|
||||
@@ -120,7 +121,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
|
||||
const urlParams = useUrlParams();
|
||||
const hostBridge = useHostBridge();
|
||||
const mediaDevices = useMediaDevices();
|
||||
const trackProcessorState$ = useTrackProcessorState$();
|
||||
const backgroundEffectsState$ = useBackgroundEffectsState$();
|
||||
// The element we have to draw the call in: the page, or the container a host
|
||||
// gave us. Its size, not the window's, decides how the call is laid out.
|
||||
const rootElement = useRootElement();
|
||||
@@ -150,7 +151,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
|
||||
},
|
||||
reactionsReader.raisedHands$,
|
||||
reactionsReader.reactions$,
|
||||
trackProcessorState$,
|
||||
backgroundEffectsState$,
|
||||
);
|
||||
// TODO move this somewhere else once we use the callViewModel in the lobby as well!
|
||||
vm.join();
|
||||
@@ -170,7 +171,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
|
||||
urlParams,
|
||||
hostBridge,
|
||||
mediaDevices,
|
||||
trackProcessorState$,
|
||||
backgroundEffectsState$,
|
||||
props.client,
|
||||
rootElement,
|
||||
]);
|
||||
@@ -184,6 +185,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
|
||||
vm,
|
||||
props.muteStates,
|
||||
mediaDevices,
|
||||
backgroundEffectsState$.pipe(map(({ settling }) => settling ?? false)),
|
||||
`${props.client.getUserId()}:${props.client.getDeviceId()}`,
|
||||
{ showControls: urlParams.showControls, header: urlParams.header },
|
||||
);
|
||||
@@ -201,7 +203,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
|
||||
props.onLeft,
|
||||
urlParams,
|
||||
mediaDevices,
|
||||
trackProcessorState$,
|
||||
backgroundEffectsState$,
|
||||
props.client,
|
||||
vm,
|
||||
]);
|
||||
|
||||
@@ -10,24 +10,33 @@ import { render, screen } from "@testing-library/react";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
import { TooltipProvider } from "@vector-im/compound-web";
|
||||
import { type MatrixClient, type RoomSummary } from "matrix-js-sdk";
|
||||
import { type Observable } from "rxjs";
|
||||
|
||||
import { KnockLobbyView } from "./KnockLobbyView";
|
||||
import { LeaveToHomeProvider } from "../LeaveToHomeContext";
|
||||
import { MediaDevicesContext } from "../MediaDevicesContext";
|
||||
import { type ProcessorState } from "../livekit/TrackProcessorContext";
|
||||
import { type BackgroundEffectsState } from "../livekit/BackgroundEffectsContext";
|
||||
import { mockMediaDevices } from "../utils/test";
|
||||
|
||||
vi.mock("@livekit/components-react", () => ({
|
||||
usePreviewTracks: (): unknown[] => [],
|
||||
}));
|
||||
|
||||
vi.mock("../livekit/TrackProcessorContext", () => ({
|
||||
useTrackProcessor: (): ProcessorState => ({
|
||||
vi.mock("../livekit/BackgroundEffectsContext", async () => {
|
||||
const { of } = await import("rxjs");
|
||||
const none: BackgroundEffectsState = {
|
||||
supported: false,
|
||||
processor: undefined,
|
||||
}),
|
||||
useTrackProcessorSync: (): void => {},
|
||||
}));
|
||||
};
|
||||
// One observable, as the real hook keeps: a fresh one each render would
|
||||
// rebuild the footer on every render.
|
||||
const none$ = of(none);
|
||||
return {
|
||||
useBackgroundEffects: (): BackgroundEffectsState => none,
|
||||
useBackgroundEffectsState$: (): Observable<BackgroundEffectsState> => none$,
|
||||
useSyncBackgroundEffects: (): void => {},
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("react-use-measure", () => ({
|
||||
default: (): [() => void, object] => [(): void => {}, {}],
|
||||
|
||||
@@ -8,7 +8,7 @@ Please see LICENSE in the repository root for full details.
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { render, screen } from "@testing-library/react";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
import { of } from "rxjs";
|
||||
import { type Observable, of } from "rxjs";
|
||||
import { LeaveToHomeProvider } from "../LeaveToHomeContext";
|
||||
import { TooltipProvider } from "@vector-im/compound-web";
|
||||
import { type MatrixClient } from "matrix-js-sdk";
|
||||
@@ -28,7 +28,7 @@ import {
|
||||
} from "../utils/test";
|
||||
import { type MediaDevices } from "../state/MediaDevices";
|
||||
import { MediaDevicesContext } from "../MediaDevicesContext";
|
||||
import { type ProcessorState } from "../livekit/TrackProcessorContext";
|
||||
import { type BackgroundEffectsState } from "../livekit/BackgroundEffectsContext";
|
||||
import { type EncryptionSystem } from "../e2ee/sharedKeyManagement";
|
||||
import lobbyStyles from "./LobbyView.module.css";
|
||||
import headerStyles from "../Header.module.css";
|
||||
@@ -41,13 +41,21 @@ vi.mock("@livekit/components-react", () => ({
|
||||
usePreviewTracks: (): unknown[] => [],
|
||||
}));
|
||||
|
||||
vi.mock("../livekit/TrackProcessorContext", () => ({
|
||||
useTrackProcessor: (): ProcessorState => ({
|
||||
vi.mock("../livekit/BackgroundEffectsContext", async () => {
|
||||
const { of } = await import("rxjs");
|
||||
const none: BackgroundEffectsState = {
|
||||
supported: false,
|
||||
processor: undefined,
|
||||
}),
|
||||
useTrackProcessorSync: (): void => {},
|
||||
}));
|
||||
};
|
||||
// One observable, as the real hook keeps: a fresh one each render would
|
||||
// rebuild the footer on every render.
|
||||
const none$ = of(none);
|
||||
return {
|
||||
useBackgroundEffects: (): BackgroundEffectsState => none,
|
||||
useBackgroundEffectsState$: (): Observable<BackgroundEffectsState> => none$,
|
||||
useSyncBackgroundEffects: (): void => {},
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("react-use-measure", () => ({
|
||||
default: (): [() => void, object] => [(): void => {}, {}],
|
||||
|
||||
+17
-6
@@ -24,6 +24,7 @@ import {
|
||||
type LocalVideoTrack,
|
||||
Track,
|
||||
} from "livekit-client";
|
||||
import { map } from "rxjs";
|
||||
|
||||
import inCallStyles from "./InCallView.module.css";
|
||||
import styles from "./LobbyView.module.css";
|
||||
@@ -40,9 +41,10 @@ import { useMediaDevices } from "../MediaDevicesContext";
|
||||
import { ObservableScope } from "../state/ObservableScope";
|
||||
import { useInitial } from "../useInitial";
|
||||
import {
|
||||
useTrackProcessor,
|
||||
useTrackProcessorSync,
|
||||
} from "../livekit/TrackProcessorContext";
|
||||
useBackgroundEffects,
|
||||
useBackgroundEffectsState$,
|
||||
useSyncBackgroundEffects,
|
||||
} from "../livekit/BackgroundEffectsContext";
|
||||
import { getValue } from "../utils/observable";
|
||||
import { useBehavior } from "../useBehavior";
|
||||
import { CallFooter, type FooterSnapshot } from "../components/CallFooter";
|
||||
@@ -136,7 +138,8 @@ export const LobbyView: FC<Props> = ({
|
||||
},
|
||||
);
|
||||
|
||||
const { processor } = useTrackProcessor();
|
||||
const { processor } = useBackgroundEffects();
|
||||
const backgroundEffectsState$ = useBackgroundEffectsState$();
|
||||
|
||||
const initialProcessor = useInitial(() => processor);
|
||||
const localTrackOptions = useMemo<CreateLocalTracksOptions>(
|
||||
@@ -183,7 +186,7 @@ export const LobbyView: FC<Props> = ({
|
||||
}
|
||||
}, [devices, videoInputId, videoTrack]);
|
||||
|
||||
useTrackProcessorSync(videoTrack);
|
||||
useSyncBackgroundEffects(videoTrack);
|
||||
|
||||
const [footerVm, setFooterVm] = useState<ViewModel<FooterSnapshot> | null>(
|
||||
null,
|
||||
@@ -195,6 +198,7 @@ export const LobbyView: FC<Props> = ({
|
||||
footerScope,
|
||||
muteStates,
|
||||
devices,
|
||||
backgroundEffectsState$.pipe(map(({ settling }) => settling ?? false)),
|
||||
openSettings,
|
||||
hangup,
|
||||
// Logo and header are connected: only show the logo in SPA with header.
|
||||
@@ -204,7 +208,14 @@ export const LobbyView: FC<Props> = ({
|
||||
return (): void => {
|
||||
footerScope.end();
|
||||
};
|
||||
}, [devices, hangup, hideHeader, muteStates, openSettings]);
|
||||
}, [
|
||||
devices,
|
||||
backgroundEffectsState$,
|
||||
hangup,
|
||||
hideHeader,
|
||||
muteStates,
|
||||
openSettings,
|
||||
]);
|
||||
|
||||
// TODO: Unify this component with InCallView, so we can get slick joining
|
||||
// animations and don't have to feel bad about reusing its CSS
|
||||
|
||||
@@ -181,7 +181,7 @@ exports[`InCallView > rendering > renders 1`] = `
|
||||
tabindex="0"
|
||||
>
|
||||
<div
|
||||
class="_indicator-icon_147l5_17"
|
||||
class="_indicator-icon_j097f_17"
|
||||
style="--cpd-icon-button-size: 100%;"
|
||||
>
|
||||
<svg
|
||||
|
||||
@@ -15,7 +15,7 @@ exports[`LobbyView > renders with AppBar android 1`] = `
|
||||
tabindex="0"
|
||||
>
|
||||
<div
|
||||
class="_indicator-icon_147l5_17"
|
||||
class="_indicator-icon_j097f_17"
|
||||
style="--cpd-icon-button-size: 100%;"
|
||||
>
|
||||
<svg
|
||||
@@ -111,7 +111,7 @@ exports[`LobbyView > renders with AppBar android 1`] = `
|
||||
tabindex="0"
|
||||
>
|
||||
<div
|
||||
class="_indicator-icon_147l5_17"
|
||||
class="_indicator-icon_j097f_17"
|
||||
style="--cpd-icon-button-size: 100%;"
|
||||
>
|
||||
<svg
|
||||
@@ -247,7 +247,7 @@ exports[`LobbyView > renders with AppBar ios 1`] = `
|
||||
tabindex="0"
|
||||
>
|
||||
<div
|
||||
class="_indicator-icon_147l5_17"
|
||||
class="_indicator-icon_j097f_17"
|
||||
style="--cpd-icon-button-size: 100%;"
|
||||
>
|
||||
<svg
|
||||
@@ -343,7 +343,7 @@ exports[`LobbyView > renders with AppBar ios 1`] = `
|
||||
tabindex="0"
|
||||
>
|
||||
<div
|
||||
class="_indicator-icon_147l5_17"
|
||||
class="_indicator-icon_j097f_17"
|
||||
style="--cpd-icon-button-size: 100%;"
|
||||
>
|
||||
<svg
|
||||
@@ -621,7 +621,7 @@ exports[`LobbyView > renders with header and participant count 1`] = `
|
||||
tabindex="0"
|
||||
>
|
||||
<div
|
||||
class="_indicator-icon_147l5_17"
|
||||
class="_indicator-icon_j097f_17"
|
||||
style="--cpd-icon-button-size: 100%;"
|
||||
>
|
||||
<svg
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { render, screen } from "@testing-library/react";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
import { type MatrixClient } from "matrix-js-sdk";
|
||||
import { TooltipProvider } from "@vector-im/compound-web";
|
||||
|
||||
import { SettingsModal } from "./SettingsModal";
|
||||
import { backgroundEffect } from "./settings";
|
||||
import { MediaDevicesContext } from "../MediaDevicesContext";
|
||||
import { type BackgroundEffectsState } from "../livekit/BackgroundEffectsContext";
|
||||
import { mockMediaDevices } from "../utils/test";
|
||||
import { constant } from "../state/Behavior";
|
||||
import { type MediaDevices } from "../state/MediaDevices";
|
||||
|
||||
vi.mock("../livekit/BackgroundEffectsContext", () => ({
|
||||
useBackgroundEffects: (): BackgroundEffectsState => ({
|
||||
supported: true,
|
||||
processor: undefined,
|
||||
}),
|
||||
}));
|
||||
|
||||
// Behaviors, as the device menus read them.
|
||||
const noDevices = {
|
||||
available$: constant(new Map()),
|
||||
selected$: constant(undefined),
|
||||
select: vi.fn(),
|
||||
};
|
||||
|
||||
function renderVideoTab(): void {
|
||||
render(
|
||||
<TooltipProvider>
|
||||
<MediaDevicesContext
|
||||
value={mockMediaDevices({
|
||||
requestDeviceNames: vi.fn(),
|
||||
audioInput: noDevices,
|
||||
audioOutput: noDevices,
|
||||
videoInput: noDevices,
|
||||
} as unknown as Partial<MediaDevices>)}
|
||||
>
|
||||
<SettingsModal
|
||||
open
|
||||
onDismiss={vi.fn()}
|
||||
tab="video"
|
||||
onTabChange={vi.fn()}
|
||||
client={
|
||||
{
|
||||
getUserId: () => "@user:example.org",
|
||||
getUser: () => null,
|
||||
} as unknown as MatrixClient
|
||||
}
|
||||
/>
|
||||
</MediaDevicesContext>
|
||||
</TooltipProvider>,
|
||||
);
|
||||
}
|
||||
|
||||
const blurControl = (): HTMLElement =>
|
||||
screen.getByRole("checkbox", { name: "Blur the background of the video" });
|
||||
|
||||
describe("SettingsModal", () => {
|
||||
afterEach(() => backgroundEffect.setValue("none"));
|
||||
|
||||
it("blur control turns blur on and off", async () => {
|
||||
const user = userEvent.setup();
|
||||
renderVideoTab();
|
||||
expect(blurControl()).not.toBeChecked();
|
||||
|
||||
await user.click(blurControl());
|
||||
expect(backgroundEffect.getValue()).toBe("blur");
|
||||
expect(blurControl()).toBeChecked();
|
||||
|
||||
await user.click(blurControl());
|
||||
expect(backgroundEffect.getValue()).toBe("none");
|
||||
expect(blurControl()).not.toBeChecked();
|
||||
});
|
||||
|
||||
it("blur control reflects and replaces an image background", async () => {
|
||||
const user = userEvent.setup();
|
||||
backgroundEffect.setValue("image:arc");
|
||||
renderVideoTab();
|
||||
expect(blurControl()).not.toBeChecked();
|
||||
|
||||
await user.click(blurControl());
|
||||
expect(backgroundEffect.getValue()).toBe("blur");
|
||||
});
|
||||
});
|
||||
@@ -22,13 +22,14 @@ import { useHostBridge } from "../HostBridge";
|
||||
import {
|
||||
useSetting,
|
||||
soundEffectVolume as soundEffectVolumeSetting,
|
||||
backgroundBlur as backgroundBlurSetting,
|
||||
backgroundEffect as backgroundEffectSetting,
|
||||
developerMode,
|
||||
} from "./settings";
|
||||
import { PreferencesSettingsTab } from "./PreferencesSettingsTab";
|
||||
import { Slider } from "../Slider";
|
||||
import { DeviceSelection } from "./DeviceSelection";
|
||||
import { useTrackProcessor } from "../livekit/TrackProcessorContext";
|
||||
import { useBackgroundEffects } from "../livekit/BackgroundEffectsContext";
|
||||
import { parseEffect, serializeEffect } from "../livekit/backgroundEffects";
|
||||
import {
|
||||
DeveloperSettingsTab,
|
||||
type DeveloperSettingsSnapshot,
|
||||
@@ -81,9 +82,11 @@ export const SettingsModal: FC<Props> = ({
|
||||
|
||||
// Generate a `Checkbox` input to turn blur on or off.
|
||||
const BlurCheckbox: React.FC = (): ReactNode => {
|
||||
const { supported } = useTrackProcessor();
|
||||
const { supported } = useBackgroundEffects();
|
||||
|
||||
const [blurActive, setBlurActive] = useSetting(backgroundBlurSetting);
|
||||
// The camera menu's setting, of which this control sees only blur.
|
||||
const [effect, setEffect] = useSetting(backgroundEffectSetting);
|
||||
const blurActive = parseEffect(effect).kind === "blur";
|
||||
|
||||
return (
|
||||
<>
|
||||
@@ -97,8 +100,12 @@ export const SettingsModal: FC<Props> = ({
|
||||
supported ? "" : t("settings.blur_not_supported_by_browser")
|
||||
}
|
||||
type="checkbox"
|
||||
checked={!!blurActive}
|
||||
onChange={(b): void => setBlurActive(b.target.checked)}
|
||||
checked={blurActive}
|
||||
onChange={(b): void =>
|
||||
setEffect(
|
||||
serializeEffect({ kind: b.target.checked ? "blur" : "none" }),
|
||||
)
|
||||
}
|
||||
disabled={!supported}
|
||||
/>
|
||||
</FieldRow>
|
||||
|
||||
@@ -12,6 +12,7 @@ import { PosthogAnalytics } from "../analytics/PosthogAnalytics";
|
||||
import { type Behavior } from "../state/Behavior";
|
||||
import { useBehavior } from "../useBehavior";
|
||||
import { MatrixRTCMode } from "../config/ConfigOptions";
|
||||
import { type EffectId } from "../livekit/backgroundEffects";
|
||||
|
||||
export class Setting<T> {
|
||||
public constructor(
|
||||
@@ -115,7 +116,14 @@ export const videoInput = new Setting<string | undefined>(
|
||||
undefined,
|
||||
);
|
||||
|
||||
export const backgroundBlur = new Setting<boolean>("background-blur", false);
|
||||
// Only read, to carry blur over for anyone who had it on.
|
||||
const backgroundBlur = new Setting<boolean>("background-blur", false);
|
||||
|
||||
/** The chosen background effect, in the form `parseEffect` reads. */
|
||||
export const backgroundEffect = new Setting<EffectId>(
|
||||
"background-effect",
|
||||
backgroundBlur.getValue() ? "blur" : "none",
|
||||
);
|
||||
|
||||
export const showHandRaisedTimer = new Setting<boolean>(
|
||||
"hand-raised-show-timer",
|
||||
|
||||
@@ -6,6 +6,7 @@ Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { type AudioOutputOptions, MediaDevices } from "./MediaDevices";
|
||||
import { createAppBackgroundEffects } from "../livekit/BackgroundEffectsContext";
|
||||
import { type ObservableScope } from "./ObservableScope";
|
||||
|
||||
/**
|
||||
@@ -16,6 +17,7 @@ export class AppViewModel {
|
||||
this.scope,
|
||||
this.audioOutputOptions,
|
||||
);
|
||||
public readonly backgroundEffects = createAppBackgroundEffects(this.scope);
|
||||
|
||||
// TODO: Move more application logic here. The CallViewModel, at the very
|
||||
// least, ought to be accessible from this object.
|
||||
|
||||
@@ -0,0 +1,240 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { BehaviorSubject, distinctUntilChanged, map } from "rxjs";
|
||||
import { type BackgroundProcessorWrapper } from "@livekit/track-processors";
|
||||
|
||||
import {
|
||||
type BackgroundEffects,
|
||||
createBackgroundEffects,
|
||||
type BackgroundEffectsOptions,
|
||||
} from "./BackgroundEffects";
|
||||
import { type BackgroundEffectsState } from "../livekit/BackgroundEffectsContext";
|
||||
import { type AddedBackground } from "../livekit/backgroundImages";
|
||||
import { shippedBackgrounds } from "../livekit/backgroundEffects";
|
||||
import { flushPromises, testScope, withTestScheduler } from "../utils/test";
|
||||
|
||||
/** A pipeline that records what it is switched to. */
|
||||
function fakePipeline(): {
|
||||
pipeline: BackgroundProcessorWrapper;
|
||||
switches: unknown[];
|
||||
/** Holds the next switch until it resolves, as a picture loading does. */
|
||||
holdNext: () => () => void;
|
||||
} {
|
||||
const switches: unknown[] = [];
|
||||
let slow: Promise<void> | undefined;
|
||||
const pipeline = {
|
||||
switchTo: vi.fn(async (options: unknown) => {
|
||||
switches.push(options);
|
||||
const held = slow;
|
||||
slow = undefined;
|
||||
await held;
|
||||
}),
|
||||
} as unknown as BackgroundProcessorWrapper;
|
||||
return {
|
||||
pipeline,
|
||||
switches,
|
||||
holdNext: () => {
|
||||
let release!: () => void;
|
||||
slow = new Promise((resolve) => (release = resolve));
|
||||
return release;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** One letter per state: idle, waiting for a frame, attached. */
|
||||
function letter(state: BackgroundEffectsState): string {
|
||||
if (state.processor === undefined) return "i";
|
||||
return state.settling ? "w" : "a";
|
||||
}
|
||||
|
||||
describe("the pipeline's state", () => {
|
||||
function testState({
|
||||
effect,
|
||||
firstFrame = "",
|
||||
expected,
|
||||
}: {
|
||||
effect: string;
|
||||
firstFrame?: string;
|
||||
expected: string;
|
||||
}): void {
|
||||
withTestScheduler(({ behavior, schedule, expectObservable }) => {
|
||||
const transformer = {
|
||||
onFirstFrame: undefined as (() => void) | undefined,
|
||||
};
|
||||
const effects = createBackgroundEffects(testScope(), {
|
||||
supported: true,
|
||||
effect$: behavior(effect, { n: "none", b: "blur" }),
|
||||
setEffect: vi.fn(),
|
||||
added$: new BehaviorSubject<AddedBackground[] | undefined>([]),
|
||||
pipeline: fakePipeline().pipeline,
|
||||
transformer,
|
||||
});
|
||||
schedule(firstFrame, { f: () => transformer.onFirstFrame?.() });
|
||||
expectObservable(
|
||||
effects.state$.pipe(map(letter), distinctUntilChanged()),
|
||||
).toBe(expected);
|
||||
});
|
||||
}
|
||||
|
||||
it("defaults to no effect", () => testState({ effect: "n", expected: "i" }));
|
||||
|
||||
it("waits from the first effect until a frame carries it", () =>
|
||||
testState({
|
||||
effect: " nb-n-b",
|
||||
firstFrame: " --f",
|
||||
expected: " iwa",
|
||||
}));
|
||||
|
||||
it("attaches on first use and stays attached", () =>
|
||||
testState({ effect: "nb-n", firstFrame: "--f", expected: "iwa" }));
|
||||
});
|
||||
|
||||
describe("background effects", () => {
|
||||
let effect$: BehaviorSubject<string>;
|
||||
let added$: BehaviorSubject<AddedBackground[] | undefined>;
|
||||
let setEffect: (raw: string) => void;
|
||||
let fake: ReturnType<typeof fakePipeline>;
|
||||
|
||||
function build(
|
||||
options: Partial<BackgroundEffectsOptions> = {},
|
||||
): BackgroundEffects {
|
||||
return createBackgroundEffects(testScope(), {
|
||||
supported: true,
|
||||
effect$,
|
||||
setEffect,
|
||||
added$,
|
||||
pipeline: fake.pipeline,
|
||||
transformer: { onFirstFrame: undefined },
|
||||
...options,
|
||||
});
|
||||
}
|
||||
const choose = async (raw: string): Promise<void> => {
|
||||
effect$.next(raw);
|
||||
await flushPromises();
|
||||
};
|
||||
const blur = async (on: boolean): Promise<void> =>
|
||||
choose(on ? "blur" : "none");
|
||||
|
||||
beforeEach(() => {
|
||||
effect$ = new BehaviorSubject("none");
|
||||
added$ = new BehaviorSubject<AddedBackground[] | undefined>(undefined);
|
||||
setEffect = vi.fn((raw: string) => effect$.next(raw));
|
||||
fake = fakePipeline();
|
||||
});
|
||||
|
||||
it("puts a shipped background on as that picture", async () => {
|
||||
build();
|
||||
await choose(`image:${shippedBackgrounds[0].id}`);
|
||||
expect(fake.switches).toEqual([
|
||||
{
|
||||
mode: "virtual-background",
|
||||
imagePath: shippedBackgrounds[0].imagePath,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("switches in place rather than reattaching", async () => {
|
||||
const effects = build();
|
||||
await blur(true);
|
||||
const pipeline = effects.state$.value.processor;
|
||||
await blur(false);
|
||||
await blur(true);
|
||||
|
||||
expect(effects.state$.value.processor).toBe(pipeline);
|
||||
expect(fake.switches).toEqual([
|
||||
{ mode: "background-blur", blurRadius: 15 },
|
||||
{ mode: "disabled" },
|
||||
{ mode: "background-blur", blurRadius: 15 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps switching after a switch fails", async () => {
|
||||
vi.mocked(fake.pipeline.switchTo).mockRejectedValueOnce(new Error("lost"));
|
||||
build();
|
||||
await blur(true);
|
||||
await blur(false);
|
||||
expect(fake.switches).toEqual([{ mode: "disabled" }]);
|
||||
});
|
||||
|
||||
it("switches one at a time, skipping those overtaken", async () => {
|
||||
const finished = fake.holdNext();
|
||||
build();
|
||||
await blur(true);
|
||||
await blur(false);
|
||||
await blur(true);
|
||||
await blur(false);
|
||||
expect(fake.switches).toHaveLength(1);
|
||||
|
||||
finished();
|
||||
await flushPromises();
|
||||
expect(fake.switches).toEqual([
|
||||
{ mode: "background-blur", blurRadius: 15 },
|
||||
{ mode: "disabled" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("clears a remembered effect when it cannot be honoured", () => {
|
||||
effect$.next(`image:${shippedBackgrounds[0].id}`);
|
||||
const effects = build({ supported: false });
|
||||
expect(effect$.value).toBe("none");
|
||||
expect(effects.state$.value.processor).toBeUndefined();
|
||||
});
|
||||
|
||||
it("puts an added background on as its picture", async () => {
|
||||
added$.next([{ id: "mine", url: "blob:mine" }]);
|
||||
build();
|
||||
await choose("added:mine");
|
||||
expect(fake.switches).toEqual([
|
||||
{ mode: "virtual-background", imagePath: "blob:mine" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("switches between any two effects in place", async () => {
|
||||
added$.next([{ id: "mine", url: "blob:mine" }]);
|
||||
const shipped = shippedBackgrounds[0];
|
||||
const options: Record<string, unknown> = {
|
||||
none: { mode: "disabled" },
|
||||
blur: { mode: "background-blur", blurRadius: 15 },
|
||||
[`image:${shipped.id}`]: {
|
||||
mode: "virtual-background",
|
||||
imagePath: shipped.imagePath,
|
||||
},
|
||||
"added:mine": { mode: "virtual-background", imagePath: "blob:mine" },
|
||||
};
|
||||
// Each kind to each other kind, and back.
|
||||
const order = [
|
||||
"blur",
|
||||
`image:${shipped.id}`,
|
||||
"added:mine",
|
||||
"blur",
|
||||
"added:mine",
|
||||
`image:${shipped.id}`,
|
||||
"none",
|
||||
"added:mine",
|
||||
"none",
|
||||
"blur",
|
||||
];
|
||||
const effects = build();
|
||||
await choose(order[0]);
|
||||
const pipeline = effects.state$.value.processor;
|
||||
for (const raw of order.slice(1)) await choose(raw);
|
||||
|
||||
expect(effects.state$.value.processor).toBe(pipeline);
|
||||
expect(fake.switches).toEqual(order.map((raw) => options[raw]));
|
||||
});
|
||||
|
||||
it("forgets an added background the device no longer keeps", () => {
|
||||
effect$.next("added:gone");
|
||||
build();
|
||||
// Not before the device has said what it keeps.
|
||||
expect(effect$.value).toBe("added:gone");
|
||||
added$.next([]);
|
||||
expect(effect$.value).toBe("none");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,165 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import {
|
||||
combineLatest,
|
||||
distinctUntilChanged,
|
||||
map,
|
||||
Observable,
|
||||
scan,
|
||||
} from "rxjs";
|
||||
import {
|
||||
type BackgroundProcessorWrapper,
|
||||
type SwitchBackgroundProcessorOptions,
|
||||
} from "@livekit/track-processors";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
import { deepCompare } from "matrix-js-sdk/lib/utils";
|
||||
|
||||
import { type Behavior } from "./Behavior";
|
||||
import { type ObservableScope } from "./ObservableScope";
|
||||
import { type BackgroundEffectsState } from "../livekit/BackgroundEffectsContext";
|
||||
import {
|
||||
type BackgroundEffect,
|
||||
blurRadius,
|
||||
type EffectId,
|
||||
imagePathFor,
|
||||
parseEffect,
|
||||
} from "../livekit/backgroundEffects";
|
||||
import { type AddedBackground } from "../livekit/backgroundImages";
|
||||
|
||||
export interface BackgroundEffectsOptions {
|
||||
/** Whether this browser can run a pipeline at all. */
|
||||
supported: boolean;
|
||||
/** The effect chosen, as the setting stores it. */
|
||||
effect$: Behavior<string>;
|
||||
/** Stores a choice, to forget one that can't be honoured. */
|
||||
setEffect: (id: EffectId) => void;
|
||||
/** The backgrounds the device keeps, undefined until it has said. */
|
||||
added$: Behavior<AddedBackground[] | undefined>;
|
||||
/** The background processor pipeline to be switched between effects. */
|
||||
pipeline: BackgroundProcessorWrapper;
|
||||
/** Tells, once, that a frame carrying an effect has been drawn. */
|
||||
transformer: { onFirstFrame: (() => void) | undefined };
|
||||
}
|
||||
|
||||
/** The background effect pipeline, as the camera tracks and the menus see it. */
|
||||
export interface BackgroundEffects {
|
||||
readonly state$: Behavior<BackgroundEffectsState>;
|
||||
}
|
||||
|
||||
/** Switches the pipeline as the choice changes, for as long as the scope lasts. */
|
||||
export function createBackgroundEffects(
|
||||
scope: ObservableScope,
|
||||
{
|
||||
supported,
|
||||
effect$,
|
||||
setEffect,
|
||||
added$,
|
||||
pipeline,
|
||||
transformer,
|
||||
}: BackgroundEffectsOptions,
|
||||
): BackgroundEffects {
|
||||
const choice$ = effect$.pipe(map(parseEffect));
|
||||
const wanted$ = choice$.pipe(
|
||||
map((effect) => effect.kind !== "none"),
|
||||
distinctUntilChanged(),
|
||||
);
|
||||
|
||||
combineLatest([choice$, added$])
|
||||
.pipe(scope.bind())
|
||||
.subscribe(([effect, added]) => {
|
||||
if (effect.kind === "none") return;
|
||||
if (
|
||||
!supported ||
|
||||
(effect.kind === "added" &&
|
||||
added !== undefined &&
|
||||
!added.some(({ id }) => id === effect.id))
|
||||
)
|
||||
setEffect("none");
|
||||
});
|
||||
|
||||
const drewAFrame$ = scope.behavior(
|
||||
new Observable<boolean>((subscriber) => {
|
||||
subscriber.next(false);
|
||||
transformer.onFirstFrame = (): void => subscriber.next(true);
|
||||
return (): void => {
|
||||
transformer.onFirstFrame = undefined;
|
||||
};
|
||||
}),
|
||||
);
|
||||
|
||||
const state$ = scope.behavior(
|
||||
combineLatest([wanted$, drewAFrame$]).pipe(
|
||||
scan<[boolean, boolean], BackgroundEffectsState>(
|
||||
(previous, [wanted, drewAFrame]) => {
|
||||
// Attached the first time an effect is wanted and never detached
|
||||
// after, so someone who never turns one on pays for none of it.
|
||||
const enable =
|
||||
previous.processor !== undefined || (supported && wanted);
|
||||
return {
|
||||
supported,
|
||||
processor: enable ? pipeline : undefined,
|
||||
settling: enable && !drewAFrame,
|
||||
};
|
||||
},
|
||||
{ supported, processor: undefined },
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
const switchOptions$ = scope.behavior<
|
||||
SwitchBackgroundProcessorOptions | undefined
|
||||
>(
|
||||
combineLatest([state$, choice$, added$]).pipe(
|
||||
map(([{ processor }, effect, added]) =>
|
||||
processor === undefined
|
||||
? undefined
|
||||
: switchOptionsFor(
|
||||
effect,
|
||||
effect.kind === "added"
|
||||
? added?.find(({ id }) => id === effect.id)?.url
|
||||
: undefined,
|
||||
),
|
||||
),
|
||||
distinctUntilChanged(deepCompare),
|
||||
),
|
||||
);
|
||||
// In turn: a picture's switch ends once it loads, so an earlier one could land last.
|
||||
scope.reconcile(switchOptions$, async (options) => {
|
||||
if (options === undefined) return;
|
||||
try {
|
||||
await pipeline.switchTo(options);
|
||||
} catch (e) {
|
||||
logger.warn("Failed to switch background effect", e);
|
||||
}
|
||||
});
|
||||
|
||||
return { state$ };
|
||||
}
|
||||
|
||||
/** `addedUrl` is the picture of the added background chosen, if still kept. */
|
||||
function switchOptionsFor(
|
||||
effect: BackgroundEffect,
|
||||
addedUrl: string | undefined,
|
||||
): SwitchBackgroundProcessorOptions {
|
||||
const withImage = (
|
||||
imagePath: string | undefined,
|
||||
): SwitchBackgroundProcessorOptions =>
|
||||
imagePath
|
||||
? { mode: "virtual-background", imagePath }
|
||||
: { mode: "disabled" };
|
||||
switch (effect.kind) {
|
||||
case "blur":
|
||||
return { mode: "background-blur", blurRadius };
|
||||
case "shipped":
|
||||
return withImage(imagePathFor(effect.id));
|
||||
case "added":
|
||||
return withImage(addedUrl);
|
||||
default:
|
||||
return { mode: "disabled" };
|
||||
}
|
||||
}
|
||||
@@ -89,7 +89,7 @@ import { E2eeType } from "../../e2ee/e2eeType";
|
||||
import { MatrixKeyProvider } from "../../e2ee/matrixKeyProvider";
|
||||
import { type MuteStates } from "../MuteStates";
|
||||
import { HeaderStyle, type UrlParams } from "../../UrlParams";
|
||||
import { type ProcessorState } from "../../livekit/TrackProcessorContext";
|
||||
import { type BackgroundEffectsState } from "../../livekit/BackgroundEffectsContext";
|
||||
import { type HostBridge, nullHostBridge } from "../../HostBridge";
|
||||
import {
|
||||
layoutShallowEquals,
|
||||
@@ -501,7 +501,7 @@ export function createCallViewModel$(
|
||||
options: CallViewModelOptions,
|
||||
handsRaisedSubject$: Observable<Record<string, RaisedHandInfo>>,
|
||||
reactionsSubject$: Observable<Record<string, ReactionInfo>>,
|
||||
trackProcessorState$: Behavior<ProcessorState>,
|
||||
backgroundEffectsState$: Behavior<BackgroundEffectsState>,
|
||||
): CallViewModel {
|
||||
const logger = rootLogger.getChild("[CallViewModel]");
|
||||
const client = matrixRoom.client;
|
||||
@@ -590,7 +590,7 @@ export function createCallViewModel$(
|
||||
client,
|
||||
matrixRoom.roomId,
|
||||
mediaDevices,
|
||||
trackProcessorState$,
|
||||
backgroundEffectsState$,
|
||||
livekitKeyProvider,
|
||||
controlledAudioDevices,
|
||||
options.livekitRoomFactory,
|
||||
@@ -645,7 +645,7 @@ export function createCallViewModel$(
|
||||
connection,
|
||||
mediaDevices,
|
||||
muteStates,
|
||||
trackProcessorState$,
|
||||
backgroundEffectsState$,
|
||||
logger.getChild(
|
||||
"[Publisher " + connection.transport.livekit_service_url + "]",
|
||||
),
|
||||
|
||||
@@ -54,7 +54,7 @@ import {
|
||||
localRtcMember,
|
||||
} from "../../utils/test-fixtures";
|
||||
import { type Behavior, constant } from "../Behavior";
|
||||
import { type ProcessorState } from "../../livekit/TrackProcessorContext";
|
||||
import { type BackgroundEffectsState } from "../../livekit/BackgroundEffectsContext";
|
||||
import { type MediaDevices } from "../MediaDevices";
|
||||
import { type MatrixRTCMode } from "../../config/ConfigOptions";
|
||||
|
||||
@@ -240,7 +240,7 @@ export function withCallViewModel(mode: MatrixRTCMode) {
|
||||
},
|
||||
raisedHands$,
|
||||
reactions$,
|
||||
new BehaviorSubject<ProcessorState>({
|
||||
new BehaviorSubject<BackgroundEffectsState>({
|
||||
processor: undefined,
|
||||
supported: undefined,
|
||||
}),
|
||||
|
||||
@@ -209,7 +209,7 @@ describe("LocalMembership", () => {
|
||||
leave: vi.fn(),
|
||||
} as unknown as MatrixRTCSession,
|
||||
muteStates: mockMuteStates(),
|
||||
trackProcessorState$: constant({
|
||||
backgroundEffectsState$: constant({
|
||||
supported: false,
|
||||
processor: undefined,
|
||||
}),
|
||||
|
||||
@@ -11,11 +11,16 @@ import {
|
||||
LocalParticipant,
|
||||
type LocalTrack,
|
||||
type LocalTrackPublication,
|
||||
LocalVideoTrack,
|
||||
ParticipantEvent,
|
||||
Track,
|
||||
} from "livekit-client";
|
||||
import { BehaviorSubject } from "rxjs";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
import {
|
||||
type BackgroundOptions,
|
||||
type ProcessorWrapper,
|
||||
} from "@livekit/track-processors";
|
||||
|
||||
import { ObservableScope } from "../../ObservableScope";
|
||||
import { constant } from "../../Behavior";
|
||||
@@ -25,6 +30,7 @@ import {
|
||||
mockMediaDevices,
|
||||
} from "../../../utils/test";
|
||||
import { Publisher } from "./Publisher";
|
||||
import { type BackgroundEffectsState } from "../../../livekit/BackgroundEffectsContext";
|
||||
import { type Connection } from "../remoteMembers/Connection";
|
||||
import { type MuteStates } from "../../MuteStates";
|
||||
|
||||
@@ -400,3 +406,67 @@ describe("Bug fix", () => {
|
||||
await publisher.destroy();
|
||||
});
|
||||
});
|
||||
|
||||
describe("turning the camera on with an effect chosen", () => {
|
||||
const processor = {} as ProcessorWrapper<BackgroundOptions>;
|
||||
let state$: BehaviorSubject<BackgroundEffectsState>;
|
||||
let publisher: Publisher;
|
||||
/** The processor the SDK would give each camera track it made. */
|
||||
let madeWith: unknown[];
|
||||
|
||||
beforeEach(() => {
|
||||
state$ = new BehaviorSubject<BackgroundEffectsState>({
|
||||
supported: true,
|
||||
processor: undefined,
|
||||
});
|
||||
publisher = new Publisher(
|
||||
connection,
|
||||
mockMediaDevices({}),
|
||||
muteStates,
|
||||
state$,
|
||||
logger,
|
||||
false,
|
||||
);
|
||||
madeWith = [];
|
||||
vi.spyOn(localParticipant, "setCameraEnabled").mockImplementation(
|
||||
async () => {
|
||||
madeWith.push(
|
||||
connection.livekitRoom.options.videoCaptureDefaults?.processor,
|
||||
);
|
||||
return Promise.resolve(undefined);
|
||||
},
|
||||
);
|
||||
vi.spyOn(localParticipant, "unpublishTrack").mockResolvedValue(undefined);
|
||||
});
|
||||
afterEach(async () => {
|
||||
await publisher.destroy();
|
||||
});
|
||||
|
||||
it("makes each camera track with the pipeline attached since", async () => {
|
||||
await publisher.createAndSetupTracks();
|
||||
state$.next({ supported: true, processor });
|
||||
videoEnabled$.next(true);
|
||||
await flushPromises();
|
||||
|
||||
expect(madeWith).toEqual([processor]);
|
||||
});
|
||||
|
||||
it("replaces a camera track turned off before the effect was chosen", async () => {
|
||||
state$.next({ supported: true, processor });
|
||||
const stopped = Object.assign(Object.create(LocalVideoTrack.prototype), {
|
||||
source: Track.Source.Camera,
|
||||
getProcessor: (): undefined => undefined,
|
||||
}) as LocalVideoTrack;
|
||||
trackPublications.push({
|
||||
track: stopped,
|
||||
source: Track.Source.Camera,
|
||||
} as Partial<LocalTrackPublication> as LocalTrackPublication);
|
||||
|
||||
await publisher.createAndSetupTracks();
|
||||
videoEnabled$.next(true);
|
||||
await flushPromises();
|
||||
|
||||
expect(localParticipant.unpublishTrack).toHaveBeenCalledWith(stopped);
|
||||
expect(madeWith).toEqual([processor]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -26,9 +26,9 @@ import type { Behavior } from "../../Behavior.ts";
|
||||
import type { MediaDevices, SelectedDevice } from "../../MediaDevices.ts";
|
||||
import type { MuteStates } from "../../MuteStates.ts";
|
||||
import {
|
||||
type ProcessorState,
|
||||
trackProcessorSync,
|
||||
} from "../../../livekit/TrackProcessorContext.tsx";
|
||||
type BackgroundEffectsState,
|
||||
syncBackgroundEffects,
|
||||
} from "../../../livekit/BackgroundEffectsContext.tsx";
|
||||
|
||||
import { observeTrackReference$ } from "../../observeTrackReference";
|
||||
import { type Connection } from "../remoteMembers/Connection.ts";
|
||||
@@ -54,7 +54,7 @@ export class Publisher {
|
||||
* @param connection - The connection to use for publishing.
|
||||
* @param devices - The media devices to use for audio and video input.
|
||||
* @param muteStates - The mute states for audio and video.
|
||||
* @param trackerProcessorState$ - The processor state for the video track processor (e.g. background blur).
|
||||
* @param backgroundEffectsState$ - The processor state for the video track processor (e.g. background blur).
|
||||
* @param logger - The logger to use for logging :D.
|
||||
* @param controlledAudioDevices - Whether the app hosting Element Call
|
||||
* controls the audio output devices, rather than the browser.
|
||||
@@ -63,7 +63,7 @@ export class Publisher {
|
||||
private connection: Pick<Connection, "livekitRoom" | "state$">, //setE2EEEnabled,
|
||||
devices: MediaDevices,
|
||||
private readonly muteStates: MuteStates,
|
||||
trackerProcessorState$: Behavior<ProcessorState>,
|
||||
private readonly backgroundEffectsState$: Behavior<BackgroundEffectsState>,
|
||||
private logger: Logger,
|
||||
controlledAudioDevices: boolean,
|
||||
) {
|
||||
@@ -73,8 +73,11 @@ export class Publisher {
|
||||
this.logger.error("Failed to set E2EE enabled on room", e);
|
||||
});
|
||||
|
||||
// Setup track processor syncing (blur)
|
||||
this.observeTrackProcessors(this.scope, room, trackerProcessorState$);
|
||||
this.observeBackgroundEffects(
|
||||
this.scope,
|
||||
room,
|
||||
this.backgroundEffectsState$,
|
||||
);
|
||||
// Observe media device changes and update LiveKit active devices accordingly
|
||||
this.observeMediaDevices(this.scope, devices, controlledAudioDevices);
|
||||
|
||||
@@ -412,6 +415,8 @@ export class Publisher {
|
||||
this.muteStates.video.setHandler(async (enable) => {
|
||||
try {
|
||||
this.logger.debug(`handler: Setting LiveKit camera enabled: ${enable}`);
|
||||
const { processor } = this.backgroundEffectsState$.value;
|
||||
if (enable && processor) await this.dropCameraWithoutEffect(lkRoom);
|
||||
await lkRoom.localParticipant.setCameraEnabled(enable);
|
||||
// Unmute will restart the track if it was paused upstream,
|
||||
// but until explicitly requested, we want to keep it paused.
|
||||
@@ -426,10 +431,23 @@ export class Publisher {
|
||||
});
|
||||
}
|
||||
|
||||
private observeTrackProcessors(
|
||||
/**
|
||||
* Unpublishes a camera track that has no effect on, so that turning the
|
||||
* camera on creates one with it. A camera turned off is stopped, and a
|
||||
* stopped track can't take an effect chosen while it was off.
|
||||
*/
|
||||
private async dropCameraWithoutEffect(lkRoom: LivekitRoom): Promise<void> {
|
||||
const track = lkRoom.localParticipant.getTrackPublication(
|
||||
Track.Source.Camera,
|
||||
)?.track;
|
||||
if (track instanceof LocalVideoTrack && !track.getProcessor())
|
||||
await lkRoom.localParticipant.unpublishTrack(track);
|
||||
}
|
||||
|
||||
private observeBackgroundEffects(
|
||||
scope: ObservableScope,
|
||||
room: LivekitRoom,
|
||||
trackerProcessorState$: Behavior<ProcessorState>,
|
||||
backgroundEffectsState$: Behavior<BackgroundEffectsState>,
|
||||
): void {
|
||||
const track$ = scope.behavior(
|
||||
observeTrackReference$(room.localParticipant, Track.Source.Camera).pipe(
|
||||
@@ -440,6 +458,15 @@ export class Publisher {
|
||||
),
|
||||
null,
|
||||
);
|
||||
trackProcessorSync(scope, track$, trackerProcessorState$);
|
||||
syncBackgroundEffects(scope, track$, backgroundEffectsState$);
|
||||
// Every camera track the SDK makes, on joining or on turning the camera
|
||||
// on, then starts with the pipeline on. One attached after publishing lets
|
||||
// the room through while it is built.
|
||||
backgroundEffectsState$.pipe(scope.bind()).subscribe(({ processor }) => {
|
||||
room.options.videoCaptureDefaults = {
|
||||
...room.options.videoCaptureDefaults,
|
||||
processor,
|
||||
};
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -655,7 +655,7 @@ describe("remote track logging", () => {
|
||||
// function createSetup(): void {
|
||||
// setUpPublishConnection();
|
||||
|
||||
// const fakeTrackProcessorSubject$ = new BehaviorSubject<ProcessorState>({
|
||||
// const fakeTrackProcessorSubject$ = new BehaviorSubject<BackgroundEffectsState>({
|
||||
// supported: true,
|
||||
// processor: undefined,
|
||||
// });
|
||||
|
||||
@@ -26,7 +26,7 @@ import type {
|
||||
} from "../../../livekit/openIDSFU.ts";
|
||||
import type { MediaDevices } from "../../MediaDevices.ts";
|
||||
import type { Behavior } from "../../Behavior.ts";
|
||||
import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx";
|
||||
import type { BackgroundEffectsState } from "../../../livekit/BackgroundEffectsContext.tsx";
|
||||
import { getLiveKitOptions } from "../../../livekit/options.ts";
|
||||
import {
|
||||
advancedCamera,
|
||||
@@ -60,7 +60,7 @@ export class ECConnectionFactory implements ConnectionFactory {
|
||||
* @param client - The OpenID client parts for authentication, needed to get openID and JWT tokens.
|
||||
* @param roomId - The current room ID.
|
||||
* @param devices - Used for video/audio out/in capture options.
|
||||
* @param processorState$ - Effects like background blur (only for publishing connection?)
|
||||
* @param backgroundEffectsState$ - Effects like background blur (only for publishing connection?)
|
||||
* @param livekitKeyProvider - Optional key provider for end-to-end encryption.
|
||||
* @param controlledAudioDevices - Option to indicate whether audio output device is controlled externally (native mobile app).
|
||||
* @param livekitRoomFactory - Optional factory function (for testing) to create LivekitRoom instances. If not provided, a default factory is used.
|
||||
@@ -69,7 +69,7 @@ export class ECConnectionFactory implements ConnectionFactory {
|
||||
private client: OpenIDClientParts,
|
||||
private readonly roomId: string,
|
||||
private devices: MediaDevices,
|
||||
private processorState$: Behavior<ProcessorState>,
|
||||
private backgroundEffectsState$: Behavior<BackgroundEffectsState>,
|
||||
livekitKeyProvider: BaseKeyProvider | undefined,
|
||||
private controlledAudioDevices: boolean,
|
||||
livekitRoomFactory?: () => LivekitRoom,
|
||||
@@ -77,7 +77,7 @@ export class ECConnectionFactory implements ConnectionFactory {
|
||||
const defaultFactory = (): LivekitRoom => {
|
||||
const roomOptions = generateRoomOption({
|
||||
devices: this.devices,
|
||||
processorState: this.processorState$.value,
|
||||
backgroundEffectsState: this.backgroundEffectsState$.value,
|
||||
e2eeLivekitOptions: livekitKeyProvider && {
|
||||
keyProvider: livekitKeyProvider,
|
||||
// It's important that every room use a separate E2EE worker.
|
||||
@@ -129,12 +129,12 @@ export class ECConnectionFactory implements ConnectionFactory {
|
||||
*/
|
||||
function generateRoomOption({
|
||||
devices,
|
||||
processorState,
|
||||
backgroundEffectsState,
|
||||
e2eeLivekitOptions,
|
||||
controlledAudioDevices,
|
||||
}: {
|
||||
devices: MediaDevices;
|
||||
processorState: ProcessorState;
|
||||
backgroundEffectsState: BackgroundEffectsState;
|
||||
e2eeLivekitOptions:
|
||||
| E2EEManagerOptions
|
||||
| { e2eeManager: BaseE2EEManager }
|
||||
@@ -147,7 +147,7 @@ function generateRoomOption({
|
||||
let videoCaptureDefaults = {
|
||||
...liveKitOptions.videoCaptureDefaults,
|
||||
deviceId: devices.videoInput.selected$.value?.id,
|
||||
processor: processorState.processor,
|
||||
processor: backgroundEffectsState.processor,
|
||||
};
|
||||
let publishDefaults = liveKitOptions.publishDefaults;
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ import {
|
||||
mockMediaDevices,
|
||||
ownMemberMock,
|
||||
} from "../../../utils/test.ts";
|
||||
import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx";
|
||||
import type { BackgroundEffectsState } from "../../../livekit/BackgroundEffectsContext.tsx";
|
||||
import { constant } from "../../Behavior";
|
||||
import {
|
||||
echoCancellationSetting,
|
||||
@@ -80,7 +80,7 @@ describe("ECConnectionFactory - Audio inputs options", () => {
|
||||
mockClient,
|
||||
"!roomid:example.org",
|
||||
mockMediaDevices({}),
|
||||
new BehaviorSubject<ProcessorState>({
|
||||
new BehaviorSubject<BackgroundEffectsState>({
|
||||
supported: true,
|
||||
processor: undefined,
|
||||
}),
|
||||
@@ -128,7 +128,7 @@ describe("ECConnectionFactory - ControlledAudioDevice", () => {
|
||||
select: () => {},
|
||||
},
|
||||
}),
|
||||
new BehaviorSubject<ProcessorState>({
|
||||
new BehaviorSubject<BackgroundEffectsState>({
|
||||
supported: true,
|
||||
processor: undefined,
|
||||
}),
|
||||
@@ -163,7 +163,7 @@ describe("ECConnectionFactory - Camera quality settings", () => {
|
||||
mockClient,
|
||||
"!roomid:example.org",
|
||||
mockMediaDevices({}),
|
||||
new BehaviorSubject<ProcessorState>({
|
||||
new BehaviorSubject<BackgroundEffectsState>({
|
||||
supported: true,
|
||||
processor: undefined,
|
||||
}),
|
||||
@@ -200,7 +200,7 @@ describe("ECConnectionFactory - Camera quality settings", () => {
|
||||
mockClient,
|
||||
"!roomid:example.org",
|
||||
mockMediaDevices({}),
|
||||
new BehaviorSubject<ProcessorState>({
|
||||
new BehaviorSubject<BackgroundEffectsState>({
|
||||
supported: true,
|
||||
processor: undefined,
|
||||
}),
|
||||
@@ -238,7 +238,7 @@ describe("ECConnectionFactory - Camera quality settings", () => {
|
||||
mockClient,
|
||||
"!roomid:example.org",
|
||||
mockMediaDevices({}),
|
||||
new BehaviorSubject<ProcessorState>({
|
||||
new BehaviorSubject<BackgroundEffectsState>({
|
||||
supported: true,
|
||||
processor: undefined,
|
||||
}),
|
||||
|
||||
@@ -26,7 +26,7 @@ import {
|
||||
ownMemberMock,
|
||||
withTestScheduler,
|
||||
} from "../../../utils/test.ts";
|
||||
import { type ProcessorState } from "../../../livekit/TrackProcessorContext.tsx";
|
||||
import { type BackgroundEffectsState } from "../../../livekit/BackgroundEffectsContext.tsx";
|
||||
import {
|
||||
areLivekitTransportsEqual,
|
||||
createRemoteMatrixLivekitMembers$,
|
||||
@@ -72,7 +72,7 @@ beforeEach(() => {
|
||||
mockClient,
|
||||
"!roomid:example.org",
|
||||
mockMediaDevices({}),
|
||||
new BehaviorSubject<ProcessorState>({
|
||||
new BehaviorSubject<BackgroundEffectsState>({
|
||||
supported: true,
|
||||
processor: undefined,
|
||||
}),
|
||||
|
||||
@@ -188,6 +188,7 @@ export function getBasicCallViewModelEnvironment(
|
||||
vm,
|
||||
muteStates,
|
||||
mediaDevices,
|
||||
constant(false),
|
||||
"reactionId",
|
||||
{ showControls: true, header: HeaderStyle.Standard },
|
||||
);
|
||||
|
||||
@@ -37,8 +37,21 @@ export default defineConfig(({ mode }) => {
|
||||
...vitePluginsConfig({ mode, html: false }),
|
||||
root: "component/dev",
|
||||
// The same scoping the library build applies, so the harness shows what a
|
||||
// host will get — including whether its own page is left alone
|
||||
css: { postcss: { plugins: [scopeStylesToRoot()] } },
|
||||
// host will get — including whether its own page is left alone. The host's
|
||||
// own stylesheets, Compound's among them, stay unscoped, as on a real host.
|
||||
css: {
|
||||
postcss: {
|
||||
plugins: [
|
||||
scopeStylesToRoot({
|
||||
exclude: [
|
||||
/component[\\/]dev[\\/]/,
|
||||
/@vector-im[\\/]compound-/,
|
||||
/@fontsource[\\/]/,
|
||||
],
|
||||
}),
|
||||
],
|
||||
},
|
||||
},
|
||||
// So that the harness can read the same config.json the standalone app
|
||||
// does, if the developer has written one
|
||||
publicDir: "../../public",
|
||||
|
||||
+35
-4
@@ -7,6 +7,7 @@ Please see LICENSE in the repository root for full details.
|
||||
|
||||
import {
|
||||
loadEnv,
|
||||
type Plugin,
|
||||
type PluginOption,
|
||||
searchForWorkspaceRoot,
|
||||
type ConfigEnv,
|
||||
@@ -42,10 +43,12 @@ export const vitePluginsConfig = ({
|
||||
}),
|
||||
react(),
|
||||
wasm(),
|
||||
nodePolyfills({
|
||||
// Enables the 'events' module, which is required by the matrix-js-sdk
|
||||
include: ["events"],
|
||||
}),
|
||||
exceptMediaPipeWasm(
|
||||
nodePolyfills({
|
||||
// Enables the 'events' module, which is required by the matrix-js-sdk
|
||||
include: ["events"],
|
||||
}),
|
||||
),
|
||||
svgrPlugin({
|
||||
svgrOptions: {
|
||||
// This enables ref forwarding on SVGR components, which is needed, for
|
||||
@@ -176,3 +179,31 @@ export default ({
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Keeps a plugin's transform off MediaPipe's WASM loader. The loader is a
|
||||
* classic script that sets `self.ModuleFactory`; transformed into a module it
|
||||
* sets nothing, and the segmenter fails with "ModuleFactory not set".
|
||||
*/
|
||||
function exceptMediaPipeWasm(plugins: PluginOption): PluginOption {
|
||||
const isLoader = (id: string): boolean =>
|
||||
id.includes("tasks-vision") && id.includes("wasm");
|
||||
return (Array.isArray(plugins) ? plugins : [plugins]).map((plugin) => {
|
||||
if (
|
||||
!plugin ||
|
||||
typeof plugin !== "object" ||
|
||||
!("transform" in plugin) ||
|
||||
typeof plugin.transform !== "function"
|
||||
)
|
||||
return plugin;
|
||||
const transform = plugin.transform;
|
||||
const wrapped: Plugin = {
|
||||
...plugin,
|
||||
transform(code, id, options) {
|
||||
if (isLoader(id)) return null;
|
||||
return transform.call(this, code, id, options);
|
||||
},
|
||||
};
|
||||
return wrapped;
|
||||
});
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user