diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index d879b2ab4..0e415724c 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -71,10 +71,27 @@ jobs: flags: unittests fail_ci_if_error: true playwright: - name: Run end-to-end tests + name: Run end-to-end tests (${{ matrix.part }}) if: ${{ !inputs.disable_e2e_tests }} timeout-minutes: 60 runs-on: ubuntu-24.04 + # We manually shard the tests across runners. Playwright's own shards cut + # the list of tests into equal counts in file order, which would leave the + # widget suite, the slowest tests by far, together at the end. + strategy: + fail-fast: false + matrix: + include: + - part: chromium-1 + args: --project=chromium --shard=1/2 + - part: chromium-2 + args: --project=chromium --shard=2/2 + - part: widget-1 + args: --project=widget --shard=1/2 + - part: widget-2 + args: --project=widget --shard=2/2 + - part: firefox-mobile + args: --project=firefox --project=mobile steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: @@ -107,17 +124,64 @@ jobs: run: pnpm playwright install --with-deps - name: Run backend components run: | + # Make sure the directory for the embedded build exists as Docker might + # create it with wrong access rights otherwise. + mkdir -p embedded/web/dist docker compose -f docker-compose-dev.yml -f docker-compose-playwright.yml pull docker compose -f docker-compose-dev.yml -f docker-compose-playwright.yml up -d docker ps - name: Run Playwright tests env: USE_DOCKER: 1 - run: pnpm exec playwright test + ARGS: ${{ matrix.args }} + # Unquoted on purpose: the part's arguments are several words + run: pnpm exec playwright test $ARGS - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: ${{ !cancelled() }} + with: + name: blob-report-${{ matrix.part }} + path: blob-report + if-no-files-found: error + # Only read by the merge below. + retention-days: 1 + + e2e_report: + # Combines the blobs into a single HTML report and blocks PRs on failure. + name: Run end-to-end tests + needs: playwright + if: ${{ always() && !inputs.disable_e2e_tests }} + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + # Inside a called workflow, github.repository is the caller so we need to make sure to still check out the EC repo here. + repository: ${{ inputs.matrix-js-sdk-sha && 'element-hq/element-call' || github.repository }} + persist-credentials: false + - name: Enable Corepack + run: corepack enable + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + cache: "pnpm" + node-version-file: ".node-version" + - name: Install dependencies + run: scripts/layered.sh --ignore-pnpmfile + env: + PR_HEAD_REPO_OWNER: ${{ github.event.pull_request.head.repo.owner.login }} + JS_SDK_GITHUB_BASE_REF: ${{ inputs.matrix-js-sdk-sha }} + - name: Download the parts' reports + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + pattern: blob-report-* + path: all-blob-reports + merge-multiple: true + - name: Merge into one HTML report + run: pnpm exec playwright merge-reports --reporter html ./all-blob-reports + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: html-report path: playwright-report if-no-files-found: error retention-days: 4 + - name: Fail on a failed part + if: ${{ needs.playwright.result != 'success' }} + run: exit 1 diff --git a/README.md b/README.md index 17eae35d9..c806c45ce 100644 --- a/README.md +++ b/README.md @@ -221,119 +221,68 @@ See also: - [Developing with linked packages](./docs/linking.md) -#### Element Call as a component (experimental) +#### Element Call as a component -Element Call can also be embedded directly into another React application -rather than being loaded in an iframe as a widget. `pnpm build:component` -builds it as a library into `component/dist` (the bundle, its stylesheet and -type declarations), and +Element Call can be rendered inside another React app instead of an iframe. +The host supplies its own Matrix client, and the call stays inside the element +it is mounted in. -```sh -pnpm dev:component -``` - -serves a harness on port 3001 that stands in for such an application: it signs -in twice against the development backend and shows two calls side by side, in -resizable boxes, with page furniture of its own around them. Use it to see how -Element Call behaves when it does not own the page — the size it is given, -whether it stays inside its container, and what it says to its host, which is -logged along the bottom. The harness is served with the same development -certificate as the app, so unless the development CA is trusted, the browser -needs a certificate exception for `https://localhost:3001` as well (see the -note under [Backend](#backend)). It reads the same `public/config.json` as -`pnpm dev` if one exists, and runs with Element Call's defaults otherwise. - -The call lays itself out for the size of the element it is mounted in, not the -window: a host that shrinks the container to a corner of its page gets the -picture-in-picture layout, just as a host that shrank the whole iframe used to. -The breakpoints in the stylesheets the component uses are -`@container element-call` queries against its root element for the same reason; -for the standalone app the root is the page, so they mean what the media queries -they replaced did. (The standalone-only views, such as the home and login pages, -still use plain media queries, since the component never shows them.) - -The component's stylesheet is confined to the element it is mounted in: the -build rewrites every selector so that it matches only Element Call's root or -what is inside it, with `html`, `body` and `:root` standing for that root (see -`component/build/scopeStylesToRoot.ts`). A host's own page keeps its styles, -and Element Call brings its own fonts and design tokens along. - -The component speaks every language the app does. English is bundled in; the -other locales are split into chunks the host's bundler loads the first time -they are needed. It starts in the browser's language, and follows the host's -own language setting through the `language` prop (`supportedLanguages` lists -the tags it accepts, and anything else falls back to its base language or to -English). The `theme` prop works the same way and takes the same values as the -widget's `theme` URL parameter: `light`, `dark`, `light-high-contrast` or -`dark-high-contrast`. Both can change while a call is running without -disturbing it. - -A host must call and await `initializeElementCall(config)` once before -rendering the component: it loads the `Intl` polyfills, applies the -deployment-wide `config.json`-style configuration and sets up translations. -The component itself takes the host's `client` and the `roomId` to call in, an -`intent` saying what the user asked for (which decides whether to show the -lobby, ring, and so on), an optional `config` overriding what the intent -implies, and an optional `hostBridge` through which Element Call tells the host -that the user has joined or hung up, that it wants to stay on screen, and so -on. The host makes its own requests (`join`, `hangUp`, `setDeviceMute`) through -the handle exposed on `ref`. The full API is documented in the type declarations -(`component/index.tsx` and `component/host.ts`). - -A few things differ from the widget on purpose: the component draws a solid -background rather than a gradient unless told otherwise, never offers to edit -the user's profile (the account is the host's), scopes its keyboard shortcuts to -its own root element so that several instances can share a page, and only shows -its own post-call and error screens when the host has not supplied a `close()` -callback; with one, it asks the host to unmount it instead. The -[global JS controls](./docs/controls.md) on `window` are unchanged and remain -page-wide, so with several instances on one page they apply to all of them. - -Each release publishes the package to npm, with the same version as the release: +##### Install ```sh npm install @element-hq/element-call-component ``` -To use an unreleased commit, a host can instead install it as a git dependency -on the `component` directory of this repository, +Releases publish the package with the same version as Element Call. For an +unreleased commit, depend on this repository directly: ```json "@element-hq/element-call-component": "github:element-hq/element-call#main&path:/component" ``` -whose `prepare` script runs the build on install. That build needs pnpm (via -Corepack) on the host's machine, runs a full `pnpm install` of this repository, -since it inherits the `--max-old-space-size` setting of -the app build; the host's pnpm also has to allow it to run at all -(`allowBuilds` in its `pnpm-workspace.yaml`). Note that `component/` is a pnpm -project of its own for this reason, so pnpm commands run from inside that -directory target it rather than the repository; run them from the repository -root. The host imports the component from -`@element-hq/element-call-component` and the stylesheet from -`@element-hq/element-call-component/style.css`, and has to provide `react`, -`react-dom`, `matrix-js-sdk`, `@vector-im/compound-web` and -`@vector-im/compound-design-tokens` itself, since the bundle leaves them -external. -Compound's stylesheet are the host's to load too: the component shares -the host's copy of Compound, (required for example for tooltips) +This builds on install, so the host needs pnpm (via Corepack) and must allow +the build (`allowBuilds` in its `pnpm-workspace.yaml`). -The component is large, and a host will usually load it lazily, only once a -call is shown. Everything a host needs in order to talk about a call before -then is also exported -from `@element-hq/element-call-component/api` (a small entry point that does not -load the component). +##### Use -- props and the types they are made of -- the handle -- the host bridge -- `UserIntent` -- `HeaderStyle` -- `BackgroundStyle` enums -- `configurationForIntent`, the defaults each intent implies - Import from it wherever a value such as - `BackgroundStyle.Solid` is needed on a path that must stay light; the main - entry point re-exports all of it too. +1. Provide the externals yourself: `react`, `react-dom`, `matrix-js-sdk`, + `@vector-im/compound-web`, `@vector-im/compound-design-tokens`, and + Compound's stylesheet. +2. Import `@element-hq/element-call-component/style.css`. +3. Await `initializeElementCall(config)` once before rendering. +4. Render the component with `client`, `roomId` and an `intent`; optionally + pass `config`, `hostBridge`, `language` and `theme` (both can change + mid-call). Control it through the `ref` handle (`join`, `hangUp`, + `setDeviceMute`). + +Load the component lazily. Types, enums and `configurationForIntent` are +available without it from `@element-hq/element-call-component/api`. The full +API is in `component/index.tsx`, `component/api.ts` and `component/host.ts`. + +##### How it differs from the widget + +- It lays out for its container, not the window: a small container gets the + picture-in-picture layout. +- Its styles and keyboard shortcuts are scoped to its root, so the host page is + untouched and several instances can share a page. The + [global JS controls](./docs/controls.md) stay page-wide. +- It draws a solid background by default and never offers profile editing. +- If the host bridge implements `close()`, it asks to be unmounted instead of + showing its own post-call and error screens. +- Sounds, images and the MediaPipe runtime are separate files the host's + bundler copies (webpack 5, Vite, Rollup); MediaPipe is fetched only when a + background effect is first used. + +##### Develop + +- `pnpm build:component` builds the library into `component/dist`. +- `pnpm dev:component` serves a test host on `https://localhost:3001`: two + calls side by side in resizable boxes, with host messages logged at the + bottom. It uses the development backend, `public/config.json` if present, + and needs the same certificate exception as `pnpm dev` (see + [Backend](#backend)). +- Run pnpm commands from the repository root; `component/` is its own pnpm + project. ### Backend diff --git a/docker-compose-playwright.yml b/docker-compose-playwright.yml index 7e58bcf64..127a7377d 100644 --- a/docker-compose-playwright.yml +++ b/docker-compose-playwright.yml @@ -19,7 +19,12 @@ services: - ./backend/playwright_homeserver-othersite.yaml:/data/cfg/homeserver.yaml:Z element-web: # Pin to a SHA so that upstream cannot break our tests. Renovate handles regular updates. - image: ghcr.io/element-hq/element-web:develop@sha256:e063472b6bdf6d0bfa1d43dfa4ef7effee1efd2c9684daee665d1bedfe746a47 + image: ghcr.io/element-hq/element-web:develop@sha256:37d88b4426ff93a884b31c63d5319a7bcee32580f189c724e6b09f793a6de390 element-web-1: # Pin to a SHA so that upstream cannot break our tests. Renovate handles regular updates. - image: ghcr.io/element-hq/element-web:develop@sha256:e063472b6bdf6d0bfa1d43dfa4ef7effee1efd2c9684daee665d1bedfe746a47 + image: ghcr.io/element-hq/element-web:develop@sha256:37d88b4426ff93a884b31c63d5319a7bcee32580f189c724e6b09f793a6de390 + volumes: + # The other site opens the embedded package under test in place of the + # copy its image bundles (the first site opens the dev server's app. + # Built by the Playwright run (see build.setup.ts). + - ./embedded/web/dist:/app/widgets/element-call:ro,Z diff --git a/docs/agents/testing.md b/docs/agents/testing.md index eb8792d67..ba203fd11 100644 --- a/docs/agents/testing.md +++ b/docs/agents/testing.md @@ -34,6 +34,10 @@ translations. The browser is Playwright's, so a fresh clone needs ## End-to-end +The Element Web instance `HOST1` opens the app the dev server serves. The instance +`HOST2` (the other site) opens the embedded package as a host ships it. This package +is built by the `build-embedded` setup project and mounted in `docker-compose-playwright.yml`. + - `playwright/*.spec.ts` — standalone, on Chromium and Firefox. - `playwright/widget/` — widget mode, where cross-context bugs surface. - `playwright/component/` — the component in a host page, via the harness on port diff --git a/playwright.config.ts b/playwright.config.ts index 2de044194..ebd5feea7 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -26,6 +26,19 @@ process.env.NODE_EXTRA_CA_CERTS = join( "backend/dev_tls_local-ca.crt", ); +const desktopChrome = { + ...devices["Desktop Chrome"], + permissions: ["clipboard-write", "clipboard-read", "microphone", "camera"], + ignoreHTTPSErrors: true, + launchOptions: { + args: [ + "--use-fake-ui-for-media-stream", + "--use-fake-device-for-media-stream", + "--mute-audio", + ], + }, +}; + /** * See https://playwright.dev/docs/test-configuration. */ @@ -43,11 +56,11 @@ export default defineConfig({ * Reporter to use. See https://playwright.dev/docs/test-reporters * * On CI, `list` prints a line per test as it finishes, which keeps the job - * log moving; `github` annotates failures on the PR; `html` is uploaded as - * an artifact and must never try to open a browser there. + * log moving; `github` annotates failures on the PR; the `blob` format is + * uploaded by sharded CI runs and combined into a HTML report by the workflow. */ reporter: process.env.CI - ? [["list"], ["github"], ["html", { open: "never" }]] + ? [["list"], ["github"], ["blob"]] : [["list"], ["html"]], /* Stop a broken run early on CI instead of waiting out every retry. */ maxFailures: process.env.CI ? 5 : undefined, @@ -63,29 +76,28 @@ export default defineConfig({ }, /* Configure projects for major browsers */ projects: [ + { + // Builds the embedded package into the directory the other site's + // Element Web container serves it from (see docker-compose-playwright.yml). + name: "build-embedded", + testMatch: "**/widget/embedded/build.setup.ts", + }, { name: "chromium", - testIgnore: "**/mobile/**", - use: { - ...devices["Desktop Chrome"], - permissions: [ - "clipboard-write", - "clipboard-read", - "microphone", - "camera", - ], - ignoreHTTPSErrors: true, - launchOptions: { - args: [ - "--use-fake-ui-for-media-stream", - "--use-fake-device-for-media-stream", - "--mute-audio", - ], - }, - }, + testIgnore: ["**/mobile/**", "**/widget/**"], + use: desktopChrome, + }, + { + // Element Call as a widget in Element Web, on Chromium. This uses its + // own project so that it can be sharded on CI. + name: "widget", + dependencies: ["build-embedded"], + testMatch: "**/widget/**/*.@(spec|test).ts", + use: desktopChrome, }, { name: "firefox", + dependencies: ["build-embedded"], testIgnore: "**/mobile/**", use: { ...devices["Desktop Firefox"], diff --git a/playwright/background-effects.spec.ts b/playwright/background-effects.spec.ts index 4d01c2d72..4c71c7a19 100644 --- a/playwright/background-effects.spec.ts +++ b/playwright/background-effects.spec.ts @@ -106,6 +106,66 @@ async function drawsPerSecond(page: Page): Promise { }); } +test.describe("the camera menu's preview", () => { + test("shows the camera with the effect in force", async ({ page }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall(page, "Preview user", "Preview in the menu"); + const camera = await cameraColour(page.locator("video").first()); + + await page.getByRole("button", { name: "Camera", exact: true }).click(); + const preview = page.getByRole("menu").locator("video"); + await expect(preview).toBeVisible(); + await expect + .poll(async () => distance(await averageColour(preview), camera), { + timeout: 20_000, + }) + .toBeLessThan(30); + + const tile = page.getByRole("menuitemradio", { name: "Background 1" }); + await tile.click(); + const picture = await averageColour(tile.locator("img")); + await expect + .poll(async () => distance(await averageColour(preview), picture), { + timeout: 60_000, + }) + .toBeLessThan(distance(camera, picture) / 2); + }); + + test("shows the call's own camera once joined", async ({ page }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall( + page, + "Call preview user", + "Preview in the call's menu", + true, + ); + await expect(page.getByTestId("incall_videomute")).toBeEnabled({ + timeout: 10_000, + }); + const camera = await cameraColour(page.locator("video").first()); + + await page.getByRole("button", { name: "Camera", exact: true }).click(); + const preview = page.getByRole("menu").locator("video"); + await expect(preview).toBeVisible(); + await expect + .poll(async () => distance(await averageColour(preview), camera), { + timeout: 20_000, + }) + .toBeLessThan(30); + + const tile = page.getByRole("menuitemradio", { name: "Background 1" }); + await tile.click(); + const picture = await averageColour(tile.locator("img")); + await expect + .poll(async () => distance(await averageColour(preview), picture), { + timeout: 60_000, + }) + .toBeLessThan(distance(camera, picture) / 2); + }); +}); + test.describe("background effects section", () => { test("section stays within the call area at every size", async ({ page }) => { test.setTimeout(120_000); @@ -151,6 +211,46 @@ test.describe("background effects section", () => { }); }); +test.describe("a pipeline that fails to build", () => { + test("offers no effects for the session, and keeps the choice", async ({ + page, + }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall(page, "Failing user", "Pipeline fails"); + const preview = page.locator("video").first(); + const camera = await cameraColour(preview); + await page.route("**/*.tflite*", async (route) => route.abort()); + + await page.getByRole("button", { name: "Camera", exact: true }).click(); + const tile = page.getByRole("menuitemradio", { name: "Background 1" }); + await tile.click(); + await expect( + page.getByText("Background effects are not supported on this platform."), + ).toBeVisible({ timeout: 30_000 }); + await expect(tile).toBeDisabled(); + await page.keyboard.press("Escape"); + // The camera and microphone carry on without it. + expect(distance(await cameraColour(preview), camera)).toBeLessThan(30); + await expect(page.getByTestId("incall_mute")).toHaveAttribute( + "aria-checked", + "true", + ); + + await page.unroute("**/*.tflite*"); + await page.reload(); + await expect(page.getByTestId("lobby_joinCall")).toBeVisible(); + await page.getByRole("button", { name: "Camera", exact: true }).click(); + await expect(tile).toHaveAttribute("aria-checked", "true"); + const picture = await averageColour(tile.locator("img")); + await expect + .poll(async () => distance(await averageColour(preview), picture), { + timeout: 60_000, + }) + .toBeLessThan(distance(camera, picture) / 2); + }); +}); + test.describe("the chosen effect", () => { test("chosen effect survives leaving and rejoining", async ({ page }) => { test.setTimeout(120_000); diff --git a/playwright/fixtures/widget-user.ts b/playwright/fixtures/widget-user.ts index b21732594..f069db1a1 100644 --- a/playwright/fixtures/widget-user.ts +++ b/playwright/fixtures/widget-user.ts @@ -127,13 +127,14 @@ export const widgetTest = test.extend({ * Provide a way to add additional users within a test. * The returned user will be registered on the default homeserver, the name will be made unique by appending a timestamp. */ - addUser: async ({ browser }, use) => { + addUser: async ({ browser }, use, testInfo) => { await use( async ( username: string, host: string = HOST1, ): Promise => { - const uniqueSuffix = Date.now(); + // Tests in parallel workers can reach this in the same millisecond. + const uniqueSuffix = `${Date.now()}_${testInfo.parallelIndex}`; const { page, clientHandle, mxId } = await TestHelpers.registerUser( browser, `${username.toLowerCase()}_${uniqueSuffix}`, diff --git a/playwright/widget/embedded/build.setup.ts b/playwright/widget/embedded/build.setup.ts new file mode 100644 index 000000000..efc1e3d73 --- /dev/null +++ b/playwright/widget/embedded/build.setup.ts @@ -0,0 +1,20 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { test as setup } from "@playwright/test"; +import { execFileSync } from "node:child_process"; +import { mkdirSync } from "node:fs"; + +const EMBEDDED_DIST = "embedded/web/dist"; + +setup("build the embedded package", () => { + setup.setTimeout(10 * 60 * 1000); + mkdirSync(EMBEDDED_DIST, { recursive: true }); + execFileSync("pnpm", ["build:embedded", "--outDir", EMBEDDED_DIST], { + stdio: "inherit", + }); +}); diff --git a/playwright/widget/embedded/config-json.spec.ts b/playwright/widget/embedded/config-json.spec.ts new file mode 100644 index 000000000..b8bba8b24 --- /dev/null +++ b/playwright/widget/embedded/config-json.spec.ts @@ -0,0 +1,56 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { expect, type Route } from "@playwright/test"; + +import { widgetTest } from "../../fixtures/widget-user.ts"; +import { HOST2, TestHelpers } from "../test-helpers.ts"; + +/** Where the other site's Element Web serves the embedded package from */ +const CONFIG_JSON = "**/widgets/element-call/config.json"; + +const HOST_ANSWERS: Record Promise> = { + "a 404": async (route) => route.fulfill({ status: 404 }), + "a failed fetch": async (route) => route.abort("failed"), + "an HTML page": async (route) => + route.fulfill({ + status: 200, + contentType: "text/html", + body: "Element", + }), +}; + +widgetTest.skip( + ({ browserName }) => browserName === "firefox", + "The widget suite does not run on Firefox, see the other specs here", +); + +for (const [answer, respond] of Object.entries(HOST_ANSWERS)) { + widgetTest( + `reaches the lobby when config.json gets ${answer}`, + async ({ addUser }) => { + const { page } = await addUser("Brooks", HOST2); + await TestHelpers.createRoom("Config Room", page); + + let answered = 0; + await page.route(CONFIG_JSON, async (route) => { + answered++; + await respond(route); + }); + + await TestHelpers.startCallInCurrentRoom(page); + + await expect( + page + .locator('iframe[title="Element Call"]') + .contentFrame() + .getByTestId("lobby_joinCall"), + ).toBeVisible({ timeout: 15000 }); + expect(answered).toBeGreaterThan(0); + }, + ); +} diff --git a/playwright/widget/test-helpers.ts b/playwright/widget/test-helpers.ts index 4e255f869..38e61f49c 100644 --- a/playwright/widget/test-helpers.ts +++ b/playwright/widget/test-helpers.ts @@ -144,7 +144,11 @@ export class TestHelpers { await this.dismissStartupToasts(page); - await TestHelpers.setDevToolElementCallDevUrl(page); + // The other site's Element Web opens the embedded package it serves itself + // (see docker-compose-playwright.yml), not the dev server's app. + if (host !== HOST2) { + await TestHelpers.setDevToolElementCallDevUrl(page); + } const clientHandle = await page.evaluateHandle(() => window.mxMatrixClientPeg.get(), diff --git a/src/components/CallFooter.stories.tsx b/src/components/CallFooter.stories.tsx index d7b1dcf90..8d6cc2a77 100644 --- a/src/components/CallFooter.stories.tsx +++ b/src/components/CallFooter.stories.tsx @@ -141,6 +141,7 @@ export const Default: Story = { backgroundImageRefusal: undefined, addBackgroundImage: fn(), removeBackgroundEffect: fn(), + cameraTrack: null, hangup: fn(), buttonSize: "lg", showFooter: true, diff --git a/src/components/CallFooter.tsx b/src/components/CallFooter.tsx index fce982347..4d13b4c6a 100644 --- a/src/components/CallFooter.tsx +++ b/src/components/CallFooter.tsx @@ -33,7 +33,9 @@ import { useBehavior } from "../useBehavior"; import { type LayoutSwitchViewModel } from "../state/LayoutSwitchViewModel"; import { LayoutSwitch } from "../room/LayoutSwitch"; import { type BackgroundEffectGridItem } from "./BackgroundEffectGrid"; +import { SelfPreview } from "./SelfPreview"; import { type UnusableReason } from "../tracks/backgroundImages"; +import { type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk"; export interface BackgroundImageRefusal { reason: UnusableReason | "not-kept"; @@ -92,6 +94,8 @@ export interface FooterState { backgroundEffectSettling: boolean; /** Why the last file offered couldn't be kept, if it couldn't. */ backgroundImageRefusal: BackgroundImageRefusal | undefined; + /** The camera to preview, with the effect in force; null while it is off. */ + cameraTrack: LocalVideoMediaTrack | null; showFooter: boolean; /* This is needed for WindowMode = "flat" */ @@ -181,6 +185,7 @@ export const CallFooter: FC = ({ const backgroundEffectSettling = useBehavior(vm.backgroundEffectSettling$); const addBackgroundImage = useBehavior(vm.addBackgroundImage$); const removeBackgroundEffect = useBehavior(vm.removeBackgroundEffect$); + const cameraTrack = useBehavior(vm.cameraTrack$); const refusal = useBehavior(vm.backgroundImageRefusal$); const refusalMessage = useMemo(() => { switch (refusal?.reason) { @@ -264,6 +269,7 @@ export const CallFooter: FC = ({ backgroundEffectSettling={backgroundEffectSettling} onAddBackgroundImage={addBackgroundImage} onRemoveBackgroundEffect={removeBackgroundEffect} + selfPreview={} backgroundImageRefusal={refusalMessage} backgroundEffectNotice={ backgroundEffectNotice === "unavailable" diff --git a/src/components/CallFooterViewModel.test.ts b/src/components/CallFooterViewModel.test.ts index a1b174462..9871bc4b0 100644 --- a/src/components/CallFooterViewModel.test.ts +++ b/src/components/CallFooterViewModel.test.ts @@ -7,14 +7,17 @@ Please see LICENSE in the repository root for full details. import { afterEach, describe, expect, it, vi } from "vitest"; import { BehaviorSubject } from "rxjs"; +import { type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk"; import { flushPromises, testScope, mockMuteStates, mockMediaDevices, + mockVideoTrack, } from "../utils/test"; -import { constant } from "../state/Behavior"; +import { type Behavior, constant } from "../state/Behavior"; +import { type BackgroundEffectsState } from "../state/BackgroundEffects"; import type { CallViewModel } from "../state/CallViewModel/CallViewModel"; import type { Layout } from "../state/layout-types"; import type { SpotlightTileViewModel } from "../state/TileViewModel"; @@ -78,6 +81,7 @@ vi.mock("livekit-client", () => ({ function buildMinimalCallViewModel(layout: Layout): CallViewModel { return { layout$: constant(layout), + camera$: constant(undefined), edgeToEdge$: constant(false), showHeader$: constant(false), hangup: (): void => {}, @@ -109,6 +113,16 @@ const pipLayout: Layout = { spotlight: {} as SpotlightTileViewModel, }; +const noPipeline = constant({ + supported: false, + processor: undefined, +}); + +/** The provider's verdict, which the SDK's check stands for here. */ +function pipelineAsTheSdkSays(): Behavior { + return constant({ supported: sdkSupportMock(), processor: undefined }); +} + const twoMicsAndOneCamMediaDevices = mockMediaDevices({ audioInput: { available$: constant( @@ -140,7 +154,7 @@ describe("createCallFooterViewModel", () => { buildMinimalCallViewModel(gridLayout), mockMuteStates(), twoMicsAndOneCamMediaDevices, - constant(false), + noPipeline, /* reactionIdentifier */ undefined, { showControls: true, header: HeaderStyle.Standard }, ); @@ -183,7 +197,7 @@ describe("createCallFooterViewModel", () => { select: vi.fn(), }, }), - constant(false), + noPipeline, /* reactionIdentifier */ undefined, { showControls: true, header: HeaderStyle.Standard }, ); @@ -202,7 +216,7 @@ describe("createCallFooterViewModel", () => { buildMinimalCallViewModel(layout), mockMuteStates(), twoMicsAndOneCamMediaDevices, - constant(false), + noPipeline, /* reactionIdentifier */ undefined, { showControls: true, header: HeaderStyle.Standard }, ); @@ -225,7 +239,7 @@ describe("createCallFooterViewModel", () => { buildMinimalCallViewModel(gridLayout), mockMuteStates(), twoMicsAndOneCamMediaDevices, - constant(false), + noPipeline, /* reactionIdentifier */ undefined, { showControls: true, header: HeaderStyle.Standard }, ); @@ -271,7 +285,8 @@ describe("createCallFooterViewModel", () => { testScope(), mockMuteStates(), twoMicsAndOneCamMediaDevices, - constant(false), + pipelineAsTheSdkSays(), + /* camera$ */ constant(null), /* openSettings */ undefined, /* hangup */ undefined, /* showLogo */ false, @@ -356,21 +371,52 @@ describe("createCallFooterViewModel", () => { it("passes on the wait for the first effect", () => { platformMock.mockReturnValue("desktop"); - const settling$ = new BehaviorSubject(true); + const pipeline$ = new BehaviorSubject({ + supported: true, + processor: undefined, + wait: "settling", + }); const vm = createLobbyFooterViewModel( testScope(), mockMuteStates(), twoMicsAndOneCamMediaDevices, - settling$, + pipeline$, + /* camera$ */ constant(null), /* openSettings */ undefined, /* hangup */ undefined, /* showLogo */ false, ); expect(vm.backgroundEffectSettling$.value).toBe(true); - settling$.next(false); + pipeline$.next({ ...pipeline$.value, wait: undefined }); expect(vm.backgroundEffectSettling$.value).toBe(false); }); + it("offers nothing once the pipeline fails to build, keeping the choice", () => { + platformMock.mockReturnValue("desktop"); + backgroundEffectSetting.setValue("blur"); + const pipeline$ = new BehaviorSubject({ + supported: true, + processor: undefined, + }); + const vm = createLobbyFooterViewModel( + testScope(), + mockMuteStates(), + twoMicsAndOneCamMediaDevices, + pipeline$, + /* camera$ */ constant(null), + /* openSettings */ undefined, + /* hangup */ undefined, + /* showLogo */ false, + ); + expect(vm.backgroundEffect$.value).toBe("blur"); + + pipeline$.next({ supported: false, processor: undefined }); + expect(vm.selectBackgroundEffect$.value).toBeUndefined(); + expect(vm.addBackgroundImage$.value).toBeUndefined(); + expect(vm.backgroundEffect$.value).toBe("none"); + expect(backgroundEffectSetting.getValue()).toBe("blur"); + }); + it("adding is unavailable at the limit", () => { sdkSupportMock.mockReturnValue(true); const added$ = addedBackgrounds.added$ as BehaviorSubject< @@ -427,6 +473,44 @@ describe("createCallFooterViewModel", () => { expect(store.remove).toHaveBeenCalledWith("spare"); }); + it("previews the lobby's camera while there is one", () => { + const track = mockVideoTrack({}); + const camera$ = new BehaviorSubject(track); + const vm = createLobbyFooterViewModel( + testScope(), + mockMuteStates(), + twoMicsAndOneCamMediaDevices, + noPipeline, + camera$, + /* openSettings */ undefined, + /* hangup */ undefined, + /* showLogo */ false, + ); + expect(vm.cameraTrack$.value).toBe(track); + // The lobby releases the camera when it is turned off + camera$.next(null); + expect(vm.cameraTrack$.value).toBeNull(); + }); + + it("previews the call's camera only while it is not muted", () => { + const muted$ = new BehaviorSubject(false); + const track = mockVideoTrack({ muted$ }); + const vm = createCallFooterViewModel( + testScope(), + { ...buildMinimalCallViewModel(gridLayout), camera$: constant(track) }, + mockMuteStates(), + twoMicsAndOneCamMediaDevices, + noPipeline, + /* reactionIdentifier */ undefined, + { showControls: true, header: HeaderStyle.Standard }, + ); + expect(vm.cameraTrack$.value).toBe(track); + muted$.next(true); + expect(vm.cameraTrack$.value).toBeNull(); + muted$.next(false); + expect(vm.cameraTrack$.value).toBe(track); + }); + it("availability is the same before and during a call", () => { for (const supported of [true, false]) { sdkSupportMock.mockReturnValue(supported); @@ -436,7 +520,7 @@ describe("createCallFooterViewModel", () => { buildMinimalCallViewModel(gridLayout), mockMuteStates(), twoMicsAndOneCamMediaDevices, - constant(false), + pipelineAsTheSdkSays(), /* reactionIdentifier */ undefined, { showControls: true, header: HeaderStyle.Standard }, ); diff --git a/src/components/CallFooterViewModel.tsx b/src/components/CallFooterViewModel.tsx index 8a22aa52e..23985e682 100644 --- a/src/components/CallFooterViewModel.tsx +++ b/src/components/CallFooterViewModel.tsx @@ -11,13 +11,12 @@ import { distinctUntilChanged, map, type Observable, + of, switchMap, } from "rxjs"; import { supportsAudioOutputSelection } from "livekit-client"; -import { - supportsBackgroundProcessors, - supportsModernBackgroundProcessors, -} from "@livekit/track-processors"; +import { type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk"; +import { supportsModernBackgroundProcessors } from "@livekit/track-processors"; import { logger } from "matrix-js-sdk/lib/logger"; import { @@ -33,6 +32,7 @@ import { } from "../tracks/backgroundImages"; import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; +import { type BackgroundEffectsState } from "../state/BackgroundEffects"; import { type MenuOptions } from "./MediaMuteAndSwitchButton"; import { type MediaDevices } from "../state/MediaDevices"; import { @@ -87,7 +87,9 @@ function buildDeviceBehaviors( mediaDevices: MediaDevices, /** return empty arrays for audioOptions and videoOptions*/ disableSwitcher$: Behavior, - backgroundEffectSettling$: Observable, + backgroundEffectsState$: Observable, + /** The camera as the pipeline has it, for the menu's preview. */ + camera$: Observable, ): Pick< ViewModel, | "audioOptions$" @@ -107,6 +109,7 @@ function buildDeviceBehaviors( | "addBackgroundImage$" | "backgroundImageRefusal$" | "removeBackgroundEffect$" + | "cameraTrack$" > { const options$ = ( available$: Behavior>, @@ -123,10 +126,14 @@ function buildDeviceBehaviors( ), ); - const supported = supportsBackgroundProcessors(); + // Also false once the pipeline has failed to build this session. + const supported$ = backgroundEffectsState$.pipe( + map(({ supported }) => supported === true), + distinctUntilChanged(), + ); const slow = !supportsModernBackgroundProcessors(); - const offered$ = disableSwitcher$.pipe( - map((switcherDisabled) => !switcherDisabled && supported), + const offered$ = combineLatest([disableSwitcher$, supported$]).pipe( + map(([switcherDisabled, supported]) => !switcherDisabled && supported), ); // A new object for each refusal, so the same one twice is shown twice. const refusal$ = new BehaviorSubject( @@ -157,8 +164,10 @@ function buildDeviceBehaviors( ), selectVideoButtonOption$: constant(mediaDevices.videoInput.select), backgroundEffect$: scope.behavior( - backgroundEffectSetting.value$.pipe( - map((raw) => (supported ? serializeEffect(parseEffect(raw)) : "none")), + combineLatest([backgroundEffectSetting.value$, supported$]).pipe( + map(([raw, supported]) => + supported ? serializeEffect(parseEffect(raw)) : "none", + ), ), ), selectBackgroundEffect$: scope.behavior( @@ -186,7 +195,10 @@ function buildDeviceBehaviors( ), ), backgroundEffectSettling$: scope.behavior( - backgroundEffectSettling$.pipe(distinctUntilChanged()), + backgroundEffectsState$.pipe( + map(({ wait }) => wait !== undefined), + distinctUntilChanged(), + ), false, ), // Kept and offered, not put on: that waits for the user to choose it. @@ -208,6 +220,17 @@ function buildDeviceBehaviors( ), ), backgroundImageRefusal$: refusal$, + // None while muted: the call mutes its track rather than releasing it. + cameraTrack$: scope.behavior( + camera$.pipe( + switchMap((camera) => + camera + ? camera.muted$.pipe(map((muted) => (muted ? null : camera))) + : of(null), + ), + ), + null, + ), // Never the one in force: taking it away would leave nothing chosen. removeBackgroundEffect$: scope.behavior( offered$.pipe( @@ -237,8 +260,8 @@ function buildDeviceBehaviors( * @param callModel - The root CallViewModel; provides layout, grid mode, reactions, etc. * @param muteStates - Audio and video mute state + toggles. * @param mediaDevices - Available and selected input devices. - * @param backgroundEffectSettling$ - Whether the first effect chosen is still - * being prepared. + * @param backgroundEffectsState$ - The background pipeline's state: whether + * effects run here, and the wait for the first. * @param reactionIdentifier - The local user's reaction identifier string, or * undefined when reactions are not supported (hides the reaction button). * @param options - `showControls`: whether the call controls should be shown. @@ -249,7 +272,7 @@ export function createCallFooterViewModel( callModel: CallViewModel, muteStates: MuteStates, mediaDevices: MediaDevices, - backgroundEffectSettling$: Observable, + backgroundEffectsState$: Observable, reactionIdentifier: string | undefined, options: { showControls: boolean; header: HeaderStyle }, ): ViewModel { @@ -268,7 +291,8 @@ export function createCallFooterViewModel( scope, mediaDevices, disableDeviceSwitcher$, - backgroundEffectSettling$, + backgroundEffectsState$, + callModel.camera$, ), // candidat to move into the FooterViewModel showFooter$: callModel.showFooter$, @@ -331,8 +355,9 @@ export function createCallFooterViewModel( * @param scope - ObservableScope that bounds the lifetime of derived behaviors. * @param muteStates - Audio and video mute state + toggles. * @param mediaDevices - Available and selected input devices. - * @param backgroundEffectSettling$ - Whether the first effect chosen is still - * being prepared. + * @param backgroundEffectsState$ - The background pipeline's state: whether + * effects run here, and the wait for the first. + * @param camera$ - The lobby's camera, for the camera menu's preview. * @param openSettings - Callback to open the settings modal, or undefined. * @param hangup - Callback to leave/cancel, or undefined (hides the button). * @param showLogo - Whether to show the Element Call logo. @@ -341,7 +366,8 @@ export function createLobbyFooterViewModel( scope: ObservableScope, muteStates: MuteStates, mediaDevices: MediaDevices, - backgroundEffectSettling$: Observable, + backgroundEffectsState$: Observable, + camera$: Behavior, openSettings: (() => void) | undefined, hangup: (() => void) | undefined, showLogo: boolean, @@ -389,7 +415,8 @@ export function createLobbyFooterViewModel( scope, mediaDevices, constant(false), - backgroundEffectSettling$, + backgroundEffectsState$, + camera$, ), }; } diff --git a/src/components/MediaMuteAndSwitchButton.module.css b/src/components/MediaMuteAndSwitchButton.module.css index 10b3267f5..2e8c60f3f 100644 --- a/src/components/MediaMuteAndSwitchButton.module.css +++ b/src/components/MediaMuteAndSwitchButton.module.css @@ -97,3 +97,35 @@ Please see LICENSE in the repository root for full details. .refusal { margin: var(--cpd-space-2x) var(--cpd-space-4x) 0; } + +.selfPreview { + position: relative; + display: flex; + align-items: center; + justify-content: center; + flex-shrink: 0; + aspect-ratio: 16 / 9; + margin-block-end: var(--cpd-space-2x); + overflow: clip; + border-start-start-radius: var(--cpd-space-3x); + border-start-end-radius: var(--cpd-space-3x); + background: var(--cpd-color-bg-subtle-secondary); + color: var(--cpd-color-icon-secondary); +} + +.selfPreview > video { + position: absolute; + inset: 0; + inline-size: 100%; + block-size: 100%; + object-fit: cover; + /* Rounded itself too: a video can be drawn on a layer of its own, which the + frame's rounded clip doesn't always reach. */ + border-radius: inherit; +} + +/* Up to the menu's frame, over the padding at its top. */ +.selfPreviewPinned, +.deviceListLeadsWithPreview { + margin-block-start: calc(-1 * var(--cpd-space-5x)); +} diff --git a/src/components/MediaMuteAndSwitchButton.stories.tsx b/src/components/MediaMuteAndSwitchButton.stories.tsx index 38ace8aee..1d9447773 100644 --- a/src/components/MediaMuteAndSwitchButton.stories.tsx +++ b/src/components/MediaMuteAndSwitchButton.stories.tsx @@ -12,6 +12,7 @@ import type { Meta, StoryObj } from "@storybook/react-vite"; import { MediaMuteAndSwitchButton } from "./MediaMuteAndSwitchButton"; import styles from "./MediaMuteAndSwitchButton.module.css"; import { type BackgroundEffectGridItem } from "./BackgroundEffectGrid"; +import { SelfPreview } from "./SelfPreview"; import { shippedBackgrounds } from "../tracks/backgroundEffectCatalog"; import meterStyles from "./MicrophoneLevelMeter.module.css"; import { MediaDevicesContext } from "../MediaDevicesContext"; @@ -60,6 +61,50 @@ const WithAMicrophone: FC<{ children: ReactNode }> = ({ children }) => { return <>{children}; }; +/** A camera for the preview: a figure against a room, drawn to a stream. */ +const FakeCamera: FC = () => { + const [stream, setStream] = useState(); + useEffect(() => { + const canvas = document.createElement("canvas"); + canvas.width = 640; + canvas.height = 360; + const context = canvas.getContext("2d")!; + const draw = (): void => { + context.fillStyle = "#6b7f8e"; + context.fillRect(0, 0, 640, 360); + context.fillStyle = "#c9a27e"; + context.beginPath(); + context.arc(320, 150, 62, 0, 2 * Math.PI); + context.fill(); + context.fillStyle = "#34495e"; + context.beginPath(); + context.ellipse(320, 380, 150, 140, 0, Math.PI, 2 * Math.PI); + context.fill(); + }; + draw(); + // Redrawn, so the stream keeps delivering frames. + const timer = setInterval(draw, 200); + const captured = canvas.captureStream(10); + setStream(captured); + return (): void => { + clearInterval(timer); + captured.getTracks().forEach((track) => track.stop()); + }; + }, []); + return ( +