diff --git a/.github/workflows/blocked.yaml b/.github/workflows/blocked.yaml index cc7db7477..39f8f9daf 100644 --- a/.github/workflows/blocked.yaml +++ b/.github/workflows/blocked.yaml @@ -14,7 +14,7 @@ permissions: jobs: prevent-blocked: name: Prevent blocked - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: pull-requests: read steps: diff --git a/.github/workflows/build-and-publish-docker.yaml b/.github/workflows/build-and-publish-docker.yaml index 3b18b133b..b6e78e754 100644 --- a/.github/workflows/build-and-publish-docker.yaml +++ b/.github/workflows/build-and-publish-docker.yaml @@ -17,7 +17,7 @@ env: jobs: build_and_deploy: name: Build & publish docker - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: contents: write # required to upload release asset packages: write # needed for publishing packages to GHCR diff --git a/.github/workflows/build-element-call.yaml b/.github/workflows/build-element-call.yaml index c885a85d7..908c5055c 100644 --- a/.github/workflows/build-element-call.yaml +++ b/.github/workflows/build-element-call.yaml @@ -29,7 +29,7 @@ on: jobs: build: name: Build Element Call - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout code uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 2afe922ab..2e24da1e3 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -25,7 +25,7 @@ jobs: # Deploy main branch to call.element.dev after build completes if: github.ref == 'refs/heads/main' needs: build_full_element_call - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Deploy to call.element.dev uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410 # v6 @@ -92,7 +92,7 @@ jobs: build_storybook: name: Build Storybook if: contains(github.event.pull_request.labels.*.name, 'storybook build') - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout code uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 diff --git a/.github/workflows/changelog-label.yml b/.github/workflows/changelog-label.yml index 84c53cdbc..3029f9d04 100644 --- a/.github/workflows/changelog-label.yml +++ b/.github/workflows/changelog-label.yml @@ -13,7 +13,7 @@ permissions: jobs: pr-changelog-label: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: yogevbd/enforce-label-action@a3c219da6b8fa73f6ba62b68ff09c469b3a1c024 # 2.2.2 with: diff --git a/.github/workflows/claude.yaml b/.github/workflows/claude.yaml index 1ddbeadaa..81058aed1 100644 --- a/.github/workflows/claude.yaml +++ b/.github/workflows/claude.yaml @@ -13,7 +13,7 @@ jobs: name: Review # Only run on labelling and skip forks. if: github.event.label.name == 'Claude-Review' && github.event.pull_request.head.repo.full_name == github.repository - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout code uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 diff --git a/.github/workflows/deploy-to-netlify.yaml b/.github/workflows/deploy-to-netlify.yaml index 554e384bc..c2bfd51a0 100644 --- a/.github/workflows/deploy-to-netlify.yaml +++ b/.github/workflows/deploy-to-netlify.yaml @@ -32,7 +32,7 @@ on: jobs: deploy: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: deployments: write environment: Netlify diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index c59ba4024..edf74b392 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -10,7 +10,7 @@ on: jobs: lint: name: Lint, format & type check - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout code uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 diff --git a/.github/workflows/pr-deploy.yaml b/.github/workflows/pr-deploy.yaml index 3d88b083f..ab6ad097c 100644 --- a/.github/workflows/pr-deploy.yaml +++ b/.github/workflows/pr-deploy.yaml @@ -17,7 +17,7 @@ jobs: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.head_repository.full_name == github.repository - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 outputs: pr_number: ${{ steps.prdetails.outputs.pr_id }} pr_data_json: ${{ steps.prdetails.outputs.data }} diff --git a/.github/workflows/publish-component.yaml b/.github/workflows/publish-component.yaml index 3976001a4..c33e3c5bc 100644 --- a/.github/workflows/publish-component.yaml +++ b/.github/workflows/publish-component.yaml @@ -17,7 +17,7 @@ permissions: {} jobs: versioning: name: Versioning - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 outputs: DRY_RUN: ${{ steps.dry_run.outputs.DRY_RUN }} PREFIXED_VERSION: ${{ steps.prefixed_version.outputs.PREFIXED_VERSION }} @@ -59,7 +59,7 @@ jobs: publish_npm: needs: versioning name: Publish NPM - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 outputs: ARTIFACT_VERSION: ${{ steps.artifact_version.outputs.ARTIFACT_VERSION }} permissions: @@ -111,7 +111,7 @@ jobs: needs: [versioning, publish_npm] if: ${{ needs.versioning.outputs.DRY_RUN == 'false' }} name: Update release notes - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # Shared with publish-embedded-packages.yaml, which appends to the same release: two # appends at once would each write back the body without the other's section concurrency: diff --git a/.github/workflows/publish-embedded-packages.yaml b/.github/workflows/publish-embedded-packages.yaml index 72454900b..1ce4a52a1 100644 --- a/.github/workflows/publish-embedded-packages.yaml +++ b/.github/workflows/publish-embedded-packages.yaml @@ -14,7 +14,7 @@ on: jobs: versioning: name: Versioning - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 outputs: DRY_RUN: ${{ steps.dry_run.outputs.DRY_RUN }} PREFIXED_VERSION: ${{ steps.prefixed_version.outputs.PREFIXED_VERSION }} @@ -76,7 +76,7 @@ jobs: needs: [build_element_call, versioning] if: always() name: Publish tarball - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: contents: write # required to upload release asset steps: @@ -107,7 +107,7 @@ jobs: needs: [build_element_call, versioning] if: always() name: Publish NPM - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 outputs: ARTIFACT_VERSION: ${{ steps.artifact_version.outputs.ARTIFACT_VERSION }} permissions: @@ -152,7 +152,7 @@ jobs: needs: [build_element_call, versioning] if: always() name: Publish Android AAR - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 outputs: ARTIFACT_VERSION: ${{ steps.artifact_version.outputs.ARTIFACT_VERSION }} permissions: @@ -212,7 +212,7 @@ jobs: needs: [build_element_call, versioning] if: always() name: Publish SwiftPM Library - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 outputs: ARTIFACT_VERSION: ${{ steps.artifact_version.outputs.ARTIFACT_VERSION }} permissions: @@ -282,7 +282,7 @@ jobs: needs: [versioning, publish_npm, publish_android, publish_ios] if: always() name: Update release notes - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # Shared with publish-component.yaml, which appends to the same release: two appends # at once would each write back the body without the other's section concurrency: diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 58e849759..d6b2c8379 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -23,7 +23,7 @@ jobs: needs: build_element_call if: always() name: Publish tarball - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: contents: write # required to upload release asset packages: write @@ -68,7 +68,7 @@ jobs: add_docker_release_note: needs: publish_docker name: Add docker release note - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Add release note uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2 diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 5e2703f67..0e415724c 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -25,7 +25,7 @@ on: jobs: vitest: name: Run unit tests - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout code uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 @@ -71,10 +71,27 @@ jobs: flags: unittests fail_ci_if_error: true playwright: - name: Run end-to-end tests + name: Run end-to-end tests (${{ matrix.part }}) if: ${{ !inputs.disable_e2e_tests }} timeout-minutes: 60 - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 + # We manually shard the tests across runners. Playwright's own shards cut + # the list of tests into equal counts in file order, which would leave the + # widget suite, the slowest tests by far, together at the end. + strategy: + fail-fast: false + matrix: + include: + - part: chromium-1 + args: --project=chromium --shard=1/2 + - part: chromium-2 + args: --project=chromium --shard=2/2 + - part: widget-1 + args: --project=widget --shard=1/2 + - part: widget-2 + args: --project=widget --shard=2/2 + - part: firefox-mobile + args: --project=firefox --project=mobile steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: @@ -107,17 +124,64 @@ jobs: run: pnpm playwright install --with-deps - name: Run backend components run: | + # Make sure the directory for the embedded build exists as Docker might + # create it with wrong access rights otherwise. + mkdir -p embedded/web/dist docker compose -f docker-compose-dev.yml -f docker-compose-playwright.yml pull docker compose -f docker-compose-dev.yml -f docker-compose-playwright.yml up -d docker ps - name: Run Playwright tests env: USE_DOCKER: 1 - run: pnpm exec playwright test + ARGS: ${{ matrix.args }} + # Unquoted on purpose: the part's arguments are several words + run: pnpm exec playwright test $ARGS - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: ${{ !cancelled() }} + with: + name: blob-report-${{ matrix.part }} + path: blob-report + if-no-files-found: error + # Only read by the merge below. + retention-days: 1 + + e2e_report: + # Combines the blobs into a single HTML report and blocks PRs on failure. + name: Run end-to-end tests + needs: playwright + if: ${{ always() && !inputs.disable_e2e_tests }} + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + # Inside a called workflow, github.repository is the caller so we need to make sure to still check out the EC repo here. + repository: ${{ inputs.matrix-js-sdk-sha && 'element-hq/element-call' || github.repository }} + persist-credentials: false + - name: Enable Corepack + run: corepack enable + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + cache: "pnpm" + node-version-file: ".node-version" + - name: Install dependencies + run: scripts/layered.sh --ignore-pnpmfile + env: + PR_HEAD_REPO_OWNER: ${{ github.event.pull_request.head.repo.owner.login }} + JS_SDK_GITHUB_BASE_REF: ${{ inputs.matrix-js-sdk-sha }} + - name: Download the parts' reports + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + pattern: blob-report-* + path: all-blob-reports + merge-multiple: true + - name: Merge into one HTML report + run: pnpm exec playwright merge-reports --reporter html ./all-blob-reports + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: html-report path: playwright-report if-no-files-found: error retention-days: 4 + - name: Fail on a failed part + if: ${{ needs.playwright.result != 'success' }} + run: exit 1 diff --git a/.github/workflows/translations-download.yaml b/.github/workflows/translations-download.yaml index f61a44c59..35b7eb3ff 100644 --- a/.github/workflows/translations-download.yaml +++ b/.github/workflows/translations-download.yaml @@ -7,7 +7,7 @@ on: jobs: download: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: contents: read diff --git a/.github/workflows/translations-upload.yaml b/.github/workflows/translations-upload.yaml index 06d9b8721..3afb857cd 100644 --- a/.github/workflows/translations-upload.yaml +++ b/.github/workflows/translations-upload.yaml @@ -8,7 +8,7 @@ on: jobs: upload: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: contents: read diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index 35ab5cb36..d78650759 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -10,7 +10,7 @@ permissions: {} jobs: zizmor: name: Run zizmor 🌈 - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: security-events: write # Required for upload-sarif (used by zizmor-action) to upload SARIF files. steps: diff --git a/README.md b/README.md index 5e75575b4..a21c5db02 100644 --- a/README.md +++ b/README.md @@ -221,124 +221,72 @@ See also: - [Developing with linked packages](./docs/linking.md) -#### Element Call as a component (experimental) +#### Element Call as a component -Element Call can also be embedded directly into another React application -rather than being loaded in an iframe as a widget. `pnpm build:component` -builds it as a library into `component/dist` (the bundle, its stylesheet and -type declarations), and +Element Call can be rendered inside another React app instead of an iframe. +The host supplies its own Matrix client, and the call stays inside the element +it is mounted in. -```sh -pnpm dev:component -``` - -serves a harness on port 3001 that stands in for such an application: it signs -in twice against the development backend and shows two calls side by side, in -resizable boxes, with page furniture of its own around them. Use it to see how -Element Call behaves when it does not own the page — the size it is given, -whether it stays inside its container, and what it says to its host, which is -logged along the bottom. The harness is served with the same development -certificate as the app, so unless the development CA is trusted, the browser -needs a certificate exception for `https://localhost:3001` as well (see the -note under [Backend](#backend)). It reads the same `public/config.json` as -`pnpm dev` if one exists, and runs with Element Call's defaults otherwise. - -`pnpm build:component` ends with `scripts/check-component-bundle.mjs`, which -fails the build on what a host must not be handed: an asset inlined into the -module as a data URL, a file referenced by an absolute path, the standalone -app's client store worker, or a module over its size budget. - -The call lays itself out for the size of the element it is mounted in, not the -window: a host that shrinks the container to a corner of its page gets the -picture-in-picture layout, just as a host that shrank the whole iframe used to. -The breakpoints in the stylesheets the component uses are -`@container element-call` queries against its root element for the same reason; -for the standalone app the root is the page, so they mean what the media queries -they replaced did. (The standalone-only views, such as the home and login pages, -still use plain media queries, since the component never shows them.) - -The component's stylesheet is confined to the element it is mounted in: the -build rewrites every selector so that it matches only Element Call's root or -what is inside it, with `html`, `body` and `:root` standing for that root (see -`component/build/scopeStylesToRoot.ts`). A host's own page keeps its styles, -and Element Call brings its own fonts and design tokens along. - -The component speaks every language the app does. English is bundled in; the -other locales are split into chunks the host's bundler loads the first time -they are needed. It starts in the browser's language, and follows the host's -own language setting through the `language` prop (`supportedLanguages` lists -the tags it accepts, and anything else falls back to its base language or to -English). The `theme` prop works the same way and takes the same values as the -widget's `theme` URL parameter: `light`, `dark`, `light-high-contrast` or -`dark-high-contrast`. Both can change while a call is running without -disturbing it. - -A host must call and await `initializeElementCall(config)` once before -rendering the component: it loads the `Intl` polyfills, applies the -deployment-wide `config.json`-style configuration and sets up translations. -The component itself takes the host's `client` and the `roomId` to call in, an -`intent` saying what the user asked for (which decides whether to show the -lobby, ring, and so on), an optional `config` overriding what the intent -implies, and an optional `hostBridge` through which Element Call tells the host -that the user has joined or hung up, that it wants to stay on screen, and so -on. The host makes its own requests (`join`, `hangUp`, `setDeviceMute`) through -the handle exposed on `ref`. The full API is documented in the type declarations -(`component/index.tsx` and `component/host.ts`). - -A few things differ from the widget on purpose: the component draws a solid -background rather than a gradient unless told otherwise, never offers to edit -the user's profile (the account is the host's), scopes its keyboard shortcuts to -its own root element so that several instances can share a page, and only shows -its own post-call and error screens when the host has not supplied a `close()` -callback; with one, it asks the host to unmount it instead. The -[global JS controls](./docs/controls.md) on `window` are unchanged and remain -page-wide, so with several instances on one page they apply to all of them. - -Each release publishes the package to npm, with the same version as the release: +##### Install ```sh npm install @element-hq/element-call-component ``` -To use an unreleased commit, a host can instead install it as a git dependency -on the `component` directory of this repository, +Releases publish the package with the same version as Element Call. For an +unreleased commit, depend on this repository directly: ```json "@element-hq/element-call-component": "github:element-hq/element-call#main&path:/component" ``` -whose `prepare` script runs the build on install. That build needs pnpm (via -Corepack) on the host's machine, runs a full `pnpm install` of this repository, -since it inherits the `--max-old-space-size` setting of -the app build; the host's pnpm also has to allow it to run at all -(`allowBuilds` in its `pnpm-workspace.yaml`). Note that `component/` is a pnpm -project of its own for this reason, so pnpm commands run from inside that -directory target it rather than the repository; run them from the repository -root. The host imports the component from -`@element-hq/element-call-component` and the stylesheet from -`@element-hq/element-call-component/style.css`, and has to provide `react`, -`react-dom`, `matrix-js-sdk`, `@vector-im/compound-web` and -`@vector-im/compound-design-tokens` itself, since the bundle leaves them -external. -Compound's stylesheet are the host's to load too: the component shares -the host's copy of Compound, (required for example for tooltips) +This builds on install, so the host needs pnpm (via Corepack) and must allow +the build (`allowBuilds` in its `pnpm-workspace.yaml`). -The component is large, and a host will usually load it lazily, only once a -call is shown. Everything a host needs in order to talk about a call before -then is also exported -from `@element-hq/element-call-component/api` (a small entry point that does not -load the component). +##### Use -- props and the types they are made of -- the handle -- the host bridge -- `UserIntent` -- `HeaderStyle` -- `BackgroundStyle` enums -- `configurationForIntent`, the defaults each intent implies - Import from it wherever a value such as - `BackgroundStyle.Solid` is needed on a path that must stay light; the main - entry point re-exports all of it too. +1. Provide the externals yourself: `react`, `react-dom`, `matrix-js-sdk`, + `@vector-im/compound-web`, `@vector-im/compound-design-tokens`, and + Compound's stylesheet. +2. Import `@element-hq/element-call-component/style.css`. +3. Await `initializeElementCall(config)` once before rendering. +4. Render the component with `client`, `roomId` and an `intent`; optionally + pass `config`, `hostBridge`, `language` and `theme` (both can change + mid-call). Control it through the `ref` handle (`join`, `hangUp`, + `setDeviceMute`). + +Load the component lazily. Types, enums and `configurationForIntent` are +available without it from `@element-hq/element-call-component/api`. The full +API is in `component/index.tsx`, `component/api.ts` and `component/host.ts`. + +##### How it differs from the widget + +- It lays out for its container, not the window: a small container gets the + picture-in-picture layout. +- Its styles and keyboard shortcuts are scoped to its root, so the host page is + untouched and several instances can share a page. The + [global JS controls](./docs/controls.md) stay page-wide. +- It draws a solid background by default and never offers profile editing. +- If the host bridge implements `close()`, it asks to be unmounted instead of + showing its own post-call and error screens. +- Sounds, images and the MediaPipe runtime are separate files the host's + bundler copies (webpack 5, Vite, Rollup); MediaPipe is fetched only when a + background effect is first used. + +##### Develop + +- `pnpm build:component` builds the library into `component/dist`. It ends + with `scripts/check-component-bundle.mjs`, which fails the build on what a + host must not be handed: an asset inlined as a data URL, a file referenced + by an absolute path, the standalone app's client store worker, or a module + over its size budget. +- `pnpm dev:component` serves a test host on `https://localhost:3001`: two + calls side by side in resizable boxes, with host messages logged at the + bottom. It uses the development backend, `public/config.json` if present, + and needs the same certificate exception as `pnpm dev` (see + [Backend](#backend)). +- Run pnpm commands from the repository root; `component/` is its own pnpm + project. ### Backend diff --git a/backend/ew.test.config.json b/backend/ew.test.config.json index 52be51b8f..f8b3ee5c4 100644 --- a/backend/ew.test.config.json +++ b/backend/ew.test.config.json @@ -26,8 +26,7 @@ "features": { "feature_element_call_video_rooms": true, "feature_video_rooms": true, - "feature_group_calls": true, - "feature_release_announcement": false + "feature_group_calls": true }, "default_federate": true, "default_theme": "light", @@ -40,7 +39,10 @@ }, "setting_defaults": { "breadcrumbs": true, - "feature_group_calls": true + "feature_group_calls": true, + "releaseAnnouncementData": { + "room_list_section": true + } }, "jitsi": { "preferred_domain": "meet.element.io" diff --git a/backend/ew.test.othersite.config.json b/backend/ew.test.othersite.config.json index 2ad3df4f3..6b3cb3a9c 100644 --- a/backend/ew.test.othersite.config.json +++ b/backend/ew.test.othersite.config.json @@ -26,8 +26,7 @@ "features": { "feature_element_call_video_rooms": true, "feature_video_rooms": true, - "feature_group_calls": true, - "feature_release_announcement": false + "feature_group_calls": true }, "default_federate": true, "default_theme": "light", @@ -40,7 +39,10 @@ }, "setting_defaults": { "breadcrumbs": true, - "feature_group_calls": true + "feature_group_calls": true, + "releaseAnnouncementData": { + "room_list_section": true + } }, "jitsi": { "preferred_domain": "meet.element.io" diff --git a/docker-compose-playwright.yml b/docker-compose-playwright.yml index e8deef680..cac217596 100644 --- a/docker-compose-playwright.yml +++ b/docker-compose-playwright.yml @@ -19,7 +19,12 @@ services: - ./backend/playwright_homeserver-othersite.yaml:/data/cfg/homeserver.yaml:Z element-web: # Pin to a SHA so that upstream cannot break our tests. Renovate handles regular updates. - image: ghcr.io/element-hq/element-web:develop@sha256:a79801206c628a912d57fa6f355d5c5ab9c6c40c2d1b6208d4192678fb6e80c7 + image: ghcr.io/element-hq/element-web:develop@sha256:72791f0b2541208f8f7edd9c6b5a7eaf88accd8ed1f0c4db1bc62bfe18eb90fe element-web-1: # Pin to a SHA so that upstream cannot break our tests. Renovate handles regular updates. - image: ghcr.io/element-hq/element-web:develop@sha256:a79801206c628a912d57fa6f355d5c5ab9c6c40c2d1b6208d4192678fb6e80c7 + image: ghcr.io/element-hq/element-web:develop@sha256:72791f0b2541208f8f7edd9c6b5a7eaf88accd8ed1f0c4db1bc62bfe18eb90fe + volumes: + # The other site opens the embedded package under test in place of the + # copy its image bundles (the first site opens the dev server's app. + # Built by the Playwright run (see build.setup.ts). + - ./embedded/web/dist:/app/widgets/element-call:ro,Z diff --git a/docs/agents/testing.md b/docs/agents/testing.md index d1a325897..ba203fd11 100644 --- a/docs/agents/testing.md +++ b/docs/agents/testing.md @@ -34,6 +34,10 @@ translations. The browser is Playwright's, so a fresh clone needs ## End-to-end +The Element Web instance `HOST1` opens the app the dev server serves. The instance +`HOST2` (the other site) opens the embedded package as a host ships it. This package +is built by the `build-embedded` setup project and mounted in `docker-compose-playwright.yml`. + - `playwright/*.spec.ts` — standalone, on Chromium and Firefox. - `playwright/widget/` — widget mode, where cross-context bugs surface. - `playwright/component/` — the component in a host page, via the harness on port @@ -48,6 +52,8 @@ Test what a user observes: the peer sees the change, it survives a reconnect, it right after a reload. Reuse `playwright/spa-helpers.ts`, `playwright/widget/test-helpers.ts`, `playwright/fixtures/`. +To mark new release announcements as viewed, add them in `backend/ew.test*.config.json`. + ```sh pnpm test # unit + storybook pnpm backend # Synapse + LiveKit, required for e2e diff --git a/locales/en/app.json b/locales/en/app.json index 3b08e8be9..aa753edc9 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -3,6 +3,7 @@ "user_menu": "User menu" }, "action": { + "add_background_image": "Add image", "close": "Close", "copy_link": "Copy link", "edit": "Edit", @@ -98,6 +99,10 @@ }, "disconnected_banner": "Connectivity to the server has been lost.", "error": { + "background_animated": "Animated images cannot be used as a background", + "background_not_an_image": "That file is not a supported image", + "background_not_kept": "That background could not be saved", + "background_undecodable": "That image could not be opened", "call_is_not_supported": "Call is not supported", "call_not_found": "Call not found", "call_not_found_description": "<0>That link doesn't appear to belong to any existing call. Check that you have the right link, or <2>create a new one.", diff --git a/package.json b/package.json index f436a6aee..6779f9ee5 100644 --- a/package.json +++ b/package.json @@ -97,6 +97,7 @@ "copy-to-clipboard": "^3.3.3", "eslint-plugin-element-call": "link:eslint", "eslint-plugin-storybook": "^10.3.6", + "fake-indexeddb": "^6.2.5", "fetch-mock": "11.1.5", "i18next": "^26.0.0", "i18next-browser-languagedetector": "^8.0.0", @@ -106,7 +107,7 @@ "livekit-client": "^2.18.1", "lodash-es": "^4.17.21", "loglevel": "^1.9.1", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#a9ef09fbf2f1644c8663d0aeaf6a921cd16e7b01", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#dc8610e796c92553c658300a4882dde961a0678a", "matrix-widget-api": "^1.18.0", "node-stdlib-browser": "^1.3.1", "normalize.css": "^8.0.1", diff --git a/playwright.config.ts b/playwright.config.ts index 2de044194..ebd5feea7 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -26,6 +26,19 @@ process.env.NODE_EXTRA_CA_CERTS = join( "backend/dev_tls_local-ca.crt", ); +const desktopChrome = { + ...devices["Desktop Chrome"], + permissions: ["clipboard-write", "clipboard-read", "microphone", "camera"], + ignoreHTTPSErrors: true, + launchOptions: { + args: [ + "--use-fake-ui-for-media-stream", + "--use-fake-device-for-media-stream", + "--mute-audio", + ], + }, +}; + /** * See https://playwright.dev/docs/test-configuration. */ @@ -43,11 +56,11 @@ export default defineConfig({ * Reporter to use. See https://playwright.dev/docs/test-reporters * * On CI, `list` prints a line per test as it finishes, which keeps the job - * log moving; `github` annotates failures on the PR; `html` is uploaded as - * an artifact and must never try to open a browser there. + * log moving; `github` annotates failures on the PR; the `blob` format is + * uploaded by sharded CI runs and combined into a HTML report by the workflow. */ reporter: process.env.CI - ? [["list"], ["github"], ["html", { open: "never" }]] + ? [["list"], ["github"], ["blob"]] : [["list"], ["html"]], /* Stop a broken run early on CI instead of waiting out every retry. */ maxFailures: process.env.CI ? 5 : undefined, @@ -63,29 +76,28 @@ export default defineConfig({ }, /* Configure projects for major browsers */ projects: [ + { + // Builds the embedded package into the directory the other site's + // Element Web container serves it from (see docker-compose-playwright.yml). + name: "build-embedded", + testMatch: "**/widget/embedded/build.setup.ts", + }, { name: "chromium", - testIgnore: "**/mobile/**", - use: { - ...devices["Desktop Chrome"], - permissions: [ - "clipboard-write", - "clipboard-read", - "microphone", - "camera", - ], - ignoreHTTPSErrors: true, - launchOptions: { - args: [ - "--use-fake-ui-for-media-stream", - "--use-fake-device-for-media-stream", - "--mute-audio", - ], - }, - }, + testIgnore: ["**/mobile/**", "**/widget/**"], + use: desktopChrome, + }, + { + // Element Call as a widget in Element Web, on Chromium. This uses its + // own project so that it can be sharded on CI. + name: "widget", + dependencies: ["build-embedded"], + testMatch: "**/widget/**/*.@(spec|test).ts", + use: desktopChrome, }, { name: "firefox", + dependencies: ["build-embedded"], testIgnore: "**/mobile/**", use: { ...devices["Desktop Firefox"], diff --git a/playwright/background-effects.spec.ts b/playwright/background-effects.spec.ts index 81d7a3ee4..4c71c7a19 100644 --- a/playwright/background-effects.spec.ts +++ b/playwright/background-effects.spec.ts @@ -5,9 +5,15 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { expect, test, type Locator, type Page } from "@playwright/test"; +import { expect, test, type Browser, type Page } from "@playwright/test"; import { SpaHelpers } from "./spa-helpers.ts"; +import { + averageColour, + cameraColour, + type Colour, + distance, +} from "./utils/colour.ts"; // Background effects need WebGL2, and headless Firefox on a CI runner has none, // so it rightly offers none of them. @@ -16,15 +22,13 @@ test.skip( "Background effects need WebGL2, which headless Firefox on CI does not have", ); -type Colour = [number, number, number]; - test.describe("background effects", () => { test("pre-join preview shows the chosen effect", async ({ page }) => { test.slow(); await page.goto("/"); await SpaHelpers.createCall(page, "Effect user", "Background effects"); const preview = page.locator("video").first(); - const camera = await averageColour(preview); + const camera = await cameraColour(preview); await page.getByRole("button", { name: "Camera", exact: true }).click(); const tile = page @@ -102,25 +106,847 @@ async function drawsPerSecond(page: Page): Promise { }); } -/** The mean colour of a video's current frame or an image, drawn small. */ -async function averageColour(element: Locator): Promise { - return element.evaluate( - async (source: HTMLVideoElement | HTMLImageElement) => { - if (source instanceof HTMLImageElement) await source.decode(); - const canvas = document.createElement("canvas"); - canvas.width = canvas.height = 32; - const context = canvas.getContext("2d")!; - context.drawImage(source, 0, 0, 32, 32); - const { data } = context.getImageData(0, 0, 32, 32); - const sum = [0, 0, 0]; - for (let i = 0; i < data.length; i += 4) - for (let c = 0; c < 3; c++) sum[c] += data[i + c]; - const pixels = data.length / 4; - return sum.map((s) => s / pixels) as Colour; - }, - ); +test.describe("the camera menu's preview", () => { + test("shows the camera with the effect in force", async ({ page }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall(page, "Preview user", "Preview in the menu"); + const camera = await cameraColour(page.locator("video").first()); + + await page.getByRole("button", { name: "Camera", exact: true }).click(); + const preview = page.getByRole("menu").locator("video"); + await expect(preview).toBeVisible(); + await expect + .poll(async () => distance(await averageColour(preview), camera), { + timeout: 20_000, + }) + .toBeLessThan(30); + + const tile = page.getByRole("menuitemradio", { name: "Background 1" }); + await tile.click(); + const picture = await averageColour(tile.locator("img")); + await expect + .poll(async () => distance(await averageColour(preview), picture), { + timeout: 60_000, + }) + .toBeLessThan(distance(camera, picture) / 2); + }); + + test("shows the call's own camera once joined", async ({ page }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall( + page, + "Call preview user", + "Preview in the call's menu", + true, + ); + await expect(page.getByTestId("incall_videomute")).toBeEnabled({ + timeout: 10_000, + }); + const camera = await cameraColour(page.locator("video").first()); + + await page.getByRole("button", { name: "Camera", exact: true }).click(); + const preview = page.getByRole("menu").locator("video"); + await expect(preview).toBeVisible(); + await expect + .poll(async () => distance(await averageColour(preview), camera), { + timeout: 20_000, + }) + .toBeLessThan(30); + + const tile = page.getByRole("menuitemradio", { name: "Background 1" }); + await tile.click(); + const picture = await averageColour(tile.locator("img")); + await expect + .poll(async () => distance(await averageColour(preview), picture), { + timeout: 60_000, + }) + .toBeLessThan(distance(camera, picture) / 2); + }); +}); + +test.describe("background effects section", () => { + test("section stays within the call area at every size", async ({ page }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall(page, "Sizes user", "Background effect sizes"); + + for (const size of [ + { width: 1280, height: 800 }, + { width: 800, height: 600 }, + { width: 600, height: 420 }, + { width: 480, height: 360 }, + { width: 360, height: 640 }, + ]) { + await page.setViewportSize(size); + await page.getByRole("button", { name: "Camera", exact: true }).click(); + const menu = page.getByRole("menu"); + await expect(menu).toBeVisible(); + + const box = (await menu.boundingBox())!; + const where = `at ${size.width}x${size.height}`; + expect(box.x, where).toBeGreaterThanOrEqual(0); + expect(box.y, where).toBeGreaterThanOrEqual(0); + expect(box.x + box.width, where).toBeLessThanOrEqual(size.width); + expect(box.y + box.height, where).toBeLessThanOrEqual(size.height); + + const last = page + .getByRole("group", { name: "Background effects" }) + .getByRole("menuitemradio") + .last(); + await last.scrollIntoViewIfNeeded(); + await expect(last, where).toBeInViewport({ ratio: 1 }); + await last.click(); + await expect(last, where).toHaveAttribute("aria-checked", "true"); + + // Back to no effect for the next size, and close. + await page + .getByRole("group", { name: "Background effects" }) + .getByRole("menuitemradio", { name: "None" }) + .click(); + await page.keyboard.press("Escape"); + await expect(menu).toBeHidden(); + } + }); +}); + +test.describe("a pipeline that fails to build", () => { + test("offers no effects for the session, and keeps the choice", async ({ + page, + }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall(page, "Failing user", "Pipeline fails"); + const preview = page.locator("video").first(); + const camera = await cameraColour(preview); + await page.route("**/*.tflite*", async (route) => route.abort()); + + await page.getByRole("button", { name: "Camera", exact: true }).click(); + const tile = page.getByRole("menuitemradio", { name: "Background 1" }); + await tile.click(); + await expect( + page.getByText("Background effects are not supported on this platform."), + ).toBeVisible({ timeout: 30_000 }); + await expect(tile).toBeDisabled(); + await page.keyboard.press("Escape"); + // The camera and microphone carry on without it. + expect(distance(await cameraColour(preview), camera)).toBeLessThan(30); + await expect(page.getByTestId("incall_mute")).toHaveAttribute( + "aria-checked", + "true", + ); + + await page.unroute("**/*.tflite*"); + await page.reload(); + await expect(page.getByTestId("lobby_joinCall")).toBeVisible(); + await page.getByRole("button", { name: "Camera", exact: true }).click(); + await expect(tile).toHaveAttribute("aria-checked", "true"); + const picture = await averageColour(tile.locator("img")); + await expect + .poll(async () => distance(await averageColour(preview), picture), { + timeout: 60_000, + }) + .toBeLessThan(distance(camera, picture) / 2); + }); +}); + +test.describe("the chosen effect", () => { + test("chosen effect survives leaving and rejoining", async ({ page }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall(page, "Returning user", "Effect remembered"); + const { camera, picture } = await chooseAndWear(page, "Background 1"); + await page.getByTestId("lobby_joinCall").click(); + await page.getByTestId("incall_leave").click(); + await expect(page.getByRole("heading")).toContainText( + "your call has ended", + ); + + await page.goto("/"); + await page.getByTestId("home_callName").fill("Effect remembered again"); + await page.getByTestId("home_go").click(); + await expect(page.getByTestId("lobby_joinCall")).toBeVisible(); + await page.getByRole("button", { name: "Camera", exact: true }).click(); + const tile = page + .getByRole("group", { name: "Background effects" }) + .getByRole("menuitemradio", { name: "Background 1" }); + await expect(tile).toHaveAttribute("aria-checked", "true"); + const preview = page.locator("video").first(); + await expect + .poll(async () => distance(await averageColour(preview), picture), { + timeout: 60_000, + }) + .toBeLessThan(distance(camera, picture) / 2); + }); + + test("second session still shows the existing error", async ({ page }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall(page, "Two tabs", "Effect in two tabs"); + await chooseAndWear(page, "Background 1"); + + const second = await page.context().newPage(); + await second.goto("/"); + await expect( + page.getByRole("heading", { name: "Opened in another tab" }), + ).toBeVisible(); + }); +}); + +/** Chooses an effect in the lobby, and waits until the preview wears it. */ +async function chooseAndWear( + page: Page, + name: string, +): Promise<{ camera: Colour; picture: Colour }> { + const preview = page.locator("video").first(); + const camera = await cameraColour(preview); + await page.getByRole("button", { name: "Camera", exact: true }).click(); + const tile = page.getByRole("menuitemradio", { name }); + await tile.click(); + const picture = await averageColour(tile.locator("img")); + await expect + .poll(async () => distance(await averageColour(preview), picture), { + timeout: 60_000, + }) + .toBeLessThan(distance(camera, picture) / 2); + await page.keyboard.press("Escape"); + return { camera, picture }; } -function distance(a: Colour, b: Colour): number { - return Math.hypot(a[0] - b[0], a[1] - b[1], a[2] - b[2]); +test.describe("a background of one's own", () => { + test("adds an image from the device", async ({ page }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall(page, "Adding user", "Own background"); + const preview = page.locator("video").first(); + const camera = await cameraColour(preview); + const red = await redImage(page); + + await addImage(page, red); + const mine = page + .getByRole("group", { name: "Background effects" }) + .getByRole("menuitemradio", { name: "Background 3" }); + await expect(mine).toBeVisible(); + await expect(mine).toHaveAttribute("aria-checked", "false"); + + await mine.click(); + await expect + .poll(async () => distance(await averageColour(preview), RED), { + timeout: 60_000, + }) + .toBeLessThan(distance(camera, RED) / 2); + }); + + test("adding an image makes no upload", async ({ page }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall(page, "Private user", "Own background kept"); + const preview = page.locator("video").first(); + const camera = await cameraColour(preview); + const red = await redImage(page); + const sent: { request: string; body: Buffer | null }[] = []; + page.on("request", (request) => + sent.push({ + request: `${request.method()} ${request.url()}`, + body: request.postDataBuffer(), + }), + ); + + await addImage(page, red); + await page + .getByRole("group", { name: "Background effects" }) + .getByRole("menuitemradio", { name: "Background 3" }) + .click(); + // Kept, chosen and in force: anything that would send it has had its turn. + await expect + .poll(async () => distance(await averageColour(preview), RED), { + timeout: 60_000, + }) + .toBeLessThan(distance(camera, RED) / 2); + // The file as given, and the copy kept, which is encoded anew. + const samples = [red, await keptBytes(page)].map((bytes) => + bytes.subarray(bytes.length - 64), + ); + const carriesIt = sent.filter( + ({ request, body }) => + request.includes("/_matrix/media/") || + samples.some((sample) => body?.includes(sample)), + ); + expect(carriesIt.map(({ request }) => request)).toEqual([]); + }); + + test("added image survives rejoining and is gone after storage is cleared", async ({ + page, + }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall(page, "Keeping user", "Own background again"); + const preview = page.locator("video").first(); + const camera = await cameraColour(preview); + const red = await redImage(page); + await addImage(page, red); + const mine = page + .getByRole("group", { name: "Background effects" }) + .getByRole("menuitemradio", { name: "Background 3" }); + await mine.click(); + await page.keyboard.press("Escape"); + + await page.reload(); + await expect(page.getByTestId("lobby_joinCall")).toBeVisible(); + await page.getByRole("button", { name: "Camera", exact: true }).click(); + await expect(mine).toHaveAttribute("aria-checked", "true"); + await page.keyboard.press("Escape"); + await expect + .poll(async () => distance(await averageColour(preview), RED), { + timeout: 60_000, + }) + .toBeLessThan(distance(camera, RED) / 2); + + await page.evaluate( + async () => + new Promise((resolve, reject) => { + const request = indexedDB.deleteDatabase( + "element-call-background-images", + ); + request.onsuccess = (): void => resolve(); + request.onerror = (): void => reject(request.error); + request.onblocked = (): void => resolve(); + }), + ); + await page.reload(); + await expect(page.getByTestId("lobby_joinCall")).toBeVisible(); + await page.getByRole("button", { name: "Camera", exact: true }).click(); + await expect( + page.getByRole("group", { name: "Background effects" }), + ).toBeVisible(); + await expect(mine).toHaveCount(0); + await page.keyboard.press("Escape"); + + await addImage(page, red); + await expect(mine).toBeVisible(); + }); + + test("a removed image stays removed", async ({ page }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall( + page, + "Removing user", + "Own background removed", + ); + await addImage(page, await redImage(page)); + const mine = page + .getByRole("group", { name: "Background effects" }) + .getByRole("menuitemradio", { name: "Background 3" }); + await expect(mine).toHaveAttribute("aria-keyshortcuts", "Delete"); + await mine.focus(); + await page.keyboard.press("Delete"); + await expect(mine).toHaveCount(0); + + await page.reload(); + await expect(page.getByTestId("lobby_joinCall")).toBeVisible(); + await page.getByRole("button", { name: "Camera", exact: true }).click(); + await expect( + page.getByRole("group", { name: "Background effects" }), + ).toBeVisible(); + await expect(mine).toHaveCount(0); + expect(await keptImages(page)).toEqual([]); + }); +}); + +test.describe("what can be added", () => { + test("refuses an animated image", async ({ page }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall(page, "Refused user", "Animated refused"); + await addImage(page, ANIMATED_GIF, { + name: "moving.gif", + type: "image/gif", + }); + + await expect( + page.getByText("Animated images cannot be used as a background"), + ).toBeVisible(); + const section = page.getByRole("group", { name: "Background effects" }); + await expect( + section.getByRole("menuitemradio", { name: "Background 3" }), + ).toHaveCount(0); + await expect( + section.getByRole("menuitemradio", { name: "None" }), + ).toHaveAttribute("aria-checked", "true"); + expect(await keptImages(page)).toEqual([]); + }); + + test("reduces an oversized image", async ({ page }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall(page, "Large user", "Oversized reduced"); + await addImage(page, await pngImage(page, { width: 4000, height: 3000 })); + + await expect( + page + .getByRole("group", { name: "Background effects" }) + .getByRole("menuitemradio", { name: "Background 3" }), + ).toBeVisible(); + await expect(page.getByRole("alert")).toHaveCount(0); + const [kept] = await keptImages(page); + expect(kept).toMatchObject({ width: 1920, height: 1440 }); + }); + + test("lays every image on an opaque ground", async ({ page }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall(page, "Logo user", "Transparent made opaque"); + await addImage( + page, + await pngImage(page, { width: 320, height: 180, transparent: true }), + ); + + await expect( + page + .getByRole("group", { name: "Background effects" }) + .getByRole("menuitemradio", { name: "Background 3" }), + ).toBeVisible(); + const [kept] = await keptImages(page); + expect(kept.minAlpha).toBe(255); + }); +}); + +/** Two frames of one pixel, red then blue. */ +const ANIMATED_GIF = Buffer.from( + "R0lGODlhAQABAPEAAP8AAAAA/wAAAAAAACH/C05FVFNDQVBFMi4wAwEAAAAh+QQACgAAACwAAAAAAQABAAACAkQBACH5BAAKAAAALAAAAAABAAEAAAICTAEAOw==", + "base64", +); + +/** A red PNG of the given size; transparent leaves its right half empty. */ +async function pngImage( + page: Page, + size: { width: number; height: number; transparent?: boolean }, +): Promise { + const dataUrl = await page.evaluate(({ width, height, transparent }) => { + const canvas = document.createElement("canvas"); + canvas.width = width; + canvas.height = height; + const context = canvas.getContext("2d")!; + context.fillStyle = "rgb(255, 0, 0)"; + context.fillRect(0, 0, transparent ? width / 2 : width, height); + return canvas.toDataURL("image/png"); + }, size); + return Buffer.from(dataUrl.split(",")[1], "base64"); +} + +/** The images the device keeps, as it keeps them. */ +async function keptImages( + page: Page, +): Promise<{ width: number; height: number; minAlpha: number }[]> { + return page.evaluate(async () => { + const db = await new Promise((resolve, reject) => { + const request = indexedDB.open("element-call-background-images"); + request.onsuccess = (): void => resolve(request.result); + request.onerror = (): void => reject(request.error); + }); + if (!db.objectStoreNames.contains("backgrounds")) return []; + const kept = await new Promise<{ image: Blob }[]>((resolve, reject) => { + const request = db + .transaction("backgrounds") + .objectStore("backgrounds") + .getAll(); + request.onsuccess = (): void => resolve(request.result); + request.onerror = (): void => reject(request.error); + }); + db.close(); + return Promise.all( + kept.map(async ({ image }) => { + const bitmap = await createImageBitmap(image); + const canvas = new OffscreenCanvas(bitmap.width, bitmap.height); + const context = canvas.getContext("2d")!; + context.drawImage(bitmap, 0, 0); + const { data } = context.getImageData( + 0, + 0, + bitmap.width, + bitmap.height, + ); + let minAlpha = 255; + for (let i = 3; i < data.length; i += 4) + minAlpha = Math.min(minAlpha, data[i]); + return { width: bitmap.width, height: bitmap.height, minAlpha }; + }), + ); + }); +} + +/** The bytes of the one image the device keeps. */ +async function keptBytes(page: Page): Promise { + const base64 = await page.evaluate( + async () => + new Promise((resolve, reject) => { + const open = indexedDB.open("element-call-background-images"); + open.onerror = (): void => reject(open.error); + open.onsuccess = (): void => { + const all = open.result + .transaction("backgrounds") + .objectStore("backgrounds") + .getAll(); + all.onerror = (): void => reject(all.error); + all.onsuccess = (): void => { + const reader = new FileReader(); + reader.onload = (): void => + resolve((reader.result as string).split(",")[1]); + reader.readAsDataURL((all.result[0] as { image: Blob }).image); + }; + }; + }), + ); + return Buffer.from(base64, "base64"); +} + +const RED: Colour = [255, 0, 0]; + +/** A solid red PNG, unlike the camera and the shipped pictures. */ +async function redImage(page: Page): Promise { + const dataUrl = await page.evaluate(() => { + const canvas = document.createElement("canvas"); + canvas.width = 320; + canvas.height = 180; + const context = canvas.getContext("2d")!; + context.fillStyle = "rgb(255, 0, 0)"; + context.fillRect(0, 0, canvas.width, canvas.height); + return canvas.toDataURL("image/png"); + }); + return Buffer.from(dataUrl.split(",")[1], "base64"); +} + +/** Adds an image from the camera menu's add tile, leaving the menu open. */ +async function addImage( + page: Page, + image: Buffer, + { name, type } = { name: "mine.png", type: "image/png" }, +): Promise { + if (!(await page.getByRole("menu").isVisible())) + await page.getByRole("button", { name: "Camera", exact: true }).click(); + const chooser = page.waitForEvent("filechooser"); + await page.getByRole("menuitem", { name: "Add image" }).click(); + await (await chooser).setFiles({ name, mimeType: type, buffer: image }); +} + +test.describe("joining with a background chosen", () => { + test("publishes no frame of the room", async ({ browser }) => { + // Two first builds of the pipeline, one of them held back. + test.setTimeout(180_000); + const hostContext = await browser.newContext(); + const host = await hostContext.newPage(); + await recordIncomingFrames(host); + await host.goto("/"); + await SpaHelpers.createCall(host, "Host", "Join with a background", true); + await expect(host.getByTestId("incall_videomute")).toBeEnabled({ + timeout: 10_000, + }); + const inviteLink = await SpaHelpers.getCallInviteLink(host); + await host.keyboard.press("Escape"); + + const guestContext = await browser.newContext(); + const guest = await guestContext.newPage(); + await guest.goto(inviteLink); + await guest.getByTestId("joincall_displayName").fill("Guest"); + await guest.getByTestId("joincall_joincall").click(); + const preview = guest.locator("video").first(); + await expect(preview).toBeVisible(); + const camera = await cameraColour(preview); + await guest.getByRole("button", { name: "Camera", exact: true }).click(); + const tile = guest.getByRole("menuitemradio", { name: "Background 1" }); + await tile.click(); + const picture = await averageColour(tile.locator("img")); + await expect + .poll(async () => distance(await averageColour(preview), picture), { + timeout: 60_000, + }) + .toBeLessThan(distance(camera, picture) / 2); + // Were the pipeline built again for the call's track, it would take as + // long as on the slower route. Routing bypasses the cache, so it waits. + let heldBack = 0; + await guest.route("**/*.tflite*", async (route) => { + heldBack += 1; + await new Promise((resolve) => setTimeout(resolve, 15_000)); + await route.continue(); + }); + await guest.keyboard.press("Escape"); + await guest.getByTestId("lobby_joinCall").click(); + + const guestTile = host + .getByTestId("videoTile") + .filter({ has: host.getByTestId("name_tag").getByText("Guest") }); + await expect(guestTile.locator("video")).toBeVisible({ timeout: 60_000 }); + // Visible can come before the stream is attached. + let trackId: string | undefined; + await expect + .poll(async () => { + trackId = await guestTile + .locator("video") + .evaluate( + (video: HTMLVideoElement) => + (video.srcObject as MediaStream | null)?.getVideoTracks()[0]?.id, + ); + return trackId; + }) + .toBeDefined(); + await expectNoFrameOfTheRoom(host, trackId!, 0, camera, picture); + // The join adopts the lobby's pipeline, so the model is never fetched again. + expect(heldBack).toBe(0); + + await hostContext.close(); + await guestContext.close(); + }); +}); + +test.describe("the first build", () => { + test("survives the camera turned off and on while it runs", async ({ + page, + }) => { + test.setTimeout(120_000); + const failures: string[] = []; + page.on("console", (m) => { + if (m.text().includes("Failed to attach video processor")) + failures.push(m.text()); + }); + await page.goto("/"); + await SpaHelpers.createCall(page, "Toggling user", "Toggle while building"); + const preview = page.locator("video").first(); + const camera = await cameraColour(preview); + + const held = await holdBackTheModel(page); + await page.getByRole("button", { name: "Camera", exact: true }).click(); + const tile = page.getByRole("menuitemradio", { name: "Background 1" }); + await tile.click(); + const picture = await averageColour(tile.locator("img")); + await page.keyboard.press("Escape"); + // New preview tracks, each attached while the first is still building. + for (let i = 0; i < 2; i++) { + await page.getByTestId("incall_videomute").click(); + await page.getByTestId("incall_videomute").click(); + } + + await expect + .poll(async () => distance(await averageColour(preview), picture), { + timeout: 60_000, + }) + .toBeLessThan(distance(camera, picture) / 2); + expect(failures).toEqual([]); + expect(held()).toBeGreaterThan(0); + }); +}); + +test.describe("turning the camera on with a background chosen", () => { + test("publishes no frame of the room when the camera started off", async ({ + browser, + }) => { + test.setTimeout(120_000); + const { host, guest, close } = await hostAndGuest(browser); + const camera = await cameraColour(guest.locator("video").first()); + await guest.getByTestId("incall_videomute").click(); + await guest.getByTestId("lobby_joinCall").click(); + + const picture = await chooseBackground(guest); + const held = await holdBackTheModel(guest); + await guest.getByTestId("incall_videomute").click(); + + const trackId = await incomingTrackId(host); + await expectNoFrameOfTheRoom(host, trackId, 0, camera, picture); + expect(held()).toBeGreaterThan(0); + await close(); + }); + + test("publishes no frame of the room when the camera was turned off", async ({ + browser, + }) => { + test.setTimeout(120_000); + const { host, guest, close } = await hostAndGuest(browser); + const camera = await cameraColour(guest.locator("video").first()); + await guest.getByTestId("lobby_joinCall").click(); + const trackId = await incomingTrackId(host); + await expect + .poll(async () => (await framesOf(host, trackId)).length, { + timeout: 20_000, + }) + .toBeGreaterThan(0); + await guest.getByTestId("incall_videomute").click(); + + const picture = await chooseBackground(guest); + const held = await holdBackTheModel(guest); + const before = (await framesOf(host, trackId)).length; + await guest.getByTestId("incall_videomute").click(); + + await expectNoFrameOfTheRoom(host, trackId, before, camera, picture); + expect(held()).toBeGreaterThan(0); + await close(); + }); +}); + +/** A host in a call, recording what it receives, and a guest in its lobby. */ +async function hostAndGuest(browser: Browser): Promise<{ + host: Page; + guest: Page; + close: () => Promise; +}> { + const hostContext = await browser.newContext(); + const host = await hostContext.newPage(); + await recordIncomingFrames(host); + await host.goto("/"); + await SpaHelpers.createCall(host, "Host", "Camera with a background", true); + await expect(host.getByTestId("incall_videomute")).toBeEnabled({ + timeout: 10_000, + }); + const inviteLink = await SpaHelpers.getCallInviteLink(host); + await host.keyboard.press("Escape"); + + const guestContext = await browser.newContext(); + const guest = await guestContext.newPage(); + await guest.goto(inviteLink); + await guest.getByTestId("joincall_displayName").fill("Guest"); + await guest.getByTestId("joincall_joincall").click(); + await expect(guest.locator("video").first()).toBeVisible(); + return { + host, + guest, + close: async (): Promise => { + await hostContext.close(); + await guestContext.close(); + }, + }; +} + +/** Chooses the first shipped picture in the call, and answers its colour. */ +async function chooseBackground(guest: Page): Promise { + // The lobby has a camera button of its own, so wait for the call's. + await expect(guest.getByTestId("incall_leave")).toBeVisible({ + timeout: 30_000, + }); + await expect(guest.getByTestId("incall_videomute")).toBeEnabled({ + timeout: 10_000, + }); + await guest.getByRole("button", { name: "Camera", exact: true }).click(); + const tile = guest.getByRole("menuitemradio", { name: "Background 1" }); + await tile.click(); + await expect(tile).toHaveAttribute("aria-checked", "true"); + const picture = await averageColour(tile.locator("img")); + await guest.keyboard.press("Escape"); + return picture; +} + +/** Makes the pipeline's first build as slow as on the slower route. */ +/** Answers how often it held the model back, which a check should see. */ +async function holdBackTheModel(page: Page): Promise<() => number> { + let held = 0; + // Routing bypasses the cache, so it waits. + await page.route("**/*.tflite*", async (route) => { + held += 1; + await new Promise((resolve) => setTimeout(resolve, 5_000)); + await route.continue(); + }); + return () => held; +} + +/** The id of the guest's camera track, as the host receives it. */ +async function incomingTrackId(host: Page): Promise { + const guestTile = host + .getByTestId("videoTile") + .filter({ has: host.getByTestId("name_tag").getByText("Guest") }); + await expect(guestTile.locator("video")).toBeVisible({ timeout: 60_000 }); + // Visible can come before the stream is attached. + let trackId: string | undefined; + await expect + .poll(async () => { + trackId = await guestTile + .locator("video") + .evaluate( + (video: HTMLVideoElement) => + (video.srcObject as MediaStream | null)?.getVideoTracks()[0]?.id, + ); + return trackId; + }) + .toBeDefined(); + return trackId!; +} + +/** Waits for the picture to arrive, then finds no frame of the room since. */ +async function expectNoFrameOfTheRoom( + host: Page, + trackId: string, + since: number, + camera: Colour, + picture: Colour, +): Promise { + await expect + .poll( + async () => { + const frames = (await framesOf(host, trackId)).slice(since); + // A black frame, as a track starts, is nearer the picture than the + // camera, but shows neither. + return frames.some( + (f) => + distance(f, picture) < + Math.min(distance(f, camera), distance(f, [0, 0, 0])), + ); + }, + { timeout: 60_000 }, + ) + .toBe(true); + const frames = (await framesOf(host, trackId)).slice(since); + expect( + frames.filter((f) => distance(f, camera) < distance(f, picture)), + ).toEqual([]); +} + +/** + * Records the mean colour of every frame of every video track the page + * receives, from the moment it arrives: a tile may attach it later than its + * first frame. + */ +async function recordIncomingFrames(page: Page): Promise { + await page.addInitScript(() => { + const frames: Record = {}; + (window as unknown as { receivedFrames: typeof frames }).receivedFrames = + frames; + const record = (track: MediaStreamTrack): void => { + if (track.kind !== "video" || frames[track.id]) return; + const seen: [number, number, number][] = (frames[track.id] = []); + const video = document.createElement("video"); + video.muted = true; + video.srcObject = new MediaStream([track]); + const canvas = document.createElement("canvas"); + canvas.width = canvas.height = 16; + const context = canvas.getContext("2d", { willReadFrequently: true })!; + const step = (): void => { + context.drawImage(video, 0, 0, 16, 16); + const { data } = context.getImageData(0, 0, 16, 16); + const sum = [0, 0, 0]; + for (let i = 0; i < data.length; i += 4) + for (let c = 0; c < 3; c++) sum[c] += data[i + c]; + seen.push(sum.map((v) => v / (data.length / 4)) as Colour); + video.requestVideoFrameCallback(step); + }; + video.requestVideoFrameCallback(step); + void video.play(); + }; + const Connection = window.RTCPeerConnection; + window.RTCPeerConnection = class extends Connection { + public constructor(...args: ConstructorParameters) { + super(...args); + this.addEventListener("track", (e) => record(e.track)); + } + }; + }); +} + +async function framesOf(page: Page, trackId: string): Promise { + return page.evaluate( + (id) => + ( + window as unknown as { + receivedFrames: Record; + } + ).receivedFrames[id] ?? [], + trackId, + ); } diff --git a/playwright/component/background-effects.spec.ts b/playwright/component/background-effects.spec.ts new file mode 100644 index 000000000..9e1a0c5c1 --- /dev/null +++ b/playwright/component/background-effects.spec.ts @@ -0,0 +1,85 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { expect, type Locator, type Page, test } from "@playwright/test"; + +import { createUserAndRoom, resizeContainer, startHarness } from "./harness.ts"; +import { averageColour, cameraColour, expectWearing } from "../utils/colour.ts"; + +// Background effects need WebGL2, and headless Firefox on a CI runner has none, +// so it rightly offers none of them. +test.skip( + ({ browserName }) => browserName === "firefox", + "Background effects need WebGL2, which headless Firefox on CI does not have", +); + +/** + * Background effects with Element Call as a component in a host page: the + * pipeline's model and WebAssembly have to load from the component's bundle, + * and the portalled menu has to be sized against the call, not the window. + * Driven from the lobby, which builds the same menu and pipeline. + */ + +// Sign-in, crypto setup and sync happen twice before anything shows. +test.describe.configure({ timeout: 180_000 }); + +test("puts a picture on the call's preview", async ({ page }) => { + const { username, roomId } = await createUserAndRoom("effects"); + const panes = await startHarness(page, username, roomId); + const pane = panes.first(); + await resizeContainer(pane.getByTestId("call-container"), { + width: 900, + height: 480, + }); + await expect(pane.getByTestId("lobby_joinCall")).toBeVisible({ + timeout: 60_000, + }); + const preview = pane.locator("video").first(); + const camera = await cameraColour(preview); + + const tile = await openEffect(page, pane, "Background 1"); + const picture = await averageColour(tile.locator("img")); + await tile.click(); + await expect(tile).toHaveAttribute("aria-checked", "true"); + await expectWearing(preview, picture, camera); +}); + +test("keeps every effect reachable in a small container", async ({ page }) => { + const { username, roomId } = await createUserAndRoom("effectsreach"); + const panes = await startHarness(page, username, roomId); + const pane = panes.first(); + const container = pane.getByTestId("call-container"); + await resizeContainer(container, { width: 400, height: 360 }); + await expect(pane.getByTestId("lobby_joinCall")).toBeVisible({ + timeout: 60_000, + }); + + const last = await openEffect(page, pane, "Background 2"); + const list = page.locator("[role='menu'] div[role='none']").first(); + expect((await list.boundingBox())!.height).toBeLessThanOrEqual( + (await container.boundingBox())!.height, + ); + await last.scrollIntoViewIfNeeded(); + await expect(last).toBeInViewport({ ratio: 1 }); + await last.click(); + await expect(last).toHaveAttribute("aria-checked", "true"); +}); + +/** Opens a component's camera menu and returns one of its effects. */ +async function openEffect( + page: Page, + pane: Locator, + name: string, +): Promise { + await pane + .getByRole("button", { name: "Camera", exact: true }) + .click({ timeout: 60_000 }); + // Portalled out of the pane, so found on the page. + return page + .getByRole("group", { name: "Background effects" }) + .getByRole("menuitemradio", { name }); +} diff --git a/playwright/fixtures/widget-user.ts b/playwright/fixtures/widget-user.ts index a60776c99..f069db1a1 100644 --- a/playwright/fixtures/widget-user.ts +++ b/playwright/fixtures/widget-user.ts @@ -29,47 +29,10 @@ export interface MyFixtures { addUser: (username: string, host: string) => Promise; } -// Minimal config.json for the local element-web instance -const CONFIG_JSON = { - default_server_config: { - "m.homeserver": { - base_url: "https://synapse.m.localhost", - server_name: "synapse.m.localhost", - }, - }, - - element_call: { - participant_limit: 8, - brand: "Element Call", - }, - - // The default language is set here for test consistency - setting_defaults: { - language: "en-GB", - feature_group_calls: true, - }, - - // the location tests want a map style url. - map_style_url: - "https://api.maptiler.com/maps/streets/style.json?key=fU3vlMsMn4Jb6dnEIFsx", - - features: { - // We don't want to go through the feature announcement during the e2e test - feature_release_announcement: false, - feature_element_call_video_rooms: true, - feature_video_rooms: true, - feature_group_calls: true, - }, -}; - export const widgetTest = test.extend({ // allow per-test override: `widgetTest.use({ callType: "dm" })` callType: ["room", { option: true }], - asWidget: async ({ browser, context, callType }, pUse) => { - await context.route(`http://localhost:8081/config.json*`, async (route) => { - await route.fulfill({ json: CONFIG_JSON }); - }); - + asWidget: async ({ browser, callType }, pUse) => { const brooksDisplayName = `brooks_${Date.now()}`; const whistlerDisplayName = `whistler_${Date.now()}`; @@ -109,15 +72,7 @@ export const widgetTest = test.extend({ await TestHelpers.dismissInviteUnknownUserModal(ewPage1); // Accept the invite - await TestHelpers.closeReleaseAnnouncement( - ewPage2, - "Introducing Sections", - ); - await TestHelpers.expandAllSections(ewPage2); - await expect( - TestHelpers.roomListItem(ewPage2, "Welcome Room"), - ).toBeVisible(); - await TestHelpers.roomListItem(ewPage2, "Welcome Room").click(); + await TestHelpers.openRoomInvite(ewPage2, "Welcome Room"); await ewPage2.getByRole("button", { name: "Accept" }).click(); await expect( ewPage2 @@ -125,17 +80,7 @@ export const widgetTest = test.extend({ .getByRole("heading", { name: "Welcome Room" }), ).toBeVisible(); } else if (callType === "dm") { - await TestHelpers.closeReleaseAnnouncement( - ewPage1, - "Introducing Sections", - ); - - await ewPage1 - .getByRole("navigation", { name: "Room list" }) - .getByRole("button", { name: "New", exact: true }) - .click(); - - await ewPage1.getByRole("menuitem", { name: "Start chat" }).click(); + await TestHelpers.chooseFromRoomListNewMenu(ewPage1, "Start chat"); await ewPage1.getByRole("textbox", { name: "Search" }).click(); await ewPage1.getByRole("textbox", { name: "Search" }).fill(whistlerMxId); await ewPage1.getByRole("button", { name: "Go" }).click(); @@ -157,12 +102,7 @@ export const widgetTest = test.extend({ ).toBeVisible(); // Accept the DM invite from brooks - await TestHelpers.closeReleaseAnnouncement( - ewPage2, - "Introducing Sections", - ); - await TestHelpers.expandAllSections(ewPage2); - await TestHelpers.roomListItem(ewPage2, brooksDisplayName).click(); + await TestHelpers.openRoomInvite(ewPage2, brooksDisplayName); await ewPage2.getByRole("button", { name: "Start chatting" }).click(); } @@ -187,13 +127,14 @@ export const widgetTest = test.extend({ * Provide a way to add additional users within a test. * The returned user will be registered on the default homeserver, the name will be made unique by appending a timestamp. */ - addUser: async ({ browser }, use) => { + addUser: async ({ browser }, use, testInfo) => { await use( async ( username: string, host: string = HOST1, ): Promise => { - const uniqueSuffix = Date.now(); + // Tests in parallel workers can reach this in the same millisecond. + const uniqueSuffix = `${Date.now()}_${testInfo.parallelIndex}`; const { page, clientHandle, mxId } = await TestHelpers.registerUser( browser, `${username.toLowerCase()}_${uniqueSuffix}`, diff --git a/playwright/utils/colour.ts b/playwright/utils/colour.ts new file mode 100644 index 000000000..cb096e15c --- /dev/null +++ b/playwright/utils/colour.ts @@ -0,0 +1,55 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { expect, type Locator } from "@playwright/test"; + +export type Colour = [number, number, number]; + +/** The camera's colour, once the preview is showing it rather than nothing. */ +export async function cameraColour(preview: Locator): Promise { + await expect + .poll(async () => Math.max(...(await averageColour(preview)))) + .toBeGreaterThan(40); + return averageColour(preview); +} + +/** The mean colour of a video's current frame or an image, drawn small. */ +export async function averageColour(element: Locator): Promise { + return element.evaluate( + async (source: HTMLVideoElement | HTMLImageElement) => { + if (source instanceof HTMLImageElement) await source.decode(); + const canvas = document.createElement("canvas"); + canvas.width = canvas.height = 32; + const context = canvas.getContext("2d")!; + context.drawImage(source, 0, 0, 32, 32); + const { data } = context.getImageData(0, 0, 32, 32); + const sum = [0, 0, 0]; + for (let i = 0; i < data.length; i += 4) + for (let c = 0; c < 3; c++) sum[c] += data[i + c]; + const pixels = data.length / 4; + return sum.map((s) => s / pixels) as Colour; + }, + ); +} + +export function distance(a: Colour, b: Colour): number { + return Math.hypot(a[0] - b[0], a[1] - b[1], a[2] - b[2]); +} + +/** Waits until a preview shows a picture rather than the camera. */ +export async function expectWearing( + preview: Locator, + picture: Colour, + camera: Colour, +): Promise { + // Long enough for the first effect of a session to be built. + await expect + .poll(async () => distance(await averageColour(preview), picture), { + timeout: 60_000, + }) + .toBeLessThan(distance(camera, picture) / 2); +} diff --git a/playwright/widget/background-effects.spec.ts b/playwright/widget/background-effects.spec.ts new file mode 100644 index 000000000..32c499bc0 --- /dev/null +++ b/playwright/widget/background-effects.spec.ts @@ -0,0 +1,43 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { expect, test } from "@playwright/test"; + +import { widgetTest } from "../fixtures/widget-user.ts"; +import { TestHelpers } from "./test-helpers.ts"; +import { averageColour, cameraColour, expectWearing } from "../utils/colour.ts"; + +// Background effects need WebGL2, and headless Firefox on a CI runner has none, +// so it rightly offers none of them. +test.skip( + ({ browserName }) => browserName === "firefox", + "Background effects need WebGL2, which headless Firefox on CI does not have", +); + +widgetTest( + "puts a picture on the preview inside Element Web", + async ({ asWidget }) => { + test.slow(); + const { brooks } = asWidget; + await TestHelpers.startCallInCurrentRoom(brooks.page, false); + const frame = brooks.page + .locator('iframe[title="Element Call"]') + .contentFrame(); + await expect(frame.getByTestId("lobby_joinCall")).toBeVisible(); + const preview = frame.locator("video").first(); + const camera = await cameraColour(preview); + + await frame.getByRole("button", { name: "Camera", exact: true }).click(); + const tile = frame + .getByRole("group", { name: "Background effects" }) + .getByRole("menuitemradio", { name: "Background 1" }); + const picture = await averageColour(tile.locator("img")); + await tile.click(); + await expect(tile).toHaveAttribute("aria-checked", "true"); + await expectWearing(preview, picture, camera); + }, +); diff --git a/playwright/widget/embedded/build.setup.ts b/playwright/widget/embedded/build.setup.ts new file mode 100644 index 000000000..efc1e3d73 --- /dev/null +++ b/playwright/widget/embedded/build.setup.ts @@ -0,0 +1,20 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { test as setup } from "@playwright/test"; +import { execFileSync } from "node:child_process"; +import { mkdirSync } from "node:fs"; + +const EMBEDDED_DIST = "embedded/web/dist"; + +setup("build the embedded package", () => { + setup.setTimeout(10 * 60 * 1000); + mkdirSync(EMBEDDED_DIST, { recursive: true }); + execFileSync("pnpm", ["build:embedded", "--outDir", EMBEDDED_DIST], { + stdio: "inherit", + }); +}); diff --git a/playwright/widget/embedded/config-json.spec.ts b/playwright/widget/embedded/config-json.spec.ts new file mode 100644 index 000000000..b8bba8b24 --- /dev/null +++ b/playwright/widget/embedded/config-json.spec.ts @@ -0,0 +1,56 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { expect, type Route } from "@playwright/test"; + +import { widgetTest } from "../../fixtures/widget-user.ts"; +import { HOST2, TestHelpers } from "../test-helpers.ts"; + +/** Where the other site's Element Web serves the embedded package from */ +const CONFIG_JSON = "**/widgets/element-call/config.json"; + +const HOST_ANSWERS: Record Promise> = { + "a 404": async (route) => route.fulfill({ status: 404 }), + "a failed fetch": async (route) => route.abort("failed"), + "an HTML page": async (route) => + route.fulfill({ + status: 200, + contentType: "text/html", + body: "Element", + }), +}; + +widgetTest.skip( + ({ browserName }) => browserName === "firefox", + "The widget suite does not run on Firefox, see the other specs here", +); + +for (const [answer, respond] of Object.entries(HOST_ANSWERS)) { + widgetTest( + `reaches the lobby when config.json gets ${answer}`, + async ({ addUser }) => { + const { page } = await addUser("Brooks", HOST2); + await TestHelpers.createRoom("Config Room", page); + + let answered = 0; + await page.route(CONFIG_JSON, async (route) => { + answered++; + await respond(route); + }); + + await TestHelpers.startCallInCurrentRoom(page); + + await expect( + page + .locator('iframe[title="Element Call"]') + .contentFrame() + .getByTestId("lobby_joinCall"), + ).toBeVisible({ timeout: 15000 }); + expect(answered).toBeGreaterThan(0); + }, + ); +} diff --git a/playwright/widget/test-helpers.ts b/playwright/widget/test-helpers.ts index bfb20080b..38e61f49c 100644 --- a/playwright/widget/test-helpers.ts +++ b/playwright/widget/test-helpers.ts @@ -144,7 +144,11 @@ export class TestHelpers { await this.dismissStartupToasts(page); - await TestHelpers.setDevToolElementCallDevUrl(page); + // The other site's Element Web opens the embedded package it serves itself + // (see docker-compose-playwright.yml), not the dev server's app. + if (host !== HOST2) { + await TestHelpers.setDevToolElementCallDevUrl(page); + } const clientHandle = await page.evaluateHandle(() => window.mxMatrixClientPeg.get(), @@ -153,29 +157,40 @@ export class TestHelpers { return { page, clientHandle, mxId }; } - // Dismisses any toasts that appear on startup, such as "Failed to load service worker" or "Back up your chats". - // Toast can be stacked, and only the top one can be dismiss, so just look at what is on top and - // dismiss (if part of expected toats) + // Dismisses the toasts a fresh device gets after sign-in. + // + // Toasts stack and only the top one can be dismissed, so look + // at what is on top each time. public static async dismissStartupToasts(page: Page): Promise { - const expectedToasts = [ + const expectedToasts: { + title: string; + button: string; + confirm?: string; + }[] = [ { title: "Failed to load service worker", button: "OK" }, + { title: "Verify this device", button: "Later" }, { title: "Back up your chats", button: "Not now" }, - { title: "Turn on key storage", button: "Dismiss" }, + { + title: "Turn on key storage", + button: "Dismiss", + confirm: "Yes, dismiss", + }, + { title: "Notifications", button: "Dismiss" }, { title: "Element does not support this browser", button: "Dismiss" }, ]; const toast = page.locator(".mx_Toast_toast"); + const toastTitle = toast.locator(".mx_Toast_title h2"); // eslint-disable-next-line no-constant-condition while (true) { try { await toast.waitFor({ state: "visible", timeout: 700 }); - const title = await toast.locator(".mx_Toast_title h2").textContent(); + const title = await toastTitle.textContent(); + if (!title) break; // Find the matching toast config - const toastConfig = expectedToasts.find((t) => - title?.includes(t.title), - ); + const toastConfig = expectedToasts.find((t) => title.includes(t.title)); if (toastConfig) { // Bounded so that a renamed button leaves the toast in place and the @@ -183,6 +198,16 @@ export class TestHelpers { await toast .getByRole("button", { name: toastConfig.button }) .click({ timeout: 5000 }); + if (toastConfig.confirm) { + const dialog = page.getByRole("dialog"); + await dialog + .getByRole("button", { name: toastConfig.confirm }) + .click({ timeout: 5000 }); + await expect(dialog).toBeHidden({ timeout: 5000 }); + } + // The same element shows the next toast a moment after the click, so + // reading it straight away acts on the dismissed one a second time + await expect(toastTitle).not.toHaveText(title, { timeout: 5000 }); } else { // Unknown toast. We don't want to act on unknown toasts break; @@ -194,20 +219,6 @@ export class TestHelpers { } } - public static async closeReleaseAnnouncement( - page: Page, - name: string, - ): Promise { - try { - await page - .getByRole("dialog", { name }) - .getByRole("button", { name: "OK" }) - .click({ timeout: 2000 }); - } catch { - // Announcement not shown; nothing to do - } - } - public static async expandAllSections(page: Page): Promise { try { await page @@ -223,16 +234,7 @@ export class TestHelpers { page: Page, andInvite: string[] = [], ): Promise { - await TestHelpers.closeReleaseAnnouncement(page, "Introducing Sections"); - - await page - .getByRole("navigation", { name: "Room list" }) - .getByRole("button", { name: "New", exact: true }) - .click(); - - await page.getByRole("menuitem", { name: "New Room" }).click({ - timeout: 5000, - }); + await TestHelpers.chooseFromRoomListNewMenu(page, "New Room"); await page.getByRole("textbox", { name: "Name" }).fill(name); await page.getByRole("button", { name: "Create room" }).click(); await expect(page.getByText("You created this room.")).toBeVisible({ @@ -259,6 +261,24 @@ export class TestHelpers { } } + /** + * Opens the room list's "New" menu and picks the given item. + * + * Driven by keyboard, as a toast arriving late sits over the button and + * swallows clicks. + */ + public static async chooseFromRoomListNewMenu( + page: Page, + itemName: string, + ): Promise { + await TestHelpers.dismissStartupToasts(page); + await page + .getByRole("navigation", { name: "Room list" }) + .getByRole("button", { name: "New", exact: true }) + .press("Enter"); + await page.getByRole("menuitem", { name: itemName }).press("Enter"); + } + /** * Accepts a room invite using the room name. * Locatest the invite in the room list. @@ -268,9 +288,7 @@ export class TestHelpers { roomName: string, page: Page, ): Promise { - await TestHelpers.closeReleaseAnnouncement(page, "Introducing Sections"); - await TestHelpers.expandAllSections(page); - await TestHelpers.roomListItem(page, roomName).click({ timeout: 10000 }); + await TestHelpers.openRoomInvite(page, roomName); await page.getByRole("button", { name: "Accept" }).click({ timeout: 5000, }); @@ -376,6 +394,28 @@ export class TestHelpers { return page.locator(`[aria-label^="Open room ${roomName}"]`); } + /** + * Opens a pending invite from the room list by its room name. + * + * Element Web collapses the Invites section every time it appears, so it is + * expanded here once the invite has arrived. + */ + public static async openRoomInvite( + page: Page, + roomName: string, + ): Promise { + const invitesSection = page.getByRole("button", { + name: "Toggle Invites section", + }); + await expect(invitesSection).toBeVisible({ timeout: 10000 }); + // Toasts sit over the top of the room list, where this header is. + await TestHelpers.dismissStartupToasts(page); + if ((await invitesSection.getAttribute("aria-expanded")) !== "true") { + await invitesSection.click(); + } + await TestHelpers.roomListItem(page, roomName).click({ timeout: 10000 }); + } + /** * Switches to a room in the room list by its name. * @param page - The EW page diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index dc9fb426b..2dc937232 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,7 +8,7 @@ overrides: '@livekit/components-core>rxjs': ^7.8.1 '@livekit/track-processors>@mediapipe/tasks-vision': ^0.10.18 tar: ^7.5.11 - minimatch: ^10.2.3 + minimatch: ^11.0.0 glob: ^10.5.0 qs: ^6.15.2 js-yaml: ^4.2.0 @@ -160,6 +160,9 @@ importers: eslint-plugin-storybook: specifier: ^10.3.6 version: 10.6.2(eslint@8.57.1(supports-color@7.2.0))(supports-color@7.2.0)(typescript@5.9.3) + fake-indexeddb: + specifier: ^6.2.5 + version: 6.2.5 fetch-mock: specifier: 11.1.5 version: 11.1.5(node-fetch@2.7.0) @@ -188,8 +191,8 @@ importers: specifier: ^1.9.1 version: 1.9.2 matrix-js-sdk: - specifier: github:matrix-org/matrix-js-sdk#a9ef09fbf2f1644c8663d0aeaf6a921cd16e7b01 - version: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/a9ef09fbf2f1644c8663d0aeaf6a921cd16e7b01 + specifier: github:matrix-org/matrix-js-sdk#dc8610e796c92553c658300a4882dde961a0678a + version: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/dc8610e796c92553c658300a4882dde961a0678a matrix-widget-api: specifier: ^1.18.0 version: 1.20.0 @@ -1142,6 +1145,9 @@ packages: resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} engines: {node: '>=12'} + '@isaacs/nonrecursive-glob-match@1.0.3': + resolution: {integrity: sha512-SI7kyKHq4XXDRozw+r9GlRZkia1ZkFrpLW98r2IC2Bpr8YkFGE3fvjzmxCG0x78A++69HmDNrDvSMYBZmHGpKA==} + '@joshwooding/vite-plugin-react-docgen-typescript@0.7.0': resolution: {integrity: sha512-qvsTEwEFefhdirGOPnu9Wp6ChfIwy2dBCRuETU3uE+4cC+PFoxMSiiEhxk4lOluA34eARHA0OxqsEUYDqRMgeQ==} peerDependencies: @@ -3527,8 +3533,8 @@ packages: boolbase@1.0.0: resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==} - brace-expansion@5.0.9: - resolution: {integrity: sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==} + brace-expansion@5.0.12: + resolution: {integrity: sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ==} engines: {node: 20 || >=22} braces@3.0.3: @@ -4048,6 +4054,10 @@ packages: resolution: {integrity: sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==} engines: {node: '>=12.0.0'} + fake-indexeddb@6.2.5: + resolution: {integrity: sha512-CGnyrvbhPlWYMngksqrSSUT1BAVP49dZocrHuK0SvtR0D5TMs5wP0o3j7jexDJW01KSadjBp1M/71o/KR3nD1w==} + engines: {node: '>=18'} + fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} @@ -4724,8 +4734,8 @@ packages: matrix-events-sdk@0.0.1: resolution: {integrity: sha512-1QEOsXO+bhyCroIe2/A5OwaxHvBm7EsSQ46DEDn8RBIfQwN5HWBpFvyWWR4QY0KHPPnnJdI99wgRiAl7Ad5qaA==} - matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/a9ef09fbf2f1644c8663d0aeaf6a921cd16e7b01: - resolution: {gitHosted: true, integrity: sha512-ZAhn/zB44MgrXqUzi9ffSFcmZY3+2nx7bEgfEJR7chYDFyT/XmlmfCJx7Ar4rOTAYZJRVnMxrEwiEHGDapUi0Q==, tarball: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/a9ef09fbf2f1644c8663d0aeaf6a921cd16e7b01} + matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/dc8610e796c92553c658300a4882dde961a0678a: + resolution: {gitHosted: true, integrity: sha512-Ldt/VkCzBtre6m1cn0FqgrzRV0GrgWIoARgQUQO58ef2Edu2ERFCINPzp/psFS5RwoN3RIy9Rm4oPVc1ATkEWA==, tarball: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/dc8610e796c92553c658300a4882dde961a0678a} version: 45.0.0 engines: {node: '>=22.0.0'} @@ -4761,9 +4771,9 @@ packages: minimalistic-crypto-utils@1.0.1: resolution: {integrity: sha512-JIYlbt6g8i5jKfJ3xz7rF0LXmv2TkDxBLUkiBeZ7bAx4GnnNMr8xFpGnOxn6GhTEHx3SjRrZEoU+j04prX1ktg==} - minimatch@10.2.6: - resolution: {integrity: sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==} - engines: {node: 18 || 20 || >=22} + minimatch@11.0.4: + resolution: {integrity: sha512-/UJ1dEoVQTjPNDh6SGlUo+ueLMmoHTas4q+8yaAR/5t6nLAIiBDJtoYUh20IYleix7ehiQkgczb84PE3uzHkxg==} + engines: {node: 20 || >=22} minimist@1.2.8: resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} @@ -4936,8 +4946,8 @@ packages: resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==} engines: {node: '>=10'} - p-retry@8.0.1: - resolution: {integrity: sha512-NAigyu8hfvJe7MsS18mnc4is0MZtau3QMdBklaJKK23oBQb6occO3UPM3vHmTckW8KhfyjMZaUOqdTFEYliHgg==} + p-retry@9.0.0: + resolution: {integrity: sha512-HuEats8gYn/UUYa5uyveyQGv75mQmwdFtuMgnZlS12dj/7bdnJSO6keaTjgcZUhvkANTj5EO3lACUmV3xmgj0Q==} engines: {node: '>=22'} p-try@2.2.0: @@ -5308,6 +5318,10 @@ packages: randomfill@1.0.4: resolution: {integrity: sha512-87lcbR8+MhcWcUiQ+9e+Rwx8MyR2P7qnt15ynUlbm3TU/fjbgz4GsvfSUDTemtCCtVCqb4ZcEFlyPNTh9bBTLw==} + re2js@2.9.1: + resolution: {integrity: sha512-JIT3bEEUZTOR2YS1walNEIfH4cF/cQeZcBUFq7ldqO6lR6g0X5CBq/S8rsa9omzZcsNf8qc36jCkWpKKtncrZg==} + engines: {node: '>=20.0.0'} + react-docgen-typescript@2.4.0: resolution: {integrity: sha512-ZtAp5XTO5HRzQctjPU0ybY0RRCQO19X/8fxn3w7y2VVTUbGHDKULPTL4ky3vB05euSgG5NpALhEhDPvQ56wvXg==} peerDependencies: @@ -6895,7 +6909,7 @@ snapshots: ignore: 5.3.2 import-fresh: 3.3.1 js-yaml: 4.3.1 - minimatch: 10.2.6 + minimatch: 11.0.4 strip-json-comments: 3.1.1 transitivePeerDependencies: - supports-color @@ -6974,7 +6988,7 @@ snapshots: dependencies: '@humanwhocodes/object-schema': 2.0.3 debug: 4.4.3(supports-color@7.2.0) - minimatch: 10.2.6 + minimatch: 11.0.4 transitivePeerDependencies: - supports-color @@ -7110,6 +7124,8 @@ snapshots: wrap-ansi: 8.1.0 wrap-ansi-cjs: wrap-ansi@7.0.0 + '@isaacs/nonrecursive-glob-match@1.0.3': {} + '@joshwooding/vite-plugin-react-docgen-typescript@0.7.0(typescript@5.9.3)(vite@8.2.4(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.103.1)(terser@5.46.1)(yaml@2.9.0))': dependencies: glob: 10.5.0 @@ -8769,7 +8785,7 @@ snapshots: '@typescript-eslint/types': 8.69.0 '@typescript-eslint/visitor-keys': 8.69.0 debug: 4.4.3(supports-color@7.2.0) - minimatch: 10.2.6 + minimatch: 11.0.4 semver: 7.8.5 tinyglobby: 0.2.17 ts-api-utils: 2.5.0(typescript@5.9.3) @@ -8784,7 +8800,7 @@ snapshots: '@typescript-eslint/types': 8.72.0 '@typescript-eslint/visitor-keys': 8.72.0 debug: 4.4.3(supports-color@7.2.0) - minimatch: 10.2.6 + minimatch: 11.0.4 semver: 7.8.5 tinyglobby: 0.2.17 ts-api-utils: 2.5.0(typescript@5.9.3) @@ -9109,7 +9125,7 @@ snapshots: boolbase@1.0.0: {} - brace-expansion@5.0.9: + brace-expansion@5.0.12: dependencies: balanced-match: 4.0.4 @@ -9654,7 +9670,7 @@ snapshots: json-stable-stringify-without-jsonify: 1.0.1 levn: 0.4.1 lodash.merge: 4.6.2 - minimatch: 10.2.6 + minimatch: 11.0.4 natural-compare: 1.4.0 optionator: 0.9.4 strip-ansi: 6.0.1 @@ -9712,6 +9728,8 @@ snapshots: expect-type@1.4.0: {} + fake-indexeddb@6.2.5: {} + fast-deep-equal@3.1.3: {} fast-glob@3.3.3: @@ -9772,7 +9790,7 @@ snapshots: filelist@1.0.6: dependencies: - minimatch: 10.2.6 + minimatch: 11.0.4 fill-range@7.1.1: dependencies: @@ -9872,7 +9890,7 @@ snapshots: dependencies: foreground-child: 3.3.1 jackspeak: 3.4.3 - minimatch: 10.2.6 + minimatch: 11.0.4 minipass: 7.1.3 package-json-from-dist: 1.0.1 path-scurry: 1.11.1 @@ -9997,7 +10015,7 @@ snapshots: jiti: 2.7.0 jsonc-parser: 3.3.1 magic-string: 1.2.3 - minimatch: 10.2.6 + minimatch: 11.0.4 ora: 9.4.1 react: 19.2.8 react-i18next: 17.0.13(i18next@26.4.1(typescript@5.9.3))(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(typescript@5.9.3) @@ -10395,7 +10413,7 @@ snapshots: matrix-events-sdk@0.0.1: {} - matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/a9ef09fbf2f1644c8663d0aeaf6a921cd16e7b01: + matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/dc8610e796c92553c658300a4882dde961a0678a: dependencies: '@babel/runtime': 8.0.8 '@matrix-org/matrix-sdk-crypto-wasm': 18.9.0 @@ -10405,7 +10423,8 @@ snapshots: loglevel: 1.9.2 matrix-events-sdk: 0.0.1 matrix-widget-api: 1.20.0 - p-retry: 8.0.1 + p-retry: 9.0.0 + re2js: 2.9.1 sdp-transform: 3.0.0 unhomoglyph: 1.0.6 @@ -10440,9 +10459,10 @@ snapshots: minimalistic-crypto-utils@1.0.1: {} - minimatch@10.2.6: + minimatch@11.0.4: dependencies: - brace-expansion: 5.0.9 + '@isaacs/nonrecursive-glob-match': 1.0.3 + brace-expansion: 5.0.12 minimist@1.2.8: {} @@ -10726,7 +10746,7 @@ snapshots: dependencies: p-limit: 3.1.0 - p-retry@8.0.1: + p-retry@9.0.0: dependencies: is-network-error: 1.3.2 @@ -11150,6 +11170,8 @@ snapshots: randombytes: 2.1.0 safe-buffer: 5.2.1 + re2js@2.9.1: {} + react-docgen-typescript@2.4.0(typescript@5.9.3): dependencies: typescript: 5.9.3 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 9e4f0fd6f..72c15d304 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -27,7 +27,7 @@ overrides: # Security fixes: # - https://github.com/advisories/GHSA-7r86-cg39-jmmj # - https://github.com/advisories/GHSA-23c5-xmqv-rm74 - "minimatch": "^10.2.3" + "minimatch": "^11.0.0" # Security fix: https://github.com/element-hq/element-call/security/dependabot/109 "glob": "^10.5.0" # Security fixes: diff --git a/sdk/src/LocalMedia.test.ts b/sdk/src/LocalMedia.test.ts index 25f8b9b70..f61be6716 100644 --- a/sdk/src/LocalMedia.test.ts +++ b/sdk/src/LocalMedia.test.ts @@ -162,6 +162,7 @@ describe("createLocalMedia", () => { media.release("camera"); expect(trackBehind(camera).stop).toHaveBeenCalled(); expect(media.tracks$.value).toEqual([microphone]); + expect(media.requests.map((r) => r.source)).toEqual(["microphone"]); media.stop(); expect(trackBehind(microphone).stop).toHaveBeenCalled(); diff --git a/sdk/src/LocalMedia.ts b/sdk/src/LocalMedia.ts index 1bb934c3d..7825566ad 100644 --- a/sdk/src/LocalMedia.ts +++ b/sdk/src/LocalMedia.ts @@ -107,6 +107,7 @@ export function createLocalMedia( if (source !== "microphone" && source !== "camera") return; capturedTracks.get(source)?.stop(); capturedTracks.delete(source); + requests.delete(source); publishCapturedTracks(); }; diff --git a/sdk/src/RTCParticipation.ts b/sdk/src/RTCParticipation.ts index a88fa55b3..e591b6210 100644 --- a/sdk/src/RTCParticipation.ts +++ b/sdk/src/RTCParticipation.ts @@ -184,6 +184,7 @@ export function createRTCParticipation( encryptMedia: encryptionSystem.kind !== E2eeType.NONE, matrixRTCMode, sendNotificationType: options.sendNotificationType, + callIntent: options.callIntent, applicationData: options.applicationData, timings, delayedLeave, diff --git a/sdk/src/api.ts b/sdk/src/api.ts index d9a7c0b75..526cf6cf1 100644 --- a/sdk/src/api.ts +++ b/sdk/src/api.ts @@ -12,6 +12,7 @@ Please see LICENSE in the repository root for full details. */ import { + type RTCCallIntent, type RTCNotificationType, type Transport, } from "matrix-js-sdk/lib/matrixrtc"; @@ -103,11 +104,14 @@ export interface RTCParticipationOptions { * to a notification (ringing, timeouts, declines) is the application's job. */ sendNotificationType?: RTCNotificationType; + /** + * The kind of call the local member intends to take part in: `"m.audio"` + * or `"m.video"`. Carried by the membership as a field of its own. + */ + callIntent?: RTCCallIntent; /** * Whatever the application wants to say about itself in the membership, - * under namespaced keys. Opaque to the SDK, apart from `m.call.intent` - * (`"audio"` or `"video"`), which the membership format carries as a - * field of its own. + * under namespaced keys. Opaque to the SDK. */ applicationData?: Record; /** Timings the host has configured; the defaults otherwise. */ @@ -287,7 +291,7 @@ export interface RTCMember { memberId: string; /** Which transport this member is on; undefined when the membership has none. */ transport$: Behavior; - /** What the member's application says about itself, e.g. `m.call.intent`. */ + /** What the member's application says about itself, e.g. `intent`. */ applicationData$: Behavior>; displayName$: Behavior; diff --git a/sdk/src/matrixrtc/joinJsSdkSession.test.ts b/sdk/src/matrixrtc/joinJsSdkSession.test.ts index 4d9213097..7bb584a53 100644 --- a/sdk/src/matrixrtc/joinJsSdkSession.test.ts +++ b/sdk/src/matrixrtc/joinJsSdkSession.test.ts @@ -60,7 +60,7 @@ describe("joinJsSdkSession", () => { expect(s.join).toHaveBeenCalledWith(ownMemberMock, [transportWithAlias], { notificationType: "ring", callIntent: undefined, - applicationData: {}, + applicationData: undefined, manageMediaKeys: true, delayedLeaveEventRestartMs: 300_000, delayedLeaveEventDelayMs: 3_600_000, @@ -95,13 +95,14 @@ describe("joinJsSdkSession", () => { ); }); - it("lifts the call intent out of the application data and sends sticky events in Matrix 2.0 mode", () => { + it("passes the call intent and application data through and sends sticky events in Matrix 2.0 mode", () => { const s = session(); joinJsSdkSession("!room:example.org", s, ownMemberMock, transport, { encryptMedia: true, matrixRTCMode: MatrixRTCMode.Matrix_2_0, - applicationData: { "m.call.intent": "audio", custom: 1 }, + callIntent: "m.audio", + applicationData: { custom: 1 }, timings: defaultSessionTimings, delayedLeave: defaultSessionTimings.delayedLeave, }); @@ -110,7 +111,7 @@ describe("joinJsSdkSession", () => { ownMemberMock, [transportWithAlias], expect.objectContaining({ - callIntent: "audio", + callIntent: "m.audio", applicationData: { custom: 1 }, unstableSendStickyEvents: true, }), @@ -124,7 +125,8 @@ describe("joinJsSdkSession", () => { joinJsSdkSession("!room:example.org", s, ownMemberMock, transport, { encryptMedia: true, matrixRTCMode: MatrixRTCMode.Matrix_2_0, - applicationData: { "m.call.intent": "audio", custom: 1 }, + callIntent: "m.audio", + applicationData: { custom: 1 }, timings: defaultSessionTimings, delayedLeave: defaultSessionTimings.delayedLeave, }); @@ -133,7 +135,7 @@ describe("joinJsSdkSession", () => { ownMemberMock, [transport], expect.objectContaining({ - callIntent: "audio", + callIntent: "m.audio", applicationData: { custom: 1 }, unstableSendStickyEvents: true, }), diff --git a/sdk/src/matrixrtc/joinJsSdkSession.ts b/sdk/src/matrixrtc/joinJsSdkSession.ts index 5c6d14049..d2dd6e9be 100644 --- a/sdk/src/matrixrtc/joinJsSdkSession.ts +++ b/sdk/src/matrixrtc/joinJsSdkSession.ts @@ -23,6 +23,7 @@ interface Options { encryptMedia: boolean; matrixRTCMode: MatrixRTCMode; sendNotificationType?: RTCNotificationType; + callIntent?: RTCCallIntent; applicationData?: Record; timings: SessionTimings; /** The delayed leave timings to use: delegated to the SFU or not. */ @@ -43,15 +44,12 @@ export function joinJsSdkSession( encryptMedia, matrixRTCMode, sendNotificationType, + callIntent, applicationData, timings, delayedLeave, }: Options, ): void { - // The membership format carries the intent as a field of its own - const { "m.call.intent": intent, ...rest } = applicationData ?? {}; - const callIntent = - typeof intent === "string" ? (intent as RTCCallIntent) : undefined; // Give up on the network as soon as either the sync has been down for the // grace period or the delayed leave has probably been sent, whichever is // sooner @@ -78,7 +76,7 @@ export function joinJsSdkSession( { notificationType: sendNotificationType, callIntent, - applicationData: rest, + applicationData, manageMediaKeys: encryptMedia, delayedLeaveEventRestartMs: delayedLeave.restart_ms, delayedLeaveEventDelayMs: delayedLeave.delay_ms, diff --git a/sdk/src/media-backend/livekit/LivekitMediaTrack.test.ts b/sdk/src/media-backend/livekit/LivekitMediaTrack.test.ts index fc11f300d..790eddd1a 100644 --- a/sdk/src/media-backend/livekit/LivekitMediaTrack.test.ts +++ b/sdk/src/media-backend/livekit/LivekitMediaTrack.test.ts @@ -225,6 +225,51 @@ describe("createLivekitMediaTrack", () => { ); }); + // LiveKit's media stream track is the processed one while a processor runs, + // and that one names no device; the camera behind it does + it.each([ + { deviceId: "cam-1", switches: false }, + { deviceId: "cam-2", switches: true }, + ])( + "leaves a processed camera alone unless its device changes: $deviceId", + async ({ deviceId, switches }) => { + vi.stubGlobal("MediaStreamTrack", class {}); + onTestFinished(() => { + vi.unstubAllGlobals(); + }); + const camera = Object.assign(Object.create(LocalVideoTrack.prototype), { + source: Track.Source.Camera, + _mediaStreamTrack: { + readyState: "live", + label: "", + getSettings: () => ({ deviceId: "cam-1" }), + }, + // What LiveKit shows as the track, with no device of its own + processor: { + processedTrack: { label: "", getSettings: () => ({}) }, + }, + }) as LocalVideoTrack; + const switchActiveDevice = vi.fn().mockResolvedValue(true); + const { participant, publication, room } = fakes({ + track: camera as unknown as FakeTrack, + }); + const track = createLocalLivekitMediaTrack( + scope, + Object.assign(participant, { + isLocal: true, + }) as unknown as LocalParticipant, + publication as unknown as TrackPublication, + Object.assign(room, { switchActiveDevice }), + vi.fn(), + ); + + await track.setDevice(deviceId); + expect(switchActiveDevice.mock.calls).toEqual( + switches ? [["videoinput", deviceId, false]] : [], + ); + }, + ); + // LiveKit stops a camera that is turned off; the processor is built when it // is turned on again it.each([ diff --git a/sdk/src/media-backend/livekit/LivekitMediaTrack.ts b/sdk/src/media-backend/livekit/LivekitMediaTrack.ts index 588c55761..c32fc2d52 100644 --- a/sdk/src/media-backend/livekit/LivekitMediaTrack.ts +++ b/sdk/src/media-backend/livekit/LivekitMediaTrack.ts @@ -199,11 +199,13 @@ export function createLocalLivekitMediaTrack( throw new MatrixRTCError(`A ${base.source} track has no device`); // LiveKit decides whether to restart by the constraints it keeps, which // a browser that does not echo the device id back (Firefox) leaves - // without one; a track already on the device is left alone here + // without one; a track already on the device is left alone here. Asked + // of the camera track itself: with a processor on, LiveKit's media + // stream track is the processed one, which names no device const current = publication.track; if ( current instanceof LocalTrack && - current.mediaStreamTrack.getSettings().deviceId === deviceId + (await current.getDeviceId(false)) === deviceId ) return; await room.switchActiveDevice( diff --git a/src/UrlConfiguration.ts b/src/UrlConfiguration.ts index 2840bad1a..4d7847095 100644 --- a/src/UrlConfiguration.ts +++ b/src/UrlConfiguration.ts @@ -18,6 +18,8 @@ Please see LICENSE in the repository root for full details. */ import { + RTC_CALL_INTENT_AUDIO, + RTC_CALL_INTENT_VIDEO, type RTCCallIntent, type RTCNotificationType, } from "matrix-js-sdk/lib/matrixrtc"; @@ -179,40 +181,40 @@ export function configurationForIntent(intent: UserIntent): UrlConfiguration { switch (intent) { case UserIntent.StartNewCall: preset.skipLobby = false; - preset.callIntent = "video"; + preset.callIntent = RTC_CALL_INTENT_VIDEO; break; case UserIntent.JoinExistingCall: // On desktop this will be overridden based on which button was used to join the call preset.skipLobby = false; - preset.callIntent = "video"; + preset.callIntent = RTC_CALL_INTENT_VIDEO; break; case UserIntent.StartNewCallVoice: preset.skipLobby = false; - preset.callIntent = "audio"; + preset.callIntent = RTC_CALL_INTENT_AUDIO; break; case UserIntent.JoinExistingCallVoice: // On desktop this will be overridden based on which button was used to join the call preset.skipLobby = false; - preset.callIntent = "audio"; + preset.callIntent = RTC_CALL_INTENT_AUDIO; break; case UserIntent.StartNewCallDMVoice: - preset.callIntent = "audio"; + preset.callIntent = RTC_CALL_INTENT_AUDIO; // Fall through case UserIntent.StartNewCallDM: preset.skipLobby = true; preset.sendNotificationType = "ring"; preset.autoLeaveWhenOthersLeft = true; preset.waitForCallPickup = true; - preset.callIntent = preset.callIntent ?? "video"; + preset.callIntent = preset.callIntent ?? RTC_CALL_INTENT_VIDEO; break; case UserIntent.JoinExistingCallDMVoice: - preset.callIntent = "audio"; + preset.callIntent = RTC_CALL_INTENT_AUDIO; // Fall through case UserIntent.JoinExistingCallDM: // On desktop this will be overridden based on which button was used to join the call preset.skipLobby = true; preset.autoLeaveWhenOthersLeft = true; - preset.callIntent = preset.callIntent ?? "video"; + preset.callIntent = preset.callIntent ?? RTC_CALL_INTENT_VIDEO; break; // Non widget usecase defaults default: diff --git a/src/UrlParams.test.ts b/src/UrlParams.test.ts index c1f1aeeb0..ee29b4c27 100644 --- a/src/UrlParams.test.ts +++ b/src/UrlParams.test.ts @@ -21,6 +21,10 @@ import { UserIntent, } from "../src/UrlParams"; import { mockConfig } from "./utils/test"; +import { + RTC_CALL_INTENT_AUDIO, + RTC_CALL_INTENT_VIDEO, +} from "matrix-js-sdk/lib/matrixrtc"; const ROOM_NAME = "roomNameHere"; const ROOM_ID = "!d45f138fsd"; @@ -275,7 +279,7 @@ describe("UrlParams", () => { ).toMatchObject({ ...startNewCallDefaults("desktop"), skipLobby: false, - callIntent: "video", + callIntent: RTC_CALL_INTENT_VIDEO, }); }); @@ -322,7 +326,7 @@ describe("UrlParams", () => { ).toMatchObject({ ...startNewCallDefaults("desktop"), skipLobby: false, - callIntent: "audio", + callIntent: RTC_CALL_INTENT_AUDIO, }); }); @@ -333,7 +337,7 @@ describe("UrlParams", () => { ), ).toMatchObject({ ...joinExistingCallDefaults("desktop"), - callIntent: "audio", + callIntent: RTC_CALL_INTENT_AUDIO, }); }); @@ -348,7 +352,7 @@ describe("UrlParams", () => { sendNotificationType: "ring", autoLeaveWhenOthersLeft: true, waitForCallPickup: true, - callIntent: "audio", + callIntent: RTC_CALL_INTENT_AUDIO, }); }); @@ -363,7 +367,7 @@ describe("UrlParams", () => { skipLobby: true, autoLeaveWhenOthersLeft: true, waitForCallPickup: false, - callIntent: "video", + callIntent: RTC_CALL_INTENT_VIDEO, }); }); @@ -377,7 +381,7 @@ describe("UrlParams", () => { skipLobby: true, autoLeaveWhenOthersLeft: true, waitForCallPickup: false, - callIntent: "audio", + callIntent: RTC_CALL_INTENT_AUDIO, }); }); }); diff --git a/src/components/BackgroundEffectGrid.module.css b/src/components/BackgroundEffectGrid.module.css index 472772c15..70c514a9e 100644 --- a/src/components/BackgroundEffectGrid.module.css +++ b/src/components/BackgroundEffectGrid.module.css @@ -90,3 +90,42 @@ Please see LICENSE in the repository root for full details. text-align: center; overflow-wrap: anywhere; } + +.removable { + position: relative; + display: flex; + min-inline-size: 0; +} + +.removable > .tile { + flex: 1; +} + +/* Clear of the target for choosing the tile. */ +.remove { + position: absolute; + inset-block-start: 0; + inset-inline-end: 0; + translate: 20% -20%; + display: none; + align-items: center; + justify-content: center; + inline-size: 28px; + block-size: 28px; + border-radius: 50%; + background: var(--cpd-color-bg-canvas-default); + color: var(--cpd-color-icon-critical-primary); + box-shadow: 0 1.2px 2.4px 0 rgb(0 0 0 / 15%); + cursor: pointer; +} + +.remove:hover { + background: var(--cpd-color-bg-critical-subtle-hovered); + box-shadow: 0 0 0 var(--cpd-border-width-1) + var(--cpd-color-border-critical-subtle); +} + +.removable:hover .remove, +.removable:focus-within .remove { + display: flex; +} diff --git a/src/components/BackgroundEffectGrid.tsx b/src/components/BackgroundEffectGrid.tsx index c75c79455..0c2803836 100644 --- a/src/components/BackgroundEffectGrid.tsx +++ b/src/components/BackgroundEffectGrid.tsx @@ -5,14 +5,16 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type FC, type ReactElement } from "react"; +import { type FC, type KeyboardEvent, type ReactElement } from "react"; import { useTranslation } from "react-i18next"; import { VisuallyHidden } from "@radix-ui/react-visually-hidden"; -import { InlineSpinner, MenuItem } from "@vector-im/compound-web"; +import { InlineSpinner, MenuItem, Tooltip } from "@vector-im/compound-web"; import { BlockIcon, BlurIcon, CheckCircleSolidIcon, + CloseIcon, + PlusIcon, } from "@vector-im/compound-design-tokens/assets/web/icons"; import styles from "./BackgroundEffectGrid.module.css"; @@ -27,6 +29,8 @@ export interface BackgroundEffectGridItem { kind: "none" | "blur" | "image"; /** For kind "image". */ imageUrl?: string; + /** Whether it is the user's to remove. */ + removable?: boolean; } interface Props { @@ -38,6 +42,12 @@ interface Props { settling?: boolean; /** Id of the text saying what the user should know before choosing. */ describedBy?: string; + /** Offers to add a background of one's own. Omit to leave that tile out. */ + onAdd?: () => void; + addLabel?: string; + /** Removes a removable one, never the one in force. Omit to offer none. */ + onRemove?: (id: string) => void; + removeLabel?: string; } /** @@ -51,6 +61,10 @@ export const BackgroundEffectGrid: FC = ({ onSelect, settling = false, describedBy, + onAdd, + addLabel, + onRemove, + removeLabel, }) => { const { t } = useTranslation(); const choose = (id: string): void => { @@ -66,7 +80,30 @@ export const BackgroundEffectGrid: FC = ({ : t("background_effects.numbered", { n: ++pictures }); const checked = effect.id === selected; const disabled = onSelect === undefined && effect.kind !== "none"; - return ( + const remove = + effect.removable && !checked && onRemove !== undefined + ? (): void => onRemove(effect.id) + : undefined; + const removeProps = + remove === undefined + ? {} + : { + "aria-keyshortcuts": "Delete", + onKeyDown: (e: KeyboardEvent): void => { + if (e.key !== "Delete" && e.key !== "Backspace") return; + e.preventDefault(); + remove(); + }, + }; + const content = ( + + ); + const tile = ( = ({ choose(effect.id); } } + {...removeProps} > - + {content} ); + if (remove === undefined) return tile; + // Beside the item, not in it: inside, the item takes the press first and + // the tile is chosen instead. + return ( +
+ {tile} + + {/* Not a control of its own, so the menu keeps one keyboard model: + the keyboard reaches the same action through Delete. */} + + + + +
+ ); }); + // A command rather than a choice, so an item beside the radios. + const addContent = ( + <> + + + + {addLabel} + + ); + const addTile = + onAdd === undefined ? null : ( + { + e.preventDefault(); + onAdd(); + }} + > + {addContent} + + ); + return ( // A group of its own, as the section's can't carry the description.
{tiles} + {addTile}
); }; diff --git a/src/components/CallFooter.stories.tsx b/src/components/CallFooter.stories.tsx index b587cc40e..8d6cc2a77 100644 --- a/src/components/CallFooter.stories.tsx +++ b/src/components/CallFooter.stories.tsx @@ -127,16 +127,21 @@ export const Default: Story = { selectBackgroundEffect: fn(), backgroundEffect: "blur", backgroundEffects: [ - { id: "none", kind: "none" }, - { id: "blur", kind: "blur" }, + { id: "none", kind: "none", removable: false }, + { id: "blur", kind: "blur", removable: false }, ...shippedBackgrounds.map((background) => ({ id: `image:${background.id}`, kind: "image" as const, imageUrl: background.imagePath, + removable: false, })), ], backgroundEffectNotice: undefined, backgroundEffectSettling: false, + backgroundImageRefusal: undefined, + addBackgroundImage: fn(), + removeBackgroundEffect: fn(), + cameraTrack: null, hangup: fn(), buttonSize: "lg", showFooter: true, diff --git a/src/components/CallFooter.tsx b/src/components/CallFooter.tsx index 8606d3a47..4d13b4c6a 100644 --- a/src/components/CallFooter.tsx +++ b/src/components/CallFooter.tsx @@ -33,6 +33,13 @@ import { useBehavior } from "../useBehavior"; import { type LayoutSwitchViewModel } from "../state/LayoutSwitchViewModel"; import { LayoutSwitch } from "../room/LayoutSwitch"; import { type BackgroundEffectGridItem } from "./BackgroundEffectGrid"; +import { SelfPreview } from "./SelfPreview"; +import { type UnusableReason } from "../tracks/backgroundImages"; +import { type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk"; + +export interface BackgroundImageRefusal { + reason: UnusableReason | "not-kept"; +} export interface AudioOutputSwitcher { targetOutput: string; @@ -60,6 +67,10 @@ export interface FooterActions { toggleVideo: (() => void) | undefined; /** Undefined where background effects can't be chosen. */ selectBackgroundEffect: ((id: string) => void) | undefined; + /** Undefined where no background can be added. */ + addBackgroundImage: ((file: File) => void) | undefined; + /** Removes an added background by its option id; never the one in force. */ + removeBackgroundEffect: ((id: string) => void) | undefined; toggleScreenSharing: (() => void) | undefined; /** Also controls if the settings button is visible */ openSettings: (() => void) | undefined; @@ -81,6 +92,10 @@ export interface FooterState { backgroundEffectNotice: "unavailable" | "slow" | undefined; /** Whether the first effect chosen is still being prepared. */ backgroundEffectSettling: boolean; + /** Why the last file offered couldn't be kept, if it couldn't. */ + backgroundImageRefusal: BackgroundImageRefusal | undefined; + /** The camera to preview, with the effect in force; null while it is off. */ + cameraTrack: LocalVideoMediaTrack | null; showFooter: boolean; /* This is needed for WindowMode = "flat" */ @@ -168,6 +183,24 @@ export const CallFooter: FC = ({ const selectBackgroundEffect = useBehavior(vm.selectBackgroundEffect$); const backgroundEffectNotice = useBehavior(vm.backgroundEffectNotice$); const backgroundEffectSettling = useBehavior(vm.backgroundEffectSettling$); + const addBackgroundImage = useBehavior(vm.addBackgroundImage$); + const removeBackgroundEffect = useBehavior(vm.removeBackgroundEffect$); + const cameraTrack = useBehavior(vm.cameraTrack$); + const refusal = useBehavior(vm.backgroundImageRefusal$); + const refusalMessage = useMemo(() => { + switch (refusal?.reason) { + case undefined: + return undefined; + case "not-an-image": + return { text: t("error.background_not_an_image") }; + case "animated": + return { text: t("error.background_animated") }; + case "undecodable": + return { text: t("error.background_undecodable") }; + case "not-kept": + return { text: t("error.background_not_kept") }; + } + }, [refusal, t]); const backgroundEffects = useBehavior(vm.backgroundEffects$); const buttonSize = useBehavior(vm.buttonSize$); const showLogo = useBehavior(vm.showLogo$); @@ -234,6 +267,10 @@ export const CallFooter: FC = ({ selectedBackgroundEffect={backgroundEffect} onSelectBackgroundEffect={selectBackgroundEffect} backgroundEffectSettling={backgroundEffectSettling} + onAddBackgroundImage={addBackgroundImage} + onRemoveBackgroundEffect={removeBackgroundEffect} + selfPreview={} + backgroundImageRefusal={refusalMessage} backgroundEffectNotice={ backgroundEffectNotice === "unavailable" ? t("background_effects.unavailable") diff --git a/src/components/CallFooterViewModel.test.ts b/src/components/CallFooterViewModel.test.ts index e061f4ef2..9871bc4b0 100644 --- a/src/components/CallFooterViewModel.test.ts +++ b/src/components/CallFooterViewModel.test.ts @@ -7,9 +7,17 @@ Please see LICENSE in the repository root for full details. import { afterEach, describe, expect, it, vi } from "vitest"; import { BehaviorSubject } from "rxjs"; +import { type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk"; -import { testScope, mockMuteStates, mockMediaDevices } from "../utils/test"; -import { constant } from "../state/Behavior"; +import { + flushPromises, + testScope, + mockMuteStates, + mockMediaDevices, + mockVideoTrack, +} from "../utils/test"; +import { type Behavior, constant } from "../state/Behavior"; +import { type BackgroundEffectsState } from "../state/BackgroundEffects"; import type { CallViewModel } from "../state/CallViewModel/CallViewModel"; import type { Layout } from "../state/layout-types"; import type { SpotlightTileViewModel } from "../state/TileViewModel"; @@ -21,6 +29,13 @@ import { import { HeaderStyle } from "../UrlParams"; import { backgroundEffect as backgroundEffectSetting } from "../settings/settings"; import { shippedBackgrounds } from "../tracks/backgroundEffectCatalog"; +import type * as BackgroundImages from "../tracks/backgroundImages"; +import { + type AddedBackground, + addedBackgrounds, + maxAddedBackgrounds, + UnusableImage, +} from "../tracks/backgroundImages"; const platformMock = vi.hoisted(() => vi.fn(() => "desktop")); vi.mock("../Platform", () => ({ @@ -37,6 +52,22 @@ vi.mock("@livekit/track-processors", () => ({ supportsModernBackgroundProcessors: (): boolean => modernRouteMock(), })); +const store = vi.hoisted(() => ({ + add: vi.fn<(file: Blob) => Promise>(), + remove: vi.fn<(id: string) => Promise>(), +})); +vi.mock("../tracks/backgroundImages", async (original) => { + const { BehaviorSubject } = await import("rxjs"); + return { + ...(await original()), + addedBackgrounds: { + added$: new BehaviorSubject([]), + add: store.add, + remove: store.remove, + }, + }; +}); + const outputSelectionMock = vi.hoisted(() => vi.fn(() => true)); vi.mock("livekit-client", () => ({ supportsAudioOutputSelection: (): boolean => outputSelectionMock(), @@ -50,6 +81,7 @@ vi.mock("livekit-client", () => ({ function buildMinimalCallViewModel(layout: Layout): CallViewModel { return { layout$: constant(layout), + camera$: constant(undefined), edgeToEdge$: constant(false), showHeader$: constant(false), hangup: (): void => {}, @@ -81,6 +113,16 @@ const pipLayout: Layout = { spotlight: {} as SpotlightTileViewModel, }; +const noPipeline = constant({ + supported: false, + processor: undefined, +}); + +/** The provider's verdict, which the SDK's check stands for here. */ +function pipelineAsTheSdkSays(): Behavior { + return constant({ supported: sdkSupportMock(), processor: undefined }); +} + const twoMicsAndOneCamMediaDevices = mockMediaDevices({ audioInput: { available$: constant( @@ -112,7 +154,7 @@ describe("createCallFooterViewModel", () => { buildMinimalCallViewModel(gridLayout), mockMuteStates(), twoMicsAndOneCamMediaDevices, - constant(false), + noPipeline, /* reactionIdentifier */ undefined, { showControls: true, header: HeaderStyle.Standard }, ); @@ -155,7 +197,7 @@ describe("createCallFooterViewModel", () => { select: vi.fn(), }, }), - constant(false), + noPipeline, /* reactionIdentifier */ undefined, { showControls: true, header: HeaderStyle.Standard }, ); @@ -174,7 +216,7 @@ describe("createCallFooterViewModel", () => { buildMinimalCallViewModel(layout), mockMuteStates(), twoMicsAndOneCamMediaDevices, - constant(false), + noPipeline, /* reactionIdentifier */ undefined, { showControls: true, header: HeaderStyle.Standard }, ); @@ -197,7 +239,7 @@ describe("createCallFooterViewModel", () => { buildMinimalCallViewModel(gridLayout), mockMuteStates(), twoMicsAndOneCamMediaDevices, - constant(false), + noPipeline, /* reactionIdentifier */ undefined, { showControls: true, header: HeaderStyle.Standard }, ); @@ -243,7 +285,8 @@ describe("createCallFooterViewModel", () => { testScope(), mockMuteStates(), twoMicsAndOneCamMediaDevices, - constant(false), + pipelineAsTheSdkSays(), + /* camera$ */ constant(null), /* openSettings */ undefined, /* hangup */ undefined, /* showLogo */ false, @@ -263,6 +306,7 @@ describe("createCallFooterViewModel", () => { it("offers nothing where the browser itself cannot run them", () => { sdkSupportMock.mockReturnValue(false); expect(lobbyOn("desktop").selectBackgroundEffect$.value).toBeUndefined(); + expect(lobbyOn("desktop").addBackgroundImage$.value).toBeUndefined(); }); it("puts no effect in force where the browser cannot run them", () => { @@ -293,17 +337,29 @@ describe("createCallFooterViewModel", () => { expect(lobbyOn("desktop").backgroundEffectNotice$.value).toBeUndefined(); }); - it("offers every effect in order", () => { + it("offers every effect in order, the added ones removable", () => { sdkSupportMock.mockReturnValue(true); + const added$ = addedBackgrounds.added$ as BehaviorSubject< + AddedBackground[] | undefined + >; + added$.next([{ id: "mine", url: "blob:mine" }]); expect(lobbyOn("desktop").backgroundEffects$.value).toEqual([ - { id: "none", kind: "none" }, - { id: "blur", kind: "blur" }, + { id: "none", kind: "none", removable: false }, + { id: "blur", kind: "blur", removable: false }, ...shippedBackgrounds.map((background) => ({ id: `image:${background.id}`, kind: "image", imageUrl: background.imagePath, + removable: false, })), + { + id: "added:mine", + kind: "image", + imageUrl: "blob:mine", + removable: true, + }, ]); + added$.next([]); }); it("says they are unavailable where they cannot be chosen", () => { @@ -315,21 +371,146 @@ describe("createCallFooterViewModel", () => { it("passes on the wait for the first effect", () => { platformMock.mockReturnValue("desktop"); - const settling$ = new BehaviorSubject(true); + const pipeline$ = new BehaviorSubject({ + supported: true, + processor: undefined, + wait: "settling", + }); const vm = createLobbyFooterViewModel( testScope(), mockMuteStates(), twoMicsAndOneCamMediaDevices, - settling$, + pipeline$, + /* camera$ */ constant(null), /* openSettings */ undefined, /* hangup */ undefined, /* showLogo */ false, ); expect(vm.backgroundEffectSettling$.value).toBe(true); - settling$.next(false); + pipeline$.next({ ...pipeline$.value, wait: undefined }); expect(vm.backgroundEffectSettling$.value).toBe(false); }); + it("offers nothing once the pipeline fails to build, keeping the choice", () => { + platformMock.mockReturnValue("desktop"); + backgroundEffectSetting.setValue("blur"); + const pipeline$ = new BehaviorSubject({ + supported: true, + processor: undefined, + }); + const vm = createLobbyFooterViewModel( + testScope(), + mockMuteStates(), + twoMicsAndOneCamMediaDevices, + pipeline$, + /* camera$ */ constant(null), + /* openSettings */ undefined, + /* hangup */ undefined, + /* showLogo */ false, + ); + expect(vm.backgroundEffect$.value).toBe("blur"); + + pipeline$.next({ supported: false, processor: undefined }); + expect(vm.selectBackgroundEffect$.value).toBeUndefined(); + expect(vm.addBackgroundImage$.value).toBeUndefined(); + expect(vm.backgroundEffect$.value).toBe("none"); + expect(backgroundEffectSetting.getValue()).toBe("blur"); + }); + + it("adding is unavailable at the limit", () => { + sdkSupportMock.mockReturnValue(true); + const added$ = addedBackgrounds.added$ as BehaviorSubject< + AddedBackground[] | undefined + >; + const vm = lobbyOn("desktop"); + expect(vm.addBackgroundImage$.value).toBeDefined(); + added$.next( + Array.from({ length: maxAddedBackgrounds }, (_, i) => ({ + id: `${i}`, + url: `blob:${i}`, + })), + ); + expect(vm.addBackgroundImage$.value).toBeUndefined(); + added$.next(added$.value!.slice(1)); + expect(vm.addBackgroundImage$.value).toBeDefined(); + added$.next([]); + }); + + it("says why each refused file was refused", async () => { + sdkSupportMock.mockReturnValue(true); + const vm = lobbyOn("desktop"); + store.add.mockRejectedValue(new UnusableImage("animated")); + vm.addBackgroundImage$.value!(new File(["x"], "a.gif")); + await flushPromises(); + const first = vm.backgroundImageRefusal$.value; + expect(first).toEqual({ reason: "animated" }); + + // The same again is a refusal of its own, so it is shown again. + vm.addBackgroundImage$.value!(new File(["x"], "a.gif")); + await flushPromises(); + expect(vm.backgroundImageRefusal$.value).toEqual({ reason: "animated" }); + expect(vm.backgroundImageRefusal$.value).not.toBe(first); + + store.add.mockRejectedValue(new Error("quota")); + vm.addBackgroundImage$.value!(new File(["x"], "b.png")); + await flushPromises(); + expect(vm.backgroundImageRefusal$.value).toEqual({ reason: "not-kept" }); + + store.add.mockResolvedValue("kept"); + vm.addBackgroundImage$.value!(new File(["x"], "c.png")); + await flushPromises(); + expect(vm.backgroundImageRefusal$.value).toBeUndefined(); + }); + + it("never removes the background in force", () => { + sdkSupportMock.mockReturnValue(true); + store.remove.mockResolvedValue(); + const vm = lobbyOn("desktop"); + backgroundEffectSetting.setValue("added:worn"); + vm.removeBackgroundEffect$.value!("added:worn"); + expect(store.remove).not.toHaveBeenCalled(); + vm.removeBackgroundEffect$.value!("added:spare"); + expect(store.remove).toHaveBeenCalledWith("spare"); + }); + + it("previews the lobby's camera while there is one", () => { + const track = mockVideoTrack({}); + const camera$ = new BehaviorSubject(track); + const vm = createLobbyFooterViewModel( + testScope(), + mockMuteStates(), + twoMicsAndOneCamMediaDevices, + noPipeline, + camera$, + /* openSettings */ undefined, + /* hangup */ undefined, + /* showLogo */ false, + ); + expect(vm.cameraTrack$.value).toBe(track); + // The lobby releases the camera when it is turned off + camera$.next(null); + expect(vm.cameraTrack$.value).toBeNull(); + }); + + it("previews the call's camera only while it is not muted", () => { + const muted$ = new BehaviorSubject(false); + const track = mockVideoTrack({ muted$ }); + const vm = createCallFooterViewModel( + testScope(), + { ...buildMinimalCallViewModel(gridLayout), camera$: constant(track) }, + mockMuteStates(), + twoMicsAndOneCamMediaDevices, + noPipeline, + /* reactionIdentifier */ undefined, + { showControls: true, header: HeaderStyle.Standard }, + ); + expect(vm.cameraTrack$.value).toBe(track); + muted$.next(true); + expect(vm.cameraTrack$.value).toBeNull(); + muted$.next(false); + expect(vm.cameraTrack$.value).toBe(track); + }); + it("availability is the same before and during a call", () => { for (const supported of [true, false]) { sdkSupportMock.mockReturnValue(supported); @@ -339,7 +520,7 @@ describe("createCallFooterViewModel", () => { buildMinimalCallViewModel(gridLayout), mockMuteStates(), twoMicsAndOneCamMediaDevices, - constant(false), + pipelineAsTheSdkSays(), /* reactionIdentifier */ undefined, { showControls: true, header: HeaderStyle.Standard }, ); diff --git a/src/components/CallFooterViewModel.tsx b/src/components/CallFooterViewModel.tsx index e9a262335..23985e682 100644 --- a/src/components/CallFooterViewModel.tsx +++ b/src/components/CallFooterViewModel.tsx @@ -6,25 +6,33 @@ Please see LICENSE in the repository root for full details. */ import { + BehaviorSubject, combineLatest, distinctUntilChanged, map, type Observable, + of, switchMap, } from "rxjs"; import { supportsAudioOutputSelection } from "livekit-client"; -import { - supportsBackgroundProcessors, - supportsModernBackgroundProcessors, -} from "@livekit/track-processors"; +import { type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk"; +import { supportsModernBackgroundProcessors } from "@livekit/track-processors"; +import { logger } from "matrix-js-sdk/lib/logger"; import { parseEffect, serializeEffect, shippedBackgrounds, } from "../tracks/backgroundEffectCatalog"; +import { + type AddedBackground, + addedBackgrounds, + maxAddedBackgrounds, + UnusableImage, +} from "../tracks/backgroundImages"; import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; +import { type BackgroundEffectsState } from "../state/BackgroundEffects"; import { type MenuOptions } from "./MediaMuteAndSwitchButton"; import { type MediaDevices } from "../state/MediaDevices"; import { @@ -37,7 +45,7 @@ import { type MuteStates } from "../state/MuteStates"; import { createStaticViewModel, type ViewModel } from "../state/ViewModel"; import { HeaderStyle } from "../UrlParams"; import { platform } from "../Platform"; -import { type FooterSnapshot } from "./CallFooter"; +import { type BackgroundImageRefusal, type FooterSnapshot } from "./CallFooter"; import { type BackgroundEffectGridItem } from "./BackgroundEffectGrid"; /** @@ -79,7 +87,9 @@ function buildDeviceBehaviors( mediaDevices: MediaDevices, /** return empty arrays for audioOptions and videoOptions*/ disableSwitcher$: Behavior, - backgroundEffectSettling$: Observable, + backgroundEffectsState$: Observable, + /** The camera as the pipeline has it, for the menu's preview. */ + camera$: Observable, ): Pick< ViewModel, | "audioOptions$" @@ -96,6 +106,10 @@ function buildDeviceBehaviors( | "backgroundEffects$" | "backgroundEffectNotice$" | "backgroundEffectSettling$" + | "addBackgroundImage$" + | "backgroundImageRefusal$" + | "removeBackgroundEffect$" + | "cameraTrack$" > { const options$ = ( available$: Behavior>, @@ -112,10 +126,18 @@ function buildDeviceBehaviors( ), ); - const supported = supportsBackgroundProcessors(); + // Also false once the pipeline has failed to build this session. + const supported$ = backgroundEffectsState$.pipe( + map(({ supported }) => supported === true), + distinctUntilChanged(), + ); const slow = !supportsModernBackgroundProcessors(); - const offered$ = disableSwitcher$.pipe( - map((switcherDisabled) => !switcherDisabled && supported), + const offered$ = combineLatest([disableSwitcher$, supported$]).pipe( + map(([switcherDisabled, supported]) => !switcherDisabled && supported), + ); + // A new object for each refusal, so the same one twice is shown twice. + const refusal$ = new BehaviorSubject( + undefined, ); return { audioOptions$: scope.behavior(options$(mediaDevices.audioInput.available$)), @@ -142,8 +164,10 @@ function buildDeviceBehaviors( ), selectVideoButtonOption$: constant(mediaDevices.videoInput.select), backgroundEffect$: scope.behavior( - backgroundEffectSetting.value$.pipe( - map((raw) => (supported ? serializeEffect(parseEffect(raw)) : "none")), + combineLatest([backgroundEffectSetting.value$, supported$]).pipe( + map(([raw, supported]) => + supported ? serializeEffect(parseEffect(raw)) : "none", + ), ), ), selectBackgroundEffect$: scope.behavior( @@ -158,7 +182,11 @@ function buildDeviceBehaviors( ), ), ), - backgroundEffects$: constant(backgroundEffectChoices()), + backgroundEffects$: scope.behavior( + addedBackgrounds.added$.pipe( + map((added) => backgroundEffectChoices(added ?? [])), + ), + ), backgroundEffectNotice$: scope.behavior( offered$.pipe( map((offered) => @@ -167,9 +195,61 @@ function buildDeviceBehaviors( ), ), backgroundEffectSettling$: scope.behavior( - backgroundEffectSettling$.pipe(distinctUntilChanged()), + backgroundEffectsState$.pipe( + map(({ wait }) => wait !== undefined), + distinctUntilChanged(), + ), false, ), + // Kept and offered, not put on: that waits for the user to choose it. + addBackgroundImage$: scope.behavior( + combineLatest([offered$, addedBackgrounds.added$]).pipe( + map(([offered, added]) => + offered && (added?.length ?? 0) < maxAddedBackgrounds + ? (file: File): void => { + refusal$.next(undefined); + addedBackgrounds.add(file).catch((e) => { + logger.warn("Could not keep that background", e); + refusal$.next({ + reason: e instanceof UnusableImage ? e.reason : "not-kept", + }); + }); + } + : undefined, + ), + ), + ), + backgroundImageRefusal$: refusal$, + // None while muted: the call mutes its track rather than releasing it. + cameraTrack$: scope.behavior( + camera$.pipe( + switchMap((camera) => + camera + ? camera.muted$.pipe(map((muted) => (muted ? null : camera))) + : of(null), + ), + ), + null, + ), + // Never the one in force: taking it away would leave nothing chosen. + removeBackgroundEffect$: scope.behavior( + offered$.pipe( + map((offered) => + offered + ? (optionId: string): void => { + const effect = parseEffect(optionId); + if (effect.kind !== "added") return; + if (backgroundEffectSetting.getValue() === optionId) return; + addedBackgrounds + .remove(effect.id) + .catch((e) => + logger.warn("Could not remove that background", e), + ); + } + : undefined, + ), + ), + ), }; } @@ -180,8 +260,8 @@ function buildDeviceBehaviors( * @param callModel - The root CallViewModel; provides layout, grid mode, reactions, etc. * @param muteStates - Audio and video mute state + toggles. * @param mediaDevices - Available and selected input devices. - * @param backgroundEffectSettling$ - Whether the first effect chosen is still - * being prepared. + * @param backgroundEffectsState$ - The background pipeline's state: whether + * effects run here, and the wait for the first. * @param reactionIdentifier - The local user's reaction identifier string, or * undefined when reactions are not supported (hides the reaction button). * @param options - `showControls`: whether the call controls should be shown. @@ -192,7 +272,7 @@ export function createCallFooterViewModel( callModel: CallViewModel, muteStates: MuteStates, mediaDevices: MediaDevices, - backgroundEffectSettling$: Observable, + backgroundEffectsState$: Observable, reactionIdentifier: string | undefined, options: { showControls: boolean; header: HeaderStyle }, ): ViewModel { @@ -211,7 +291,8 @@ export function createCallFooterViewModel( scope, mediaDevices, disableDeviceSwitcher$, - backgroundEffectSettling$, + backgroundEffectsState$, + callModel.camera$, ), // candidat to move into the FooterViewModel showFooter$: callModel.showFooter$, @@ -274,8 +355,9 @@ export function createCallFooterViewModel( * @param scope - ObservableScope that bounds the lifetime of derived behaviors. * @param muteStates - Audio and video mute state + toggles. * @param mediaDevices - Available and selected input devices. - * @param backgroundEffectSettling$ - Whether the first effect chosen is still - * being prepared. + * @param backgroundEffectsState$ - The background pipeline's state: whether + * effects run here, and the wait for the first. + * @param camera$ - The lobby's camera, for the camera menu's preview. * @param openSettings - Callback to open the settings modal, or undefined. * @param hangup - Callback to leave/cancel, or undefined (hides the button). * @param showLogo - Whether to show the Element Call logo. @@ -284,7 +366,8 @@ export function createLobbyFooterViewModel( scope: ObservableScope, muteStates: MuteStates, mediaDevices: MediaDevices, - backgroundEffectSettling$: Observable, + backgroundEffectsState$: Observable, + camera$: Behavior, openSettings: (() => void) | undefined, hangup: (() => void) | undefined, showLogo: boolean, @@ -332,19 +415,29 @@ export function createLobbyFooterViewModel( scope, mediaDevices, constant(false), - backgroundEffectSettling$, + backgroundEffectsState$, + camera$, ), }; } -function backgroundEffectChoices(): BackgroundEffectGridItem[] { +function backgroundEffectChoices( + added: AddedBackground[], +): BackgroundEffectGridItem[] { return [ - { id: serializeEffect({ kind: "none" }), kind: "none" }, - { id: serializeEffect({ kind: "blur" }), kind: "blur" }, + { id: serializeEffect({ kind: "none" }), kind: "none", removable: false }, + { id: serializeEffect({ kind: "blur" }), kind: "blur", removable: false }, ...shippedBackgrounds.map((background) => ({ id: serializeEffect({ kind: "shipped", id: background.id }), kind: "image" as const, imageUrl: background.imagePath, + removable: false, + })), + ...added.map((background) => ({ + id: serializeEffect({ kind: "added", id: background.id }), + kind: "image" as const, + imageUrl: background.url, + removable: true, })), ]; } diff --git a/src/components/MediaMuteAndSwitchButton.module.css b/src/components/MediaMuteAndSwitchButton.module.css index 6e860db3c..2e8c60f3f 100644 --- a/src/components/MediaMuteAndSwitchButton.module.css +++ b/src/components/MediaMuteAndSwitchButton.module.css @@ -93,3 +93,39 @@ Please see LICENSE in the repository root for full details. flex-shrink: 0; color: var(--cpd-color-icon-secondary); } + +.refusal { + margin: var(--cpd-space-2x) var(--cpd-space-4x) 0; +} + +.selfPreview { + position: relative; + display: flex; + align-items: center; + justify-content: center; + flex-shrink: 0; + aspect-ratio: 16 / 9; + margin-block-end: var(--cpd-space-2x); + overflow: clip; + border-start-start-radius: var(--cpd-space-3x); + border-start-end-radius: var(--cpd-space-3x); + background: var(--cpd-color-bg-subtle-secondary); + color: var(--cpd-color-icon-secondary); +} + +.selfPreview > video { + position: absolute; + inset: 0; + inline-size: 100%; + block-size: 100%; + object-fit: cover; + /* Rounded itself too: a video can be drawn on a layer of its own, which the + frame's rounded clip doesn't always reach. */ + border-radius: inherit; +} + +/* Up to the menu's frame, over the padding at its top. */ +.selfPreviewPinned, +.deviceListLeadsWithPreview { + margin-block-start: calc(-1 * var(--cpd-space-5x)); +} diff --git a/src/components/MediaMuteAndSwitchButton.stories.tsx b/src/components/MediaMuteAndSwitchButton.stories.tsx index ba9163234..1d9447773 100644 --- a/src/components/MediaMuteAndSwitchButton.stories.tsx +++ b/src/components/MediaMuteAndSwitchButton.stories.tsx @@ -12,6 +12,7 @@ import type { Meta, StoryObj } from "@storybook/react-vite"; import { MediaMuteAndSwitchButton } from "./MediaMuteAndSwitchButton"; import styles from "./MediaMuteAndSwitchButton.module.css"; import { type BackgroundEffectGridItem } from "./BackgroundEffectGrid"; +import { SelfPreview } from "./SelfPreview"; import { shippedBackgrounds } from "../tracks/backgroundEffectCatalog"; import meterStyles from "./MicrophoneLevelMeter.module.css"; import { MediaDevicesContext } from "../MediaDevicesContext"; @@ -60,6 +61,50 @@ const WithAMicrophone: FC<{ children: ReactNode }> = ({ children }) => { return <>{children}; }; +/** A camera for the preview: a figure against a room, drawn to a stream. */ +const FakeCamera: FC = () => { + const [stream, setStream] = useState(); + useEffect(() => { + const canvas = document.createElement("canvas"); + canvas.width = 640; + canvas.height = 360; + const context = canvas.getContext("2d")!; + const draw = (): void => { + context.fillStyle = "#6b7f8e"; + context.fillRect(0, 0, 640, 360); + context.fillStyle = "#c9a27e"; + context.beginPath(); + context.arc(320, 150, 62, 0, 2 * Math.PI); + context.fill(); + context.fillStyle = "#34495e"; + context.beginPath(); + context.ellipse(320, 380, 150, 140, 0, Math.PI, 2 * Math.PI); + context.fill(); + }; + draw(); + // Redrawn, so the stream keeps delivering frames. + const timer = setInterval(draw, 200); + const captured = canvas.captureStream(10); + setStream(captured); + return (): void => { + clearInterval(timer); + captured.getTracks().forEach((track) => track.stop()); + }; + }, []); + return ( +