mirror of
https://github.com/vector-im/element-call.git
synced 2026-11-13 01:12:29 +00:00
Compare commits
63
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d5f72f464d | ||
|
|
96949a57da | ||
|
|
2e05ef77a0 | ||
|
|
73826ec4f9 | ||
|
|
1135c02e6f | ||
|
|
9dc425b5f8 | ||
|
|
c1b94b62dd | ||
|
|
fa6efd9dad | ||
|
|
6906404a64 | ||
|
|
98e8343d0b | ||
|
|
88659d6498 | ||
|
|
704cbd65a6 | ||
|
|
5f59283b4a | ||
|
|
07dc08cd1c | ||
|
|
60d6029184 | ||
|
|
f41bc2589a | ||
|
|
ceeb26b271 | ||
|
|
e822f967c1 | ||
|
|
ec46b04b71 | ||
|
|
4f3dc9af75 | ||
|
|
cfd8312e46 | ||
|
|
38edc47f64 | ||
|
|
ab4f0f0f77 | ||
|
|
9411a56afc | ||
|
|
a5374791e4 | ||
|
|
a0601876d8 | ||
|
|
eef86fc06e | ||
|
|
cd5c367ce9 | ||
|
|
190902eec0 | ||
|
|
88f748ec21 | ||
|
|
9639effd28 | ||
|
|
f2f67019b4 | ||
|
|
51fac690f4 | ||
|
|
797a86815b | ||
|
|
1ae57e9647 | ||
|
|
4b05ab503e | ||
|
|
ea977cecd9 | ||
|
|
aadbda39b1 | ||
|
|
53c1e4c30f | ||
|
|
6b56c7e683 | ||
|
|
fb4cbc72e2 | ||
|
|
bac9811829 | ||
|
|
a2cfc2af93 | ||
|
|
dcb2b9e8f6 | ||
|
|
373b1e0fc0 | ||
|
|
1bc372b587 | ||
|
|
82ba51ea6a | ||
|
|
07aaa326d2 | ||
|
|
e9cffb60dd | ||
|
|
741dab6dbb | ||
|
|
4f184f504d | ||
|
|
ecf2cbc71e | ||
|
|
fb38a1e6e0 | ||
|
|
06f07fb808 | ||
|
|
eee101ffba | ||
|
|
4e779ac4f5 | ||
|
|
7fb222ce60 | ||
|
|
5fdb920ea8 | ||
|
|
813dd4a49e | ||
|
|
21b54501de | ||
|
|
3d57920556 | ||
|
|
11e143c8ab | ||
|
|
d68e761f7a |
@@ -0,0 +1,35 @@
|
||||
---
|
||||
name: handoff
|
||||
description: Implement the first cut of a task and stop for a direction check before the quality pass.
|
||||
argument-hint: <the task>
|
||||
disable-model-invocation: true
|
||||
---
|
||||
|
||||
Task: $ARGUMENTS
|
||||
|
||||
Stop after the first implementation and hand it back before the quality pass.
|
||||
|
||||
## Before the hand-off
|
||||
|
||||
- Run the narrowest check that rules out an obviously broken hand-off: the test
|
||||
file for the touched code, `tsc` on its own, a single story. Not the full
|
||||
`pnpm lint`, not the whole suite.
|
||||
- If a wider check is needed before feedback can be useful, keep it to the touched
|
||||
code and say why it was needed.
|
||||
- Leave wider refactors, extra tests and documentation polish out. They are
|
||||
follow-up work, not part of the first hand-off.
|
||||
|
||||
## The hand-off
|
||||
|
||||
Stop and ask whether the direction is right. The message says:
|
||||
|
||||
- what was built and where, in a few lines,
|
||||
- which check ran and what it returned, verbatim if it failed,
|
||||
- what was left out on purpose,
|
||||
- the decisions that could have gone another way.
|
||||
|
||||
Do not continue past this point on your own.
|
||||
|
||||
## After the user confirms
|
||||
|
||||
The skill ends here. Continue with the quality pass as usual.
|
||||
@@ -14,7 +14,7 @@ permissions:
|
||||
jobs:
|
||||
prevent-blocked:
|
||||
name: Prevent blocked
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
pull-requests: read
|
||||
steps:
|
||||
|
||||
@@ -17,7 +17,7 @@ env:
|
||||
jobs:
|
||||
build_and_deploy:
|
||||
name: Build & publish docker
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
contents: write # required to upload release asset
|
||||
packages: write # needed for publishing packages to GHCR
|
||||
|
||||
@@ -29,7 +29,7 @@ on:
|
||||
jobs:
|
||||
build:
|
||||
name: Build Element Call
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
# Deploy main branch to call.element.dev after build completes
|
||||
if: github.ref == 'refs/heads/main'
|
||||
needs: build_full_element_call
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Deploy to call.element.dev
|
||||
uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410 # v6
|
||||
@@ -92,7 +92,7 @@ jobs:
|
||||
build_storybook:
|
||||
name: Build Storybook
|
||||
if: contains(github.event.pull_request.labels.*.name, 'storybook build')
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
|
||||
@@ -13,7 +13,7 @@ permissions:
|
||||
|
||||
jobs:
|
||||
pr-changelog-label:
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: yogevbd/enforce-label-action@a3c219da6b8fa73f6ba62b68ff09c469b3a1c024 # 2.2.2
|
||||
with:
|
||||
|
||||
@@ -13,7 +13,7 @@ jobs:
|
||||
name: Review
|
||||
# Only run on labelling and skip forks.
|
||||
if: github.event.label.name == 'Claude-Review' && github.event.pull_request.head.repo.full_name == github.repository
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
||||
|
||||
@@ -32,7 +32,7 @@ on:
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
deployments: write
|
||||
environment: Netlify
|
||||
|
||||
@@ -10,7 +10,7 @@ on:
|
||||
jobs:
|
||||
lint:
|
||||
name: Lint, format & type check
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
|
||||
@@ -17,7 +17,7 @@ jobs:
|
||||
github.event.workflow_run.conclusion == 'success' &&
|
||||
github.event.workflow_run.event == 'pull_request' &&
|
||||
github.event.workflow_run.head_repository.full_name == github.repository
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-24.04
|
||||
outputs:
|
||||
pr_number: ${{ steps.prdetails.outputs.pr_id }}
|
||||
pr_data_json: ${{ steps.prdetails.outputs.data }}
|
||||
|
||||
@@ -17,7 +17,7 @@ permissions: {}
|
||||
jobs:
|
||||
versioning:
|
||||
name: Versioning
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-24.04
|
||||
outputs:
|
||||
DRY_RUN: ${{ steps.dry_run.outputs.DRY_RUN }}
|
||||
PREFIXED_VERSION: ${{ steps.prefixed_version.outputs.PREFIXED_VERSION }}
|
||||
@@ -59,7 +59,7 @@ jobs:
|
||||
publish_npm:
|
||||
needs: versioning
|
||||
name: Publish NPM
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-24.04
|
||||
outputs:
|
||||
ARTIFACT_VERSION: ${{ steps.artifact_version.outputs.ARTIFACT_VERSION }}
|
||||
permissions:
|
||||
@@ -111,7 +111,7 @@ jobs:
|
||||
needs: [versioning, publish_npm]
|
||||
if: ${{ needs.versioning.outputs.DRY_RUN == 'false' }}
|
||||
name: Update release notes
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-24.04
|
||||
# Shared with publish-embedded-packages.yaml, which appends to the same release: two
|
||||
# appends at once would each write back the body without the other's section
|
||||
concurrency:
|
||||
|
||||
@@ -14,7 +14,7 @@ on:
|
||||
jobs:
|
||||
versioning:
|
||||
name: Versioning
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-24.04
|
||||
outputs:
|
||||
DRY_RUN: ${{ steps.dry_run.outputs.DRY_RUN }}
|
||||
PREFIXED_VERSION: ${{ steps.prefixed_version.outputs.PREFIXED_VERSION }}
|
||||
@@ -76,7 +76,7 @@ jobs:
|
||||
needs: [build_element_call, versioning]
|
||||
if: always()
|
||||
name: Publish tarball
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
contents: write # required to upload release asset
|
||||
steps:
|
||||
@@ -107,7 +107,7 @@ jobs:
|
||||
needs: [build_element_call, versioning]
|
||||
if: always()
|
||||
name: Publish NPM
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-24.04
|
||||
outputs:
|
||||
ARTIFACT_VERSION: ${{ steps.artifact_version.outputs.ARTIFACT_VERSION }}
|
||||
permissions:
|
||||
@@ -152,7 +152,7 @@ jobs:
|
||||
needs: [build_element_call, versioning]
|
||||
if: always()
|
||||
name: Publish Android AAR
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-24.04
|
||||
outputs:
|
||||
ARTIFACT_VERSION: ${{ steps.artifact_version.outputs.ARTIFACT_VERSION }}
|
||||
permissions:
|
||||
@@ -212,7 +212,7 @@ jobs:
|
||||
needs: [build_element_call, versioning]
|
||||
if: always()
|
||||
name: Publish SwiftPM Library
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-24.04
|
||||
outputs:
|
||||
ARTIFACT_VERSION: ${{ steps.artifact_version.outputs.ARTIFACT_VERSION }}
|
||||
permissions:
|
||||
@@ -282,7 +282,7 @@ jobs:
|
||||
needs: [versioning, publish_npm, publish_android, publish_ios]
|
||||
if: always()
|
||||
name: Update release notes
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-24.04
|
||||
# Shared with publish-component.yaml, which appends to the same release: two appends
|
||||
# at once would each write back the body without the other's section
|
||||
concurrency:
|
||||
|
||||
@@ -23,7 +23,7 @@ jobs:
|
||||
needs: build_element_call
|
||||
if: always()
|
||||
name: Publish tarball
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
contents: write # required to upload release asset
|
||||
packages: write
|
||||
@@ -68,7 +68,7 @@ jobs:
|
||||
add_docker_release_note:
|
||||
needs: publish_docker
|
||||
name: Add docker release note
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Add release note
|
||||
uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2
|
||||
|
||||
@@ -25,7 +25,7 @@ on:
|
||||
jobs:
|
||||
vitest:
|
||||
name: Run unit tests
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
@@ -71,10 +71,27 @@ jobs:
|
||||
flags: unittests
|
||||
fail_ci_if_error: true
|
||||
playwright:
|
||||
name: Run end-to-end tests
|
||||
name: Run end-to-end tests (${{ matrix.part }})
|
||||
if: ${{ !inputs.disable_e2e_tests }}
|
||||
timeout-minutes: 60
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-24.04
|
||||
# We manually shard the tests across runners. Playwright's own shards cut
|
||||
# the list of tests into equal counts in file order, which would leave the
|
||||
# widget suite, the slowest tests by far, together at the end.
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- part: chromium-1
|
||||
args: --project=chromium --shard=1/2
|
||||
- part: chromium-2
|
||||
args: --project=chromium --shard=2/2
|
||||
- part: widget-1
|
||||
args: --project=widget --shard=1/2
|
||||
- part: widget-2
|
||||
args: --project=widget --shard=2/2
|
||||
- part: firefox-mobile
|
||||
args: --project=firefox --project=mobile
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
with:
|
||||
@@ -107,17 +124,64 @@ jobs:
|
||||
run: pnpm playwright install --with-deps
|
||||
- name: Run backend components
|
||||
run: |
|
||||
# Make sure the directory for the embedded build exists as Docker might
|
||||
# create it with wrong access rights otherwise.
|
||||
mkdir -p embedded/web/dist
|
||||
docker compose -f docker-compose-dev.yml -f docker-compose-playwright.yml pull
|
||||
docker compose -f docker-compose-dev.yml -f docker-compose-playwright.yml up -d
|
||||
docker ps
|
||||
- name: Run Playwright tests
|
||||
env:
|
||||
USE_DOCKER: 1
|
||||
run: pnpm exec playwright test
|
||||
ARGS: ${{ matrix.args }}
|
||||
# Unquoted on purpose: the part's arguments are several words
|
||||
run: pnpm exec playwright test $ARGS
|
||||
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
if: ${{ !cancelled() }}
|
||||
with:
|
||||
name: blob-report-${{ matrix.part }}
|
||||
path: blob-report
|
||||
if-no-files-found: error
|
||||
# Only read by the merge below.
|
||||
retention-days: 1
|
||||
|
||||
e2e_report:
|
||||
# Combines the blobs into a single HTML report and blocks PRs on failure.
|
||||
name: Run end-to-end tests
|
||||
needs: playwright
|
||||
if: ${{ always() && !inputs.disable_e2e_tests }}
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
|
||||
with:
|
||||
# Inside a called workflow, github.repository is the caller so we need to make sure to still check out the EC repo here.
|
||||
repository: ${{ inputs.matrix-js-sdk-sha && 'element-hq/element-call' || github.repository }}
|
||||
persist-credentials: false
|
||||
- name: Enable Corepack
|
||||
run: corepack enable
|
||||
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
|
||||
with:
|
||||
cache: "pnpm"
|
||||
node-version-file: ".node-version"
|
||||
- name: Install dependencies
|
||||
run: scripts/layered.sh --ignore-pnpmfile
|
||||
env:
|
||||
PR_HEAD_REPO_OWNER: ${{ github.event.pull_request.head.repo.owner.login }}
|
||||
JS_SDK_GITHUB_BASE_REF: ${{ inputs.matrix-js-sdk-sha }}
|
||||
- name: Download the parts' reports
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
|
||||
with:
|
||||
pattern: blob-report-*
|
||||
path: all-blob-reports
|
||||
merge-multiple: true
|
||||
- name: Merge into one HTML report
|
||||
run: pnpm exec playwright merge-reports --reporter html ./all-blob-reports
|
||||
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: html-report
|
||||
path: playwright-report
|
||||
if-no-files-found: error
|
||||
retention-days: 4
|
||||
- name: Fail on a failed part
|
||||
if: ${{ needs.playwright.result != 'success' }}
|
||||
run: exit 1
|
||||
|
||||
@@ -7,7 +7,7 @@ on:
|
||||
|
||||
jobs:
|
||||
download:
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ on:
|
||||
|
||||
jobs:
|
||||
upload:
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ permissions: {}
|
||||
jobs:
|
||||
zizmor:
|
||||
name: Run zizmor 🌈
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
security-events: write # Required for upload-sarif (used by zizmor-action) to upload SARIF files.
|
||||
steps:
|
||||
|
||||
@@ -44,3 +44,7 @@ storybook-static
|
||||
|
||||
# Nested checkout made by scripts/layered.sh
|
||||
/matrix-js-sdk/
|
||||
|
||||
# Claude
|
||||
/.claude/*
|
||||
!/.claude/skills/
|
||||
|
||||
@@ -16,19 +16,35 @@ host's page. It is the MatrixRTC reference implementation.
|
||||
`@media`, global `i18next` or the `widget` global — take it from a provider.
|
||||
- The change works standalone, as a widget and as a component. Say what you checked.
|
||||
- Every gate below is green before you push.
|
||||
- Hand off after the first implementation, before the quality pass. Then commit on
|
||||
the user's word where the setup allows it, or hand them the message.
|
||||
|
||||
## Read before you
|
||||
|
||||
| … | … |
|
||||
| -------------------------------- | ---------------------------------------------------------- |
|
||||
| start a task of any size | [docs/agents/workflow.md](docs/agents/workflow.md) |
|
||||
| touch call logic or a view model | [docs/agents/architecture.md](docs/agents/architecture.md) |
|
||||
| write any code | [docs/agents/code-style.md](docs/agents/code-style.md) |
|
||||
| write a test, story or e2e spec | [docs/agents/testing.md](docs/agents/testing.md) |
|
||||
| open a PR | [CONTRIBUTING.md](CONTRIBUTING.md) |
|
||||
| implement from a feature spec | [FEATURES_SPEC/AGENTS.md](FEATURES_SPEC/AGENTS.md) |
|
||||
|
||||
## Scratch files live in `agent-workspace/`
|
||||
|
||||
Git-ignored. One kebab-case subfolder per task, matching the branch topic:
|
||||
`agent-workspace/<slug>/`.
|
||||
|
||||
| File | Purpose |
|
||||
| --------------------------- | -------------------------------------------------------------------------------- |
|
||||
| `plan.md` | The concrete approach — files, symbols, edit order — once direction is confirmed |
|
||||
| `commit-msg.txt` | `git commit -F agent-workspace/<slug>/commit-msg.txt` |
|
||||
| `pr-body.md` | `gh pr create --body-file agent-workspace/<slug>/pr-body.md` |
|
||||
| `implementation-summary.md` | What was built, decisions, trade-offs |
|
||||
| `NN-prompt.md` | Raw prompt text, numbered, when worth keeping |
|
||||
|
||||
- Nothing here is durable. The folder is ignored, so anything worth keeping is
|
||||
promoted into the PR body or a doc before the task ends.
|
||||
- Never write scratch files to the repo root. Stray `load_test_summary.md` and
|
||||
`config.json_` files are what this folder prevents.
|
||||
- Start `pr-body.md` from `.github/PULL_REQUEST_TEMPLATE.md` and fill every
|
||||
section.
|
||||
|
||||
## Gates
|
||||
|
||||
@@ -42,6 +58,11 @@ pnpm i18n:check
|
||||
`knip` fails on dead code; a file deliberately inert ahead of its consumer goes in
|
||||
`knip.ts` `ignoreFiles` with a reason.
|
||||
|
||||
Ready means every gate is green and the change is covered at the layers
|
||||
[testing.md](docs/agents/testing.md) asks for. Read the diff against
|
||||
[code-style.md](docs/agents/code-style.md) first. Re-run the whole checklist after
|
||||
any fix.
|
||||
|
||||
## A good PR
|
||||
|
||||
- One slice, under ~400 changed lines, green on its own, linked to its issue.
|
||||
|
||||
@@ -221,119 +221,68 @@ See also:
|
||||
|
||||
- [Developing with linked packages](./docs/linking.md)
|
||||
|
||||
#### Element Call as a component (experimental)
|
||||
#### Element Call as a component
|
||||
|
||||
Element Call can also be embedded directly into another React application
|
||||
rather than being loaded in an iframe as a widget. `pnpm build:component`
|
||||
builds it as a library into `component/dist` (the bundle, its stylesheet and
|
||||
type declarations), and
|
||||
Element Call can be rendered inside another React app instead of an iframe.
|
||||
The host supplies its own Matrix client, and the call stays inside the element
|
||||
it is mounted in.
|
||||
|
||||
```sh
|
||||
pnpm dev:component
|
||||
```
|
||||
|
||||
serves a harness on port 3001 that stands in for such an application: it signs
|
||||
in twice against the development backend and shows two calls side by side, in
|
||||
resizable boxes, with page furniture of its own around them. Use it to see how
|
||||
Element Call behaves when it does not own the page — the size it is given,
|
||||
whether it stays inside its container, and what it says to its host, which is
|
||||
logged along the bottom. The harness is served with the same development
|
||||
certificate as the app, so unless the development CA is trusted, the browser
|
||||
needs a certificate exception for `https://localhost:3001` as well (see the
|
||||
note under [Backend](#backend)). It reads the same `public/config.json` as
|
||||
`pnpm dev` if one exists, and runs with Element Call's defaults otherwise.
|
||||
|
||||
The call lays itself out for the size of the element it is mounted in, not the
|
||||
window: a host that shrinks the container to a corner of its page gets the
|
||||
picture-in-picture layout, just as a host that shrank the whole iframe used to.
|
||||
The breakpoints in the stylesheets the component uses are
|
||||
`@container element-call` queries against its root element for the same reason;
|
||||
for the standalone app the root is the page, so they mean what the media queries
|
||||
they replaced did. (The standalone-only views, such as the home and login pages,
|
||||
still use plain media queries, since the component never shows them.)
|
||||
|
||||
The component's stylesheet is confined to the element it is mounted in: the
|
||||
build rewrites every selector so that it matches only Element Call's root or
|
||||
what is inside it, with `html`, `body` and `:root` standing for that root (see
|
||||
`component/build/scopeStylesToRoot.ts`). A host's own page keeps its styles,
|
||||
and Element Call brings its own fonts and design tokens along.
|
||||
|
||||
The component speaks every language the app does. English is bundled in; the
|
||||
other locales are split into chunks the host's bundler loads the first time
|
||||
they are needed. It starts in the browser's language, and follows the host's
|
||||
own language setting through the `language` prop (`supportedLanguages` lists
|
||||
the tags it accepts, and anything else falls back to its base language or to
|
||||
English). The `theme` prop works the same way and takes the same values as the
|
||||
widget's `theme` URL parameter: `light`, `dark`, `light-high-contrast` or
|
||||
`dark-high-contrast`. Both can change while a call is running without
|
||||
disturbing it.
|
||||
|
||||
A host must call and await `initializeElementCall(config)` once before
|
||||
rendering the component: it loads the `Intl` polyfills, applies the
|
||||
deployment-wide `config.json`-style configuration and sets up translations.
|
||||
The component itself takes the host's `client` and the `roomId` to call in, an
|
||||
`intent` saying what the user asked for (which decides whether to show the
|
||||
lobby, ring, and so on), an optional `config` overriding what the intent
|
||||
implies, and an optional `hostBridge` through which Element Call tells the host
|
||||
that the user has joined or hung up, that it wants to stay on screen, and so
|
||||
on. The host makes its own requests (`join`, `hangUp`, `setDeviceMute`) through
|
||||
the handle exposed on `ref`. The full API is documented in the type declarations
|
||||
(`component/index.tsx` and `component/host.ts`).
|
||||
|
||||
A few things differ from the widget on purpose: the component draws a solid
|
||||
background rather than a gradient unless told otherwise, never offers to edit
|
||||
the user's profile (the account is the host's), scopes its keyboard shortcuts to
|
||||
its own root element so that several instances can share a page, and only shows
|
||||
its own post-call and error screens when the host has not supplied a `close()`
|
||||
callback; with one, it asks the host to unmount it instead. The
|
||||
[global JS controls](./docs/controls.md) on `window` are unchanged and remain
|
||||
page-wide, so with several instances on one page they apply to all of them.
|
||||
|
||||
Each release publishes the package to npm, with the same version as the release:
|
||||
##### Install
|
||||
|
||||
```sh
|
||||
npm install @element-hq/element-call-component
|
||||
```
|
||||
|
||||
To use an unreleased commit, a host can instead install it as a git dependency
|
||||
on the `component` directory of this repository,
|
||||
Releases publish the package with the same version as Element Call. For an
|
||||
unreleased commit, depend on this repository directly:
|
||||
|
||||
```json
|
||||
"@element-hq/element-call-component": "github:element-hq/element-call#main&path:/component"
|
||||
```
|
||||
|
||||
whose `prepare` script runs the build on install. That build needs pnpm (via
|
||||
Corepack) on the host's machine, runs a full `pnpm install` of this repository,
|
||||
since it inherits the `--max-old-space-size` setting of
|
||||
the app build; the host's pnpm also has to allow it to run at all
|
||||
(`allowBuilds` in its `pnpm-workspace.yaml`). Note that `component/` is a pnpm
|
||||
project of its own for this reason, so pnpm commands run from inside that
|
||||
directory target it rather than the repository; run them from the repository
|
||||
root. The host imports the component from
|
||||
`@element-hq/element-call-component` and the stylesheet from
|
||||
`@element-hq/element-call-component/style.css`, and has to provide `react`,
|
||||
`react-dom`, `matrix-js-sdk`, `@vector-im/compound-web` and
|
||||
`@vector-im/compound-design-tokens` itself, since the bundle leaves them
|
||||
external.
|
||||
Compound's stylesheet are the host's to load too: the component shares
|
||||
the host's copy of Compound, (required for example for tooltips)
|
||||
This builds on install, so the host needs pnpm (via Corepack) and must allow
|
||||
the build (`allowBuilds` in its `pnpm-workspace.yaml`).
|
||||
|
||||
The component is large, and a host will usually load it lazily, only once a
|
||||
call is shown. Everything a host needs in order to talk about a call before
|
||||
then is also exported
|
||||
from `@element-hq/element-call-component/api` (a small entry point that does not
|
||||
load the component).
|
||||
##### Use
|
||||
|
||||
- props and the types they are made of
|
||||
- the handle
|
||||
- the host bridge
|
||||
- `UserIntent`
|
||||
- `HeaderStyle`
|
||||
- `BackgroundStyle` enums
|
||||
- `configurationForIntent`, the defaults each intent implies
|
||||
Import from it wherever a value such as
|
||||
`BackgroundStyle.Solid` is needed on a path that must stay light; the main
|
||||
entry point re-exports all of it too.
|
||||
1. Provide the externals yourself: `react`, `react-dom`, `matrix-js-sdk`,
|
||||
`@vector-im/compound-web`, `@vector-im/compound-design-tokens`, and
|
||||
Compound's stylesheet.
|
||||
2. Import `@element-hq/element-call-component/style.css`.
|
||||
3. Await `initializeElementCall(config)` once before rendering.
|
||||
4. Render the component with `client`, `roomId` and an `intent`; optionally
|
||||
pass `config`, `hostBridge`, `language` and `theme` (both can change
|
||||
mid-call). Control it through the `ref` handle (`join`, `hangUp`,
|
||||
`setDeviceMute`).
|
||||
|
||||
Load the component lazily. Types, enums and `configurationForIntent` are
|
||||
available without it from `@element-hq/element-call-component/api`. The full
|
||||
API is in `component/index.tsx`, `component/api.ts` and `component/host.ts`.
|
||||
|
||||
##### How it differs from the widget
|
||||
|
||||
- It lays out for its container, not the window: a small container gets the
|
||||
picture-in-picture layout.
|
||||
- Its styles and keyboard shortcuts are scoped to its root, so the host page is
|
||||
untouched and several instances can share a page. The
|
||||
[global JS controls](./docs/controls.md) stay page-wide.
|
||||
- It draws a solid background by default and never offers profile editing.
|
||||
- If the host bridge implements `close()`, it asks to be unmounted instead of
|
||||
showing its own post-call and error screens.
|
||||
- Sounds, images and the MediaPipe runtime are separate files the host's
|
||||
bundler copies (webpack 5, Vite, Rollup); MediaPipe is fetched only when a
|
||||
background effect is first used.
|
||||
|
||||
##### Develop
|
||||
|
||||
- `pnpm build:component` builds the library into `component/dist`.
|
||||
- `pnpm dev:component` serves a test host on `https://localhost:3001`: two
|
||||
calls side by side in resizable boxes, with host messages logged at the
|
||||
bottom. It uses the development backend, `public/config.json` if present,
|
||||
and needs the same certificate exception as `pnpm dev` (see
|
||||
[Backend](#backend)).
|
||||
- Run pnpm commands from the repository root; `component/` is its own pnpm
|
||||
project.
|
||||
|
||||
### Backend
|
||||
|
||||
|
||||
@@ -26,8 +26,7 @@
|
||||
"features": {
|
||||
"feature_element_call_video_rooms": true,
|
||||
"feature_video_rooms": true,
|
||||
"feature_group_calls": true,
|
||||
"feature_release_announcement": false
|
||||
"feature_group_calls": true
|
||||
},
|
||||
"default_federate": true,
|
||||
"default_theme": "light",
|
||||
@@ -40,7 +39,10 @@
|
||||
},
|
||||
"setting_defaults": {
|
||||
"breadcrumbs": true,
|
||||
"feature_group_calls": true
|
||||
"feature_group_calls": true,
|
||||
"releaseAnnouncementData": {
|
||||
"room_list_section": true
|
||||
}
|
||||
},
|
||||
"jitsi": {
|
||||
"preferred_domain": "meet.element.io"
|
||||
|
||||
@@ -26,8 +26,7 @@
|
||||
"features": {
|
||||
"feature_element_call_video_rooms": true,
|
||||
"feature_video_rooms": true,
|
||||
"feature_group_calls": true,
|
||||
"feature_release_announcement": false
|
||||
"feature_group_calls": true
|
||||
},
|
||||
"default_federate": true,
|
||||
"default_theme": "light",
|
||||
@@ -40,7 +39,10 @@
|
||||
},
|
||||
"setting_defaults": {
|
||||
"breadcrumbs": true,
|
||||
"feature_group_calls": true
|
||||
"feature_group_calls": true,
|
||||
"releaseAnnouncementData": {
|
||||
"room_list_section": true
|
||||
}
|
||||
},
|
||||
"jitsi": {
|
||||
"preferred_domain": "meet.element.io"
|
||||
|
||||
@@ -19,7 +19,12 @@ services:
|
||||
- ./backend/playwright_homeserver-othersite.yaml:/data/cfg/homeserver.yaml:Z
|
||||
element-web:
|
||||
# Pin to a SHA so that upstream cannot break our tests. Renovate handles regular updates.
|
||||
image: ghcr.io/element-hq/element-web:develop@sha256:a79801206c628a912d57fa6f355d5c5ab9c6c40c2d1b6208d4192678fb6e80c7
|
||||
image: ghcr.io/element-hq/element-web:develop@sha256:72791f0b2541208f8f7edd9c6b5a7eaf88accd8ed1f0c4db1bc62bfe18eb90fe
|
||||
element-web-1:
|
||||
# Pin to a SHA so that upstream cannot break our tests. Renovate handles regular updates.
|
||||
image: ghcr.io/element-hq/element-web:develop@sha256:a79801206c628a912d57fa6f355d5c5ab9c6c40c2d1b6208d4192678fb6e80c7
|
||||
image: ghcr.io/element-hq/element-web:develop@sha256:72791f0b2541208f8f7edd9c6b5a7eaf88accd8ed1f0c4db1bc62bfe18eb90fe
|
||||
volumes:
|
||||
# The other site opens the embedded package under test in place of the
|
||||
# copy its image bundles (the first site opens the dev server's app.
|
||||
# Built by the Playwright run (see build.setup.ts).
|
||||
- ./embedded/web/dist:/app/widgets/element-call:ro,Z
|
||||
|
||||
+5
-1
@@ -14,7 +14,11 @@ This folder contains documentation for setup, usage, and development of Element
|
||||
|
||||
Entry point: [AGENTS.md](../AGENTS.md).
|
||||
|
||||
- [Agent workflow](./agents/workflow.md)
|
||||
- [Architecture](./agents/architecture.md)
|
||||
- [Code style](./agents/code-style.md)
|
||||
- [Testing](./agents/testing.md)
|
||||
|
||||
Opt-in skills in [`.claude/skills/`](../.claude/skills/):
|
||||
|
||||
- `/handoff <task>` — get the first implementation back for a direction check
|
||||
before lint, the full suite and polish.
|
||||
|
||||
@@ -34,6 +34,10 @@ translations. The browser is Playwright's, so a fresh clone needs
|
||||
|
||||
## End-to-end
|
||||
|
||||
The Element Web instance `HOST1` opens the app the dev server serves. The instance
|
||||
`HOST2` (the other site) opens the embedded package as a host ships it. This package
|
||||
is built by the `build-embedded` setup project and mounted in `docker-compose-playwright.yml`.
|
||||
|
||||
- `playwright/*.spec.ts` — standalone, on Chromium and Firefox.
|
||||
- `playwright/widget/` — widget mode, where cross-context bugs surface.
|
||||
- `playwright/component/` — the component in a host page, via the harness on port
|
||||
@@ -48,6 +52,8 @@ Test what a user observes: the peer sees the change, it survives a reconnect, it
|
||||
right after a reload. Reuse `playwright/spa-helpers.ts`,
|
||||
`playwright/widget/test-helpers.ts`, `playwright/fixtures/`.
|
||||
|
||||
To mark new release announcements as viewed, add them in `backend/ew.test*.config.json`.
|
||||
|
||||
```sh
|
||||
pnpm test # unit + storybook
|
||||
pnpm backend # Synapse + LiveKit, required for e2e
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
# Agent workflow
|
||||
|
||||
## Scratch files live in `agent-workspace/`
|
||||
|
||||
Git-ignored. One kebab-case subfolder per task, matching the branch topic:
|
||||
`agent-workspace/<slug>/`.
|
||||
|
||||
| File | Purpose |
|
||||
| --------------------------- | -------------------------------------------------------------------------------- |
|
||||
| `plan.md` | The concrete approach — files, symbols, edit order — once direction is confirmed |
|
||||
| `commit-msg.txt` | `git commit -F agent-workspace/<slug>/commit-msg.txt` |
|
||||
| `pr-body.md` | `gh pr create --body-file agent-workspace/<slug>/pr-body.md` |
|
||||
| `implementation-summary.md` | What was built, decisions, trade-offs |
|
||||
| `NN-prompt.md` | Raw prompt text, numbered, when worth keeping |
|
||||
|
||||
- Nothing here is durable. The folder is ignored, so anything worth keeping is
|
||||
promoted into the PR body, a doc, or a feature spec before the task ends.
|
||||
- Never write scratch files to the repo root. Stray `load_test_summary.md` and
|
||||
`config.json_` files are what this folder prevents.
|
||||
- Start `pr-body.md` from `.github/PULL_REQUEST_TEMPLATE.md` and fill every
|
||||
section.
|
||||
- Commit subjects are plain imperative English. No conventional-commits prefixes.
|
||||
- A `plan.md` and a feature spec sit at different altitudes; neither replaces the
|
||||
other. A spec in `FEATURES_SPEC/` is durable and deliberately abstract —
|
||||
behaviour, decisions, acceptance criteria — so the feature can be rebuilt against
|
||||
a `main` nobody has seen yet. A plan is one slice of it landing on today's
|
||||
`main`: the paths, symbols and edit order the spec's decisions and criteria must
|
||||
not name. `FEATURES_SPEC/AGENTS.md` wins wherever the two genuinely overlap.
|
||||
|
||||
## Hand off before the quality pass
|
||||
|
||||
- Implement the change, run the narrowest check that rules out an obviously broken
|
||||
handoff, then stop and ask whether the direction is right.
|
||||
- Full `pnpm lint`, the whole suite, coverage and benchmarks come after the
|
||||
direction is confirmed.
|
||||
- Wider refactors, extra tests and documentation polish are follow-up work, not
|
||||
part of the first handoff.
|
||||
- If a check is needed before feedback, keep it to the touched code and say why.
|
||||
|
||||
## Commit and PR readiness
|
||||
|
||||
- Commit once the user confirms direction, or asks for one. Not before. Where the
|
||||
setup has no git identity, a sandbox included, write the message to
|
||||
`commit-msg.txt` and hand it over instead.
|
||||
- Ready means every gate in [AGENTS.md](../../AGENTS.md#gates) is green and the
|
||||
change is covered at the layers [testing.md](./testing.md) asks for. Read the diff
|
||||
against [code-style.md](./code-style.md) first.
|
||||
- Re-run the whole checklist after any fix. Only on green, either way.
|
||||
- Once a human has started reviewing, fix forward — never force-push a
|
||||
regeneration over a review in progress.
|
||||
+13
-1
@@ -3,7 +3,7 @@
|
||||
"user_menu": "User menu"
|
||||
},
|
||||
"action": {
|
||||
"blur_background": "Blur background",
|
||||
"add_background_image": "Add image",
|
||||
"close": "Close",
|
||||
"copy_link": "Copy link",
|
||||
"edit": "Edit",
|
||||
@@ -23,6 +23,13 @@
|
||||
"upload_file": "Upload file"
|
||||
},
|
||||
"analytics_notice": "By participating in this beta, you consent to the collection of anonymous data, which we use to improve the product. You can find more information about which data we track in our <2>Privacy Policy</2> and our <6>Cookie Policy</6>.",
|
||||
"background_effects": {
|
||||
"blur": "Blur",
|
||||
"none": "None",
|
||||
"numbered": "Background {{n}}",
|
||||
"slow": "Background effects run slowly on this platform, which may cause your video to stutter.",
|
||||
"unavailable": "Background effects are not supported on this platform."
|
||||
},
|
||||
"call_ended_view": {
|
||||
"create_account_button": "Create account",
|
||||
"create_account_prompt": "<0>Why not finish by setting up a password to keep your account?</0><1>You'll be able to keep your name and set an avatar for use on future calls</1>",
|
||||
@@ -92,6 +99,10 @@
|
||||
},
|
||||
"disconnected_banner": "Connectivity to the server has been lost.",
|
||||
"error": {
|
||||
"background_animated": "Animated images cannot be used as a background",
|
||||
"background_not_an_image": "That file is not a supported image",
|
||||
"background_not_kept": "That background could not be saved",
|
||||
"background_undecodable": "That image could not be opened",
|
||||
"call_is_not_supported": "Call is not supported",
|
||||
"call_not_found": "Call not found",
|
||||
"call_not_found_description": "<0>That link doesn't appear to belong to any existing call. Check that you have the right link, or <2>create a new one</2>.</0>",
|
||||
@@ -212,6 +223,7 @@
|
||||
"auto_gain_control_label": "Automatic gain control",
|
||||
"background_blur_header": "Background",
|
||||
"background_blur_label": "Blur the background of the video",
|
||||
"background_effects_header": "Background effects",
|
||||
"bitrate_label": "Bitrate",
|
||||
"blur_not_supported_by_browser": "(Background blur is not supported by this device.)",
|
||||
"camera_header": "Camera quality",
|
||||
|
||||
+2
-1
@@ -97,6 +97,7 @@
|
||||
"copy-to-clipboard": "^3.3.3",
|
||||
"eslint-plugin-element-call": "link:eslint",
|
||||
"eslint-plugin-storybook": "^10.3.6",
|
||||
"fake-indexeddb": "^6.2.5",
|
||||
"fetch-mock": "11.1.5",
|
||||
"i18next": "^26.0.0",
|
||||
"i18next-browser-languagedetector": "^8.0.0",
|
||||
@@ -106,7 +107,7 @@
|
||||
"livekit-client": "^2.18.1",
|
||||
"lodash-es": "^4.17.21",
|
||||
"loglevel": "^1.9.1",
|
||||
"matrix-js-sdk": "github:matrix-org/matrix-js-sdk#a9ef09fbf2f1644c8663d0aeaf6a921cd16e7b01",
|
||||
"matrix-js-sdk": "github:matrix-org/matrix-js-sdk#dc8610e796c92553c658300a4882dde961a0678a",
|
||||
"matrix-widget-api": "^1.18.0",
|
||||
"node-stdlib-browser": "^1.3.1",
|
||||
"normalize.css": "^8.0.1",
|
||||
|
||||
+33
-21
@@ -26,6 +26,19 @@ process.env.NODE_EXTRA_CA_CERTS = join(
|
||||
"backend/dev_tls_local-ca.crt",
|
||||
);
|
||||
|
||||
const desktopChrome = {
|
||||
...devices["Desktop Chrome"],
|
||||
permissions: ["clipboard-write", "clipboard-read", "microphone", "camera"],
|
||||
ignoreHTTPSErrors: true,
|
||||
launchOptions: {
|
||||
args: [
|
||||
"--use-fake-ui-for-media-stream",
|
||||
"--use-fake-device-for-media-stream",
|
||||
"--mute-audio",
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* See https://playwright.dev/docs/test-configuration.
|
||||
*/
|
||||
@@ -43,11 +56,11 @@ export default defineConfig({
|
||||
* Reporter to use. See https://playwright.dev/docs/test-reporters
|
||||
*
|
||||
* On CI, `list` prints a line per test as it finishes, which keeps the job
|
||||
* log moving; `github` annotates failures on the PR; `html` is uploaded as
|
||||
* an artifact and must never try to open a browser there.
|
||||
* log moving; `github` annotates failures on the PR; the `blob` format is
|
||||
* uploaded by sharded CI runs and combined into a HTML report by the workflow.
|
||||
*/
|
||||
reporter: process.env.CI
|
||||
? [["list"], ["github"], ["html", { open: "never" }]]
|
||||
? [["list"], ["github"], ["blob"]]
|
||||
: [["list"], ["html"]],
|
||||
/* Stop a broken run early on CI instead of waiting out every retry. */
|
||||
maxFailures: process.env.CI ? 5 : undefined,
|
||||
@@ -63,29 +76,28 @@ export default defineConfig({
|
||||
},
|
||||
/* Configure projects for major browsers */
|
||||
projects: [
|
||||
{
|
||||
// Builds the embedded package into the directory the other site's
|
||||
// Element Web container serves it from (see docker-compose-playwright.yml).
|
||||
name: "build-embedded",
|
||||
testMatch: "**/widget/embedded/build.setup.ts",
|
||||
},
|
||||
{
|
||||
name: "chromium",
|
||||
testIgnore: "**/mobile/**",
|
||||
use: {
|
||||
...devices["Desktop Chrome"],
|
||||
permissions: [
|
||||
"clipboard-write",
|
||||
"clipboard-read",
|
||||
"microphone",
|
||||
"camera",
|
||||
],
|
||||
ignoreHTTPSErrors: true,
|
||||
launchOptions: {
|
||||
args: [
|
||||
"--use-fake-ui-for-media-stream",
|
||||
"--use-fake-device-for-media-stream",
|
||||
"--mute-audio",
|
||||
],
|
||||
},
|
||||
},
|
||||
testIgnore: ["**/mobile/**", "**/widget/**"],
|
||||
use: desktopChrome,
|
||||
},
|
||||
{
|
||||
// Element Call as a widget in Element Web, on Chromium. This uses its
|
||||
// own project so that it can be sharded on CI.
|
||||
name: "widget",
|
||||
dependencies: ["build-embedded"],
|
||||
testMatch: "**/widget/**/*.@(spec|test).ts",
|
||||
use: desktopChrome,
|
||||
},
|
||||
{
|
||||
name: "firefox",
|
||||
dependencies: ["build-embedded"],
|
||||
testIgnore: "**/mobile/**",
|
||||
use: {
|
||||
...devices["Desktop Firefox"],
|
||||
|
||||
@@ -0,0 +1,952 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { expect, test, type Browser, type Page } from "@playwright/test";
|
||||
|
||||
import { SpaHelpers } from "./spa-helpers.ts";
|
||||
import {
|
||||
averageColour,
|
||||
cameraColour,
|
||||
type Colour,
|
||||
distance,
|
||||
} from "./utils/colour.ts";
|
||||
|
||||
// Background effects need WebGL2, and headless Firefox on a CI runner has none,
|
||||
// so it rightly offers none of them.
|
||||
test.skip(
|
||||
({ browserName }) => browserName === "firefox",
|
||||
"Background effects need WebGL2, which headless Firefox on CI does not have",
|
||||
);
|
||||
|
||||
test.describe("background effects", () => {
|
||||
test("pre-join preview shows the chosen effect", async ({ page }) => {
|
||||
test.slow();
|
||||
await page.goto("/");
|
||||
await SpaHelpers.createCall(page, "Effect user", "Background effects");
|
||||
const preview = page.locator("video").first();
|
||||
const camera = await cameraColour(preview);
|
||||
|
||||
await page.getByRole("button", { name: "Camera", exact: true }).click();
|
||||
const tile = page
|
||||
.getByRole("group", { name: "Background effects" })
|
||||
.getByRole("menuitemradio", { name: "Background 1" });
|
||||
await tile.click();
|
||||
await expect(tile).toHaveAttribute("aria-checked", "true");
|
||||
|
||||
// The fake camera shows no person, so the whole frame is background: the
|
||||
// preview takes on the picture's colours and leaves the camera's.
|
||||
const picture = await averageColour(tile.locator("img"));
|
||||
// Long enough for the first effect of a session to be built.
|
||||
await expect
|
||||
.poll(async () => distance(await averageColour(preview), picture), {
|
||||
timeout: 60_000,
|
||||
})
|
||||
.toBeLessThan(distance(camera, picture) / 2);
|
||||
});
|
||||
});
|
||||
|
||||
test.describe("no effect", () => {
|
||||
test("draws nothing, even after blurring", async ({ page }) => {
|
||||
test.setTimeout(120_000);
|
||||
await countDraws(page);
|
||||
await page.goto("/");
|
||||
await SpaHelpers.createCall(page, "Idle user", "No effect after blur");
|
||||
|
||||
await page.getByRole("button", { name: "Camera", exact: true }).click();
|
||||
const effects = page.getByRole("group", { name: "Background effects" });
|
||||
await effects.getByRole("menuitemradio", { name: "Blur" }).click();
|
||||
// Long enough for the first effect of a session to be built.
|
||||
await expect
|
||||
.poll(async () => drawsPerSecond(page), { timeout: 60_000 })
|
||||
.toBeGreaterThan(0);
|
||||
const blurred = await drawsPerSecond(page);
|
||||
|
||||
await effects.getByRole("menuitemradio", { name: "None" }).click();
|
||||
// A frame already on its way may still be drawn.
|
||||
await expect.poll(async () => drawsPerSecond(page)).toBe(0);
|
||||
test.info().annotations.push({
|
||||
type: "WebGL draws per second",
|
||||
description: `blurred ${blurred}, none 0`,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
/** Counts every WebGL draw on the page, the segmenter's and the effect's. */
|
||||
async function countDraws(page: Page): Promise<void> {
|
||||
await page.addInitScript(() => {
|
||||
const counter = { draws: 0 };
|
||||
(window as unknown as { webglDraws: typeof counter }).webglDraws = counter;
|
||||
for (const name of ["drawArrays", "drawElements"] as const) {
|
||||
const draw = WebGL2RenderingContext.prototype[name] as (
|
||||
...args: unknown[]
|
||||
) => void;
|
||||
WebGL2RenderingContext.prototype[name] = function (
|
||||
this: WebGL2RenderingContext,
|
||||
...args: unknown[]
|
||||
): void {
|
||||
counter.draws += 1;
|
||||
draw.apply(this, args);
|
||||
} as never;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/** WebGL draws over the next second. */
|
||||
async function drawsPerSecond(page: Page): Promise<number> {
|
||||
return page.evaluate(async () => {
|
||||
const counter = (window as unknown as { webglDraws: { draws: number } })
|
||||
.webglDraws;
|
||||
const before = counter.draws;
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
return counter.draws - before;
|
||||
});
|
||||
}
|
||||
|
||||
test.describe("the camera menu's preview", () => {
|
||||
test("shows the camera with the effect in force", async ({ page }) => {
|
||||
test.setTimeout(120_000);
|
||||
await page.goto("/");
|
||||
await SpaHelpers.createCall(page, "Preview user", "Preview in the menu");
|
||||
const camera = await cameraColour(page.locator("video").first());
|
||||
|
||||
await page.getByRole("button", { name: "Camera", exact: true }).click();
|
||||
const preview = page.getByRole("menu").locator("video");
|
||||
await expect(preview).toBeVisible();
|
||||
await expect
|
||||
.poll(async () => distance(await averageColour(preview), camera), {
|
||||
timeout: 20_000,
|
||||
})
|
||||
.toBeLessThan(30);
|
||||
|
||||
const tile = page.getByRole("menuitemradio", { name: "Background 1" });
|
||||
await tile.click();
|
||||
const picture = await averageColour(tile.locator("img"));
|
||||
await expect
|
||||
.poll(async () => distance(await averageColour(preview), picture), {
|
||||
timeout: 60_000,
|
||||
})
|
||||
.toBeLessThan(distance(camera, picture) / 2);
|
||||
});
|
||||
|
||||
test("shows the call's own camera once joined", async ({ page }) => {
|
||||
test.setTimeout(120_000);
|
||||
await page.goto("/");
|
||||
await SpaHelpers.createCall(
|
||||
page,
|
||||
"Call preview user",
|
||||
"Preview in the call's menu",
|
||||
true,
|
||||
);
|
||||
await expect(page.getByTestId("incall_videomute")).toBeEnabled({
|
||||
timeout: 10_000,
|
||||
});
|
||||
const camera = await cameraColour(page.locator("video").first());
|
||||
|
||||
await page.getByRole("button", { name: "Camera", exact: true }).click();
|
||||
const preview = page.getByRole("menu").locator("video");
|
||||
await expect(preview).toBeVisible();
|
||||
await expect
|
||||
.poll(async () => distance(await averageColour(preview), camera), {
|
||||
timeout: 20_000,
|
||||
})
|
||||
.toBeLessThan(30);
|
||||
|
||||
const tile = page.getByRole("menuitemradio", { name: "Background 1" });
|
||||
await tile.click();
|
||||
const picture = await averageColour(tile.locator("img"));
|
||||
await expect
|
||||
.poll(async () => distance(await averageColour(preview), picture), {
|
||||
timeout: 60_000,
|
||||
})
|
||||
.toBeLessThan(distance(camera, picture) / 2);
|
||||
});
|
||||
});
|
||||
|
||||
test.describe("background effects section", () => {
|
||||
test("section stays within the call area at every size", async ({ page }) => {
|
||||
test.setTimeout(120_000);
|
||||
await page.goto("/");
|
||||
await SpaHelpers.createCall(page, "Sizes user", "Background effect sizes");
|
||||
|
||||
for (const size of [
|
||||
{ width: 1280, height: 800 },
|
||||
{ width: 800, height: 600 },
|
||||
{ width: 600, height: 420 },
|
||||
{ width: 480, height: 360 },
|
||||
{ width: 360, height: 640 },
|
||||
]) {
|
||||
await page.setViewportSize(size);
|
||||
await page.getByRole("button", { name: "Camera", exact: true }).click();
|
||||
const menu = page.getByRole("menu");
|
||||
await expect(menu).toBeVisible();
|
||||
|
||||
const box = (await menu.boundingBox())!;
|
||||
const where = `at ${size.width}x${size.height}`;
|
||||
expect(box.x, where).toBeGreaterThanOrEqual(0);
|
||||
expect(box.y, where).toBeGreaterThanOrEqual(0);
|
||||
expect(box.x + box.width, where).toBeLessThanOrEqual(size.width);
|
||||
expect(box.y + box.height, where).toBeLessThanOrEqual(size.height);
|
||||
|
||||
const last = page
|
||||
.getByRole("group", { name: "Background effects" })
|
||||
.getByRole("menuitemradio")
|
||||
.last();
|
||||
await last.scrollIntoViewIfNeeded();
|
||||
await expect(last, where).toBeInViewport({ ratio: 1 });
|
||||
await last.click();
|
||||
await expect(last, where).toHaveAttribute("aria-checked", "true");
|
||||
|
||||
// Back to no effect for the next size, and close.
|
||||
await page
|
||||
.getByRole("group", { name: "Background effects" })
|
||||
.getByRole("menuitemradio", { name: "None" })
|
||||
.click();
|
||||
await page.keyboard.press("Escape");
|
||||
await expect(menu).toBeHidden();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test.describe("a pipeline that fails to build", () => {
|
||||
test("offers no effects for the session, and keeps the choice", async ({
|
||||
page,
|
||||
}) => {
|
||||
test.setTimeout(120_000);
|
||||
await page.goto("/");
|
||||
await SpaHelpers.createCall(page, "Failing user", "Pipeline fails");
|
||||
const preview = page.locator("video").first();
|
||||
const camera = await cameraColour(preview);
|
||||
await page.route("**/*.tflite*", async (route) => route.abort());
|
||||
|
||||
await page.getByRole("button", { name: "Camera", exact: true }).click();
|
||||
const tile = page.getByRole("menuitemradio", { name: "Background 1" });
|
||||
await tile.click();
|
||||
await expect(
|
||||
page.getByText("Background effects are not supported on this platform."),
|
||||
).toBeVisible({ timeout: 30_000 });
|
||||
await expect(tile).toBeDisabled();
|
||||
await page.keyboard.press("Escape");
|
||||
// The camera and microphone carry on without it.
|
||||
expect(distance(await cameraColour(preview), camera)).toBeLessThan(30);
|
||||
await expect(page.getByTestId("incall_mute")).toHaveAttribute(
|
||||
"aria-checked",
|
||||
"true",
|
||||
);
|
||||
|
||||
await page.unroute("**/*.tflite*");
|
||||
await page.reload();
|
||||
await expect(page.getByTestId("lobby_joinCall")).toBeVisible();
|
||||
await page.getByRole("button", { name: "Camera", exact: true }).click();
|
||||
await expect(tile).toHaveAttribute("aria-checked", "true");
|
||||
const picture = await averageColour(tile.locator("img"));
|
||||
await expect
|
||||
.poll(async () => distance(await averageColour(preview), picture), {
|
||||
timeout: 60_000,
|
||||
})
|
||||
.toBeLessThan(distance(camera, picture) / 2);
|
||||
});
|
||||
});
|
||||
|
||||
test.describe("the chosen effect", () => {
|
||||
test("chosen effect survives leaving and rejoining", async ({ page }) => {
|
||||
test.setTimeout(120_000);
|
||||
await page.goto("/");
|
||||
await SpaHelpers.createCall(page, "Returning user", "Effect remembered");
|
||||
const { camera, picture } = await chooseAndWear(page, "Background 1");
|
||||
await page.getByTestId("lobby_joinCall").click();
|
||||
await page.getByTestId("incall_leave").click();
|
||||
await expect(page.getByRole("heading")).toContainText(
|
||||
"your call has ended",
|
||||
);
|
||||
|
||||
await page.goto("/");
|
||||
await page.getByTestId("home_callName").fill("Effect remembered again");
|
||||
await page.getByTestId("home_go").click();
|
||||
await expect(page.getByTestId("lobby_joinCall")).toBeVisible();
|
||||
await page.getByRole("button", { name: "Camera", exact: true }).click();
|
||||
const tile = page
|
||||
.getByRole("group", { name: "Background effects" })
|
||||
.getByRole("menuitemradio", { name: "Background 1" });
|
||||
await expect(tile).toHaveAttribute("aria-checked", "true");
|
||||
const preview = page.locator("video").first();
|
||||
await expect
|
||||
.poll(async () => distance(await averageColour(preview), picture), {
|
||||
timeout: 60_000,
|
||||
})
|
||||
.toBeLessThan(distance(camera, picture) / 2);
|
||||
});
|
||||
|
||||
test("second session still shows the existing error", async ({ page }) => {
|
||||
test.setTimeout(120_000);
|
||||
await page.goto("/");
|
||||
await SpaHelpers.createCall(page, "Two tabs", "Effect in two tabs");
|
||||
await chooseAndWear(page, "Background 1");
|
||||
|
||||
const second = await page.context().newPage();
|
||||
await second.goto("/");
|
||||
await expect(
|
||||
page.getByRole("heading", { name: "Opened in another tab" }),
|
||||
).toBeVisible();
|
||||
});
|
||||
});
|
||||
|
||||
/** Chooses an effect in the lobby, and waits until the preview wears it. */
|
||||
async function chooseAndWear(
|
||||
page: Page,
|
||||
name: string,
|
||||
): Promise<{ camera: Colour; picture: Colour }> {
|
||||
const preview = page.locator("video").first();
|
||||
const camera = await cameraColour(preview);
|
||||
await page.getByRole("button", { name: "Camera", exact: true }).click();
|
||||
const tile = page.getByRole("menuitemradio", { name });
|
||||
await tile.click();
|
||||
const picture = await averageColour(tile.locator("img"));
|
||||
await expect
|
||||
.poll(async () => distance(await averageColour(preview), picture), {
|
||||
timeout: 60_000,
|
||||
})
|
||||
.toBeLessThan(distance(camera, picture) / 2);
|
||||
await page.keyboard.press("Escape");
|
||||
return { camera, picture };
|
||||
}
|
||||
|
||||
test.describe("a background of one's own", () => {
|
||||
test("adds an image from the device", async ({ page }) => {
|
||||
test.setTimeout(120_000);
|
||||
await page.goto("/");
|
||||
await SpaHelpers.createCall(page, "Adding user", "Own background");
|
||||
const preview = page.locator("video").first();
|
||||
const camera = await cameraColour(preview);
|
||||
const red = await redImage(page);
|
||||
|
||||
await addImage(page, red);
|
||||
const mine = page
|
||||
.getByRole("group", { name: "Background effects" })
|
||||
.getByRole("menuitemradio", { name: "Background 3" });
|
||||
await expect(mine).toBeVisible();
|
||||
await expect(mine).toHaveAttribute("aria-checked", "false");
|
||||
|
||||
await mine.click();
|
||||
await expect
|
||||
.poll(async () => distance(await averageColour(preview), RED), {
|
||||
timeout: 60_000,
|
||||
})
|
||||
.toBeLessThan(distance(camera, RED) / 2);
|
||||
});
|
||||
|
||||
test("adding an image makes no upload", async ({ page }) => {
|
||||
test.setTimeout(120_000);
|
||||
await page.goto("/");
|
||||
await SpaHelpers.createCall(page, "Private user", "Own background kept");
|
||||
const preview = page.locator("video").first();
|
||||
const camera = await cameraColour(preview);
|
||||
const red = await redImage(page);
|
||||
const sent: { request: string; body: Buffer | null }[] = [];
|
||||
page.on("request", (request) =>
|
||||
sent.push({
|
||||
request: `${request.method()} ${request.url()}`,
|
||||
body: request.postDataBuffer(),
|
||||
}),
|
||||
);
|
||||
|
||||
await addImage(page, red);
|
||||
await page
|
||||
.getByRole("group", { name: "Background effects" })
|
||||
.getByRole("menuitemradio", { name: "Background 3" })
|
||||
.click();
|
||||
// Kept, chosen and in force: anything that would send it has had its turn.
|
||||
await expect
|
||||
.poll(async () => distance(await averageColour(preview), RED), {
|
||||
timeout: 60_000,
|
||||
})
|
||||
.toBeLessThan(distance(camera, RED) / 2);
|
||||
// The file as given, and the copy kept, which is encoded anew.
|
||||
const samples = [red, await keptBytes(page)].map((bytes) =>
|
||||
bytes.subarray(bytes.length - 64),
|
||||
);
|
||||
const carriesIt = sent.filter(
|
||||
({ request, body }) =>
|
||||
request.includes("/_matrix/media/") ||
|
||||
samples.some((sample) => body?.includes(sample)),
|
||||
);
|
||||
expect(carriesIt.map(({ request }) => request)).toEqual([]);
|
||||
});
|
||||
|
||||
test("added image survives rejoining and is gone after storage is cleared", async ({
|
||||
page,
|
||||
}) => {
|
||||
test.setTimeout(120_000);
|
||||
await page.goto("/");
|
||||
await SpaHelpers.createCall(page, "Keeping user", "Own background again");
|
||||
const preview = page.locator("video").first();
|
||||
const camera = await cameraColour(preview);
|
||||
const red = await redImage(page);
|
||||
await addImage(page, red);
|
||||
const mine = page
|
||||
.getByRole("group", { name: "Background effects" })
|
||||
.getByRole("menuitemradio", { name: "Background 3" });
|
||||
await mine.click();
|
||||
await page.keyboard.press("Escape");
|
||||
|
||||
await page.reload();
|
||||
await expect(page.getByTestId("lobby_joinCall")).toBeVisible();
|
||||
await page.getByRole("button", { name: "Camera", exact: true }).click();
|
||||
await expect(mine).toHaveAttribute("aria-checked", "true");
|
||||
await page.keyboard.press("Escape");
|
||||
await expect
|
||||
.poll(async () => distance(await averageColour(preview), RED), {
|
||||
timeout: 60_000,
|
||||
})
|
||||
.toBeLessThan(distance(camera, RED) / 2);
|
||||
|
||||
await page.evaluate(
|
||||
async () =>
|
||||
new Promise<void>((resolve, reject) => {
|
||||
const request = indexedDB.deleteDatabase(
|
||||
"element-call-background-images",
|
||||
);
|
||||
request.onsuccess = (): void => resolve();
|
||||
request.onerror = (): void => reject(request.error);
|
||||
request.onblocked = (): void => resolve();
|
||||
}),
|
||||
);
|
||||
await page.reload();
|
||||
await expect(page.getByTestId("lobby_joinCall")).toBeVisible();
|
||||
await page.getByRole("button", { name: "Camera", exact: true }).click();
|
||||
await expect(
|
||||
page.getByRole("group", { name: "Background effects" }),
|
||||
).toBeVisible();
|
||||
await expect(mine).toHaveCount(0);
|
||||
await page.keyboard.press("Escape");
|
||||
|
||||
await addImage(page, red);
|
||||
await expect(mine).toBeVisible();
|
||||
});
|
||||
|
||||
test("a removed image stays removed", async ({ page }) => {
|
||||
test.setTimeout(120_000);
|
||||
await page.goto("/");
|
||||
await SpaHelpers.createCall(
|
||||
page,
|
||||
"Removing user",
|
||||
"Own background removed",
|
||||
);
|
||||
await addImage(page, await redImage(page));
|
||||
const mine = page
|
||||
.getByRole("group", { name: "Background effects" })
|
||||
.getByRole("menuitemradio", { name: "Background 3" });
|
||||
await expect(mine).toHaveAttribute("aria-keyshortcuts", "Delete");
|
||||
await mine.focus();
|
||||
await page.keyboard.press("Delete");
|
||||
await expect(mine).toHaveCount(0);
|
||||
|
||||
await page.reload();
|
||||
await expect(page.getByTestId("lobby_joinCall")).toBeVisible();
|
||||
await page.getByRole("button", { name: "Camera", exact: true }).click();
|
||||
await expect(
|
||||
page.getByRole("group", { name: "Background effects" }),
|
||||
).toBeVisible();
|
||||
await expect(mine).toHaveCount(0);
|
||||
expect(await keptImages(page)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
test.describe("what can be added", () => {
|
||||
test("refuses an animated image", async ({ page }) => {
|
||||
test.setTimeout(120_000);
|
||||
await page.goto("/");
|
||||
await SpaHelpers.createCall(page, "Refused user", "Animated refused");
|
||||
await addImage(page, ANIMATED_GIF, {
|
||||
name: "moving.gif",
|
||||
type: "image/gif",
|
||||
});
|
||||
|
||||
await expect(
|
||||
page.getByText("Animated images cannot be used as a background"),
|
||||
).toBeVisible();
|
||||
const section = page.getByRole("group", { name: "Background effects" });
|
||||
await expect(
|
||||
section.getByRole("menuitemradio", { name: "Background 3" }),
|
||||
).toHaveCount(0);
|
||||
await expect(
|
||||
section.getByRole("menuitemradio", { name: "None" }),
|
||||
).toHaveAttribute("aria-checked", "true");
|
||||
expect(await keptImages(page)).toEqual([]);
|
||||
});
|
||||
|
||||
test("reduces an oversized image", async ({ page }) => {
|
||||
test.setTimeout(120_000);
|
||||
await page.goto("/");
|
||||
await SpaHelpers.createCall(page, "Large user", "Oversized reduced");
|
||||
await addImage(page, await pngImage(page, { width: 4000, height: 3000 }));
|
||||
|
||||
await expect(
|
||||
page
|
||||
.getByRole("group", { name: "Background effects" })
|
||||
.getByRole("menuitemradio", { name: "Background 3" }),
|
||||
).toBeVisible();
|
||||
await expect(page.getByRole("alert")).toHaveCount(0);
|
||||
const [kept] = await keptImages(page);
|
||||
expect(kept).toMatchObject({ width: 1920, height: 1440 });
|
||||
});
|
||||
|
||||
test("lays every image on an opaque ground", async ({ page }) => {
|
||||
test.setTimeout(120_000);
|
||||
await page.goto("/");
|
||||
await SpaHelpers.createCall(page, "Logo user", "Transparent made opaque");
|
||||
await addImage(
|
||||
page,
|
||||
await pngImage(page, { width: 320, height: 180, transparent: true }),
|
||||
);
|
||||
|
||||
await expect(
|
||||
page
|
||||
.getByRole("group", { name: "Background effects" })
|
||||
.getByRole("menuitemradio", { name: "Background 3" }),
|
||||
).toBeVisible();
|
||||
const [kept] = await keptImages(page);
|
||||
expect(kept.minAlpha).toBe(255);
|
||||
});
|
||||
});
|
||||
|
||||
/** Two frames of one pixel, red then blue. */
|
||||
const ANIMATED_GIF = Buffer.from(
|
||||
"R0lGODlhAQABAPEAAP8AAAAA/wAAAAAAACH/C05FVFNDQVBFMi4wAwEAAAAh+QQACgAAACwAAAAAAQABAAACAkQBACH5BAAKAAAALAAAAAABAAEAAAICTAEAOw==",
|
||||
"base64",
|
||||
);
|
||||
|
||||
/** A red PNG of the given size; transparent leaves its right half empty. */
|
||||
async function pngImage(
|
||||
page: Page,
|
||||
size: { width: number; height: number; transparent?: boolean },
|
||||
): Promise<Buffer> {
|
||||
const dataUrl = await page.evaluate(({ width, height, transparent }) => {
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = width;
|
||||
canvas.height = height;
|
||||
const context = canvas.getContext("2d")!;
|
||||
context.fillStyle = "rgb(255, 0, 0)";
|
||||
context.fillRect(0, 0, transparent ? width / 2 : width, height);
|
||||
return canvas.toDataURL("image/png");
|
||||
}, size);
|
||||
return Buffer.from(dataUrl.split(",")[1], "base64");
|
||||
}
|
||||
|
||||
/** The images the device keeps, as it keeps them. */
|
||||
async function keptImages(
|
||||
page: Page,
|
||||
): Promise<{ width: number; height: number; minAlpha: number }[]> {
|
||||
return page.evaluate(async () => {
|
||||
const db = await new Promise<IDBDatabase>((resolve, reject) => {
|
||||
const request = indexedDB.open("element-call-background-images");
|
||||
request.onsuccess = (): void => resolve(request.result);
|
||||
request.onerror = (): void => reject(request.error);
|
||||
});
|
||||
if (!db.objectStoreNames.contains("backgrounds")) return [];
|
||||
const kept = await new Promise<{ image: Blob }[]>((resolve, reject) => {
|
||||
const request = db
|
||||
.transaction("backgrounds")
|
||||
.objectStore("backgrounds")
|
||||
.getAll();
|
||||
request.onsuccess = (): void => resolve(request.result);
|
||||
request.onerror = (): void => reject(request.error);
|
||||
});
|
||||
db.close();
|
||||
return Promise.all(
|
||||
kept.map(async ({ image }) => {
|
||||
const bitmap = await createImageBitmap(image);
|
||||
const canvas = new OffscreenCanvas(bitmap.width, bitmap.height);
|
||||
const context = canvas.getContext("2d")!;
|
||||
context.drawImage(bitmap, 0, 0);
|
||||
const { data } = context.getImageData(
|
||||
0,
|
||||
0,
|
||||
bitmap.width,
|
||||
bitmap.height,
|
||||
);
|
||||
let minAlpha = 255;
|
||||
for (let i = 3; i < data.length; i += 4)
|
||||
minAlpha = Math.min(minAlpha, data[i]);
|
||||
return { width: bitmap.width, height: bitmap.height, minAlpha };
|
||||
}),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/** The bytes of the one image the device keeps. */
|
||||
async function keptBytes(page: Page): Promise<Buffer> {
|
||||
const base64 = await page.evaluate(
|
||||
async () =>
|
||||
new Promise<string>((resolve, reject) => {
|
||||
const open = indexedDB.open("element-call-background-images");
|
||||
open.onerror = (): void => reject(open.error);
|
||||
open.onsuccess = (): void => {
|
||||
const all = open.result
|
||||
.transaction("backgrounds")
|
||||
.objectStore("backgrounds")
|
||||
.getAll();
|
||||
all.onerror = (): void => reject(all.error);
|
||||
all.onsuccess = (): void => {
|
||||
const reader = new FileReader();
|
||||
reader.onload = (): void =>
|
||||
resolve((reader.result as string).split(",")[1]);
|
||||
reader.readAsDataURL((all.result[0] as { image: Blob }).image);
|
||||
};
|
||||
};
|
||||
}),
|
||||
);
|
||||
return Buffer.from(base64, "base64");
|
||||
}
|
||||
|
||||
const RED: Colour = [255, 0, 0];
|
||||
|
||||
/** A solid red PNG, unlike the camera and the shipped pictures. */
|
||||
async function redImage(page: Page): Promise<Buffer> {
|
||||
const dataUrl = await page.evaluate(() => {
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = 320;
|
||||
canvas.height = 180;
|
||||
const context = canvas.getContext("2d")!;
|
||||
context.fillStyle = "rgb(255, 0, 0)";
|
||||
context.fillRect(0, 0, canvas.width, canvas.height);
|
||||
return canvas.toDataURL("image/png");
|
||||
});
|
||||
return Buffer.from(dataUrl.split(",")[1], "base64");
|
||||
}
|
||||
|
||||
/** Adds an image from the camera menu's add tile, leaving the menu open. */
|
||||
async function addImage(
|
||||
page: Page,
|
||||
image: Buffer,
|
||||
{ name, type } = { name: "mine.png", type: "image/png" },
|
||||
): Promise<void> {
|
||||
if (!(await page.getByRole("menu").isVisible()))
|
||||
await page.getByRole("button", { name: "Camera", exact: true }).click();
|
||||
const chooser = page.waitForEvent("filechooser");
|
||||
await page.getByRole("menuitem", { name: "Add image" }).click();
|
||||
await (await chooser).setFiles({ name, mimeType: type, buffer: image });
|
||||
}
|
||||
|
||||
test.describe("joining with a background chosen", () => {
|
||||
test("publishes no frame of the room", async ({ browser }) => {
|
||||
// Two first builds of the pipeline, one of them held back.
|
||||
test.setTimeout(180_000);
|
||||
const hostContext = await browser.newContext();
|
||||
const host = await hostContext.newPage();
|
||||
await recordIncomingFrames(host);
|
||||
await host.goto("/");
|
||||
await SpaHelpers.createCall(host, "Host", "Join with a background", true);
|
||||
await expect(host.getByTestId("incall_videomute")).toBeEnabled({
|
||||
timeout: 10_000,
|
||||
});
|
||||
const inviteLink = await SpaHelpers.getCallInviteLink(host);
|
||||
await host.keyboard.press("Escape");
|
||||
|
||||
const guestContext = await browser.newContext();
|
||||
const guest = await guestContext.newPage();
|
||||
await guest.goto(inviteLink);
|
||||
await guest.getByTestId("joincall_displayName").fill("Guest");
|
||||
await guest.getByTestId("joincall_joincall").click();
|
||||
const preview = guest.locator("video").first();
|
||||
await expect(preview).toBeVisible();
|
||||
const camera = await cameraColour(preview);
|
||||
await guest.getByRole("button", { name: "Camera", exact: true }).click();
|
||||
const tile = guest.getByRole("menuitemradio", { name: "Background 1" });
|
||||
await tile.click();
|
||||
const picture = await averageColour(tile.locator("img"));
|
||||
await expect
|
||||
.poll(async () => distance(await averageColour(preview), picture), {
|
||||
timeout: 60_000,
|
||||
})
|
||||
.toBeLessThan(distance(camera, picture) / 2);
|
||||
// Were the pipeline built again for the call's track, it would take as
|
||||
// long as on the slower route. Routing bypasses the cache, so it waits.
|
||||
let heldBack = 0;
|
||||
await guest.route("**/*.tflite*", async (route) => {
|
||||
heldBack += 1;
|
||||
await new Promise((resolve) => setTimeout(resolve, 15_000));
|
||||
await route.continue();
|
||||
});
|
||||
await guest.keyboard.press("Escape");
|
||||
await guest.getByTestId("lobby_joinCall").click();
|
||||
|
||||
const guestTile = host
|
||||
.getByTestId("videoTile")
|
||||
.filter({ has: host.getByTestId("name_tag").getByText("Guest") });
|
||||
await expect(guestTile.locator("video")).toBeVisible({ timeout: 60_000 });
|
||||
// Visible can come before the stream is attached.
|
||||
let trackId: string | undefined;
|
||||
await expect
|
||||
.poll(async () => {
|
||||
trackId = await guestTile
|
||||
.locator("video")
|
||||
.evaluate(
|
||||
(video: HTMLVideoElement) =>
|
||||
(video.srcObject as MediaStream | null)?.getVideoTracks()[0]?.id,
|
||||
);
|
||||
return trackId;
|
||||
})
|
||||
.toBeDefined();
|
||||
await expectNoFrameOfTheRoom(host, trackId!, 0, camera, picture);
|
||||
// The join adopts the lobby's pipeline, so the model is never fetched again.
|
||||
expect(heldBack).toBe(0);
|
||||
|
||||
await hostContext.close();
|
||||
await guestContext.close();
|
||||
});
|
||||
});
|
||||
|
||||
test.describe("the first build", () => {
|
||||
test("survives the camera turned off and on while it runs", async ({
|
||||
page,
|
||||
}) => {
|
||||
test.setTimeout(120_000);
|
||||
const failures: string[] = [];
|
||||
page.on("console", (m) => {
|
||||
if (m.text().includes("Failed to attach video processor"))
|
||||
failures.push(m.text());
|
||||
});
|
||||
await page.goto("/");
|
||||
await SpaHelpers.createCall(page, "Toggling user", "Toggle while building");
|
||||
const preview = page.locator("video").first();
|
||||
const camera = await cameraColour(preview);
|
||||
|
||||
const held = await holdBackTheModel(page);
|
||||
await page.getByRole("button", { name: "Camera", exact: true }).click();
|
||||
const tile = page.getByRole("menuitemradio", { name: "Background 1" });
|
||||
await tile.click();
|
||||
const picture = await averageColour(tile.locator("img"));
|
||||
await page.keyboard.press("Escape");
|
||||
// New preview tracks, each attached while the first is still building.
|
||||
for (let i = 0; i < 2; i++) {
|
||||
await page.getByTestId("incall_videomute").click();
|
||||
await page.getByTestId("incall_videomute").click();
|
||||
}
|
||||
|
||||
await expect
|
||||
.poll(async () => distance(await averageColour(preview), picture), {
|
||||
timeout: 60_000,
|
||||
})
|
||||
.toBeLessThan(distance(camera, picture) / 2);
|
||||
expect(failures).toEqual([]);
|
||||
expect(held()).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
test.describe("turning the camera on with a background chosen", () => {
|
||||
test("publishes no frame of the room when the camera started off", async ({
|
||||
browser,
|
||||
}) => {
|
||||
test.setTimeout(120_000);
|
||||
const { host, guest, close } = await hostAndGuest(browser);
|
||||
const camera = await cameraColour(guest.locator("video").first());
|
||||
await guest.getByTestId("incall_videomute").click();
|
||||
await guest.getByTestId("lobby_joinCall").click();
|
||||
|
||||
const picture = await chooseBackground(guest);
|
||||
const held = await holdBackTheModel(guest);
|
||||
await guest.getByTestId("incall_videomute").click();
|
||||
|
||||
const trackId = await incomingTrackId(host);
|
||||
await expectNoFrameOfTheRoom(host, trackId, 0, camera, picture);
|
||||
expect(held()).toBeGreaterThan(0);
|
||||
await close();
|
||||
});
|
||||
|
||||
test("publishes no frame of the room when the camera was turned off", async ({
|
||||
browser,
|
||||
}) => {
|
||||
test.setTimeout(120_000);
|
||||
const { host, guest, close } = await hostAndGuest(browser);
|
||||
const camera = await cameraColour(guest.locator("video").first());
|
||||
await guest.getByTestId("lobby_joinCall").click();
|
||||
const trackId = await incomingTrackId(host);
|
||||
await expect
|
||||
.poll(async () => (await framesOf(host, trackId)).length, {
|
||||
timeout: 20_000,
|
||||
})
|
||||
.toBeGreaterThan(0);
|
||||
await guest.getByTestId("incall_videomute").click();
|
||||
|
||||
const picture = await chooseBackground(guest);
|
||||
const held = await holdBackTheModel(guest);
|
||||
const before = (await framesOf(host, trackId)).length;
|
||||
await guest.getByTestId("incall_videomute").click();
|
||||
|
||||
await expectNoFrameOfTheRoom(host, trackId, before, camera, picture);
|
||||
expect(held()).toBeGreaterThan(0);
|
||||
await close();
|
||||
});
|
||||
});
|
||||
|
||||
/** A host in a call, recording what it receives, and a guest in its lobby. */
|
||||
async function hostAndGuest(browser: Browser): Promise<{
|
||||
host: Page;
|
||||
guest: Page;
|
||||
close: () => Promise<void>;
|
||||
}> {
|
||||
const hostContext = await browser.newContext();
|
||||
const host = await hostContext.newPage();
|
||||
await recordIncomingFrames(host);
|
||||
await host.goto("/");
|
||||
await SpaHelpers.createCall(host, "Host", "Camera with a background", true);
|
||||
await expect(host.getByTestId("incall_videomute")).toBeEnabled({
|
||||
timeout: 10_000,
|
||||
});
|
||||
const inviteLink = await SpaHelpers.getCallInviteLink(host);
|
||||
await host.keyboard.press("Escape");
|
||||
|
||||
const guestContext = await browser.newContext();
|
||||
const guest = await guestContext.newPage();
|
||||
await guest.goto(inviteLink);
|
||||
await guest.getByTestId("joincall_displayName").fill("Guest");
|
||||
await guest.getByTestId("joincall_joincall").click();
|
||||
await expect(guest.locator("video").first()).toBeVisible();
|
||||
return {
|
||||
host,
|
||||
guest,
|
||||
close: async (): Promise<void> => {
|
||||
await hostContext.close();
|
||||
await guestContext.close();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Chooses the first shipped picture in the call, and answers its colour. */
|
||||
async function chooseBackground(guest: Page): Promise<Colour> {
|
||||
// The lobby has a camera button of its own, so wait for the call's.
|
||||
await expect(guest.getByTestId("incall_leave")).toBeVisible({
|
||||
timeout: 30_000,
|
||||
});
|
||||
await expect(guest.getByTestId("incall_videomute")).toBeEnabled({
|
||||
timeout: 10_000,
|
||||
});
|
||||
await guest.getByRole("button", { name: "Camera", exact: true }).click();
|
||||
const tile = guest.getByRole("menuitemradio", { name: "Background 1" });
|
||||
await tile.click();
|
||||
await expect(tile).toHaveAttribute("aria-checked", "true");
|
||||
const picture = await averageColour(tile.locator("img"));
|
||||
await guest.keyboard.press("Escape");
|
||||
return picture;
|
||||
}
|
||||
|
||||
/** Makes the pipeline's first build as slow as on the slower route. */
|
||||
/** Answers how often it held the model back, which a check should see. */
|
||||
async function holdBackTheModel(page: Page): Promise<() => number> {
|
||||
let held = 0;
|
||||
// Routing bypasses the cache, so it waits.
|
||||
await page.route("**/*.tflite*", async (route) => {
|
||||
held += 1;
|
||||
await new Promise((resolve) => setTimeout(resolve, 5_000));
|
||||
await route.continue();
|
||||
});
|
||||
return () => held;
|
||||
}
|
||||
|
||||
/** The id of the guest's camera track, as the host receives it. */
|
||||
async function incomingTrackId(host: Page): Promise<string> {
|
||||
const guestTile = host
|
||||
.getByTestId("videoTile")
|
||||
.filter({ has: host.getByTestId("name_tag").getByText("Guest") });
|
||||
await expect(guestTile.locator("video")).toBeVisible({ timeout: 60_000 });
|
||||
// Visible can come before the stream is attached.
|
||||
let trackId: string | undefined;
|
||||
await expect
|
||||
.poll(async () => {
|
||||
trackId = await guestTile
|
||||
.locator("video")
|
||||
.evaluate(
|
||||
(video: HTMLVideoElement) =>
|
||||
(video.srcObject as MediaStream | null)?.getVideoTracks()[0]?.id,
|
||||
);
|
||||
return trackId;
|
||||
})
|
||||
.toBeDefined();
|
||||
return trackId!;
|
||||
}
|
||||
|
||||
/** Waits for the picture to arrive, then finds no frame of the room since. */
|
||||
async function expectNoFrameOfTheRoom(
|
||||
host: Page,
|
||||
trackId: string,
|
||||
since: number,
|
||||
camera: Colour,
|
||||
picture: Colour,
|
||||
): Promise<void> {
|
||||
await expect
|
||||
.poll(
|
||||
async () => {
|
||||
const frames = (await framesOf(host, trackId)).slice(since);
|
||||
// A black frame, as a track starts, is nearer the picture than the
|
||||
// camera, but shows neither.
|
||||
return frames.some(
|
||||
(f) =>
|
||||
distance(f, picture) <
|
||||
Math.min(distance(f, camera), distance(f, [0, 0, 0])),
|
||||
);
|
||||
},
|
||||
{ timeout: 60_000 },
|
||||
)
|
||||
.toBe(true);
|
||||
const frames = (await framesOf(host, trackId)).slice(since);
|
||||
expect(
|
||||
frames.filter((f) => distance(f, camera) < distance(f, picture)),
|
||||
).toEqual([]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Records the mean colour of every frame of every video track the page
|
||||
* receives, from the moment it arrives: a tile may attach it later than its
|
||||
* first frame.
|
||||
*/
|
||||
async function recordIncomingFrames(page: Page): Promise<void> {
|
||||
await page.addInitScript(() => {
|
||||
const frames: Record<string, [number, number, number][]> = {};
|
||||
(window as unknown as { receivedFrames: typeof frames }).receivedFrames =
|
||||
frames;
|
||||
const record = (track: MediaStreamTrack): void => {
|
||||
if (track.kind !== "video" || frames[track.id]) return;
|
||||
const seen: [number, number, number][] = (frames[track.id] = []);
|
||||
const video = document.createElement("video");
|
||||
video.muted = true;
|
||||
video.srcObject = new MediaStream([track]);
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = canvas.height = 16;
|
||||
const context = canvas.getContext("2d", { willReadFrequently: true })!;
|
||||
const step = (): void => {
|
||||
context.drawImage(video, 0, 0, 16, 16);
|
||||
const { data } = context.getImageData(0, 0, 16, 16);
|
||||
const sum = [0, 0, 0];
|
||||
for (let i = 0; i < data.length; i += 4)
|
||||
for (let c = 0; c < 3; c++) sum[c] += data[i + c];
|
||||
seen.push(sum.map((v) => v / (data.length / 4)) as Colour);
|
||||
video.requestVideoFrameCallback(step);
|
||||
};
|
||||
video.requestVideoFrameCallback(step);
|
||||
void video.play();
|
||||
};
|
||||
const Connection = window.RTCPeerConnection;
|
||||
window.RTCPeerConnection = class extends Connection {
|
||||
public constructor(...args: ConstructorParameters<typeof Connection>) {
|
||||
super(...args);
|
||||
this.addEventListener("track", (e) => record(e.track));
|
||||
}
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function framesOf(page: Page, trackId: string): Promise<Colour[]> {
|
||||
return page.evaluate(
|
||||
(id) =>
|
||||
(
|
||||
window as unknown as {
|
||||
receivedFrames: Record<string, [number, number, number][]>;
|
||||
}
|
||||
).receivedFrames[id] ?? [],
|
||||
trackId,
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { expect, type Locator, type Page, test } from "@playwright/test";
|
||||
|
||||
import { createUserAndRoom, resizeContainer, startHarness } from "./harness.ts";
|
||||
import { averageColour, cameraColour, expectWearing } from "../utils/colour.ts";
|
||||
|
||||
// Background effects need WebGL2, and headless Firefox on a CI runner has none,
|
||||
// so it rightly offers none of them.
|
||||
test.skip(
|
||||
({ browserName }) => browserName === "firefox",
|
||||
"Background effects need WebGL2, which headless Firefox on CI does not have",
|
||||
);
|
||||
|
||||
/**
|
||||
* Background effects with Element Call as a component in a host page: the
|
||||
* pipeline's model and WebAssembly have to load from the component's bundle,
|
||||
* and the portalled menu has to be sized against the call, not the window.
|
||||
* Driven from the lobby, which builds the same menu and pipeline.
|
||||
*/
|
||||
|
||||
// Sign-in, crypto setup and sync happen twice before anything shows.
|
||||
test.describe.configure({ timeout: 180_000 });
|
||||
|
||||
test("puts a picture on the call's preview", async ({ page }) => {
|
||||
const { username, roomId } = await createUserAndRoom("effects");
|
||||
const panes = await startHarness(page, username, roomId);
|
||||
const pane = panes.first();
|
||||
await resizeContainer(pane.getByTestId("call-container"), {
|
||||
width: 900,
|
||||
height: 480,
|
||||
});
|
||||
await expect(pane.getByTestId("lobby_joinCall")).toBeVisible({
|
||||
timeout: 60_000,
|
||||
});
|
||||
const preview = pane.locator("video").first();
|
||||
const camera = await cameraColour(preview);
|
||||
|
||||
const tile = await openEffect(page, pane, "Background 1");
|
||||
const picture = await averageColour(tile.locator("img"));
|
||||
await tile.click();
|
||||
await expect(tile).toHaveAttribute("aria-checked", "true");
|
||||
await expectWearing(preview, picture, camera);
|
||||
});
|
||||
|
||||
test("keeps every effect reachable in a small container", async ({ page }) => {
|
||||
const { username, roomId } = await createUserAndRoom("effectsreach");
|
||||
const panes = await startHarness(page, username, roomId);
|
||||
const pane = panes.first();
|
||||
const container = pane.getByTestId("call-container");
|
||||
await resizeContainer(container, { width: 400, height: 360 });
|
||||
await expect(pane.getByTestId("lobby_joinCall")).toBeVisible({
|
||||
timeout: 60_000,
|
||||
});
|
||||
|
||||
const last = await openEffect(page, pane, "Background 2");
|
||||
const list = page.locator("[role='menu'] div[role='none']").first();
|
||||
expect((await list.boundingBox())!.height).toBeLessThanOrEqual(
|
||||
(await container.boundingBox())!.height,
|
||||
);
|
||||
await last.scrollIntoViewIfNeeded();
|
||||
await expect(last).toBeInViewport({ ratio: 1 });
|
||||
await last.click();
|
||||
await expect(last).toHaveAttribute("aria-checked", "true");
|
||||
});
|
||||
|
||||
/** Opens a component's camera menu and returns one of its effects. */
|
||||
async function openEffect(
|
||||
page: Page,
|
||||
pane: Locator,
|
||||
name: string,
|
||||
): Promise<Locator> {
|
||||
await pane
|
||||
.getByRole("button", { name: "Camera", exact: true })
|
||||
.click({ timeout: 60_000 });
|
||||
// Portalled out of the pane, so found on the page.
|
||||
return page
|
||||
.getByRole("group", { name: "Background effects" })
|
||||
.getByRole("menuitemradio", { name });
|
||||
}
|
||||
@@ -29,47 +29,10 @@ export interface MyFixtures {
|
||||
addUser: (username: string, host: string) => Promise<UserBaseFixture>;
|
||||
}
|
||||
|
||||
// Minimal config.json for the local element-web instance
|
||||
const CONFIG_JSON = {
|
||||
default_server_config: {
|
||||
"m.homeserver": {
|
||||
base_url: "https://synapse.m.localhost",
|
||||
server_name: "synapse.m.localhost",
|
||||
},
|
||||
},
|
||||
|
||||
element_call: {
|
||||
participant_limit: 8,
|
||||
brand: "Element Call",
|
||||
},
|
||||
|
||||
// The default language is set here for test consistency
|
||||
setting_defaults: {
|
||||
language: "en-GB",
|
||||
feature_group_calls: true,
|
||||
},
|
||||
|
||||
// the location tests want a map style url.
|
||||
map_style_url:
|
||||
"https://api.maptiler.com/maps/streets/style.json?key=fU3vlMsMn4Jb6dnEIFsx",
|
||||
|
||||
features: {
|
||||
// We don't want to go through the feature announcement during the e2e test
|
||||
feature_release_announcement: false,
|
||||
feature_element_call_video_rooms: true,
|
||||
feature_video_rooms: true,
|
||||
feature_group_calls: true,
|
||||
},
|
||||
};
|
||||
|
||||
export const widgetTest = test.extend<MyFixtures>({
|
||||
// allow per-test override: `widgetTest.use({ callType: "dm" })`
|
||||
callType: ["room", { option: true }],
|
||||
asWidget: async ({ browser, context, callType }, pUse) => {
|
||||
await context.route(`http://localhost:8081/config.json*`, async (route) => {
|
||||
await route.fulfill({ json: CONFIG_JSON });
|
||||
});
|
||||
|
||||
asWidget: async ({ browser, callType }, pUse) => {
|
||||
const brooksDisplayName = `brooks_${Date.now()}`;
|
||||
const whistlerDisplayName = `whistler_${Date.now()}`;
|
||||
|
||||
@@ -109,15 +72,7 @@ export const widgetTest = test.extend<MyFixtures>({
|
||||
await TestHelpers.dismissInviteUnknownUserModal(ewPage1);
|
||||
|
||||
// Accept the invite
|
||||
await TestHelpers.closeReleaseAnnouncement(
|
||||
ewPage2,
|
||||
"Introducing Sections",
|
||||
);
|
||||
await TestHelpers.expandAllSections(ewPage2);
|
||||
await expect(
|
||||
TestHelpers.roomListItem(ewPage2, "Welcome Room"),
|
||||
).toBeVisible();
|
||||
await TestHelpers.roomListItem(ewPage2, "Welcome Room").click();
|
||||
await TestHelpers.openRoomInvite(ewPage2, "Welcome Room");
|
||||
await ewPage2.getByRole("button", { name: "Accept" }).click();
|
||||
await expect(
|
||||
ewPage2
|
||||
@@ -125,17 +80,7 @@ export const widgetTest = test.extend<MyFixtures>({
|
||||
.getByRole("heading", { name: "Welcome Room" }),
|
||||
).toBeVisible();
|
||||
} else if (callType === "dm") {
|
||||
await TestHelpers.closeReleaseAnnouncement(
|
||||
ewPage1,
|
||||
"Introducing Sections",
|
||||
);
|
||||
|
||||
await ewPage1
|
||||
.getByRole("navigation", { name: "Room list" })
|
||||
.getByRole("button", { name: "New", exact: true })
|
||||
.click();
|
||||
|
||||
await ewPage1.getByRole("menuitem", { name: "Start chat" }).click();
|
||||
await TestHelpers.chooseFromRoomListNewMenu(ewPage1, "Start chat");
|
||||
await ewPage1.getByRole("textbox", { name: "Search" }).click();
|
||||
await ewPage1.getByRole("textbox", { name: "Search" }).fill(whistlerMxId);
|
||||
await ewPage1.getByRole("button", { name: "Go" }).click();
|
||||
@@ -157,12 +102,7 @@ export const widgetTest = test.extend<MyFixtures>({
|
||||
).toBeVisible();
|
||||
|
||||
// Accept the DM invite from brooks
|
||||
await TestHelpers.closeReleaseAnnouncement(
|
||||
ewPage2,
|
||||
"Introducing Sections",
|
||||
);
|
||||
await TestHelpers.expandAllSections(ewPage2);
|
||||
await TestHelpers.roomListItem(ewPage2, brooksDisplayName).click();
|
||||
await TestHelpers.openRoomInvite(ewPage2, brooksDisplayName);
|
||||
await ewPage2.getByRole("button", { name: "Start chatting" }).click();
|
||||
}
|
||||
|
||||
@@ -187,13 +127,14 @@ export const widgetTest = test.extend<MyFixtures>({
|
||||
* Provide a way to add additional users within a test.
|
||||
* The returned user will be registered on the default homeserver, the name will be made unique by appending a timestamp.
|
||||
*/
|
||||
addUser: async ({ browser }, use) => {
|
||||
addUser: async ({ browser }, use, testInfo) => {
|
||||
await use(
|
||||
async (
|
||||
username: string,
|
||||
host: string = HOST1,
|
||||
): Promise<UserBaseFixture> => {
|
||||
const uniqueSuffix = Date.now();
|
||||
// Tests in parallel workers can reach this in the same millisecond.
|
||||
const uniqueSuffix = `${Date.now()}_${testInfo.parallelIndex}`;
|
||||
const { page, clientHandle, mxId } = await TestHelpers.registerUser(
|
||||
browser,
|
||||
`${username.toLowerCase()}_${uniqueSuffix}`,
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { expect, type Locator } from "@playwright/test";
|
||||
|
||||
export type Colour = [number, number, number];
|
||||
|
||||
/** The camera's colour, once the preview is showing it rather than nothing. */
|
||||
export async function cameraColour(preview: Locator): Promise<Colour> {
|
||||
await expect
|
||||
.poll(async () => Math.max(...(await averageColour(preview))))
|
||||
.toBeGreaterThan(40);
|
||||
return averageColour(preview);
|
||||
}
|
||||
|
||||
/** The mean colour of a video's current frame or an image, drawn small. */
|
||||
export async function averageColour(element: Locator): Promise<Colour> {
|
||||
return element.evaluate(
|
||||
async (source: HTMLVideoElement | HTMLImageElement) => {
|
||||
if (source instanceof HTMLImageElement) await source.decode();
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = canvas.height = 32;
|
||||
const context = canvas.getContext("2d")!;
|
||||
context.drawImage(source, 0, 0, 32, 32);
|
||||
const { data } = context.getImageData(0, 0, 32, 32);
|
||||
const sum = [0, 0, 0];
|
||||
for (let i = 0; i < data.length; i += 4)
|
||||
for (let c = 0; c < 3; c++) sum[c] += data[i + c];
|
||||
const pixels = data.length / 4;
|
||||
return sum.map((s) => s / pixels) as Colour;
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
export function distance(a: Colour, b: Colour): number {
|
||||
return Math.hypot(a[0] - b[0], a[1] - b[1], a[2] - b[2]);
|
||||
}
|
||||
|
||||
/** Waits until a preview shows a picture rather than the camera. */
|
||||
export async function expectWearing(
|
||||
preview: Locator,
|
||||
picture: Colour,
|
||||
camera: Colour,
|
||||
): Promise<void> {
|
||||
// Long enough for the first effect of a session to be built.
|
||||
await expect
|
||||
.poll(async () => distance(await averageColour(preview), picture), {
|
||||
timeout: 60_000,
|
||||
})
|
||||
.toBeLessThan(distance(camera, picture) / 2);
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { expect, test } from "@playwright/test";
|
||||
|
||||
import { widgetTest } from "../fixtures/widget-user.ts";
|
||||
import { TestHelpers } from "./test-helpers.ts";
|
||||
import { averageColour, cameraColour, expectWearing } from "../utils/colour.ts";
|
||||
|
||||
// Background effects need WebGL2, and headless Firefox on a CI runner has none,
|
||||
// so it rightly offers none of them.
|
||||
test.skip(
|
||||
({ browserName }) => browserName === "firefox",
|
||||
"Background effects need WebGL2, which headless Firefox on CI does not have",
|
||||
);
|
||||
|
||||
widgetTest(
|
||||
"puts a picture on the preview inside Element Web",
|
||||
async ({ asWidget }) => {
|
||||
test.slow();
|
||||
const { brooks } = asWidget;
|
||||
await TestHelpers.startCallInCurrentRoom(brooks.page, false);
|
||||
const frame = brooks.page
|
||||
.locator('iframe[title="Element Call"]')
|
||||
.contentFrame();
|
||||
await expect(frame.getByTestId("lobby_joinCall")).toBeVisible();
|
||||
const preview = frame.locator("video").first();
|
||||
const camera = await cameraColour(preview);
|
||||
|
||||
await frame.getByRole("button", { name: "Camera", exact: true }).click();
|
||||
const tile = frame
|
||||
.getByRole("group", { name: "Background effects" })
|
||||
.getByRole("menuitemradio", { name: "Background 1" });
|
||||
const picture = await averageColour(tile.locator("img"));
|
||||
await tile.click();
|
||||
await expect(tile).toHaveAttribute("aria-checked", "true");
|
||||
await expectWearing(preview, picture, camera);
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,20 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { test as setup } from "@playwright/test";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { mkdirSync } from "node:fs";
|
||||
|
||||
const EMBEDDED_DIST = "embedded/web/dist";
|
||||
|
||||
setup("build the embedded package", () => {
|
||||
setup.setTimeout(10 * 60 * 1000);
|
||||
mkdirSync(EMBEDDED_DIST, { recursive: true });
|
||||
execFileSync("pnpm", ["build:embedded", "--outDir", EMBEDDED_DIST], {
|
||||
stdio: "inherit",
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,56 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { expect, type Route } from "@playwright/test";
|
||||
|
||||
import { widgetTest } from "../../fixtures/widget-user.ts";
|
||||
import { HOST2, TestHelpers } from "../test-helpers.ts";
|
||||
|
||||
/** Where the other site's Element Web serves the embedded package from */
|
||||
const CONFIG_JSON = "**/widgets/element-call/config.json";
|
||||
|
||||
const HOST_ANSWERS: Record<string, (route: Route) => Promise<void>> = {
|
||||
"a 404": async (route) => route.fulfill({ status: 404 }),
|
||||
"a failed fetch": async (route) => route.abort("failed"),
|
||||
"an HTML page": async (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: "text/html",
|
||||
body: "<!doctype html><title>Element</title>",
|
||||
}),
|
||||
};
|
||||
|
||||
widgetTest.skip(
|
||||
({ browserName }) => browserName === "firefox",
|
||||
"The widget suite does not run on Firefox, see the other specs here",
|
||||
);
|
||||
|
||||
for (const [answer, respond] of Object.entries(HOST_ANSWERS)) {
|
||||
widgetTest(
|
||||
`reaches the lobby when config.json gets ${answer}`,
|
||||
async ({ addUser }) => {
|
||||
const { page } = await addUser("Brooks", HOST2);
|
||||
await TestHelpers.createRoom("Config Room", page);
|
||||
|
||||
let answered = 0;
|
||||
await page.route(CONFIG_JSON, async (route) => {
|
||||
answered++;
|
||||
await respond(route);
|
||||
});
|
||||
|
||||
await TestHelpers.startCallInCurrentRoom(page);
|
||||
|
||||
await expect(
|
||||
page
|
||||
.locator('iframe[title="Element Call"]')
|
||||
.contentFrame()
|
||||
.getByTestId("lobby_joinCall"),
|
||||
).toBeVisible({ timeout: 15000 });
|
||||
expect(answered).toBeGreaterThan(0);
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -144,7 +144,11 @@ export class TestHelpers {
|
||||
|
||||
await this.dismissStartupToasts(page);
|
||||
|
||||
await TestHelpers.setDevToolElementCallDevUrl(page);
|
||||
// The other site's Element Web opens the embedded package it serves itself
|
||||
// (see docker-compose-playwright.yml), not the dev server's app.
|
||||
if (host !== HOST2) {
|
||||
await TestHelpers.setDevToolElementCallDevUrl(page);
|
||||
}
|
||||
|
||||
const clientHandle = await page.evaluateHandle(() =>
|
||||
window.mxMatrixClientPeg.get(),
|
||||
@@ -153,29 +157,40 @@ export class TestHelpers {
|
||||
return { page, clientHandle, mxId };
|
||||
}
|
||||
|
||||
// Dismisses any toasts that appear on startup, such as "Failed to load service worker" or "Back up your chats".
|
||||
// Toast can be stacked, and only the top one can be dismiss, so just look at what is on top and
|
||||
// dismiss (if part of expected toats)
|
||||
// Dismisses the toasts a fresh device gets after sign-in.
|
||||
//
|
||||
// Toasts stack and only the top one can be dismissed, so look
|
||||
// at what is on top each time.
|
||||
public static async dismissStartupToasts(page: Page): Promise<void> {
|
||||
const expectedToasts = [
|
||||
const expectedToasts: {
|
||||
title: string;
|
||||
button: string;
|
||||
confirm?: string;
|
||||
}[] = [
|
||||
{ title: "Failed to load service worker", button: "OK" },
|
||||
{ title: "Verify this device", button: "Later" },
|
||||
{ title: "Back up your chats", button: "Not now" },
|
||||
{ title: "Turn on key storage", button: "Dismiss" },
|
||||
{
|
||||
title: "Turn on key storage",
|
||||
button: "Dismiss",
|
||||
confirm: "Yes, dismiss",
|
||||
},
|
||||
{ title: "Notifications", button: "Dismiss" },
|
||||
{ title: "Element does not support this browser", button: "Dismiss" },
|
||||
];
|
||||
|
||||
const toast = page.locator(".mx_Toast_toast");
|
||||
const toastTitle = toast.locator(".mx_Toast_title h2");
|
||||
|
||||
// eslint-disable-next-line no-constant-condition
|
||||
while (true) {
|
||||
try {
|
||||
await toast.waitFor({ state: "visible", timeout: 700 });
|
||||
const title = await toast.locator(".mx_Toast_title h2").textContent();
|
||||
const title = await toastTitle.textContent();
|
||||
if (!title) break;
|
||||
|
||||
// Find the matching toast config
|
||||
const toastConfig = expectedToasts.find((t) =>
|
||||
title?.includes(t.title),
|
||||
);
|
||||
const toastConfig = expectedToasts.find((t) => title.includes(t.title));
|
||||
|
||||
if (toastConfig) {
|
||||
// Bounded so that a renamed button leaves the toast in place and the
|
||||
@@ -183,6 +198,16 @@ export class TestHelpers {
|
||||
await toast
|
||||
.getByRole("button", { name: toastConfig.button })
|
||||
.click({ timeout: 5000 });
|
||||
if (toastConfig.confirm) {
|
||||
const dialog = page.getByRole("dialog");
|
||||
await dialog
|
||||
.getByRole("button", { name: toastConfig.confirm })
|
||||
.click({ timeout: 5000 });
|
||||
await expect(dialog).toBeHidden({ timeout: 5000 });
|
||||
}
|
||||
// The same element shows the next toast a moment after the click, so
|
||||
// reading it straight away acts on the dismissed one a second time
|
||||
await expect(toastTitle).not.toHaveText(title, { timeout: 5000 });
|
||||
} else {
|
||||
// Unknown toast. We don't want to act on unknown toasts
|
||||
break;
|
||||
@@ -194,20 +219,6 @@ export class TestHelpers {
|
||||
}
|
||||
}
|
||||
|
||||
public static async closeReleaseAnnouncement(
|
||||
page: Page,
|
||||
name: string,
|
||||
): Promise<void> {
|
||||
try {
|
||||
await page
|
||||
.getByRole("dialog", { name })
|
||||
.getByRole("button", { name: "OK" })
|
||||
.click({ timeout: 2000 });
|
||||
} catch {
|
||||
// Announcement not shown; nothing to do
|
||||
}
|
||||
}
|
||||
|
||||
public static async expandAllSections(page: Page): Promise<void> {
|
||||
try {
|
||||
await page
|
||||
@@ -223,16 +234,7 @@ export class TestHelpers {
|
||||
page: Page,
|
||||
andInvite: string[] = [],
|
||||
): Promise<void> {
|
||||
await TestHelpers.closeReleaseAnnouncement(page, "Introducing Sections");
|
||||
|
||||
await page
|
||||
.getByRole("navigation", { name: "Room list" })
|
||||
.getByRole("button", { name: "New", exact: true })
|
||||
.click();
|
||||
|
||||
await page.getByRole("menuitem", { name: "New Room" }).click({
|
||||
timeout: 5000,
|
||||
});
|
||||
await TestHelpers.chooseFromRoomListNewMenu(page, "New Room");
|
||||
await page.getByRole("textbox", { name: "Name" }).fill(name);
|
||||
await page.getByRole("button", { name: "Create room" }).click();
|
||||
await expect(page.getByText("You created this room.")).toBeVisible({
|
||||
@@ -259,6 +261,24 @@ export class TestHelpers {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Opens the room list's "New" menu and picks the given item.
|
||||
*
|
||||
* Driven by keyboard, as a toast arriving late sits over the button and
|
||||
* swallows clicks.
|
||||
*/
|
||||
public static async chooseFromRoomListNewMenu(
|
||||
page: Page,
|
||||
itemName: string,
|
||||
): Promise<void> {
|
||||
await TestHelpers.dismissStartupToasts(page);
|
||||
await page
|
||||
.getByRole("navigation", { name: "Room list" })
|
||||
.getByRole("button", { name: "New", exact: true })
|
||||
.press("Enter");
|
||||
await page.getByRole("menuitem", { name: itemName }).press("Enter");
|
||||
}
|
||||
|
||||
/**
|
||||
* Accepts a room invite using the room name.
|
||||
* Locatest the invite in the room list.
|
||||
@@ -268,9 +288,7 @@ export class TestHelpers {
|
||||
roomName: string,
|
||||
page: Page,
|
||||
): Promise<void> {
|
||||
await TestHelpers.closeReleaseAnnouncement(page, "Introducing Sections");
|
||||
await TestHelpers.expandAllSections(page);
|
||||
await TestHelpers.roomListItem(page, roomName).click({ timeout: 10000 });
|
||||
await TestHelpers.openRoomInvite(page, roomName);
|
||||
await page.getByRole("button", { name: "Accept" }).click({
|
||||
timeout: 5000,
|
||||
});
|
||||
@@ -376,6 +394,28 @@ export class TestHelpers {
|
||||
return page.locator(`[aria-label^="Open room ${roomName}"]`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Opens a pending invite from the room list by its room name.
|
||||
*
|
||||
* Element Web collapses the Invites section every time it appears, so it is
|
||||
* expanded here once the invite has arrived.
|
||||
*/
|
||||
public static async openRoomInvite(
|
||||
page: Page,
|
||||
roomName: string,
|
||||
): Promise<void> {
|
||||
const invitesSection = page.getByRole("button", {
|
||||
name: "Toggle Invites section",
|
||||
});
|
||||
await expect(invitesSection).toBeVisible({ timeout: 10000 });
|
||||
// Toasts sit over the top of the room list, where this header is.
|
||||
await TestHelpers.dismissStartupToasts(page);
|
||||
if ((await invitesSection.getAttribute("aria-expanded")) !== "true") {
|
||||
await invitesSection.click();
|
||||
}
|
||||
await TestHelpers.roomListItem(page, roomName).click({ timeout: 10000 });
|
||||
}
|
||||
|
||||
/**
|
||||
* Switches to a room in the room list by its name.
|
||||
* @param page - The EW page
|
||||
|
||||
Generated
+48
-26
@@ -8,7 +8,7 @@ overrides:
|
||||
'@livekit/components-core>rxjs': ^7.8.1
|
||||
'@livekit/track-processors>@mediapipe/tasks-vision': ^0.10.18
|
||||
tar: ^7.5.11
|
||||
minimatch: ^10.2.3
|
||||
minimatch: ^11.0.0
|
||||
glob: ^10.5.0
|
||||
qs: ^6.15.2
|
||||
js-yaml: ^4.2.0
|
||||
@@ -160,6 +160,9 @@ importers:
|
||||
eslint-plugin-storybook:
|
||||
specifier: ^10.3.6
|
||||
version: 10.6.2(eslint@8.57.1(supports-color@7.2.0))(supports-color@7.2.0)(typescript@5.9.3)
|
||||
fake-indexeddb:
|
||||
specifier: ^6.2.5
|
||||
version: 6.2.5
|
||||
fetch-mock:
|
||||
specifier: 11.1.5
|
||||
version: 11.1.5(node-fetch@2.7.0)
|
||||
@@ -188,8 +191,8 @@ importers:
|
||||
specifier: ^1.9.1
|
||||
version: 1.9.2
|
||||
matrix-js-sdk:
|
||||
specifier: github:matrix-org/matrix-js-sdk#a9ef09fbf2f1644c8663d0aeaf6a921cd16e7b01
|
||||
version: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/a9ef09fbf2f1644c8663d0aeaf6a921cd16e7b01
|
||||
specifier: github:matrix-org/matrix-js-sdk#dc8610e796c92553c658300a4882dde961a0678a
|
||||
version: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/dc8610e796c92553c658300a4882dde961a0678a
|
||||
matrix-widget-api:
|
||||
specifier: ^1.18.0
|
||||
version: 1.20.0
|
||||
@@ -1142,6 +1145,9 @@ packages:
|
||||
resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==}
|
||||
engines: {node: '>=12'}
|
||||
|
||||
'@isaacs/nonrecursive-glob-match@1.0.3':
|
||||
resolution: {integrity: sha512-SI7kyKHq4XXDRozw+r9GlRZkia1ZkFrpLW98r2IC2Bpr8YkFGE3fvjzmxCG0x78A++69HmDNrDvSMYBZmHGpKA==}
|
||||
|
||||
'@joshwooding/vite-plugin-react-docgen-typescript@0.7.0':
|
||||
resolution: {integrity: sha512-qvsTEwEFefhdirGOPnu9Wp6ChfIwy2dBCRuETU3uE+4cC+PFoxMSiiEhxk4lOluA34eARHA0OxqsEUYDqRMgeQ==}
|
||||
peerDependencies:
|
||||
@@ -3527,8 +3533,8 @@ packages:
|
||||
boolbase@1.0.0:
|
||||
resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==}
|
||||
|
||||
brace-expansion@5.0.9:
|
||||
resolution: {integrity: sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==}
|
||||
brace-expansion@5.0.12:
|
||||
resolution: {integrity: sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ==}
|
||||
engines: {node: 20 || >=22}
|
||||
|
||||
braces@3.0.3:
|
||||
@@ -4048,6 +4054,10 @@ packages:
|
||||
resolution: {integrity: sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==}
|
||||
engines: {node: '>=12.0.0'}
|
||||
|
||||
fake-indexeddb@6.2.5:
|
||||
resolution: {integrity: sha512-CGnyrvbhPlWYMngksqrSSUT1BAVP49dZocrHuK0SvtR0D5TMs5wP0o3j7jexDJW01KSadjBp1M/71o/KR3nD1w==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
fast-deep-equal@3.1.3:
|
||||
resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==}
|
||||
|
||||
@@ -4724,8 +4734,8 @@ packages:
|
||||
matrix-events-sdk@0.0.1:
|
||||
resolution: {integrity: sha512-1QEOsXO+bhyCroIe2/A5OwaxHvBm7EsSQ46DEDn8RBIfQwN5HWBpFvyWWR4QY0KHPPnnJdI99wgRiAl7Ad5qaA==}
|
||||
|
||||
matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/a9ef09fbf2f1644c8663d0aeaf6a921cd16e7b01:
|
||||
resolution: {gitHosted: true, integrity: sha512-ZAhn/zB44MgrXqUzi9ffSFcmZY3+2nx7bEgfEJR7chYDFyT/XmlmfCJx7Ar4rOTAYZJRVnMxrEwiEHGDapUi0Q==, tarball: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/a9ef09fbf2f1644c8663d0aeaf6a921cd16e7b01}
|
||||
matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/dc8610e796c92553c658300a4882dde961a0678a:
|
||||
resolution: {gitHosted: true, integrity: sha512-Ldt/VkCzBtre6m1cn0FqgrzRV0GrgWIoARgQUQO58ef2Edu2ERFCINPzp/psFS5RwoN3RIy9Rm4oPVc1ATkEWA==, tarball: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/dc8610e796c92553c658300a4882dde961a0678a}
|
||||
version: 45.0.0
|
||||
engines: {node: '>=22.0.0'}
|
||||
|
||||
@@ -4761,9 +4771,9 @@ packages:
|
||||
minimalistic-crypto-utils@1.0.1:
|
||||
resolution: {integrity: sha512-JIYlbt6g8i5jKfJ3xz7rF0LXmv2TkDxBLUkiBeZ7bAx4GnnNMr8xFpGnOxn6GhTEHx3SjRrZEoU+j04prX1ktg==}
|
||||
|
||||
minimatch@10.2.6:
|
||||
resolution: {integrity: sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==}
|
||||
engines: {node: 18 || 20 || >=22}
|
||||
minimatch@11.0.4:
|
||||
resolution: {integrity: sha512-/UJ1dEoVQTjPNDh6SGlUo+ueLMmoHTas4q+8yaAR/5t6nLAIiBDJtoYUh20IYleix7ehiQkgczb84PE3uzHkxg==}
|
||||
engines: {node: 20 || >=22}
|
||||
|
||||
minimist@1.2.8:
|
||||
resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==}
|
||||
@@ -4936,8 +4946,8 @@ packages:
|
||||
resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==}
|
||||
engines: {node: '>=10'}
|
||||
|
||||
p-retry@8.0.1:
|
||||
resolution: {integrity: sha512-NAigyu8hfvJe7MsS18mnc4is0MZtau3QMdBklaJKK23oBQb6occO3UPM3vHmTckW8KhfyjMZaUOqdTFEYliHgg==}
|
||||
p-retry@9.0.0:
|
||||
resolution: {integrity: sha512-HuEats8gYn/UUYa5uyveyQGv75mQmwdFtuMgnZlS12dj/7bdnJSO6keaTjgcZUhvkANTj5EO3lACUmV3xmgj0Q==}
|
||||
engines: {node: '>=22'}
|
||||
|
||||
p-try@2.2.0:
|
||||
@@ -5308,6 +5318,10 @@ packages:
|
||||
randomfill@1.0.4:
|
||||
resolution: {integrity: sha512-87lcbR8+MhcWcUiQ+9e+Rwx8MyR2P7qnt15ynUlbm3TU/fjbgz4GsvfSUDTemtCCtVCqb4ZcEFlyPNTh9bBTLw==}
|
||||
|
||||
re2js@2.9.1:
|
||||
resolution: {integrity: sha512-JIT3bEEUZTOR2YS1walNEIfH4cF/cQeZcBUFq7ldqO6lR6g0X5CBq/S8rsa9omzZcsNf8qc36jCkWpKKtncrZg==}
|
||||
engines: {node: '>=20.0.0'}
|
||||
|
||||
react-docgen-typescript@2.4.0:
|
||||
resolution: {integrity: sha512-ZtAp5XTO5HRzQctjPU0ybY0RRCQO19X/8fxn3w7y2VVTUbGHDKULPTL4ky3vB05euSgG5NpALhEhDPvQ56wvXg==}
|
||||
peerDependencies:
|
||||
@@ -6895,7 +6909,7 @@ snapshots:
|
||||
ignore: 5.3.2
|
||||
import-fresh: 3.3.1
|
||||
js-yaml: 4.3.1
|
||||
minimatch: 10.2.6
|
||||
minimatch: 11.0.4
|
||||
strip-json-comments: 3.1.1
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
@@ -6974,7 +6988,7 @@ snapshots:
|
||||
dependencies:
|
||||
'@humanwhocodes/object-schema': 2.0.3
|
||||
debug: 4.4.3(supports-color@7.2.0)
|
||||
minimatch: 10.2.6
|
||||
minimatch: 11.0.4
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
@@ -7110,6 +7124,8 @@ snapshots:
|
||||
wrap-ansi: 8.1.0
|
||||
wrap-ansi-cjs: wrap-ansi@7.0.0
|
||||
|
||||
'@isaacs/nonrecursive-glob-match@1.0.3': {}
|
||||
|
||||
'@joshwooding/vite-plugin-react-docgen-typescript@0.7.0(typescript@5.9.3)(vite@8.2.4(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.103.1)(terser@5.46.1)(yaml@2.9.0))':
|
||||
dependencies:
|
||||
glob: 10.5.0
|
||||
@@ -8769,7 +8785,7 @@ snapshots:
|
||||
'@typescript-eslint/types': 8.69.0
|
||||
'@typescript-eslint/visitor-keys': 8.69.0
|
||||
debug: 4.4.3(supports-color@7.2.0)
|
||||
minimatch: 10.2.6
|
||||
minimatch: 11.0.4
|
||||
semver: 7.8.5
|
||||
tinyglobby: 0.2.17
|
||||
ts-api-utils: 2.5.0(typescript@5.9.3)
|
||||
@@ -8784,7 +8800,7 @@ snapshots:
|
||||
'@typescript-eslint/types': 8.72.0
|
||||
'@typescript-eslint/visitor-keys': 8.72.0
|
||||
debug: 4.4.3(supports-color@7.2.0)
|
||||
minimatch: 10.2.6
|
||||
minimatch: 11.0.4
|
||||
semver: 7.8.5
|
||||
tinyglobby: 0.2.17
|
||||
ts-api-utils: 2.5.0(typescript@5.9.3)
|
||||
@@ -9109,7 +9125,7 @@ snapshots:
|
||||
|
||||
boolbase@1.0.0: {}
|
||||
|
||||
brace-expansion@5.0.9:
|
||||
brace-expansion@5.0.12:
|
||||
dependencies:
|
||||
balanced-match: 4.0.4
|
||||
|
||||
@@ -9654,7 +9670,7 @@ snapshots:
|
||||
json-stable-stringify-without-jsonify: 1.0.1
|
||||
levn: 0.4.1
|
||||
lodash.merge: 4.6.2
|
||||
minimatch: 10.2.6
|
||||
minimatch: 11.0.4
|
||||
natural-compare: 1.4.0
|
||||
optionator: 0.9.4
|
||||
strip-ansi: 6.0.1
|
||||
@@ -9712,6 +9728,8 @@ snapshots:
|
||||
|
||||
expect-type@1.4.0: {}
|
||||
|
||||
fake-indexeddb@6.2.5: {}
|
||||
|
||||
fast-deep-equal@3.1.3: {}
|
||||
|
||||
fast-glob@3.3.3:
|
||||
@@ -9772,7 +9790,7 @@ snapshots:
|
||||
|
||||
filelist@1.0.6:
|
||||
dependencies:
|
||||
minimatch: 10.2.6
|
||||
minimatch: 11.0.4
|
||||
|
||||
fill-range@7.1.1:
|
||||
dependencies:
|
||||
@@ -9872,7 +9890,7 @@ snapshots:
|
||||
dependencies:
|
||||
foreground-child: 3.3.1
|
||||
jackspeak: 3.4.3
|
||||
minimatch: 10.2.6
|
||||
minimatch: 11.0.4
|
||||
minipass: 7.1.3
|
||||
package-json-from-dist: 1.0.1
|
||||
path-scurry: 1.11.1
|
||||
@@ -9997,7 +10015,7 @@ snapshots:
|
||||
jiti: 2.7.0
|
||||
jsonc-parser: 3.3.1
|
||||
magic-string: 1.2.3
|
||||
minimatch: 10.2.6
|
||||
minimatch: 11.0.4
|
||||
ora: 9.4.1
|
||||
react: 19.2.8
|
||||
react-i18next: 17.0.13(i18next@26.4.1(typescript@5.9.3))(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(typescript@5.9.3)
|
||||
@@ -10395,7 +10413,7 @@ snapshots:
|
||||
|
||||
matrix-events-sdk@0.0.1: {}
|
||||
|
||||
matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/a9ef09fbf2f1644c8663d0aeaf6a921cd16e7b01:
|
||||
matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/dc8610e796c92553c658300a4882dde961a0678a:
|
||||
dependencies:
|
||||
'@babel/runtime': 8.0.8
|
||||
'@matrix-org/matrix-sdk-crypto-wasm': 18.9.0
|
||||
@@ -10405,7 +10423,8 @@ snapshots:
|
||||
loglevel: 1.9.2
|
||||
matrix-events-sdk: 0.0.1
|
||||
matrix-widget-api: 1.20.0
|
||||
p-retry: 8.0.1
|
||||
p-retry: 9.0.0
|
||||
re2js: 2.9.1
|
||||
sdp-transform: 3.0.0
|
||||
unhomoglyph: 1.0.6
|
||||
|
||||
@@ -10440,9 +10459,10 @@ snapshots:
|
||||
|
||||
minimalistic-crypto-utils@1.0.1: {}
|
||||
|
||||
minimatch@10.2.6:
|
||||
minimatch@11.0.4:
|
||||
dependencies:
|
||||
brace-expansion: 5.0.9
|
||||
'@isaacs/nonrecursive-glob-match': 1.0.3
|
||||
brace-expansion: 5.0.12
|
||||
|
||||
minimist@1.2.8: {}
|
||||
|
||||
@@ -10726,7 +10746,7 @@ snapshots:
|
||||
dependencies:
|
||||
p-limit: 3.1.0
|
||||
|
||||
p-retry@8.0.1:
|
||||
p-retry@9.0.0:
|
||||
dependencies:
|
||||
is-network-error: 1.3.2
|
||||
|
||||
@@ -11150,6 +11170,8 @@ snapshots:
|
||||
randombytes: 2.1.0
|
||||
safe-buffer: 5.2.1
|
||||
|
||||
re2js@2.9.1: {}
|
||||
|
||||
react-docgen-typescript@2.4.0(typescript@5.9.3):
|
||||
dependencies:
|
||||
typescript: 5.9.3
|
||||
|
||||
+1
-1
@@ -27,7 +27,7 @@ overrides:
|
||||
# Security fixes:
|
||||
# - https://github.com/advisories/GHSA-7r86-cg39-jmmj
|
||||
# - https://github.com/advisories/GHSA-23c5-xmqv-rm74
|
||||
"minimatch": "^10.2.3"
|
||||
"minimatch": "^11.0.0"
|
||||
# Security fix: https://github.com/element-hq/element-call/security/dependabot/109
|
||||
"glob": "^10.5.0"
|
||||
# Security fixes:
|
||||
|
||||
@@ -162,6 +162,7 @@ describe("createLocalMedia", () => {
|
||||
media.release("camera");
|
||||
expect(trackBehind(camera).stop).toHaveBeenCalled();
|
||||
expect(media.tracks$.value).toEqual([microphone]);
|
||||
expect(media.requests.map((r) => r.source)).toEqual(["microphone"]);
|
||||
|
||||
media.stop();
|
||||
expect(trackBehind(microphone).stop).toHaveBeenCalled();
|
||||
|
||||
@@ -107,6 +107,7 @@ export function createLocalMedia(
|
||||
if (source !== "microphone" && source !== "camera") return;
|
||||
capturedTracks.get(source)?.stop();
|
||||
capturedTracks.delete(source);
|
||||
requests.delete(source);
|
||||
publishCapturedTracks();
|
||||
};
|
||||
|
||||
|
||||
@@ -184,6 +184,7 @@ export function createRTCParticipation(
|
||||
encryptMedia: encryptionSystem.kind !== E2eeType.NONE,
|
||||
matrixRTCMode,
|
||||
sendNotificationType: options.sendNotificationType,
|
||||
callIntent: options.callIntent,
|
||||
applicationData: options.applicationData,
|
||||
timings,
|
||||
delayedLeave,
|
||||
|
||||
+8
-4
@@ -12,6 +12,7 @@ Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import {
|
||||
type RTCCallIntent,
|
||||
type RTCNotificationType,
|
||||
type Transport,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
@@ -103,11 +104,14 @@ export interface RTCParticipationOptions {
|
||||
* to a notification (ringing, timeouts, declines) is the application's job.
|
||||
*/
|
||||
sendNotificationType?: RTCNotificationType;
|
||||
/**
|
||||
* The kind of call the local member intends to take part in: `"m.audio"`
|
||||
* or `"m.video"`. Carried by the membership as a field of its own.
|
||||
*/
|
||||
callIntent?: RTCCallIntent;
|
||||
/**
|
||||
* Whatever the application wants to say about itself in the membership,
|
||||
* under namespaced keys. Opaque to the SDK, apart from `m.call.intent`
|
||||
* (`"audio"` or `"video"`), which the membership format carries as a
|
||||
* field of its own.
|
||||
* under namespaced keys. Opaque to the SDK.
|
||||
*/
|
||||
applicationData?: Record<string, unknown>;
|
||||
/** Timings the host has configured; the defaults otherwise. */
|
||||
@@ -287,7 +291,7 @@ export interface RTCMember {
|
||||
memberId: string;
|
||||
/** Which transport this member is on; undefined when the membership has none. */
|
||||
transport$: Behavior<TransportMetadata | undefined>;
|
||||
/** What the member's application says about itself, e.g. `m.call.intent`. */
|
||||
/** What the member's application says about itself, e.g. `intent`. */
|
||||
applicationData$: Behavior<Record<string, unknown>>;
|
||||
|
||||
displayName$: Behavior<string>;
|
||||
|
||||
@@ -60,7 +60,7 @@ describe("joinJsSdkSession", () => {
|
||||
expect(s.join).toHaveBeenCalledWith(ownMemberMock, [transportWithAlias], {
|
||||
notificationType: "ring",
|
||||
callIntent: undefined,
|
||||
applicationData: {},
|
||||
applicationData: undefined,
|
||||
manageMediaKeys: true,
|
||||
delayedLeaveEventRestartMs: 300_000,
|
||||
delayedLeaveEventDelayMs: 3_600_000,
|
||||
@@ -95,13 +95,14 @@ describe("joinJsSdkSession", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("lifts the call intent out of the application data and sends sticky events in Matrix 2.0 mode", () => {
|
||||
it("passes the call intent and application data through and sends sticky events in Matrix 2.0 mode", () => {
|
||||
const s = session();
|
||||
|
||||
joinJsSdkSession("!room:example.org", s, ownMemberMock, transport, {
|
||||
encryptMedia: true,
|
||||
matrixRTCMode: MatrixRTCMode.Matrix_2_0,
|
||||
applicationData: { "m.call.intent": "audio", custom: 1 },
|
||||
callIntent: "m.audio",
|
||||
applicationData: { custom: 1 },
|
||||
timings: defaultSessionTimings,
|
||||
delayedLeave: defaultSessionTimings.delayedLeave,
|
||||
});
|
||||
@@ -110,7 +111,7 @@ describe("joinJsSdkSession", () => {
|
||||
ownMemberMock,
|
||||
[transportWithAlias],
|
||||
expect.objectContaining({
|
||||
callIntent: "audio",
|
||||
callIntent: "m.audio",
|
||||
applicationData: { custom: 1 },
|
||||
unstableSendStickyEvents: true,
|
||||
}),
|
||||
@@ -124,7 +125,8 @@ describe("joinJsSdkSession", () => {
|
||||
joinJsSdkSession("!room:example.org", s, ownMemberMock, transport, {
|
||||
encryptMedia: true,
|
||||
matrixRTCMode: MatrixRTCMode.Matrix_2_0,
|
||||
applicationData: { "m.call.intent": "audio", custom: 1 },
|
||||
callIntent: "m.audio",
|
||||
applicationData: { custom: 1 },
|
||||
timings: defaultSessionTimings,
|
||||
delayedLeave: defaultSessionTimings.delayedLeave,
|
||||
});
|
||||
@@ -133,7 +135,7 @@ describe("joinJsSdkSession", () => {
|
||||
ownMemberMock,
|
||||
[transport],
|
||||
expect.objectContaining({
|
||||
callIntent: "audio",
|
||||
callIntent: "m.audio",
|
||||
applicationData: { custom: 1 },
|
||||
unstableSendStickyEvents: true,
|
||||
}),
|
||||
|
||||
@@ -23,6 +23,7 @@ interface Options {
|
||||
encryptMedia: boolean;
|
||||
matrixRTCMode: MatrixRTCMode;
|
||||
sendNotificationType?: RTCNotificationType;
|
||||
callIntent?: RTCCallIntent;
|
||||
applicationData?: Record<string, unknown>;
|
||||
timings: SessionTimings;
|
||||
/** The delayed leave timings to use: delegated to the SFU or not. */
|
||||
@@ -43,15 +44,12 @@ export function joinJsSdkSession(
|
||||
encryptMedia,
|
||||
matrixRTCMode,
|
||||
sendNotificationType,
|
||||
callIntent,
|
||||
applicationData,
|
||||
timings,
|
||||
delayedLeave,
|
||||
}: Options,
|
||||
): void {
|
||||
// The membership format carries the intent as a field of its own
|
||||
const { "m.call.intent": intent, ...rest } = applicationData ?? {};
|
||||
const callIntent =
|
||||
typeof intent === "string" ? (intent as RTCCallIntent) : undefined;
|
||||
// Give up on the network as soon as either the sync has been down for the
|
||||
// grace period or the delayed leave has probably been sent, whichever is
|
||||
// sooner
|
||||
@@ -78,7 +76,7 @@ export function joinJsSdkSession(
|
||||
{
|
||||
notificationType: sendNotificationType,
|
||||
callIntent,
|
||||
applicationData: rest,
|
||||
applicationData,
|
||||
manageMediaKeys: encryptMedia,
|
||||
delayedLeaveEventRestartMs: delayedLeave.restart_ms,
|
||||
delayedLeaveEventDelayMs: delayedLeave.delay_ms,
|
||||
|
||||
@@ -225,6 +225,51 @@ describe("createLivekitMediaTrack", () => {
|
||||
);
|
||||
});
|
||||
|
||||
// LiveKit's media stream track is the processed one while a processor runs,
|
||||
// and that one names no device; the camera behind it does
|
||||
it.each([
|
||||
{ deviceId: "cam-1", switches: false },
|
||||
{ deviceId: "cam-2", switches: true },
|
||||
])(
|
||||
"leaves a processed camera alone unless its device changes: $deviceId",
|
||||
async ({ deviceId, switches }) => {
|
||||
vi.stubGlobal("MediaStreamTrack", class {});
|
||||
onTestFinished(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
const camera = Object.assign(Object.create(LocalVideoTrack.prototype), {
|
||||
source: Track.Source.Camera,
|
||||
_mediaStreamTrack: {
|
||||
readyState: "live",
|
||||
label: "",
|
||||
getSettings: () => ({ deviceId: "cam-1" }),
|
||||
},
|
||||
// What LiveKit shows as the track, with no device of its own
|
||||
processor: {
|
||||
processedTrack: { label: "", getSettings: () => ({}) },
|
||||
},
|
||||
}) as LocalVideoTrack;
|
||||
const switchActiveDevice = vi.fn().mockResolvedValue(true);
|
||||
const { participant, publication, room } = fakes({
|
||||
track: camera as unknown as FakeTrack,
|
||||
});
|
||||
const track = createLocalLivekitMediaTrack(
|
||||
scope,
|
||||
Object.assign(participant, {
|
||||
isLocal: true,
|
||||
}) as unknown as LocalParticipant,
|
||||
publication as unknown as TrackPublication,
|
||||
Object.assign(room, { switchActiveDevice }),
|
||||
vi.fn(),
|
||||
);
|
||||
|
||||
await track.setDevice(deviceId);
|
||||
expect(switchActiveDevice.mock.calls).toEqual(
|
||||
switches ? [["videoinput", deviceId, false]] : [],
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
// LiveKit stops a camera that is turned off; the processor is built when it
|
||||
// is turned on again
|
||||
it.each([
|
||||
|
||||
@@ -199,11 +199,13 @@ export function createLocalLivekitMediaTrack(
|
||||
throw new MatrixRTCError(`A ${base.source} track has no device`);
|
||||
// LiveKit decides whether to restart by the constraints it keeps, which
|
||||
// a browser that does not echo the device id back (Firefox) leaves
|
||||
// without one; a track already on the device is left alone here
|
||||
// without one; a track already on the device is left alone here. Asked
|
||||
// of the camera track itself: with a processor on, LiveKit's media
|
||||
// stream track is the processed one, which names no device
|
||||
const current = publication.track;
|
||||
if (
|
||||
current instanceof LocalTrack &&
|
||||
current.mediaStreamTrack.getSettings().deviceId === deviceId
|
||||
(await current.getDeviceId(false)) === deviceId
|
||||
)
|
||||
return;
|
||||
await room.switchActiveDevice(
|
||||
|
||||
+10
-8
@@ -18,6 +18,8 @@ Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import {
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
RTC_CALL_INTENT_VIDEO,
|
||||
type RTCCallIntent,
|
||||
type RTCNotificationType,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
@@ -179,40 +181,40 @@ export function configurationForIntent(intent: UserIntent): UrlConfiguration {
|
||||
switch (intent) {
|
||||
case UserIntent.StartNewCall:
|
||||
preset.skipLobby = false;
|
||||
preset.callIntent = "video";
|
||||
preset.callIntent = RTC_CALL_INTENT_VIDEO;
|
||||
break;
|
||||
case UserIntent.JoinExistingCall:
|
||||
// On desktop this will be overridden based on which button was used to join the call
|
||||
preset.skipLobby = false;
|
||||
preset.callIntent = "video";
|
||||
preset.callIntent = RTC_CALL_INTENT_VIDEO;
|
||||
break;
|
||||
case UserIntent.StartNewCallVoice:
|
||||
preset.skipLobby = false;
|
||||
preset.callIntent = "audio";
|
||||
preset.callIntent = RTC_CALL_INTENT_AUDIO;
|
||||
break;
|
||||
case UserIntent.JoinExistingCallVoice:
|
||||
// On desktop this will be overridden based on which button was used to join the call
|
||||
preset.skipLobby = false;
|
||||
preset.callIntent = "audio";
|
||||
preset.callIntent = RTC_CALL_INTENT_AUDIO;
|
||||
break;
|
||||
case UserIntent.StartNewCallDMVoice:
|
||||
preset.callIntent = "audio";
|
||||
preset.callIntent = RTC_CALL_INTENT_AUDIO;
|
||||
// Fall through
|
||||
case UserIntent.StartNewCallDM:
|
||||
preset.skipLobby = true;
|
||||
preset.sendNotificationType = "ring";
|
||||
preset.autoLeaveWhenOthersLeft = true;
|
||||
preset.waitForCallPickup = true;
|
||||
preset.callIntent = preset.callIntent ?? "video";
|
||||
preset.callIntent = preset.callIntent ?? RTC_CALL_INTENT_VIDEO;
|
||||
break;
|
||||
case UserIntent.JoinExistingCallDMVoice:
|
||||
preset.callIntent = "audio";
|
||||
preset.callIntent = RTC_CALL_INTENT_AUDIO;
|
||||
// Fall through
|
||||
case UserIntent.JoinExistingCallDM:
|
||||
// On desktop this will be overridden based on which button was used to join the call
|
||||
preset.skipLobby = true;
|
||||
preset.autoLeaveWhenOthersLeft = true;
|
||||
preset.callIntent = preset.callIntent ?? "video";
|
||||
preset.callIntent = preset.callIntent ?? RTC_CALL_INTENT_VIDEO;
|
||||
break;
|
||||
// Non widget usecase defaults
|
||||
default:
|
||||
|
||||
+10
-6
@@ -21,6 +21,10 @@ import {
|
||||
UserIntent,
|
||||
} from "../src/UrlParams";
|
||||
import { mockConfig } from "./utils/test";
|
||||
import {
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
RTC_CALL_INTENT_VIDEO,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
|
||||
const ROOM_NAME = "roomNameHere";
|
||||
const ROOM_ID = "!d45f138fsd";
|
||||
@@ -275,7 +279,7 @@ describe("UrlParams", () => {
|
||||
).toMatchObject({
|
||||
...startNewCallDefaults("desktop"),
|
||||
skipLobby: false,
|
||||
callIntent: "video",
|
||||
callIntent: RTC_CALL_INTENT_VIDEO,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -322,7 +326,7 @@ describe("UrlParams", () => {
|
||||
).toMatchObject({
|
||||
...startNewCallDefaults("desktop"),
|
||||
skipLobby: false,
|
||||
callIntent: "audio",
|
||||
callIntent: RTC_CALL_INTENT_AUDIO,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -333,7 +337,7 @@ describe("UrlParams", () => {
|
||||
),
|
||||
).toMatchObject({
|
||||
...joinExistingCallDefaults("desktop"),
|
||||
callIntent: "audio",
|
||||
callIntent: RTC_CALL_INTENT_AUDIO,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -348,7 +352,7 @@ describe("UrlParams", () => {
|
||||
sendNotificationType: "ring",
|
||||
autoLeaveWhenOthersLeft: true,
|
||||
waitForCallPickup: true,
|
||||
callIntent: "audio",
|
||||
callIntent: RTC_CALL_INTENT_AUDIO,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -363,7 +367,7 @@ describe("UrlParams", () => {
|
||||
skipLobby: true,
|
||||
autoLeaveWhenOthersLeft: true,
|
||||
waitForCallPickup: false,
|
||||
callIntent: "video",
|
||||
callIntent: RTC_CALL_INTENT_VIDEO,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -377,7 +381,7 @@ describe("UrlParams", () => {
|
||||
skipLobby: true,
|
||||
autoLeaveWhenOthersLeft: true,
|
||||
waitForCallPickup: false,
|
||||
callIntent: "audio",
|
||||
callIntent: RTC_CALL_INTENT_AUDIO,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
.grid {
|
||||
display: grid;
|
||||
/* A zero floor, so a long label wraps inside its tile rather than widening
|
||||
the grid past the menu. */
|
||||
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||
grid-auto-rows: 1fr;
|
||||
gap: var(--cpd-space-3x);
|
||||
padding: var(--cpd-space-2x) var(--cpd-space-4x) 0;
|
||||
}
|
||||
|
||||
/* Within the grid, to outweigh the menu item styles the tiles are built on. */
|
||||
.grid .tile {
|
||||
position: relative;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
gap: var(--cpd-space-1x);
|
||||
min-inline-size: 0;
|
||||
padding: var(--cpd-space-3x) var(--cpd-space-2x);
|
||||
/* Not hidden: that makes the tile a scroll container, which focus scrolls
|
||||
into view without its border, and so without the focus ring. */
|
||||
overflow: clip;
|
||||
border: var(--cpd-border-width-2) solid
|
||||
var(--cpd-color-border-interactive-secondary);
|
||||
border-radius: var(--cpd-space-2x);
|
||||
background: none;
|
||||
color: var(--cpd-color-icon-secondary);
|
||||
font: inherit;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
/* As the menu items highlight on hover. */
|
||||
.grid .tile[data-highlighted] {
|
||||
background: var(--cpd-color-bg-action-secondary-hovered);
|
||||
}
|
||||
|
||||
.grid .tile[aria-checked="true"] {
|
||||
border-color: var(--cpd-color-border-accent);
|
||||
}
|
||||
|
||||
.grid .tile[data-disabled] {
|
||||
cursor: default;
|
||||
opacity: 0.55;
|
||||
}
|
||||
|
||||
.tile[data-disabled] .picture {
|
||||
filter: grayscale(1);
|
||||
}
|
||||
|
||||
.swatch {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.picture {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
inline-size: 100%;
|
||||
block-size: 100%;
|
||||
object-fit: cover;
|
||||
}
|
||||
|
||||
/* Positioned, or the picture, positioned itself, would cover it. */
|
||||
.mark {
|
||||
position: relative;
|
||||
display: inline-flex;
|
||||
color: var(--cpd-color-icon-accent-primary);
|
||||
}
|
||||
|
||||
/* On a picture the mark needs a ground of its own to be seen. */
|
||||
.picture ~ .mark {
|
||||
border-radius: 50%;
|
||||
background: var(--cpd-color-bg-canvas-default);
|
||||
box-shadow: 0 0 0 var(--cpd-border-width-2) var(--cpd-color-bg-canvas-default);
|
||||
}
|
||||
|
||||
.label {
|
||||
max-inline-size: 100%;
|
||||
font: var(--cpd-font-body-sm-medium);
|
||||
color: var(--cpd-color-text-primary);
|
||||
text-align: center;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
.removable {
|
||||
position: relative;
|
||||
display: flex;
|
||||
min-inline-size: 0;
|
||||
}
|
||||
|
||||
.removable > .tile {
|
||||
flex: 1;
|
||||
}
|
||||
|
||||
/* Clear of the target for choosing the tile. */
|
||||
.remove {
|
||||
position: absolute;
|
||||
inset-block-start: 0;
|
||||
inset-inline-end: 0;
|
||||
translate: 20% -20%;
|
||||
display: none;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
inline-size: 28px;
|
||||
block-size: 28px;
|
||||
border-radius: 50%;
|
||||
background: var(--cpd-color-bg-canvas-default);
|
||||
color: var(--cpd-color-icon-critical-primary);
|
||||
box-shadow: 0 1.2px 2.4px 0 rgb(0 0 0 / 15%);
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.remove:hover {
|
||||
background: var(--cpd-color-bg-critical-subtle-hovered);
|
||||
box-shadow: 0 0 0 var(--cpd-border-width-1)
|
||||
var(--cpd-color-border-critical-subtle);
|
||||
}
|
||||
|
||||
.removable:hover .remove,
|
||||
.removable:focus-within .remove {
|
||||
display: flex;
|
||||
}
|
||||
@@ -0,0 +1,224 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { type FC, type KeyboardEvent, type ReactElement } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { VisuallyHidden } from "@radix-ui/react-visually-hidden";
|
||||
import { InlineSpinner, MenuItem, Tooltip } from "@vector-im/compound-web";
|
||||
import {
|
||||
BlockIcon,
|
||||
BlurIcon,
|
||||
CheckCircleSolidIcon,
|
||||
CloseIcon,
|
||||
PlusIcon,
|
||||
} from "@vector-im/compound-design-tokens/assets/web/icons";
|
||||
|
||||
import styles from "./BackgroundEffectGrid.module.css";
|
||||
|
||||
/**
|
||||
* One choice in the camera menu's Background effects section, as the view
|
||||
* model offers it. The grid names it: pictures are numbered in order.
|
||||
*/
|
||||
export interface BackgroundEffectGridItem {
|
||||
/** Its option id, as `backgroundEffect` gives the one in force. */
|
||||
id: string;
|
||||
kind: "none" | "blur" | "image";
|
||||
/** For kind "image". */
|
||||
imageUrl?: string;
|
||||
/** Whether it is the user's to remove. */
|
||||
removable?: boolean;
|
||||
}
|
||||
|
||||
interface Props {
|
||||
effects: BackgroundEffectGridItem[];
|
||||
selected: string | undefined;
|
||||
/** Undefined where effects can't run: all but no effect are disabled. */
|
||||
onSelect: ((id: string) => void) | undefined;
|
||||
/** Shown on the selected tile until its effect is on screen. */
|
||||
settling?: boolean;
|
||||
/** Id of the text saying what the user should know before choosing. */
|
||||
describedBy?: string;
|
||||
/** Offers to add a background of one's own. Omit to leave that tile out. */
|
||||
onAdd?: () => void;
|
||||
addLabel?: string;
|
||||
/** Removes a removable one, never the one in force. Omit to offer none. */
|
||||
onRemove?: (id: string) => void;
|
||||
removeLabel?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* The background effects as a grid of tiles: one radio choice, walked by the
|
||||
* menu's arrow keys like the device rows. Sits in a menu section, which names
|
||||
* it.
|
||||
*/
|
||||
export const BackgroundEffectGrid: FC<Props> = ({
|
||||
effects,
|
||||
selected,
|
||||
onSelect,
|
||||
settling = false,
|
||||
describedBy,
|
||||
onAdd,
|
||||
addLabel,
|
||||
onRemove,
|
||||
removeLabel,
|
||||
}) => {
|
||||
const { t } = useTranslation();
|
||||
const choose = (id: string): void => {
|
||||
if (id !== selected) onSelect?.(id);
|
||||
};
|
||||
let pictures = 0;
|
||||
const tiles = effects.map((effect) => {
|
||||
const label =
|
||||
effect.kind === "none"
|
||||
? t("background_effects.none")
|
||||
: effect.kind === "blur"
|
||||
? t("background_effects.blur")
|
||||
: t("background_effects.numbered", { n: ++pictures });
|
||||
const checked = effect.id === selected;
|
||||
const disabled = onSelect === undefined && effect.kind !== "none";
|
||||
const remove =
|
||||
effect.removable && !checked && onRemove !== undefined
|
||||
? (): void => onRemove(effect.id)
|
||||
: undefined;
|
||||
const removeProps =
|
||||
remove === undefined
|
||||
? {}
|
||||
: {
|
||||
"aria-keyshortcuts": "Delete",
|
||||
onKeyDown: (e: KeyboardEvent): void => {
|
||||
if (e.key !== "Delete" && e.key !== "Backspace") return;
|
||||
e.preventDefault();
|
||||
remove();
|
||||
},
|
||||
};
|
||||
const content = (
|
||||
<TileContent
|
||||
effect={effect}
|
||||
label={label}
|
||||
checked={checked}
|
||||
settling={checked && settling}
|
||||
/>
|
||||
);
|
||||
const tile = (
|
||||
<MenuItem
|
||||
key={effect.id}
|
||||
role="menuitemradio"
|
||||
aria-checked={checked}
|
||||
aria-busy={checked && settling}
|
||||
disabled={disabled}
|
||||
data-disabled={disabled ? "" : undefined}
|
||||
label={null}
|
||||
hideChevron
|
||||
className={styles.tile}
|
||||
// Without a handler it isn't one of the menu's items, so the keyboard
|
||||
// passes over it.
|
||||
onSelect={
|
||||
disabled
|
||||
? null
|
||||
: (e): void => {
|
||||
// Kept open, so the user sees what they chose take effect.
|
||||
e.preventDefault();
|
||||
choose(effect.id);
|
||||
}
|
||||
}
|
||||
{...removeProps}
|
||||
>
|
||||
{content}
|
||||
</MenuItem>
|
||||
);
|
||||
if (remove === undefined) return tile;
|
||||
// Beside the item, not in it: inside, the item takes the press first and
|
||||
// the tile is chosen instead.
|
||||
return (
|
||||
<div key={effect.id} className={styles.removable}>
|
||||
{tile}
|
||||
<Tooltip label={removeLabel ?? ""}>
|
||||
{/* Not a control of its own, so the menu keeps one keyboard model:
|
||||
the keyboard reaches the same action through Delete. */}
|
||||
<span aria-hidden className={styles.remove} onClick={remove}>
|
||||
<CloseIcon width={20} height={20} />
|
||||
</span>
|
||||
</Tooltip>
|
||||
</div>
|
||||
);
|
||||
});
|
||||
|
||||
// A command rather than a choice, so an item beside the radios.
|
||||
const addContent = (
|
||||
<>
|
||||
<span aria-hidden className={styles.swatch}>
|
||||
<PlusIcon width={24} height={24} />
|
||||
</span>
|
||||
<VisuallyHidden>{addLabel}</VisuallyHidden>
|
||||
</>
|
||||
);
|
||||
const addTile =
|
||||
onAdd === undefined ? null : (
|
||||
<MenuItem
|
||||
label={null}
|
||||
hideChevron
|
||||
className={styles.tile}
|
||||
onSelect={(e) => {
|
||||
e.preventDefault();
|
||||
onAdd();
|
||||
}}
|
||||
>
|
||||
{addContent}
|
||||
</MenuItem>
|
||||
);
|
||||
|
||||
return (
|
||||
// A group of its own, as the section's can't carry the description.
|
||||
<div role="group" aria-describedby={describedBy} className={styles.grid}>
|
||||
{tiles}
|
||||
{addTile}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
function TileContent({
|
||||
effect,
|
||||
label,
|
||||
checked,
|
||||
settling,
|
||||
}: {
|
||||
effect: BackgroundEffectGridItem;
|
||||
/** The tile's accessible name; drawn only where the tile has no picture. */
|
||||
label: string;
|
||||
checked: boolean;
|
||||
settling: boolean;
|
||||
}): ReactElement {
|
||||
return (
|
||||
<>
|
||||
<span aria-hidden className={styles.swatch}>
|
||||
{effect.kind === "image" && (
|
||||
<img className={styles.picture} src={effect.imageUrl} alt="" />
|
||||
)}
|
||||
{settling ? (
|
||||
<span className={styles.mark}>
|
||||
<InlineSpinner size={20} />
|
||||
</span>
|
||||
) : checked ? (
|
||||
<CheckCircleSolidIcon
|
||||
className={styles.mark}
|
||||
width={20}
|
||||
height={20}
|
||||
/>
|
||||
) : effect.kind === "none" ? (
|
||||
<BlockIcon width={20} height={20} />
|
||||
) : effect.kind === "blur" ? (
|
||||
<BlurIcon width={20} height={20} />
|
||||
) : null}
|
||||
</span>
|
||||
{effect.kind === "image" ? (
|
||||
<VisuallyHidden>{label}</VisuallyHidden>
|
||||
) : (
|
||||
<span className={styles.label}>{label}</span>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -22,6 +22,7 @@ import { MediaDevices } from "../state/MediaDevices";
|
||||
import { globalScope } from "../state/ObservableScope";
|
||||
import { constant } from "../state/Behavior";
|
||||
import { type LayoutMode } from "../state/LayoutSwitchViewModel";
|
||||
import { shippedBackgrounds } from "../tracks/backgroundEffectCatalog";
|
||||
|
||||
// consts for tests
|
||||
const reactionIdentifier = "@user:example.com:DEVICE";
|
||||
@@ -123,8 +124,24 @@ export const Default: Story = {
|
||||
toggleAudio: fn(),
|
||||
toggleVideo: fn(),
|
||||
toggleScreenSharing: fn(),
|
||||
toggleBlur: fn(),
|
||||
videoBlurEnabled: true,
|
||||
selectBackgroundEffect: fn(),
|
||||
backgroundEffect: "blur",
|
||||
backgroundEffects: [
|
||||
{ id: "none", kind: "none", removable: false },
|
||||
{ id: "blur", kind: "blur", removable: false },
|
||||
...shippedBackgrounds.map((background) => ({
|
||||
id: `image:${background.id}`,
|
||||
kind: "image" as const,
|
||||
imageUrl: background.imagePath,
|
||||
removable: false,
|
||||
})),
|
||||
],
|
||||
backgroundEffectNotice: undefined,
|
||||
backgroundEffectSettling: false,
|
||||
backgroundImageRefusal: undefined,
|
||||
addBackgroundImage: fn(),
|
||||
removeBackgroundEffect: fn(),
|
||||
cameraTrack: null,
|
||||
hangup: fn(),
|
||||
buttonSize: "lg",
|
||||
showFooter: true,
|
||||
|
||||
@@ -7,6 +7,7 @@ Please see LICENSE in the repository root for full details.
|
||||
|
||||
import { type FC, type JSX, type Ref, useMemo } from "react";
|
||||
import classNames from "classnames";
|
||||
import { useTranslation } from "react-i18next";
|
||||
|
||||
import LogoMark from "../icons/LogoMark.svg?react";
|
||||
import LogoType from "../icons/LogoType.svg?react";
|
||||
@@ -31,6 +32,14 @@ import { type ViewModel } from "../state/ViewModel";
|
||||
import { useBehavior } from "../useBehavior";
|
||||
import { type LayoutSwitchViewModel } from "../state/LayoutSwitchViewModel";
|
||||
import { LayoutSwitch } from "../room/LayoutSwitch";
|
||||
import { type BackgroundEffectGridItem } from "./BackgroundEffectGrid";
|
||||
import { SelfPreview } from "./SelfPreview";
|
||||
import { type UnusableReason } from "../tracks/backgroundImages";
|
||||
import { type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk";
|
||||
|
||||
export interface BackgroundImageRefusal {
|
||||
reason: UnusableReason | "not-kept";
|
||||
}
|
||||
|
||||
export interface AudioOutputSwitcher {
|
||||
targetOutput: string;
|
||||
@@ -56,20 +65,37 @@ export interface FooterActions {
|
||||
toggleAudio: (() => void) | undefined;
|
||||
/** Also controls if the videoMute button is disabled */
|
||||
toggleVideo: (() => void) | undefined;
|
||||
toggleBlur: (() => void) | undefined;
|
||||
/** Undefined where background effects can't be chosen. */
|
||||
selectBackgroundEffect: ((id: string) => void) | undefined;
|
||||
/** Undefined where no background can be added. */
|
||||
addBackgroundImage: ((file: File) => void) | undefined;
|
||||
/** Removes an added background by its option id; never the one in force. */
|
||||
removeBackgroundEffect: ((id: string) => void) | undefined;
|
||||
toggleScreenSharing: (() => void) | undefined;
|
||||
/** Also controls if the settings button is visible */
|
||||
openSettings: (() => void) | undefined;
|
||||
/** Also controls if the hangup button is visible */
|
||||
hangup: (() => void) | undefined;
|
||||
}
|
||||
|
||||
// we do not use any ? optional properties so that the vm type is including all fields.
|
||||
export interface FooterState {
|
||||
audioEnabled: boolean;
|
||||
audioBusy: boolean;
|
||||
videoEnabled: boolean;
|
||||
videoBusy: boolean;
|
||||
videoBlurEnabled: boolean;
|
||||
/** The background effect in force, as its option id. */
|
||||
backgroundEffect: string;
|
||||
/** Every effect on offer, in the order they are shown. */
|
||||
backgroundEffects: BackgroundEffectGridItem[];
|
||||
/** What the user is told about effects here, if anything. */
|
||||
backgroundEffectNotice: "unavailable" | "slow" | undefined;
|
||||
/** Whether the first effect chosen is still being prepared. */
|
||||
backgroundEffectSettling: boolean;
|
||||
/** Why the last file offered couldn't be kept, if it couldn't. */
|
||||
backgroundImageRefusal: BackgroundImageRefusal | undefined;
|
||||
/** The camera to preview, with the effect in force; null while it is off. */
|
||||
cameraTrack: LocalVideoMediaTrack | null;
|
||||
showFooter: boolean;
|
||||
|
||||
/* This is needed for WindowMode = "flat" */
|
||||
@@ -117,12 +143,14 @@ export interface FooterProps {
|
||||
children?: JSX.Element | JSX.Element[] | false;
|
||||
vm: ViewModel<FooterSnapshot>;
|
||||
}
|
||||
|
||||
export const CallFooter: FC<FooterProps> = ({
|
||||
className,
|
||||
ref,
|
||||
children,
|
||||
vm,
|
||||
}) => {
|
||||
const { t } = useTranslation();
|
||||
const asOverlay = useBehavior(vm.asOverlay$);
|
||||
const showFooter = useBehavior(vm.showFooter$);
|
||||
const hideControls = useBehavior(vm.hideControls$);
|
||||
@@ -151,8 +179,29 @@ export const CallFooter: FC<FooterProps> = ({
|
||||
const selectedAudioOutput = useBehavior(vm.selectedAudioOutput$);
|
||||
const selectAudioOutputOption = useBehavior(vm.selectAudioOutputOption$);
|
||||
const selectVideoButtonOption = useBehavior(vm.selectVideoButtonOption$);
|
||||
const toggleBlur = useBehavior(vm.toggleBlur$);
|
||||
const videoBlurEnabled = useBehavior(vm.videoBlurEnabled$);
|
||||
const backgroundEffect = useBehavior(vm.backgroundEffect$);
|
||||
const selectBackgroundEffect = useBehavior(vm.selectBackgroundEffect$);
|
||||
const backgroundEffectNotice = useBehavior(vm.backgroundEffectNotice$);
|
||||
const backgroundEffectSettling = useBehavior(vm.backgroundEffectSettling$);
|
||||
const addBackgroundImage = useBehavior(vm.addBackgroundImage$);
|
||||
const removeBackgroundEffect = useBehavior(vm.removeBackgroundEffect$);
|
||||
const cameraTrack = useBehavior(vm.cameraTrack$);
|
||||
const refusal = useBehavior(vm.backgroundImageRefusal$);
|
||||
const refusalMessage = useMemo(() => {
|
||||
switch (refusal?.reason) {
|
||||
case undefined:
|
||||
return undefined;
|
||||
case "not-an-image":
|
||||
return { text: t("error.background_not_an_image") };
|
||||
case "animated":
|
||||
return { text: t("error.background_animated") };
|
||||
case "undecodable":
|
||||
return { text: t("error.background_undecodable") };
|
||||
case "not-kept":
|
||||
return { text: t("error.background_not_kept") };
|
||||
}
|
||||
}, [refusal, t]);
|
||||
const backgroundEffects = useBehavior(vm.backgroundEffects$);
|
||||
const buttonSize = useBehavior(vm.buttonSize$);
|
||||
const showLogo = useBehavior(vm.showLogo$);
|
||||
|
||||
@@ -214,8 +263,21 @@ export const CallFooter: FC<FooterProps> = ({
|
||||
options={videoOptions}
|
||||
selectedOption={selectedVideo}
|
||||
onSelect={selectVideoButtonOption}
|
||||
videoBlurToggleClick={toggleBlur}
|
||||
videoBlurEnabled={videoBlurEnabled}
|
||||
backgroundEffects={backgroundEffects}
|
||||
selectedBackgroundEffect={backgroundEffect}
|
||||
onSelectBackgroundEffect={selectBackgroundEffect}
|
||||
backgroundEffectSettling={backgroundEffectSettling}
|
||||
onAddBackgroundImage={addBackgroundImage}
|
||||
onRemoveBackgroundEffect={removeBackgroundEffect}
|
||||
selfPreview={<SelfPreview track={cameraTrack} />}
|
||||
backgroundImageRefusal={refusalMessage}
|
||||
backgroundEffectNotice={
|
||||
backgroundEffectNotice === "unavailable"
|
||||
? t("background_effects.unavailable")
|
||||
: backgroundEffectNotice === "slow"
|
||||
? t("background_effects.slow")
|
||||
: undefined
|
||||
}
|
||||
/>,
|
||||
);
|
||||
} else {
|
||||
|
||||
@@ -5,10 +5,19 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { BehaviorSubject } from "rxjs";
|
||||
import { type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk";
|
||||
|
||||
import { testScope, mockMuteStates, mockMediaDevices } from "../utils/test";
|
||||
import { constant } from "../state/Behavior";
|
||||
import {
|
||||
flushPromises,
|
||||
testScope,
|
||||
mockMuteStates,
|
||||
mockMediaDevices,
|
||||
mockVideoTrack,
|
||||
} from "../utils/test";
|
||||
import { type Behavior, constant } from "../state/Behavior";
|
||||
import { type BackgroundEffectsState } from "../state/BackgroundEffects";
|
||||
import type { CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import type { Layout } from "../state/layout-types";
|
||||
import type { SpotlightTileViewModel } from "../state/TileViewModel";
|
||||
@@ -18,6 +27,15 @@ import {
|
||||
createLobbyFooterViewModel,
|
||||
} from "./CallFooterViewModel";
|
||||
import { HeaderStyle } from "../UrlParams";
|
||||
import { backgroundEffect as backgroundEffectSetting } from "../settings/settings";
|
||||
import { shippedBackgrounds } from "../tracks/backgroundEffectCatalog";
|
||||
import type * as BackgroundImages from "../tracks/backgroundImages";
|
||||
import {
|
||||
type AddedBackground,
|
||||
addedBackgrounds,
|
||||
maxAddedBackgrounds,
|
||||
UnusableImage,
|
||||
} from "../tracks/backgroundImages";
|
||||
|
||||
const platformMock = vi.hoisted(() => vi.fn(() => "desktop"));
|
||||
vi.mock("../Platform", () => ({
|
||||
@@ -28,10 +46,28 @@ vi.mock("../Platform", () => ({
|
||||
|
||||
// The SDK's own check needs WebGL and canvas APIs jsdom does not have.
|
||||
const sdkSupportMock = vi.hoisted(() => vi.fn(() => false));
|
||||
const modernRouteMock = vi.hoisted(() => vi.fn(() => true));
|
||||
vi.mock("@livekit/track-processors", () => ({
|
||||
supportsBackgroundProcessors: (): boolean => sdkSupportMock(),
|
||||
supportsModernBackgroundProcessors: (): boolean => modernRouteMock(),
|
||||
}));
|
||||
|
||||
const store = vi.hoisted(() => ({
|
||||
add: vi.fn<(file: Blob) => Promise<string>>(),
|
||||
remove: vi.fn<(id: string) => Promise<void>>(),
|
||||
}));
|
||||
vi.mock("../tracks/backgroundImages", async (original) => {
|
||||
const { BehaviorSubject } = await import("rxjs");
|
||||
return {
|
||||
...(await original<typeof BackgroundImages>()),
|
||||
addedBackgrounds: {
|
||||
added$: new BehaviorSubject<AddedBackground[] | undefined>([]),
|
||||
add: store.add,
|
||||
remove: store.remove,
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
const outputSelectionMock = vi.hoisted(() => vi.fn(() => true));
|
||||
vi.mock("livekit-client", () => ({
|
||||
supportsAudioOutputSelection: (): boolean => outputSelectionMock(),
|
||||
@@ -45,6 +81,7 @@ vi.mock("livekit-client", () => ({
|
||||
function buildMinimalCallViewModel(layout: Layout): CallViewModel {
|
||||
return {
|
||||
layout$: constant(layout),
|
||||
camera$: constant(undefined),
|
||||
edgeToEdge$: constant(false),
|
||||
showHeader$: constant(false),
|
||||
hangup: (): void => {},
|
||||
@@ -76,6 +113,16 @@ const pipLayout: Layout = {
|
||||
spotlight: {} as SpotlightTileViewModel,
|
||||
};
|
||||
|
||||
const noPipeline = constant<BackgroundEffectsState>({
|
||||
supported: false,
|
||||
processor: undefined,
|
||||
});
|
||||
|
||||
/** The provider's verdict, which the SDK's check stands for here. */
|
||||
function pipelineAsTheSdkSays(): Behavior<BackgroundEffectsState> {
|
||||
return constant({ supported: sdkSupportMock(), processor: undefined });
|
||||
}
|
||||
|
||||
const twoMicsAndOneCamMediaDevices = mockMediaDevices({
|
||||
audioInput: {
|
||||
available$: constant(
|
||||
@@ -107,6 +154,7 @@ describe("createCallFooterViewModel", () => {
|
||||
buildMinimalCallViewModel(gridLayout),
|
||||
mockMuteStates(),
|
||||
twoMicsAndOneCamMediaDevices,
|
||||
noPipeline,
|
||||
/* reactionIdentifier */ undefined,
|
||||
{ showControls: true, header: HeaderStyle.Standard },
|
||||
);
|
||||
@@ -149,6 +197,7 @@ describe("createCallFooterViewModel", () => {
|
||||
select: vi.fn(),
|
||||
},
|
||||
}),
|
||||
noPipeline,
|
||||
/* reactionIdentifier */ undefined,
|
||||
{ showControls: true, header: HeaderStyle.Standard },
|
||||
);
|
||||
@@ -167,6 +216,7 @@ describe("createCallFooterViewModel", () => {
|
||||
buildMinimalCallViewModel(layout),
|
||||
mockMuteStates(),
|
||||
twoMicsAndOneCamMediaDevices,
|
||||
noPipeline,
|
||||
/* reactionIdentifier */ undefined,
|
||||
{ showControls: true, header: HeaderStyle.Standard },
|
||||
);
|
||||
@@ -189,6 +239,7 @@ describe("createCallFooterViewModel", () => {
|
||||
buildMinimalCallViewModel(gridLayout),
|
||||
mockMuteStates(),
|
||||
twoMicsAndOneCamMediaDevices,
|
||||
noPipeline,
|
||||
/* reactionIdentifier */ undefined,
|
||||
{ showControls: true, header: HeaderStyle.Standard },
|
||||
);
|
||||
@@ -222,6 +273,8 @@ describe("createCallFooterViewModel", () => {
|
||||
});
|
||||
|
||||
describe("background effects", () => {
|
||||
afterEach(() => backgroundEffectSetting.setValue("none"));
|
||||
|
||||
// The lobby, because in a call the whole device switcher is already
|
||||
// withheld on a phone, whatever the verdict says.
|
||||
function lobbyOn(
|
||||
@@ -232,25 +285,230 @@ describe("createCallFooterViewModel", () => {
|
||||
testScope(),
|
||||
mockMuteStates(),
|
||||
twoMicsAndOneCamMediaDevices,
|
||||
pipelineAsTheSdkSays(),
|
||||
/* camera$ */ constant(null),
|
||||
/* openSettings */ undefined,
|
||||
/* hangup */ undefined,
|
||||
/* showLogo */ false,
|
||||
);
|
||||
}
|
||||
|
||||
it("offers nothing on a phone, where the pipeline would refuse them", () => {
|
||||
it("offers them on a phone whose browser can run them", () => {
|
||||
sdkSupportMock.mockReturnValue(true);
|
||||
expect(lobbyOn("ios").toggleBlur$.value).toBeUndefined();
|
||||
expect(lobbyOn("ios").selectBackgroundEffect$.value).toBeDefined();
|
||||
});
|
||||
|
||||
it("offers them where the pipeline will honour them", () => {
|
||||
it("offers them on a desktop whose browser can run them", () => {
|
||||
sdkSupportMock.mockReturnValue(true);
|
||||
expect(lobbyOn("desktop").toggleBlur$.value).toBeDefined();
|
||||
expect(lobbyOn("desktop").selectBackgroundEffect$.value).toBeDefined();
|
||||
});
|
||||
|
||||
it("offers nothing where the browser itself cannot run them", () => {
|
||||
sdkSupportMock.mockReturnValue(false);
|
||||
expect(lobbyOn("desktop").toggleBlur$.value).toBeUndefined();
|
||||
expect(lobbyOn("desktop").selectBackgroundEffect$.value).toBeUndefined();
|
||||
expect(lobbyOn("desktop").addBackgroundImage$.value).toBeUndefined();
|
||||
});
|
||||
|
||||
it("puts no effect in force where the browser cannot run them", () => {
|
||||
backgroundEffectSetting.setValue("blur");
|
||||
sdkSupportMock.mockReturnValue(false);
|
||||
expect(lobbyOn("desktop").backgroundEffect$.value).toBe("none");
|
||||
sdkSupportMock.mockReturnValue(true);
|
||||
expect(lobbyOn("desktop").backgroundEffect$.value).toBe("blur");
|
||||
});
|
||||
|
||||
it("stores the effect chosen", () => {
|
||||
sdkSupportMock.mockReturnValue(true);
|
||||
const vm = lobbyOn("desktop");
|
||||
vm.selectBackgroundEffect$.value?.("image:arc");
|
||||
expect(vm.backgroundEffect$.value).toBe("image:arc");
|
||||
// What isn't an effect on offer is stored as none.
|
||||
vm.selectBackgroundEffect$.value?.("image:gone");
|
||||
expect(backgroundEffectSetting.getValue()).toBe("none");
|
||||
});
|
||||
|
||||
it("says they run slowly where only the slower route exists", () => {
|
||||
sdkSupportMock.mockReturnValue(true);
|
||||
modernRouteMock.mockReturnValue(false);
|
||||
const slow = lobbyOn("desktop");
|
||||
expect(slow.backgroundEffectNotice$.value).toBe("slow");
|
||||
expect(slow.selectBackgroundEffect$.value).toBeDefined();
|
||||
modernRouteMock.mockReturnValue(true);
|
||||
expect(lobbyOn("desktop").backgroundEffectNotice$.value).toBeUndefined();
|
||||
});
|
||||
|
||||
it("offers every effect in order, the added ones removable", () => {
|
||||
sdkSupportMock.mockReturnValue(true);
|
||||
const added$ = addedBackgrounds.added$ as BehaviorSubject<
|
||||
AddedBackground[] | undefined
|
||||
>;
|
||||
added$.next([{ id: "mine", url: "blob:mine" }]);
|
||||
expect(lobbyOn("desktop").backgroundEffects$.value).toEqual([
|
||||
{ id: "none", kind: "none", removable: false },
|
||||
{ id: "blur", kind: "blur", removable: false },
|
||||
...shippedBackgrounds.map((background) => ({
|
||||
id: `image:${background.id}`,
|
||||
kind: "image",
|
||||
imageUrl: background.imagePath,
|
||||
removable: false,
|
||||
})),
|
||||
{
|
||||
id: "added:mine",
|
||||
kind: "image",
|
||||
imageUrl: "blob:mine",
|
||||
removable: true,
|
||||
},
|
||||
]);
|
||||
added$.next([]);
|
||||
});
|
||||
|
||||
it("says they are unavailable where they cannot be chosen", () => {
|
||||
sdkSupportMock.mockReturnValue(false);
|
||||
expect(lobbyOn("desktop").backgroundEffectNotice$.value).toBe(
|
||||
"unavailable",
|
||||
);
|
||||
});
|
||||
|
||||
it("passes on the wait for the first effect", () => {
|
||||
platformMock.mockReturnValue("desktop");
|
||||
const pipeline$ = new BehaviorSubject<BackgroundEffectsState>({
|
||||
supported: true,
|
||||
processor: undefined,
|
||||
wait: "settling",
|
||||
});
|
||||
const vm = createLobbyFooterViewModel(
|
||||
testScope(),
|
||||
mockMuteStates(),
|
||||
twoMicsAndOneCamMediaDevices,
|
||||
pipeline$,
|
||||
/* camera$ */ constant(null),
|
||||
/* openSettings */ undefined,
|
||||
/* hangup */ undefined,
|
||||
/* showLogo */ false,
|
||||
);
|
||||
expect(vm.backgroundEffectSettling$.value).toBe(true);
|
||||
pipeline$.next({ ...pipeline$.value, wait: undefined });
|
||||
expect(vm.backgroundEffectSettling$.value).toBe(false);
|
||||
});
|
||||
|
||||
it("offers nothing once the pipeline fails to build, keeping the choice", () => {
|
||||
platformMock.mockReturnValue("desktop");
|
||||
backgroundEffectSetting.setValue("blur");
|
||||
const pipeline$ = new BehaviorSubject<BackgroundEffectsState>({
|
||||
supported: true,
|
||||
processor: undefined,
|
||||
});
|
||||
const vm = createLobbyFooterViewModel(
|
||||
testScope(),
|
||||
mockMuteStates(),
|
||||
twoMicsAndOneCamMediaDevices,
|
||||
pipeline$,
|
||||
/* camera$ */ constant(null),
|
||||
/* openSettings */ undefined,
|
||||
/* hangup */ undefined,
|
||||
/* showLogo */ false,
|
||||
);
|
||||
expect(vm.backgroundEffect$.value).toBe("blur");
|
||||
|
||||
pipeline$.next({ supported: false, processor: undefined });
|
||||
expect(vm.selectBackgroundEffect$.value).toBeUndefined();
|
||||
expect(vm.addBackgroundImage$.value).toBeUndefined();
|
||||
expect(vm.backgroundEffect$.value).toBe("none");
|
||||
expect(backgroundEffectSetting.getValue()).toBe("blur");
|
||||
});
|
||||
|
||||
it("adding is unavailable at the limit", () => {
|
||||
sdkSupportMock.mockReturnValue(true);
|
||||
const added$ = addedBackgrounds.added$ as BehaviorSubject<
|
||||
AddedBackground[] | undefined
|
||||
>;
|
||||
const vm = lobbyOn("desktop");
|
||||
expect(vm.addBackgroundImage$.value).toBeDefined();
|
||||
added$.next(
|
||||
Array.from({ length: maxAddedBackgrounds }, (_, i) => ({
|
||||
id: `${i}`,
|
||||
url: `blob:${i}`,
|
||||
})),
|
||||
);
|
||||
expect(vm.addBackgroundImage$.value).toBeUndefined();
|
||||
added$.next(added$.value!.slice(1));
|
||||
expect(vm.addBackgroundImage$.value).toBeDefined();
|
||||
added$.next([]);
|
||||
});
|
||||
|
||||
it("says why each refused file was refused", async () => {
|
||||
sdkSupportMock.mockReturnValue(true);
|
||||
const vm = lobbyOn("desktop");
|
||||
store.add.mockRejectedValue(new UnusableImage("animated"));
|
||||
vm.addBackgroundImage$.value!(new File(["x"], "a.gif"));
|
||||
await flushPromises();
|
||||
const first = vm.backgroundImageRefusal$.value;
|
||||
expect(first).toEqual({ reason: "animated" });
|
||||
|
||||
// The same again is a refusal of its own, so it is shown again.
|
||||
vm.addBackgroundImage$.value!(new File(["x"], "a.gif"));
|
||||
await flushPromises();
|
||||
expect(vm.backgroundImageRefusal$.value).toEqual({ reason: "animated" });
|
||||
expect(vm.backgroundImageRefusal$.value).not.toBe(first);
|
||||
|
||||
store.add.mockRejectedValue(new Error("quota"));
|
||||
vm.addBackgroundImage$.value!(new File(["x"], "b.png"));
|
||||
await flushPromises();
|
||||
expect(vm.backgroundImageRefusal$.value).toEqual({ reason: "not-kept" });
|
||||
|
||||
store.add.mockResolvedValue("kept");
|
||||
vm.addBackgroundImage$.value!(new File(["x"], "c.png"));
|
||||
await flushPromises();
|
||||
expect(vm.backgroundImageRefusal$.value).toBeUndefined();
|
||||
});
|
||||
|
||||
it("never removes the background in force", () => {
|
||||
sdkSupportMock.mockReturnValue(true);
|
||||
store.remove.mockResolvedValue();
|
||||
const vm = lobbyOn("desktop");
|
||||
backgroundEffectSetting.setValue("added:worn");
|
||||
vm.removeBackgroundEffect$.value!("added:worn");
|
||||
expect(store.remove).not.toHaveBeenCalled();
|
||||
vm.removeBackgroundEffect$.value!("added:spare");
|
||||
expect(store.remove).toHaveBeenCalledWith("spare");
|
||||
});
|
||||
|
||||
it("previews the lobby's camera while there is one", () => {
|
||||
const track = mockVideoTrack({});
|
||||
const camera$ = new BehaviorSubject<LocalVideoMediaTrack | null>(track);
|
||||
const vm = createLobbyFooterViewModel(
|
||||
testScope(),
|
||||
mockMuteStates(),
|
||||
twoMicsAndOneCamMediaDevices,
|
||||
noPipeline,
|
||||
camera$,
|
||||
/* openSettings */ undefined,
|
||||
/* hangup */ undefined,
|
||||
/* showLogo */ false,
|
||||
);
|
||||
expect(vm.cameraTrack$.value).toBe(track);
|
||||
// The lobby releases the camera when it is turned off
|
||||
camera$.next(null);
|
||||
expect(vm.cameraTrack$.value).toBeNull();
|
||||
});
|
||||
|
||||
it("previews the call's camera only while it is not muted", () => {
|
||||
const muted$ = new BehaviorSubject(false);
|
||||
const track = mockVideoTrack({ muted$ });
|
||||
const vm = createCallFooterViewModel(
|
||||
testScope(),
|
||||
{ ...buildMinimalCallViewModel(gridLayout), camera$: constant(track) },
|
||||
mockMuteStates(),
|
||||
twoMicsAndOneCamMediaDevices,
|
||||
noPipeline,
|
||||
/* reactionIdentifier */ undefined,
|
||||
{ showControls: true, header: HeaderStyle.Standard },
|
||||
);
|
||||
expect(vm.cameraTrack$.value).toBe(track);
|
||||
muted$.next(true);
|
||||
expect(vm.cameraTrack$.value).toBeNull();
|
||||
muted$.next(false);
|
||||
expect(vm.cameraTrack$.value).toBe(track);
|
||||
});
|
||||
|
||||
it("availability is the same before and during a call", () => {
|
||||
@@ -262,11 +520,15 @@ describe("createCallFooterViewModel", () => {
|
||||
buildMinimalCallViewModel(gridLayout),
|
||||
mockMuteStates(),
|
||||
twoMicsAndOneCamMediaDevices,
|
||||
pipelineAsTheSdkSays(),
|
||||
/* reactionIdentifier */ undefined,
|
||||
{ showControls: true, header: HeaderStyle.Standard },
|
||||
);
|
||||
const offeredInLobby = lobby.toggleBlur$.value !== undefined;
|
||||
expect(inCall.toggleBlur$.value !== undefined).toBe(offeredInLobby);
|
||||
const offeredInLobby =
|
||||
lobby.selectBackgroundEffect$.value !== undefined;
|
||||
expect(inCall.selectBackgroundEffect$.value !== undefined).toBe(
|
||||
offeredInLobby,
|
||||
);
|
||||
expect(offeredInLobby).toBe(supported);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -5,13 +5,34 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { combineLatest, map, type Observable, switchMap } from "rxjs";
|
||||
import {
|
||||
BehaviorSubject,
|
||||
combineLatest,
|
||||
distinctUntilChanged,
|
||||
map,
|
||||
type Observable,
|
||||
of,
|
||||
switchMap,
|
||||
} from "rxjs";
|
||||
import { supportsAudioOutputSelection } from "livekit-client";
|
||||
import { type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk";
|
||||
import { supportsModernBackgroundProcessors } from "@livekit/track-processors";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
|
||||
import { supportsBackgroundProcessors } from "../tracks/backgroundProcessing";
|
||||
import { parseEffect } from "../tracks/backgroundEffectCatalog";
|
||||
import {
|
||||
parseEffect,
|
||||
serializeEffect,
|
||||
shippedBackgrounds,
|
||||
} from "../tracks/backgroundEffectCatalog";
|
||||
import {
|
||||
type AddedBackground,
|
||||
addedBackgrounds,
|
||||
maxAddedBackgrounds,
|
||||
UnusableImage,
|
||||
} from "../tracks/backgroundImages";
|
||||
|
||||
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import { type BackgroundEffectsState } from "../state/BackgroundEffects";
|
||||
import { type MenuOptions } from "./MediaMuteAndSwitchButton";
|
||||
import { type MediaDevices } from "../state/MediaDevices";
|
||||
import {
|
||||
@@ -24,7 +45,8 @@ import { type MuteStates } from "../state/MuteStates";
|
||||
import { createStaticViewModel, type ViewModel } from "../state/ViewModel";
|
||||
import { HeaderStyle } from "../UrlParams";
|
||||
import { platform } from "../Platform";
|
||||
import { type FooterSnapshot } from "./CallFooter";
|
||||
import { type BackgroundImageRefusal, type FooterSnapshot } from "./CallFooter";
|
||||
import { type BackgroundEffectGridItem } from "./BackgroundEffectGrid";
|
||||
|
||||
/**
|
||||
* Shared helper: maps MuteStates into the audio/video enabled + toggle behaviors
|
||||
@@ -58,13 +80,16 @@ function buildMuteBehaviors(
|
||||
|
||||
/**
|
||||
* Shared helper: maps MediaDevices into the audio/video device-list behaviors
|
||||
* needed by FooterSnapshot (options, selection, callbacks, blur toggle).
|
||||
* needed by FooterSnapshot (options, selection, callbacks, background effect).
|
||||
*/
|
||||
function buildDeviceBehaviors(
|
||||
scope: ObservableScope,
|
||||
mediaDevices: MediaDevices,
|
||||
/** return empty arrays for audioOptions and videoOptions*/
|
||||
disableSwitcher$: Behavior<boolean>,
|
||||
backgroundEffectsState$: Observable<BackgroundEffectsState>,
|
||||
/** The camera as the pipeline has it, for the menu's preview. */
|
||||
camera$: Observable<LocalVideoMediaTrack | null | undefined>,
|
||||
): Pick<
|
||||
ViewModel<FooterSnapshot>,
|
||||
| "audioOptions$"
|
||||
@@ -76,8 +101,15 @@ function buildDeviceBehaviors(
|
||||
| "videoOptions$"
|
||||
| "selectedVideo$"
|
||||
| "selectVideoButtonOption$"
|
||||
| "toggleBlur$"
|
||||
| "videoBlurEnabled$"
|
||||
| "backgroundEffect$"
|
||||
| "selectBackgroundEffect$"
|
||||
| "backgroundEffects$"
|
||||
| "backgroundEffectNotice$"
|
||||
| "backgroundEffectSettling$"
|
||||
| "addBackgroundImage$"
|
||||
| "backgroundImageRefusal$"
|
||||
| "removeBackgroundEffect$"
|
||||
| "cameraTrack$"
|
||||
> {
|
||||
const options$ = (
|
||||
available$: Behavior<Map<string, MenuOptions["label"]>>,
|
||||
@@ -94,10 +126,18 @@ function buildDeviceBehaviors(
|
||||
),
|
||||
);
|
||||
|
||||
const blurOn$ = scope.behavior(
|
||||
backgroundEffectSetting.value$.pipe(
|
||||
map((raw) => parseEffect(raw).kind === "blur"),
|
||||
),
|
||||
// Also false once the pipeline has failed to build this session.
|
||||
const supported$ = backgroundEffectsState$.pipe(
|
||||
map(({ supported }) => supported === true),
|
||||
distinctUntilChanged(),
|
||||
);
|
||||
const slow = !supportsModernBackgroundProcessors();
|
||||
const offered$ = combineLatest([disableSwitcher$, supported$]).pipe(
|
||||
map(([switcherDisabled, supported]) => !switcherDisabled && supported),
|
||||
);
|
||||
// A new object for each refusal, so the same one twice is shown twice.
|
||||
const refusal$ = new BehaviorSubject<BackgroundImageRefusal | undefined>(
|
||||
undefined,
|
||||
);
|
||||
return {
|
||||
audioOptions$: scope.behavior(options$(mediaDevices.audioInput.available$)),
|
||||
@@ -123,18 +163,93 @@ function buildDeviceBehaviors(
|
||||
mediaDevices.videoInput.selected$.pipe(map((s) => s?.id)),
|
||||
),
|
||||
selectVideoButtonOption$: constant(mediaDevices.videoInput.select),
|
||||
toggleBlur$: scope.behavior(
|
||||
combineLatest([blurOn$, disableSwitcher$]).pipe(
|
||||
map(([current, switcherDisabled]) => {
|
||||
return !switcherDisabled && supportsBackgroundProcessors()
|
||||
? (): void => {
|
||||
backgroundEffectSetting.setValue(current ? "none" : "blur");
|
||||
backgroundEffect$: scope.behavior(
|
||||
combineLatest([backgroundEffectSetting.value$, supported$]).pipe(
|
||||
map(([raw, supported]) =>
|
||||
supported ? serializeEffect(parseEffect(raw)) : "none",
|
||||
),
|
||||
),
|
||||
),
|
||||
selectBackgroundEffect$: scope.behavior(
|
||||
offered$.pipe(
|
||||
map((offered) =>
|
||||
offered
|
||||
? (id: string): void =>
|
||||
backgroundEffectSetting.setValue(
|
||||
serializeEffect(parseEffect(id)),
|
||||
)
|
||||
: undefined,
|
||||
),
|
||||
),
|
||||
),
|
||||
backgroundEffects$: scope.behavior(
|
||||
addedBackgrounds.added$.pipe(
|
||||
map((added) => backgroundEffectChoices(added ?? [])),
|
||||
),
|
||||
),
|
||||
backgroundEffectNotice$: scope.behavior(
|
||||
offered$.pipe(
|
||||
map((offered) =>
|
||||
!offered ? "unavailable" : slow ? "slow" : undefined,
|
||||
),
|
||||
),
|
||||
),
|
||||
backgroundEffectSettling$: scope.behavior(
|
||||
backgroundEffectsState$.pipe(
|
||||
map(({ wait }) => wait !== undefined),
|
||||
distinctUntilChanged(),
|
||||
),
|
||||
false,
|
||||
),
|
||||
// Kept and offered, not put on: that waits for the user to choose it.
|
||||
addBackgroundImage$: scope.behavior(
|
||||
combineLatest([offered$, addedBackgrounds.added$]).pipe(
|
||||
map(([offered, added]) =>
|
||||
offered && (added?.length ?? 0) < maxAddedBackgrounds
|
||||
? (file: File): void => {
|
||||
refusal$.next(undefined);
|
||||
addedBackgrounds.add(file).catch((e) => {
|
||||
logger.warn("Could not keep that background", e);
|
||||
refusal$.next({
|
||||
reason: e instanceof UnusableImage ? e.reason : "not-kept",
|
||||
});
|
||||
});
|
||||
}
|
||||
: undefined;
|
||||
}),
|
||||
: undefined,
|
||||
),
|
||||
),
|
||||
),
|
||||
backgroundImageRefusal$: refusal$,
|
||||
// None while muted: the call mutes its track rather than releasing it.
|
||||
cameraTrack$: scope.behavior(
|
||||
camera$.pipe(
|
||||
switchMap((camera) =>
|
||||
camera
|
||||
? camera.muted$.pipe(map((muted) => (muted ? null : camera)))
|
||||
: of(null),
|
||||
),
|
||||
),
|
||||
null,
|
||||
),
|
||||
// Never the one in force: taking it away would leave nothing chosen.
|
||||
removeBackgroundEffect$: scope.behavior(
|
||||
offered$.pipe(
|
||||
map((offered) =>
|
||||
offered
|
||||
? (optionId: string): void => {
|
||||
const effect = parseEffect(optionId);
|
||||
if (effect.kind !== "added") return;
|
||||
if (backgroundEffectSetting.getValue() === optionId) return;
|
||||
addedBackgrounds
|
||||
.remove(effect.id)
|
||||
.catch((e) =>
|
||||
logger.warn("Could not remove that background", e),
|
||||
);
|
||||
}
|
||||
: undefined,
|
||||
),
|
||||
),
|
||||
),
|
||||
videoBlurEnabled$: blurOn$,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -145,6 +260,8 @@ function buildDeviceBehaviors(
|
||||
* @param callModel - The root CallViewModel; provides layout, grid mode, reactions, etc.
|
||||
* @param muteStates - Audio and video mute state + toggles.
|
||||
* @param mediaDevices - Available and selected input devices.
|
||||
* @param backgroundEffectsState$ - The background pipeline's state: whether
|
||||
* effects run here, and the wait for the first.
|
||||
* @param reactionIdentifier - The local user's reaction identifier string, or
|
||||
* undefined when reactions are not supported (hides the reaction button).
|
||||
* @param options - `showControls`: whether the call controls should be shown.
|
||||
@@ -155,6 +272,7 @@ export function createCallFooterViewModel(
|
||||
callModel: CallViewModel,
|
||||
muteStates: MuteStates,
|
||||
mediaDevices: MediaDevices,
|
||||
backgroundEffectsState$: Observable<BackgroundEffectsState>,
|
||||
reactionIdentifier: string | undefined,
|
||||
options: { showControls: boolean; header: HeaderStyle },
|
||||
): ViewModel<FooterSnapshot> {
|
||||
@@ -169,7 +287,13 @@ export function createCallFooterViewModel(
|
||||
);
|
||||
return {
|
||||
...buildMuteBehaviors(scope, muteStates),
|
||||
...buildDeviceBehaviors(scope, mediaDevices, disableDeviceSwitcher$),
|
||||
...buildDeviceBehaviors(
|
||||
scope,
|
||||
mediaDevices,
|
||||
disableDeviceSwitcher$,
|
||||
backgroundEffectsState$,
|
||||
callModel.camera$,
|
||||
),
|
||||
// candidat to move into the FooterViewModel
|
||||
showFooter$: callModel.showFooter$,
|
||||
hideControls$: constant(!showControls),
|
||||
@@ -231,6 +355,9 @@ export function createCallFooterViewModel(
|
||||
* @param scope - ObservableScope that bounds the lifetime of derived behaviors.
|
||||
* @param muteStates - Audio and video mute state + toggles.
|
||||
* @param mediaDevices - Available and selected input devices.
|
||||
* @param backgroundEffectsState$ - The background pipeline's state: whether
|
||||
* effects run here, and the wait for the first.
|
||||
* @param camera$ - The lobby's camera, for the camera menu's preview.
|
||||
* @param openSettings - Callback to open the settings modal, or undefined.
|
||||
* @param hangup - Callback to leave/cancel, or undefined (hides the button).
|
||||
* @param showLogo - Whether to show the Element Call logo.
|
||||
@@ -239,6 +366,8 @@ export function createLobbyFooterViewModel(
|
||||
scope: ObservableScope,
|
||||
muteStates: MuteStates,
|
||||
mediaDevices: MediaDevices,
|
||||
backgroundEffectsState$: Observable<BackgroundEffectsState>,
|
||||
camera$: Behavior<LocalVideoMediaTrack | null>,
|
||||
openSettings: (() => void) | undefined,
|
||||
hangup: (() => void) | undefined,
|
||||
showLogo: boolean,
|
||||
@@ -282,6 +411,33 @@ export function createLobbyFooterViewModel(
|
||||
selectVideoButtonOption: undefined,
|
||||
}),
|
||||
...buildMuteBehaviors(scope, muteStates),
|
||||
...buildDeviceBehaviors(scope, mediaDevices, constant(false)),
|
||||
...buildDeviceBehaviors(
|
||||
scope,
|
||||
mediaDevices,
|
||||
constant(false),
|
||||
backgroundEffectsState$,
|
||||
camera$,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
function backgroundEffectChoices(
|
||||
added: AddedBackground[],
|
||||
): BackgroundEffectGridItem[] {
|
||||
return [
|
||||
{ id: serializeEffect({ kind: "none" }), kind: "none", removable: false },
|
||||
{ id: serializeEffect({ kind: "blur" }), kind: "blur", removable: false },
|
||||
...shippedBackgrounds.map((background) => ({
|
||||
id: serializeEffect({ kind: "shipped", id: background.id }),
|
||||
kind: "image" as const,
|
||||
imageUrl: background.imagePath,
|
||||
removable: false,
|
||||
})),
|
||||
...added.map((background) => ({
|
||||
id: serializeEffect({ kind: "added", id: background.id }),
|
||||
kind: "image" as const,
|
||||
imageUrl: background.url,
|
||||
removable: true,
|
||||
})),
|
||||
];
|
||||
}
|
||||
|
||||
@@ -45,6 +45,8 @@ Please see LICENSE in the repository root for full details.
|
||||
so CSS can't size it against the call. */
|
||||
.deviceList {
|
||||
max-block-size: var(--device-list-max-height);
|
||||
/* The raised headings stay raised within the list, under the menu's frame. */
|
||||
isolation: isolate;
|
||||
}
|
||||
|
||||
/* Compound's heading rule heads each section; the design adds space below it. */
|
||||
@@ -57,6 +59,12 @@ Please see LICENSE in the repository root for full details.
|
||||
margin-block-start: var(--cpd-space-7x);
|
||||
}
|
||||
|
||||
/* Raised, or the positioned effect tiles scroll over the heading they sit
|
||||
under. Compound's sticky heading is the section's aria-hidden first child. */
|
||||
.deviceList [role="group"] > [aria-hidden="true"]:first-child {
|
||||
z-index: 1;
|
||||
}
|
||||
|
||||
/* Radix focuses whatever the pointer is over, so the browser's ring would
|
||||
follow the mouse. Replaced by one shown only for keyboard focus. */
|
||||
.menu [role^="menuitem"]:focus,
|
||||
@@ -68,3 +76,56 @@ Please see LICENSE in the repository root for full details.
|
||||
outline: var(--cpd-border-width-2) solid var(--cpd-color-border-focused);
|
||||
outline-offset: calc(-1 * var(--cpd-border-width-2));
|
||||
}
|
||||
|
||||
.notice {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
gap: var(--cpd-space-3x);
|
||||
margin: var(--cpd-space-2x) var(--cpd-space-4x) 0;
|
||||
padding: var(--cpd-space-3x);
|
||||
border-radius: var(--cpd-space-2x);
|
||||
background: var(--cpd-color-bg-subtle-secondary);
|
||||
color: var(--cpd-color-text-primary);
|
||||
font: var(--cpd-font-body-md-regular);
|
||||
}
|
||||
|
||||
.notice > svg {
|
||||
flex-shrink: 0;
|
||||
color: var(--cpd-color-icon-secondary);
|
||||
}
|
||||
|
||||
.refusal {
|
||||
margin: var(--cpd-space-2x) var(--cpd-space-4x) 0;
|
||||
}
|
||||
|
||||
.selfPreview {
|
||||
position: relative;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
flex-shrink: 0;
|
||||
aspect-ratio: 16 / 9;
|
||||
margin-block-end: var(--cpd-space-2x);
|
||||
overflow: clip;
|
||||
border-start-start-radius: var(--cpd-space-3x);
|
||||
border-start-end-radius: var(--cpd-space-3x);
|
||||
background: var(--cpd-color-bg-subtle-secondary);
|
||||
color: var(--cpd-color-icon-secondary);
|
||||
}
|
||||
|
||||
.selfPreview > video {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
inline-size: 100%;
|
||||
block-size: 100%;
|
||||
object-fit: cover;
|
||||
/* Rounded itself too: a video can be drawn on a layer of its own, which the
|
||||
frame's rounded clip doesn't always reach. */
|
||||
border-radius: inherit;
|
||||
}
|
||||
|
||||
/* Up to the menu's frame, over the padding at its top. */
|
||||
.selfPreviewPinned,
|
||||
.deviceListLeadsWithPreview {
|
||||
margin-block-start: calc(-1 * var(--cpd-space-5x));
|
||||
}
|
||||
|
||||
@@ -11,6 +11,9 @@ import { useEffect, useState, type FC, type JSX, type ReactNode } from "react";
|
||||
import type { Meta, StoryObj } from "@storybook/react-vite";
|
||||
import { MediaMuteAndSwitchButton } from "./MediaMuteAndSwitchButton";
|
||||
import styles from "./MediaMuteAndSwitchButton.module.css";
|
||||
import { type BackgroundEffectGridItem } from "./BackgroundEffectGrid";
|
||||
import { SelfPreview } from "./SelfPreview";
|
||||
import { shippedBackgrounds } from "../tracks/backgroundEffectCatalog";
|
||||
import meterStyles from "./MicrophoneLevelMeter.module.css";
|
||||
import { MediaDevicesContext } from "../MediaDevicesContext";
|
||||
import { RootElementProvider } from "../RootElementContext";
|
||||
@@ -58,16 +61,63 @@ const WithAMicrophone: FC<{ children: ReactNode }> = ({ children }) => {
|
||||
return <>{children}</>;
|
||||
};
|
||||
|
||||
/** A camera for the preview: a figure against a room, drawn to a stream. */
|
||||
const FakeCamera: FC = () => {
|
||||
const [stream, setStream] = useState<MediaStream>();
|
||||
useEffect(() => {
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = 640;
|
||||
canvas.height = 360;
|
||||
const context = canvas.getContext("2d")!;
|
||||
const draw = (): void => {
|
||||
context.fillStyle = "#6b7f8e";
|
||||
context.fillRect(0, 0, 640, 360);
|
||||
context.fillStyle = "#c9a27e";
|
||||
context.beginPath();
|
||||
context.arc(320, 150, 62, 0, 2 * Math.PI);
|
||||
context.fill();
|
||||
context.fillStyle = "#34495e";
|
||||
context.beginPath();
|
||||
context.ellipse(320, 380, 150, 140, 0, Math.PI, 2 * Math.PI);
|
||||
context.fill();
|
||||
};
|
||||
draw();
|
||||
// Redrawn, so the stream keeps delivering frames.
|
||||
const timer = setInterval(draw, 200);
|
||||
const captured = canvas.captureStream(10);
|
||||
setStream(captured);
|
||||
return (): void => {
|
||||
clearInterval(timer);
|
||||
captured.getTracks().forEach((track) => track.stop());
|
||||
};
|
||||
}, []);
|
||||
return (
|
||||
<video
|
||||
data-testid="preview-camera"
|
||||
autoPlay
|
||||
muted
|
||||
playsInline
|
||||
ref={(video): void => {
|
||||
if (video && stream && video.srcObject !== stream)
|
||||
video.srcObject = stream;
|
||||
}}
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
||||
/** Supplies a call-sized root. Without one the list is bounded by the whole Storybook frame. */
|
||||
const WithACallArea: FC<{ children: ReactNode }> = ({ children }) => {
|
||||
const WithACallArea: FC<{ children: ReactNode; blockSize?: number }> = ({
|
||||
children,
|
||||
// A call's size: anything smaller makes a short device list scroll, and a
|
||||
// narrower one narrows the menu.
|
||||
blockSize = 720,
|
||||
}) => {
|
||||
const [callArea, setCallArea] = useState<HTMLElement | null>(null);
|
||||
return (
|
||||
<div
|
||||
ref={setCallArea}
|
||||
style={{
|
||||
// A call's size: anything smaller makes a short device list scroll, and
|
||||
// a narrower one narrows the menu.
|
||||
blockSize: 720,
|
||||
blockSize,
|
||||
inlineSize: 1024,
|
||||
display: "flex",
|
||||
alignItems: "flex-end",
|
||||
@@ -133,8 +183,6 @@ export const AudioMute: Story = {
|
||||
{ label: { type: "name", name: "Microphone 1" }, id: "1" },
|
||||
{ label: { type: "name", name: "Microphone 2" }, id: "2" },
|
||||
],
|
||||
videoBlurEnabled: true,
|
||||
videoBlurToggleClick: fn(),
|
||||
selectedOption: "2",
|
||||
},
|
||||
play: async ({ args, canvasElement }) => {
|
||||
@@ -168,11 +216,21 @@ export const VideoMute: Story = {
|
||||
{ label: { type: "name", name: "Camera 1" }, id: "1" },
|
||||
{ label: { type: "name", name: "Camera 2" }, id: "2" },
|
||||
],
|
||||
|
||||
selectedOption: "1",
|
||||
selfPreview: <SelfPreview track={null} />,
|
||||
},
|
||||
};
|
||||
|
||||
const effects: BackgroundEffectGridItem[] = [
|
||||
{ id: "none", kind: "none" },
|
||||
{ id: "blur", kind: "blur" },
|
||||
...shippedBackgrounds.map((background) => ({
|
||||
id: `image:${background.id}`,
|
||||
kind: "image" as const,
|
||||
imageUrl: background.imagePath,
|
||||
})),
|
||||
];
|
||||
|
||||
export const VideoUnmute: Story = {
|
||||
args: {
|
||||
iconsAndLabels: "video",
|
||||
@@ -181,9 +239,486 @@ export const VideoUnmute: Story = {
|
||||
{ label: { type: "name", name: "Camera 1" }, id: "1" },
|
||||
{ label: { type: "name", name: "Camera 2" }, id: "2" },
|
||||
],
|
||||
videoBlurEnabled: true,
|
||||
videoBlurToggleClick: fn(),
|
||||
selectedOption: "2",
|
||||
backgroundEffects: effects,
|
||||
selectedBackgroundEffect: "none",
|
||||
onSelectBackgroundEffect: fn(),
|
||||
onAddBackgroundImage: fn(),
|
||||
// As the footer always gives the camera menu one.
|
||||
selfPreview: <FakeCamera />,
|
||||
},
|
||||
};
|
||||
|
||||
/** One choice among no effect, blur and the shipped images, under the cameras. */
|
||||
export const BackgroundEffects: Story = {
|
||||
args: VideoUnmute.args,
|
||||
play: async ({ args, canvasElement }) => {
|
||||
const canvas = within(canvasElement);
|
||||
await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
|
||||
const menu = within(document.body);
|
||||
const section = await menu.findByRole("group", {
|
||||
name: "Background effects",
|
||||
});
|
||||
const tiles = within(section).getAllByRole("menuitemradio");
|
||||
await expect(tiles.map((tile) => tile.textContent)).toEqual([
|
||||
"None",
|
||||
"Blur",
|
||||
"Background 1",
|
||||
"Background 2",
|
||||
]);
|
||||
await expect(
|
||||
within(section).getByRole("menuitemradio", { checked: true }),
|
||||
).toHaveTextContent("None");
|
||||
|
||||
const cameras = menu.getByRole("group", { name: "Camera" });
|
||||
await expect(
|
||||
cameras.compareDocumentPosition(section) &
|
||||
Node.DOCUMENT_POSITION_FOLLOWING,
|
||||
).toBeTruthy();
|
||||
const [first, second, third, fourth] = tiles.map((tile) =>
|
||||
tile.getBoundingClientRect(),
|
||||
);
|
||||
await expect(second.top).toBe(first.top);
|
||||
await expect(third.top).toBe(first.top);
|
||||
await expect(fourth.top).toBeGreaterThan(first.bottom);
|
||||
const frame = document.body
|
||||
.querySelector("[role='menu']")!
|
||||
.getBoundingClientRect();
|
||||
for (const tile of [first, second, third])
|
||||
await expect(tile.left >= frame.left && tile.right <= frame.right).toBe(
|
||||
true,
|
||||
);
|
||||
|
||||
const add = within(section).getByRole("menuitem", { name: "Add image" });
|
||||
await expect(
|
||||
tiles[3].compareDocumentPosition(add) & Node.DOCUMENT_POSITION_FOLLOWING,
|
||||
).toBeTruthy();
|
||||
|
||||
await userEvent.click(menu.getByRole("menuitemradio", { name: "Blur" }));
|
||||
await expect(args.onSelectBackgroundEffect).toHaveBeenCalledWith("blur");
|
||||
await expect(menu.getByRole("menu")).toBeVisible();
|
||||
},
|
||||
};
|
||||
|
||||
/** Where effects can't run: shown, disabled and explained, except no effect. */
|
||||
export const BackgroundEffectsUnavailable: Story = {
|
||||
args: {
|
||||
...VideoUnmute.args,
|
||||
onSelectBackgroundEffect: undefined,
|
||||
backgroundEffectNotice:
|
||||
"Background effects are not supported on this platform.",
|
||||
},
|
||||
play: async ({ canvasElement }) => {
|
||||
const canvas = within(canvasElement);
|
||||
await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
|
||||
const menu = within(document.body);
|
||||
const section = await menu.findByRole("group", {
|
||||
name: "Background effects",
|
||||
});
|
||||
// On the tiles' own group: the section's can't carry a description.
|
||||
await expect(
|
||||
within(section).getByRole("group"),
|
||||
).toHaveAccessibleDescription(
|
||||
"Background effects are not supported on this platform.",
|
||||
);
|
||||
for (const tile of within(section).getAllByRole("menuitemradio"))
|
||||
if (tile.textContent === "None") await expect(tile).toBeEnabled();
|
||||
else await expect(tile).toBeDisabled();
|
||||
},
|
||||
};
|
||||
|
||||
/** Where only the slower route exists: offered and choosable, with the cost said. */
|
||||
export const BackgroundEffectsSlowInThisBrowser: Story = {
|
||||
args: {
|
||||
...VideoUnmute.args,
|
||||
backgroundEffectNotice:
|
||||
"Background effects run slowly on this platform, which may cause your video to stutter.",
|
||||
},
|
||||
play: async ({ args, canvasElement }) => {
|
||||
const canvas = within(canvasElement);
|
||||
await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
|
||||
const menu = within(document.body);
|
||||
const section = await menu.findByRole("group", {
|
||||
name: "Background effects",
|
||||
});
|
||||
// On the tiles' own group: the section's can't carry a description.
|
||||
await expect(
|
||||
within(section).getByRole("group"),
|
||||
).toHaveAccessibleDescription(
|
||||
"Background effects run slowly on this platform, which may cause your video to stutter.",
|
||||
);
|
||||
for (const tile of within(section).getAllByRole("menuitemradio"))
|
||||
await expect(tile).not.toHaveAttribute("aria-disabled");
|
||||
await userEvent.click(
|
||||
within(section).getByRole("menuitemradio", { name: "Background 1" }),
|
||||
);
|
||||
await expect(args.onSelectBackgroundEffect).toHaveBeenCalledWith(
|
||||
"image:arc",
|
||||
);
|
||||
|
||||
const notice = document.body.querySelector(`.${styles.notice}`)!;
|
||||
const frame = document.body
|
||||
.querySelector("[role='menu']")!
|
||||
.getBoundingClientRect();
|
||||
await expect(frame.width).toBeLessThanOrEqual(296);
|
||||
await expect(notice.getBoundingClientRect().right).toBeLessThanOrEqual(
|
||||
frame.right,
|
||||
);
|
||||
},
|
||||
};
|
||||
|
||||
/** The first effect of a session shows its wait on its own tile. */
|
||||
export const BackgroundEffectsSettling: Story = {
|
||||
args: {
|
||||
...VideoUnmute.args,
|
||||
selectedBackgroundEffect: "blur",
|
||||
backgroundEffectSettling: true,
|
||||
},
|
||||
play: async ({ canvasElement }) => {
|
||||
const canvas = within(canvasElement);
|
||||
await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
|
||||
const menu = within(document.body);
|
||||
const chosen = await menu.findByRole("menuitemradio", {
|
||||
name: "Blur",
|
||||
checked: true,
|
||||
});
|
||||
await expect(chosen).toHaveAttribute("aria-busy", "true");
|
||||
const marks = chosen.querySelectorAll("svg");
|
||||
await expect(marks).toHaveLength(1);
|
||||
await expect(getComputedStyle(marks[0]).animationName).not.toBe("none");
|
||||
const section = menu.getByRole("group", { name: "Background effects" });
|
||||
for (const other of within(section).getAllByRole("menuitemradio", {
|
||||
checked: false,
|
||||
}))
|
||||
await expect(other).toHaveAttribute("aria-busy", "false");
|
||||
await expect(menu.queryByTestId("preview-camera")).toBeNull();
|
||||
const frames = document.body.querySelectorAll(`.${styles.selfPreview} svg`);
|
||||
await expect(frames).toHaveLength(1);
|
||||
},
|
||||
};
|
||||
|
||||
/** A file that can't be used: said where it was chosen, and nothing changes. */
|
||||
export const BackgroundImageRefused: Story = {
|
||||
args: {
|
||||
...VideoUnmute.args,
|
||||
selectedBackgroundEffect: "blur",
|
||||
backgroundEffectNotice:
|
||||
"Background effects run slowly on this platform, which may cause your video to stutter.",
|
||||
backgroundImageRefusal: { text: "That file is not a supported image" },
|
||||
},
|
||||
decorators: [
|
||||
(Story): JSX.Element => (
|
||||
<WithACallArea blockSize={300}>
|
||||
<Story />
|
||||
</WithACallArea>
|
||||
),
|
||||
],
|
||||
play: async ({ canvasElement }) => {
|
||||
const canvas = within(canvasElement);
|
||||
await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
|
||||
const menu = within(document.body);
|
||||
const message = await menu.findByText("That file is not a supported image");
|
||||
const list = document.body.querySelector<HTMLElement>(
|
||||
`.${styles.deviceList}`,
|
||||
)!;
|
||||
await expect(list.scrollHeight).toBeGreaterThan(list.clientHeight);
|
||||
const scrollport = list.getBoundingClientRect();
|
||||
const shown = message.getBoundingClientRect();
|
||||
await expect(shown.bottom).toBeLessThanOrEqual(scrollport.bottom + 1);
|
||||
await expect(shown.top).toBeGreaterThanOrEqual(scrollport.top - 1);
|
||||
await expect(menu.queryByText(/run slowly/)).toBeNull();
|
||||
await expect(
|
||||
menu.getByRole("menuitemradio", { name: "Blur" }),
|
||||
).toHaveAttribute("aria-checked", "true");
|
||||
},
|
||||
};
|
||||
|
||||
/** A flat picture, for backgrounds of one's own. */
|
||||
function swatch(colour: string): string {
|
||||
return `data:image/svg+xml,${encodeURIComponent(
|
||||
`<svg xmlns="http://www.w3.org/2000/svg" width="16" height="9"><rect width="16" height="9" fill="${colour}"/></svg>`,
|
||||
)}`;
|
||||
}
|
||||
|
||||
/** One's own backgrounds are removable, never the one in force. */
|
||||
export const AddedBackgroundsCanBeRemoved: Story = {
|
||||
args: {
|
||||
...VideoUnmute.args,
|
||||
backgroundEffects: [
|
||||
...effects,
|
||||
{
|
||||
id: "added:one",
|
||||
kind: "image",
|
||||
imageUrl: swatch("#c2410c"),
|
||||
removable: true,
|
||||
},
|
||||
{
|
||||
id: "added:two",
|
||||
kind: "image",
|
||||
imageUrl: swatch("#1d4ed8"),
|
||||
removable: true,
|
||||
},
|
||||
],
|
||||
selectedBackgroundEffect: "added:one",
|
||||
onRemoveBackgroundEffect: fn(),
|
||||
},
|
||||
play: async ({ args, canvasElement }) => {
|
||||
const canvas = within(canvasElement);
|
||||
await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
|
||||
const body = within(document.body);
|
||||
|
||||
const inForce = await body.findByRole("menuitemradio", {
|
||||
name: "Background 3",
|
||||
});
|
||||
await expect(inForce).not.toHaveAttribute("aria-keyshortcuts");
|
||||
|
||||
const other = body.getByRole("menuitemradio", { name: "Background 4" });
|
||||
await expect(other).toHaveAttribute("aria-keyshortcuts", "Delete");
|
||||
other.focus();
|
||||
await userEvent.keyboard("{Delete}");
|
||||
await expect(args.onRemoveBackgroundEffect).toHaveBeenCalledWith(
|
||||
"added:two",
|
||||
);
|
||||
|
||||
const wrapper = other.parentElement!;
|
||||
await userEvent.hover(wrapper);
|
||||
const cross = wrapper.querySelector<HTMLElement>(
|
||||
"[aria-hidden]:last-child",
|
||||
)!;
|
||||
const probe = document.createElement("span");
|
||||
probe.style.color = "var(--cpd-color-icon-critical-primary)";
|
||||
cross.append(probe);
|
||||
const critical = getComputedStyle(probe).color;
|
||||
probe.remove();
|
||||
await expect(getComputedStyle(cross).color).toBe(critical);
|
||||
const tile = other.getBoundingClientRect();
|
||||
const box = cross.getBoundingClientRect();
|
||||
await expect(box.right).toBeGreaterThan(tile.right);
|
||||
await expect(box.top).toBeLessThan(tile.top);
|
||||
|
||||
await userEvent.hover(cross);
|
||||
await waitFor(async () =>
|
||||
expect(
|
||||
[...document.body.querySelectorAll("div")].some(
|
||||
(d) => d.textContent === "Remove" && d.offsetParent !== null,
|
||||
),
|
||||
).toBe(true),
|
||||
);
|
||||
},
|
||||
};
|
||||
|
||||
/** Removing by pointer while the selection really changes. */
|
||||
export const RemovingWithLiveSelection: Story = {
|
||||
args: AddedBackgroundsCanBeRemoved.args,
|
||||
render: function WithLiveSelection(args): JSX.Element {
|
||||
const [selected, setSelected] = useState("none");
|
||||
const [offered, setOffered] = useState(args.backgroundEffects ?? []);
|
||||
return (
|
||||
<MediaMuteAndSwitchButton
|
||||
{...args}
|
||||
backgroundEffects={offered}
|
||||
selectedBackgroundEffect={selected}
|
||||
onSelectBackgroundEffect={setSelected}
|
||||
onRemoveBackgroundEffect={(id): void =>
|
||||
setOffered((current) => current.filter((o) => o.id !== id))
|
||||
}
|
||||
/>
|
||||
);
|
||||
},
|
||||
play: async ({ canvasElement }) => {
|
||||
const canvas = within(canvasElement);
|
||||
await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
|
||||
const body = within(document.body);
|
||||
|
||||
const tile = await body.findByRole("menuitemradio", {
|
||||
name: "Background 4",
|
||||
});
|
||||
await userEvent.hover(tile);
|
||||
const cross = tile.parentElement!.querySelector<HTMLElement>(
|
||||
"[aria-hidden]:last-child",
|
||||
)!;
|
||||
await expect(cross).toBeVisible();
|
||||
await userEvent.click(cross);
|
||||
|
||||
await waitFor(async () =>
|
||||
expect(
|
||||
body.queryByRole("menuitemradio", { name: "Background 4" }),
|
||||
).toBeNull(),
|
||||
);
|
||||
await expect(
|
||||
body.getByRole("menuitemradio", { name: "None" }),
|
||||
).toHaveAttribute("aria-checked", "true");
|
||||
},
|
||||
};
|
||||
|
||||
/** Where the call leaves room, the camera holds the top while the options scroll. */
|
||||
export const BackgroundEffectsWithPreview: Story = {
|
||||
args: {
|
||||
...VideoUnmute.args,
|
||||
// Enough that the list fills its share, so the preview's cost counts.
|
||||
options: Array.from({ length: 8 }, (_, i) => ({
|
||||
label: { type: "name" as const, name: `Camera ${i + 1}` },
|
||||
id: `${i + 1}`,
|
||||
})),
|
||||
},
|
||||
decorators: [
|
||||
(Story): JSX.Element => (
|
||||
<WithACallArea blockSize={640}>
|
||||
<Story />
|
||||
</WithACallArea>
|
||||
),
|
||||
],
|
||||
play: async ({ canvasElement }) => {
|
||||
const canvas = within(canvasElement);
|
||||
await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
|
||||
const picture = await within(document.body).findByTestId("preview-camera");
|
||||
const preview = picture.parentElement!;
|
||||
const list = document.body.querySelector<HTMLElement>(
|
||||
`.${styles.deviceList}`,
|
||||
)!;
|
||||
const frame = document.body
|
||||
.querySelector("[role='menu']")!
|
||||
.getBoundingClientRect();
|
||||
|
||||
const box = preview.getBoundingClientRect();
|
||||
await expect(list.contains(preview)).toBe(false);
|
||||
await expect(box.top - frame.top).toBeLessThanOrEqual(2);
|
||||
await expect(frame.width - box.width).toBeLessThanOrEqual(2);
|
||||
await expect(box.width / box.height).toBeCloseTo(16 / 9, 1);
|
||||
await expect(box.bottom).toBeLessThanOrEqual(
|
||||
list.getBoundingClientRect().top,
|
||||
);
|
||||
|
||||
const listShare = Math.round(640 * 0.6);
|
||||
// The list runs on through the menu's padding at its foot, which isn't
|
||||
// part of its share.
|
||||
const listEnd =
|
||||
list.getBoundingClientRect().bottom -
|
||||
Number.parseFloat(getComputedStyle(list).paddingBlockEnd);
|
||||
await expect(listEnd - box.top).toBeLessThanOrEqual(listShare + 1);
|
||||
|
||||
await expect(list.scrollHeight).toBeGreaterThan(list.clientHeight);
|
||||
list.scrollTop = list.scrollHeight;
|
||||
await expect(preview.getBoundingClientRect().top).toBe(box.top);
|
||||
},
|
||||
};
|
||||
|
||||
/** With the camera off the preview says so, and effects can still be chosen. */
|
||||
export const BackgroundEffectsPreviewWithTheCameraOff: Story = {
|
||||
args: {
|
||||
...VideoUnmute.args,
|
||||
enabled: false,
|
||||
selfPreview: <SelfPreview track={null} />,
|
||||
},
|
||||
play: async ({ args, canvasElement }) => {
|
||||
const canvas = within(canvasElement);
|
||||
await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
|
||||
const menu = within(document.body);
|
||||
await menu.findByRole("group", { name: "Background effects" });
|
||||
const preview = document.body.querySelector(`.${styles.selfPreview}`)!;
|
||||
await expect(preview.querySelector("video")).toBeNull();
|
||||
await expect(preview.querySelector("svg")).not.toBeNull();
|
||||
|
||||
await userEvent.click(menu.getByRole("menuitemradio", { name: "Blur" }));
|
||||
await expect(args.onSelectBackgroundEffect).toHaveBeenCalledWith("blur");
|
||||
},
|
||||
};
|
||||
|
||||
/** In a call too short to hold it, the camera leads the options and scrolls away. */
|
||||
export const BackgroundEffectsPreviewScrollsWithTheList: Story = {
|
||||
args: VideoUnmute.args,
|
||||
decorators: [
|
||||
(Story): JSX.Element => (
|
||||
<WithACallArea blockSize={300}>
|
||||
<Story />
|
||||
</WithACallArea>
|
||||
),
|
||||
],
|
||||
play: async ({ canvasElement }) => {
|
||||
const canvas = within(canvasElement);
|
||||
await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
|
||||
const picture = await within(document.body).findByTestId("preview-camera");
|
||||
const preview = picture.parentElement!;
|
||||
const list = document.body.querySelector<HTMLElement>(
|
||||
`.${styles.deviceList}`,
|
||||
)!;
|
||||
await expect(list.contains(preview)).toBe(true);
|
||||
const cameras = within(document.body).getByRole("group", {
|
||||
name: "Camera",
|
||||
});
|
||||
await expect(
|
||||
preview.compareDocumentPosition(cameras) &
|
||||
Node.DOCUMENT_POSITION_FOLLOWING,
|
||||
).toBeTruthy();
|
||||
|
||||
list.scrollTop = list.scrollHeight;
|
||||
await expect(preview.getBoundingClientRect().bottom).toBeLessThan(
|
||||
list.getBoundingClientRect().top,
|
||||
);
|
||||
},
|
||||
};
|
||||
|
||||
/** In a short call the effects scroll into view with the list. */
|
||||
export const BackgroundEffectsScrollWhenTheyDoNotFit: Story = {
|
||||
args: {
|
||||
...VideoUnmute.args,
|
||||
// It sits with the effects, so it must fit the same share of the call.
|
||||
backgroundEffectNotice:
|
||||
"Background effects run slowly on this platform, which may cause your video to stutter.",
|
||||
},
|
||||
decorators: [
|
||||
(Story): JSX.Element => (
|
||||
<WithACallArea blockSize={300}>
|
||||
<Story />
|
||||
</WithACallArea>
|
||||
),
|
||||
],
|
||||
play: async ({ canvasElement }) => {
|
||||
const canvas = within(canvasElement);
|
||||
await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
|
||||
const menu = within(document.body);
|
||||
const last = await menu.findByRole("menuitemradio", {
|
||||
name: "Background 2",
|
||||
});
|
||||
const list = document.body.querySelector<HTMLElement>(
|
||||
`.${styles.deviceList}`,
|
||||
)!;
|
||||
await expect(list.scrollHeight).toBeGreaterThan(list.clientHeight);
|
||||
await expect(last.getBoundingClientRect().top).toBeGreaterThan(
|
||||
list.getBoundingClientRect().bottom,
|
||||
);
|
||||
|
||||
// Scrolled under the section's heading, the tiles pass beneath it. By
|
||||
// position: Compound's heading is the section's hidden first child.
|
||||
const heading = menu.getByRole("group", { name: "Background effects" })
|
||||
.firstElementChild as HTMLElement;
|
||||
list.scrollTop = heading.offsetTop + 30;
|
||||
await waitFor(async () => {
|
||||
const box = heading.getBoundingClientRect();
|
||||
const drawn = document.elementFromPoint(
|
||||
box.left + box.width / 2,
|
||||
box.bottom - 2,
|
||||
);
|
||||
await expect(heading.contains(drawn)).toBe(true);
|
||||
});
|
||||
list.scrollTop = 0;
|
||||
|
||||
for (let i = 0; i < 10 && document.activeElement !== last; i++)
|
||||
await userEvent.keyboard("{ArrowDown}");
|
||||
await expect(document.activeElement).toBe(last);
|
||||
const scrollport = list.getBoundingClientRect();
|
||||
const reached = last.getBoundingClientRect();
|
||||
await expect(reached.bottom).toBeLessThanOrEqual(scrollport.bottom + 1);
|
||||
await expect(reached.top).toBeGreaterThanOrEqual(scrollport.top - 1);
|
||||
|
||||
const callArea = canvasElement
|
||||
.querySelector<HTMLElement>("[style*='block-size: 300px']")!
|
||||
.getBoundingClientRect();
|
||||
const frame = document.body
|
||||
.querySelector("[role='menu']")!
|
||||
.getBoundingClientRect();
|
||||
await expect(frame.top).toBeGreaterThanOrEqual(callArea.top);
|
||||
await expect(frame.bottom).toBeLessThanOrEqual(callArea.bottom);
|
||||
},
|
||||
};
|
||||
|
||||
@@ -648,31 +1183,23 @@ function overlapping(element: Element, overlays: Element[]): number {
|
||||
}, 0);
|
||||
}
|
||||
|
||||
/** The blur toggle gets the keyboard ring too: it is the menu's child, not the list's. */
|
||||
export const FocusRingCoversTheBlurToggle: Story = {
|
||||
args: {
|
||||
...VideoUnmute.args,
|
||||
iconsAndLabels: "video",
|
||||
videoBlurEnabled: false,
|
||||
videoBlurToggleClick: fn(),
|
||||
},
|
||||
/** The effect tiles get the keyboard ring the device rows get. */
|
||||
export const FocusRingCoversTheEffectTiles: Story = {
|
||||
args: VideoUnmute.args,
|
||||
play: async ({ canvasElement }) => {
|
||||
const canvas = within(canvasElement);
|
||||
await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
|
||||
const toggle = await within(document.body).findByRole("menuitemcheckbox", {
|
||||
name: /Blur background/,
|
||||
const tile = await within(document.body).findByRole("menuitemradio", {
|
||||
name: "Blur",
|
||||
});
|
||||
|
||||
// Arrow down past the cameras to the toggle.
|
||||
for (let i = 0; i < 6 && document.activeElement !== toggle; i++)
|
||||
for (let i = 0; i < 6 && document.activeElement !== tile; i++)
|
||||
await userEvent.keyboard("{ArrowDown}");
|
||||
await expect(document.activeElement).toBe(toggle);
|
||||
// The same ring the device rows get.
|
||||
await expect(outlineWidth(toggle)).toBeGreaterThan(0);
|
||||
await expect(document.activeElement).toBe(tile);
|
||||
await expect(outlineWidth(tile)).toBeGreaterThan(0);
|
||||
|
||||
// And the pointer takes it away, with the toggle still focused.
|
||||
await userEvent.hover(toggle);
|
||||
await expect(document.activeElement).toBe(toggle);
|
||||
await expect(outlineWidth(toggle)).toBe(0);
|
||||
await userEvent.hover(tile);
|
||||
await expect(document.activeElement).toBe(tile);
|
||||
await expect(outlineWidth(tile)).toBe(0);
|
||||
},
|
||||
};
|
||||
|
||||
@@ -5,10 +5,11 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { afterEach, describe, expect, test, vi } from "vitest";
|
||||
import { afterEach, describe, expect, onTestFinished, test, vi } from "vitest";
|
||||
import { axe } from "vitest-axe";
|
||||
import {
|
||||
act,
|
||||
fireEvent,
|
||||
render,
|
||||
screen,
|
||||
within,
|
||||
@@ -22,10 +23,18 @@ import {
|
||||
MediaMuteAndSwitchButton,
|
||||
type MenuOptions,
|
||||
} from "./MediaMuteAndSwitchButton";
|
||||
import { type BackgroundEffectGridItem } from "./BackgroundEffectGrid";
|
||||
import { MediaDevicesContext } from "../MediaDevicesContext";
|
||||
import { type MediaDevices } from "../state/MediaDevices";
|
||||
import { restoreAudioCapture, stubAudioCapture } from "../utils/test";
|
||||
|
||||
const effects: BackgroundEffectGridItem[] = [
|
||||
{ id: "none", kind: "none" },
|
||||
{ id: "blur", kind: "blur" },
|
||||
{ id: "image:arc", kind: "image", imageUrl: "" },
|
||||
{ id: "image:glow", kind: "image", imageUrl: "" },
|
||||
];
|
||||
|
||||
interface RenderOptions {
|
||||
requestDeviceNames: () => void;
|
||||
}
|
||||
@@ -305,30 +314,202 @@ describe("MediaMuteAndSwitchButton", () => {
|
||||
expect(mic2ItemAfter.querySelector(".rotate")).toBeNull();
|
||||
});
|
||||
|
||||
test("renders menu with toggle control and calls toggle callback", async () => {
|
||||
test("background effects are one keyboard-operable labelled choice", async () => {
|
||||
const user = userEvent.setup();
|
||||
const onSelect = vi.fn();
|
||||
const onVideoBlurToggle = vi.fn();
|
||||
const onSelectBackgroundEffect = vi.fn();
|
||||
const { getByRole } = renderComponent(
|
||||
<MediaMuteAndSwitchButton
|
||||
iconsAndLabels="video"
|
||||
enabled={true}
|
||||
videoBlurToggleClick={onVideoBlurToggle}
|
||||
onSelect={onSelect}
|
||||
options={[{ label: { type: "name", name: "Camera 1" }, id: "cam1" }]}
|
||||
selectedOption="cam1"
|
||||
onSelect={vi.fn()}
|
||||
backgroundEffects={effects}
|
||||
selectedBackgroundEffect="none"
|
||||
onSelectBackgroundEffect={onSelectBackgroundEffect}
|
||||
/>,
|
||||
);
|
||||
|
||||
await user.click(getByRole("button", { name: "Camera" }));
|
||||
|
||||
const toggle = screen.getByRole("menuitemcheckbox", {
|
||||
name: "Blur background",
|
||||
const section = screen.getByRole("group", { name: "Background effects" });
|
||||
within(section).getByRole("menuitemradio", { name: "None", checked: true });
|
||||
within(section).getByRole("menuitemradio", {
|
||||
name: "Background 1",
|
||||
checked: false,
|
||||
});
|
||||
expect(toggle).toBeInTheDocument();
|
||||
expect(toggle).toHaveAttribute("aria-checked", "false");
|
||||
|
||||
await user.click(toggle);
|
||||
const blur = within(section).getByRole("menuitemradio", { name: "Blur" });
|
||||
for (let i = 0; i < 6 && document.activeElement !== blur; i++)
|
||||
await user.keyboard("{ArrowDown}");
|
||||
expect(document.activeElement).toBe(blur);
|
||||
await user.keyboard("{Enter}");
|
||||
expect(onSelectBackgroundEffect).toHaveBeenCalledWith("blur");
|
||||
expect(screen.getByRole("menu")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
expect(onVideoBlurToggle).toHaveBeenCalled();
|
||||
test("does not choose the effect already in force again", async () => {
|
||||
const user = userEvent.setup();
|
||||
const onSelectBackgroundEffect = vi.fn();
|
||||
const { getByRole } = renderComponent(
|
||||
<MediaMuteAndSwitchButton
|
||||
iconsAndLabels="video"
|
||||
enabled={true}
|
||||
options={[{ label: { type: "name", name: "Camera 1" }, id: "cam1" }]}
|
||||
selectedOption="cam1"
|
||||
backgroundEffects={effects}
|
||||
selectedBackgroundEffect="blur"
|
||||
onSelectBackgroundEffect={onSelectBackgroundEffect}
|
||||
/>,
|
||||
);
|
||||
|
||||
await user.click(getByRole("button", { name: "Camera" }));
|
||||
await user.click(screen.getByRole("menuitemradio", { name: "Blur" }));
|
||||
expect(onSelectBackgroundEffect).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("added images are chosen like shipped ones", async () => {
|
||||
const user = userEvent.setup();
|
||||
const onSelectBackgroundEffect = vi.fn();
|
||||
const { getByRole } = renderComponent(
|
||||
<MediaMuteAndSwitchButton
|
||||
iconsAndLabels="video"
|
||||
enabled={true}
|
||||
options={[{ label: { type: "name", name: "Camera 1" }, id: "cam1" }]}
|
||||
selectedOption="cam1"
|
||||
backgroundEffects={[
|
||||
...effects,
|
||||
{
|
||||
id: "added:mine",
|
||||
kind: "image",
|
||||
imageUrl: "",
|
||||
},
|
||||
]}
|
||||
selectedBackgroundEffect="none"
|
||||
onSelectBackgroundEffect={onSelectBackgroundEffect}
|
||||
/>,
|
||||
);
|
||||
|
||||
await user.click(getByRole("button", { name: "Camera" }));
|
||||
const section = screen.getByRole("group", { name: "Background effects" });
|
||||
await user.click(
|
||||
within(section).getByRole("menuitemradio", { name: "Background 3" }),
|
||||
);
|
||||
expect(onSelectBackgroundEffect).toHaveBeenCalledWith("added:mine");
|
||||
});
|
||||
|
||||
test("adds the file chosen from the add tile, and stays open", async () => {
|
||||
const user = userEvent.setup();
|
||||
const onAddBackgroundImage = vi.fn();
|
||||
const pick = vi
|
||||
.spyOn(HTMLInputElement.prototype, "click")
|
||||
.mockImplementation(() => {});
|
||||
try {
|
||||
const { getByRole, container } = renderComponent(
|
||||
<MediaMuteAndSwitchButton
|
||||
iconsAndLabels="video"
|
||||
enabled={true}
|
||||
options={[{ label: { type: "name", name: "Camera 1" }, id: "cam1" }]}
|
||||
selectedOption="cam1"
|
||||
backgroundEffects={effects}
|
||||
selectedBackgroundEffect="none"
|
||||
onSelectBackgroundEffect={vi.fn()}
|
||||
onAddBackgroundImage={onAddBackgroundImage}
|
||||
/>,
|
||||
);
|
||||
|
||||
await user.click(getByRole("button", { name: "Camera" }));
|
||||
await user.click(screen.getByRole("menuitem", { name: "Add image" }));
|
||||
expect(pick).toHaveBeenCalled();
|
||||
// The picker takes the focus, which would otherwise close the menu.
|
||||
await user.keyboard("{Escape}");
|
||||
expect(screen.getByRole("menu")).toBeInTheDocument();
|
||||
|
||||
const file = new File(["x"], "mine.png", { type: "image/png" });
|
||||
// What the picker hands back; the open menu blocks pointer events.
|
||||
fireEvent.change(
|
||||
container.querySelector<HTMLInputElement>("input[type=file]")!,
|
||||
{ target: { files: [file] } },
|
||||
);
|
||||
expect(onAddBackgroundImage).toHaveBeenCalledWith(file);
|
||||
screen.getByRole("menuitemradio", { name: "None", checked: true });
|
||||
} finally {
|
||||
pick.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
test("a refusal can be dismissed and does not outlive the menu", async () => {
|
||||
const user = userEvent.setup();
|
||||
// jsdom lays nothing out, so has nothing to scroll.
|
||||
const scroll = Element.prototype.scrollIntoView;
|
||||
Element.prototype.scrollIntoView = vi.fn();
|
||||
onTestFinished(() => {
|
||||
Element.prototype.scrollIntoView = scroll;
|
||||
});
|
||||
const menu = (refusal?: { text: string }): JSX.Element =>
|
||||
withProviders(
|
||||
<MediaMuteAndSwitchButton
|
||||
iconsAndLabels="video"
|
||||
enabled={true}
|
||||
options={[{ label: { type: "name", name: "Camera 1" }, id: "cam1" }]}
|
||||
selectedOption="cam1"
|
||||
backgroundEffects={effects}
|
||||
selectedBackgroundEffect="blur"
|
||||
onSelectBackgroundEffect={vi.fn()}
|
||||
onAddBackgroundImage={vi.fn()}
|
||||
backgroundImageRefusal={refusal}
|
||||
/>,
|
||||
);
|
||||
const { rerender, getByRole } = render(menu());
|
||||
await user.click(getByRole("button", { name: "Camera" }));
|
||||
|
||||
rerender(menu({ text: "That file is not a supported image" }));
|
||||
expect(
|
||||
screen.getByText("That file is not a supported image"),
|
||||
).toBeVisible();
|
||||
screen.getByRole("menuitemradio", { name: "Blur", checked: true });
|
||||
await user.click(screen.getByRole("button", { name: "Close" }));
|
||||
expect(screen.queryByText("That file is not a supported image")).toBeNull();
|
||||
|
||||
// The next refusal is shown, even if it says the same.
|
||||
rerender(menu({ text: "That file is not a supported image" }));
|
||||
expect(
|
||||
screen.getByText("That file is not a supported image"),
|
||||
).toBeVisible();
|
||||
|
||||
await user.keyboard("{Escape}");
|
||||
await user.click(getByRole("button", { name: "Camera" }));
|
||||
expect(screen.queryByText("That file is not a supported image")).toBeNull();
|
||||
});
|
||||
|
||||
test("offers the background effects in a phone's drawer", async () => {
|
||||
const userAgent = vi
|
||||
.spyOn(navigator, "userAgent", "get")
|
||||
.mockReturnValue("Mozilla/5.0 (Linux; Android 14)");
|
||||
try {
|
||||
const user = userEvent.setup();
|
||||
const onSelectBackgroundEffect = vi.fn();
|
||||
const { getByRole } = renderComponent(
|
||||
<MediaMuteAndSwitchButton
|
||||
iconsAndLabels="video"
|
||||
enabled={true}
|
||||
options={[{ label: { type: "name", name: "Camera 1" }, id: "cam1" }]}
|
||||
selectedOption="cam1"
|
||||
backgroundEffects={effects}
|
||||
selectedBackgroundEffect="none"
|
||||
onSelectBackgroundEffect={onSelectBackgroundEffect}
|
||||
/>,
|
||||
);
|
||||
|
||||
await user.click(getByRole("button", { name: "Camera" }));
|
||||
const section = screen.getByRole("group", { name: "Background effects" });
|
||||
// A click alone: the drawer's drag handling reads layout jsdom lacks.
|
||||
fireEvent.click(
|
||||
within(section).getByRole("menuitemradio", { name: "Background 2" }),
|
||||
);
|
||||
expect(onSelectBackgroundEffect).toHaveBeenCalledWith("image:glow");
|
||||
} finally {
|
||||
userAgent.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
test("marks the selected menu item as checked", async () => {
|
||||
@@ -552,7 +733,9 @@ describe("MediaMuteAndSwitchButton", () => {
|
||||
]}
|
||||
selectedOption="cam1"
|
||||
onSelect={vi.fn()}
|
||||
videoBlurToggleClick={vi.fn()}
|
||||
backgroundEffects={effects}
|
||||
selectedBackgroundEffect="none"
|
||||
onSelectBackgroundEffect={vi.fn()}
|
||||
/>,
|
||||
);
|
||||
|
||||
@@ -560,9 +743,10 @@ describe("MediaMuteAndSwitchButton", () => {
|
||||
|
||||
screen.getByRole("menuitemradio", { name: "Camera 1", checked: true });
|
||||
screen.getByRole("menuitemradio", { name: "Camera 2", checked: false });
|
||||
expect(
|
||||
screen.getByRole("menuitemcheckbox", { name: "Blur background" }),
|
||||
).toBeInTheDocument();
|
||||
within(screen.getByRole("group", { name: "Background effects" })).getByRole(
|
||||
"menuitemradio",
|
||||
{ name: "Blur" },
|
||||
);
|
||||
});
|
||||
|
||||
test("marks focus as keyboard-driven only when the keyboard moved it", async () => {
|
||||
|
||||
@@ -7,24 +7,29 @@ Please see LICENSE in the repository root for full details.
|
||||
|
||||
import {
|
||||
useCallback,
|
||||
useId,
|
||||
useRef,
|
||||
useState,
|
||||
type CSSProperties,
|
||||
type FC,
|
||||
useEffect,
|
||||
type ReactElement,
|
||||
type ReactNode,
|
||||
} from "react";
|
||||
import {
|
||||
Alert,
|
||||
Button,
|
||||
InlineSpinner,
|
||||
Menu,
|
||||
MenuItem,
|
||||
MenuScrollArea,
|
||||
MenuScrollSection,
|
||||
RadioInput,
|
||||
ToggleMenuItem,
|
||||
} from "@vector-im/compound-web";
|
||||
import {
|
||||
ChevronUpIcon,
|
||||
ChevronDownIcon,
|
||||
InfoIcon,
|
||||
SpinnerIcon,
|
||||
} from "@vector-im/compound-design-tokens/assets/web/icons";
|
||||
import classNames from "classnames";
|
||||
@@ -41,6 +46,10 @@ import { useMediaDevices } from "../MediaDevicesContext";
|
||||
import { useRootElement } from "../RootElementContext";
|
||||
import { observeElementSize$ } from "../utils/elementSize";
|
||||
import { LiveMicrophoneLevelMeter } from "./MicrophoneLevelMeter";
|
||||
import {
|
||||
BackgroundEffectGrid,
|
||||
type BackgroundEffectGridItem,
|
||||
} from "./BackgroundEffectGrid";
|
||||
|
||||
export interface MenuOptions {
|
||||
label: DeviceLabel | AudioOutputDeviceLabel;
|
||||
@@ -65,8 +74,23 @@ export interface MediaMuteAndSwitchButtonProps {
|
||||
selectedOutputOption?: string;
|
||||
/** Picks an output device. Undefined disables the speaker section. */
|
||||
onSelectOutput?: (id: string) => void;
|
||||
videoBlurToggleClick?: () => void;
|
||||
videoBlurEnabled?: boolean;
|
||||
/** Camera menu only. None or an empty list leaves the section out. */
|
||||
backgroundEffects?: BackgroundEffectGridItem[];
|
||||
selectedBackgroundEffect?: string;
|
||||
/** Undefined where effects can't run: all but no effect are disabled. */
|
||||
onSelectBackgroundEffect?: (id: string) => void;
|
||||
/** What the user should know before choosing an effect, if anything. */
|
||||
backgroundEffectNotice?: string;
|
||||
/** Whether the first effect chosen is still being prepared. */
|
||||
backgroundEffectSettling?: boolean;
|
||||
/** Called with the file chosen from the add tile. Omit to leave it out. */
|
||||
onAddBackgroundImage?: (file: File) => void;
|
||||
/** Removes a removable effect, never the one in force. */
|
||||
onRemoveBackgroundEffect?: (id: string) => void;
|
||||
/** Why the last file offered couldn't be used; a new object each time. */
|
||||
backgroundImageRefusal?: { text: string };
|
||||
/** The user's own camera, shown at the top of the camera menu. */
|
||||
selfPreview?: ReactNode;
|
||||
/**
|
||||
* For any toggle and option this method will be called.
|
||||
* So toggles need to be implemented by listening here and setting the right toggle item to `enabled`
|
||||
@@ -74,8 +98,6 @@ export interface MediaMuteAndSwitchButtonProps {
|
||||
onSelect?: (id: string) => void;
|
||||
}
|
||||
|
||||
const BLUR_ID = "blur";
|
||||
|
||||
/** Id of the placeholder "Default" row, shown when the platform lists no outputs. */
|
||||
const DEFAULT_OUTPUT_ID = "default";
|
||||
|
||||
@@ -85,9 +107,16 @@ const LIST_SHARE_OF_CALL = 0.6;
|
||||
/** Smallest device list height in px, so a short call still shows more than one device. */
|
||||
const MIN_LIST_HEIGHT = 160;
|
||||
|
||||
/** The menu's width as its stylesheet sets it, to place the preview before
|
||||
the menu is drawn. */
|
||||
const MENU_WIDTH = 296;
|
||||
|
||||
/** Space kept between the menu and the call area's sides. */
|
||||
const MENU_MARGIN = 16;
|
||||
|
||||
/** Below the preview, as its stylesheet draws it. */
|
||||
const PREVIEW_GAP = 12;
|
||||
|
||||
export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
|
||||
enabled,
|
||||
busy,
|
||||
@@ -98,8 +127,15 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
|
||||
outputOptions,
|
||||
selectedOutputOption,
|
||||
onSelectOutput,
|
||||
videoBlurEnabled,
|
||||
videoBlurToggleClick,
|
||||
backgroundEffects,
|
||||
selectedBackgroundEffect,
|
||||
onSelectBackgroundEffect,
|
||||
backgroundEffectNotice,
|
||||
backgroundEffectSettling,
|
||||
onAddBackgroundImage,
|
||||
onRemoveBackgroundEffect,
|
||||
backgroundImageRefusal,
|
||||
selfPreview,
|
||||
onSelect,
|
||||
}) => {
|
||||
// Requested but not yet selected. Keyed by kind too, since Chrome uses
|
||||
@@ -109,14 +145,46 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
|
||||
id: string;
|
||||
} | null>(null);
|
||||
const [menuOpen, setMenuOpen] = useState(false);
|
||||
const onOpenChange = useCallback((open: boolean): void => {
|
||||
setMenuOpen(open);
|
||||
// Drop a request that never arrived.
|
||||
if (!open) setPlannedSelection(null);
|
||||
// The file picker takes the focus, which the menu reads as a click
|
||||
// elsewhere; it is held open until the picker is done.
|
||||
const choosingFile = useRef(false);
|
||||
const fileInput = useRef<HTMLInputElement | null>(null);
|
||||
// A refusal is about the attempt that caused it: shown until dismissed or
|
||||
// until the menu closes, and again for the next one.
|
||||
const [seenRefusal, setSeenRefusal] = useState<object>();
|
||||
const refusal =
|
||||
backgroundImageRefusal !== seenRefusal ? backgroundImageRefusal : undefined;
|
||||
const onOpenChange = useCallback(
|
||||
(open: boolean): void => {
|
||||
if (!open && choosingFile.current) return;
|
||||
setMenuOpen(open);
|
||||
// Drop a request that never arrived.
|
||||
if (!open) setPlannedSelection(null);
|
||||
if (!open) setSeenRefusal(backgroundImageRefusal);
|
||||
},
|
||||
[backgroundImageRefusal],
|
||||
);
|
||||
// Scrolled to as it appears, as the list may be scrolled away from it.
|
||||
const scrollIntoView = useCallback((element: HTMLElement | null): void => {
|
||||
element?.scrollIntoView({ block: "nearest" });
|
||||
}, []);
|
||||
const watchFileInput = useCallback(
|
||||
(input: HTMLInputElement | null): (() => void) | undefined => {
|
||||
fileInput.current = input;
|
||||
if (input === null) return;
|
||||
// Dismissing the picker fires cancel, which React doesn't type.
|
||||
const done = (): void => {
|
||||
choosingFile.current = false;
|
||||
};
|
||||
input.addEventListener("cancel", done);
|
||||
return (): void => input.removeEventListener("cancel", done);
|
||||
},
|
||||
[],
|
||||
);
|
||||
const isBusy = busy ?? false;
|
||||
const { t } = useTranslation();
|
||||
const devices = useMediaDevices();
|
||||
const noticeId = useId();
|
||||
|
||||
/**
|
||||
* Records on the menu whether the keyboard or the pointer moved focus, for
|
||||
@@ -146,21 +214,53 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
|
||||
|
||||
// Measured on the call area: CSS can't size the portalled menu against it.
|
||||
const rootElement = useRootElement();
|
||||
const [listMaxHeight, setListMaxHeight] = useState<number>();
|
||||
const [listShare, setListShare] = useState<number>();
|
||||
const [previewBlock, setPreviewBlock] = useState(0);
|
||||
useEffect(() => {
|
||||
if (!menuOpen) return;
|
||||
// Followed, since a host can resize the call while the menu is open.
|
||||
const subscription = observeElementSize$(rootElement)
|
||||
.pipe(
|
||||
map(({ height }) =>
|
||||
Math.max(MIN_LIST_HEIGHT, Math.round(height * LIST_SHARE_OF_CALL)),
|
||||
map(({ width, height }) => ({
|
||||
height: Math.round(height * LIST_SHARE_OF_CALL),
|
||||
width: Math.min(MENU_WIDTH, Math.round(width - 2 * MENU_MARGIN)),
|
||||
})),
|
||||
distinctUntilChanged(
|
||||
(a, b) => a.height === b.height && a.width === b.width,
|
||||
),
|
||||
distinctUntilChanged(),
|
||||
)
|
||||
.subscribe(setListMaxHeight);
|
||||
.subscribe(({ height, width }) => {
|
||||
setListShare(height);
|
||||
setPreviewBlock(Math.round((width * 9) / 16) + PREVIEW_GAP);
|
||||
});
|
||||
return (): void => subscription.unsubscribe();
|
||||
}, [menuOpen, rootElement]);
|
||||
|
||||
// Paid for out of the list's share, so a preview makes the menu no taller.
|
||||
const hasPreview =
|
||||
iconsAndLabels === "video" &&
|
||||
selfPreview !== undefined &&
|
||||
listShare !== undefined;
|
||||
const previewPinned =
|
||||
hasPreview && listShare - previewBlock >= MIN_LIST_HEIGHT;
|
||||
const listMaxHeight =
|
||||
listShare === undefined
|
||||
? undefined
|
||||
: Math.max(
|
||||
MIN_LIST_HEIGHT,
|
||||
listShare - (previewPinned ? previewBlock : 0),
|
||||
);
|
||||
const preview = hasPreview && (
|
||||
<div
|
||||
aria-hidden
|
||||
className={classNames(styles.selfPreview, {
|
||||
[styles.selfPreviewPinned]: previewPinned,
|
||||
})}
|
||||
>
|
||||
{backgroundEffectSettling ? <InlineSpinner size={32} /> : selfPreview}
|
||||
</div>
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
if (menuOpen) devices.requestDeviceNames(); // No-op after the first call
|
||||
}, [menuOpen, devices]);
|
||||
@@ -187,16 +287,8 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
|
||||
/>
|
||||
);
|
||||
|
||||
const toggles =
|
||||
iconsAndLabels === "video" && videoBlurToggleClick !== undefined
|
||||
? [
|
||||
{
|
||||
label: t("action.blur_background"),
|
||||
enabled: videoBlurEnabled ?? false,
|
||||
id: BLUR_ID,
|
||||
},
|
||||
]
|
||||
: [];
|
||||
const showEffects =
|
||||
iconsAndLabels === "video" && (backgroundEffects?.length ?? 0) > 0;
|
||||
|
||||
let optionsButtonLabel: string;
|
||||
let menuTitle: string;
|
||||
@@ -320,6 +412,21 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
|
||||
>
|
||||
{/* The mute button lives inside */}
|
||||
{button}
|
||||
{onAddBackgroundImage !== undefined && (
|
||||
<input
|
||||
ref={watchFileInput}
|
||||
type="file"
|
||||
accept="image/*"
|
||||
hidden
|
||||
onChange={(e) => {
|
||||
const file = e.target.files?.[0];
|
||||
// Cleared, so the same file can be chosen twice in a row.
|
||||
e.target.value = "";
|
||||
choosingFile.current = false;
|
||||
if (file) onAddBackgroundImage(file);
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
<Menu
|
||||
className={styles.menu}
|
||||
// Named for screen readers only: each section has its own heading.
|
||||
@@ -344,9 +451,12 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
|
||||
/>
|
||||
}
|
||||
>
|
||||
{previewPinned && preview}
|
||||
<MenuScrollArea
|
||||
ref={trackFocusSource}
|
||||
className={styles.deviceList}
|
||||
className={classNames(styles.deviceList, {
|
||||
[styles.deviceListLeadsWithPreview]: hasPreview && !previewPinned,
|
||||
})}
|
||||
data-testid="device-list"
|
||||
style={
|
||||
{
|
||||
@@ -355,6 +465,7 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
|
||||
} as CSSProperties
|
||||
}
|
||||
>
|
||||
{hasPreview && !previewPinned && preview}
|
||||
{audioMeter && speakerOptions && (
|
||||
<MenuScrollSection title={t("settings.devices.speaker")}>
|
||||
{deviceItems(
|
||||
@@ -378,19 +489,52 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
|
||||
numberedLabel,
|
||||
)}
|
||||
</MenuScrollSection>
|
||||
{showEffects && (
|
||||
<MenuScrollSection title={t("settings.background_effects_header")}>
|
||||
<BackgroundEffectGrid
|
||||
effects={backgroundEffects ?? []}
|
||||
selected={selectedBackgroundEffect}
|
||||
onSelect={onSelectBackgroundEffect}
|
||||
settling={backgroundEffectSettling}
|
||||
describedBy={
|
||||
backgroundEffectNotice === undefined ? undefined : noticeId
|
||||
}
|
||||
onAdd={
|
||||
onAddBackgroundImage === undefined
|
||||
? undefined
|
||||
: (): void => {
|
||||
choosingFile.current = true;
|
||||
fileInput.current?.click();
|
||||
}
|
||||
}
|
||||
addLabel={t("action.add_background_image")}
|
||||
onRemove={onRemoveBackgroundEffect}
|
||||
removeLabel={t("action.remove")}
|
||||
/>
|
||||
{/* In the list, so the menu grows no taller for it. */}
|
||||
{refusal !== undefined && (
|
||||
<div
|
||||
ref={scrollIntoView}
|
||||
role="none"
|
||||
className={styles.refusal}
|
||||
>
|
||||
<Alert
|
||||
type="critical"
|
||||
title={refusal.text}
|
||||
onClose={(): void => setSeenRefusal(refusal)}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
{refusal === undefined &&
|
||||
backgroundEffectNotice !== undefined && (
|
||||
<div id={noticeId} role="none" className={styles.notice}>
|
||||
<InfoIcon width={20} height={20} />
|
||||
<span>{backgroundEffectNotice}</span>
|
||||
</div>
|
||||
)}
|
||||
</MenuScrollSection>
|
||||
)}
|
||||
</MenuScrollArea>
|
||||
{toggles.length > 0 && <hr />}
|
||||
{toggles.map((toggle) => (
|
||||
<ToggleMenuItem
|
||||
label={toggle.label}
|
||||
onSelect={(e) => {
|
||||
videoBlurToggleClick?.();
|
||||
e.preventDefault();
|
||||
}}
|
||||
checked={toggle.enabled}
|
||||
key={toggle.id}
|
||||
/>
|
||||
))}
|
||||
</Menu>
|
||||
</div>
|
||||
);
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
.mirrored {
|
||||
transform: scaleX(-1);
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { act, render } from "@testing-library/react";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { BehaviorSubject } from "rxjs";
|
||||
|
||||
import { SelfPreview } from "./SelfPreview";
|
||||
import styles from "./SelfPreview.module.css";
|
||||
import { mockVideoTrack } from "../utils/test";
|
||||
|
||||
describe("SelfPreview", () => {
|
||||
it("detaches only its own element from a camera shown twice", () => {
|
||||
const track = mockVideoTrack({});
|
||||
const first = render(<SelfPreview track={track} />);
|
||||
const second = render(<SelfPreview track={track} />);
|
||||
const [a, b] = [first, second].map((r) =>
|
||||
r.container.querySelector("video"),
|
||||
);
|
||||
expect(track.attach).toHaveBeenCalledWith(a);
|
||||
expect(track.attach).toHaveBeenCalledWith(b);
|
||||
|
||||
first.unmount();
|
||||
expect(track.detach).toHaveBeenCalledExactlyOnceWith(a);
|
||||
});
|
||||
|
||||
it("mirrors the camera while it faces the user, and only then", () => {
|
||||
const facingMode$ = new BehaviorSubject<"user" | "environment">("user");
|
||||
const { container } = render(
|
||||
<SelfPreview track={mockVideoTrack({ facingMode$ })} />,
|
||||
);
|
||||
const video = (): HTMLVideoElement => container.querySelector("video")!;
|
||||
expect(video().classList.contains(styles.mirrored)).toBe(true);
|
||||
act(() => facingMode$.next("environment"));
|
||||
expect(video().classList.contains(styles.mirrored)).toBe(false);
|
||||
});
|
||||
|
||||
it("says the camera is off while there is none", () => {
|
||||
const { container } = render(<SelfPreview track={null} />);
|
||||
expect(container.querySelector("video")).toBeNull();
|
||||
expect(container.querySelector("svg")).not.toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,48 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { type FC, useEffect, useRef } from "react";
|
||||
import { constant, type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk";
|
||||
import { VideoCallOffIcon } from "@vector-im/compound-design-tokens/assets/web/icons";
|
||||
|
||||
import styles from "./SelfPreview.module.css";
|
||||
import { useBehavior } from "../useBehavior";
|
||||
|
||||
const noFacingMode = constant(undefined);
|
||||
|
||||
interface Props {
|
||||
/** The camera as the pipeline has it, or none while the camera is off. */
|
||||
track: LocalVideoMediaTrack | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The user's own camera with the effect in force: the pipeline's own track,
|
||||
* shown a second time rather than captured again.
|
||||
*/
|
||||
export const SelfPreview: FC<Props> = ({ track }) => {
|
||||
const videoRef = useRef<HTMLVideoElement | null>(null);
|
||||
useEffect(() => {
|
||||
const element = videoRef.current;
|
||||
if (!track || !element) return;
|
||||
track.attach(element);
|
||||
// This element only: the pre-join preview may be showing the same camera
|
||||
return (): void => {
|
||||
track.detach(element);
|
||||
};
|
||||
}, [track]);
|
||||
const facingMode = useBehavior(track?.facingMode$ ?? noFacingMode);
|
||||
|
||||
if (!track) return <VideoCallOffIcon width={32} height={32} />;
|
||||
return (
|
||||
<video
|
||||
ref={videoRef}
|
||||
muted
|
||||
playsInline
|
||||
className={facingMode === "user" ? styles.mirrored : undefined}
|
||||
/>
|
||||
);
|
||||
};
|
||||
@@ -9,7 +9,7 @@ exports[`MediaMuteAndSwitchButton > renders 1`] = `
|
||||
aria-busy="false"
|
||||
aria-checked="false"
|
||||
aria-disabled="true"
|
||||
aria-labelledby="_r_0_"
|
||||
aria-labelledby="_r_1_"
|
||||
class="_button_1nw83_8 _has-icon_1nw83_60 _icon-only_1nw83_53"
|
||||
data-kind="primary"
|
||||
data-size="lg"
|
||||
@@ -39,7 +39,7 @@ exports[`MediaMuteAndSwitchButton > renders 1`] = `
|
||||
data-kind="tertiary"
|
||||
data-size="lg"
|
||||
data-state="closed"
|
||||
id="radix-_r_5_"
|
||||
id="radix-_r_6_"
|
||||
role="button"
|
||||
tabindex="0"
|
||||
type="button"
|
||||
|
||||
@@ -95,6 +95,35 @@ describe("Config.initWith", () => {
|
||||
expect(Config.get().ssla).toBe("https://example.invalid/ssla");
|
||||
});
|
||||
|
||||
it("falls back to the defaults when fetching config.json rejects", async () => {
|
||||
// A rejected promise is what a webview gives for a missing local file.
|
||||
vi.spyOn(globalThis, "fetch").mockRejectedValue(
|
||||
new TypeError("Load failed"),
|
||||
);
|
||||
vi.spyOn(logger, "info").mockImplementation(() => {});
|
||||
Reflect.set(Config, "internalInstance", undefined);
|
||||
|
||||
await expect(Config.init()).resolves.toBeUndefined();
|
||||
|
||||
expect(Config.get().media_quality).toEqual(DEFAULT_CONFIG.media_quality);
|
||||
});
|
||||
|
||||
it("falls back to the defaults when config.json is not JSON", async () => {
|
||||
vi.spyOn(globalThis, "fetch").mockResolvedValue(
|
||||
new Response("<!doctype html><title>Element</title>", {
|
||||
status: 200,
|
||||
headers: { "content-type": "text/html" },
|
||||
}),
|
||||
);
|
||||
const warnSpy = vi.spyOn(logger, "warn").mockImplementation(() => {});
|
||||
Reflect.set(Config, "internalInstance", undefined);
|
||||
|
||||
await expect(Config.init()).resolves.toBeUndefined();
|
||||
|
||||
expect(Config.get().media_quality).toEqual(DEFAULT_CONFIG.media_quality);
|
||||
expect(warnSpy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("replaces a config initialized earlier", () => {
|
||||
Config.initWith({ ssla: "https://first.invalid/ssla" });
|
||||
Config.initWith({ ssla: "https://second.invalid/ssla" });
|
||||
|
||||
+20
-8
@@ -140,14 +140,26 @@ export function validateConfig(config: ConfigOptions): ConfigOptions {
|
||||
}
|
||||
|
||||
async function downloadConfig(fetchTarget: string): Promise<ConfigOptions> {
|
||||
const response = await fetch(fetchTarget);
|
||||
|
||||
if (isFailure(response)) {
|
||||
// Lack of a config isn't an error, we should just use the defaults.
|
||||
// Also treat a blank config as no config, assuming the status code is 0, because we don't get 404s from file:
|
||||
// URIs so this is the only way we can not fail if the file doesn't exist when loading from a file:// URI.
|
||||
return DEFAULT_CONFIG;
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetch(fetchTarget);
|
||||
} catch (e) {
|
||||
// The embedded package ships no config.json, and some webviews (Element X
|
||||
// iOS) reject the fetch of a missing local file rather than answering 404.
|
||||
logger.info(`No config at ${fetchTarget}, using defaults`, e);
|
||||
// resolveConfig merges in the defaults.
|
||||
return {};
|
||||
}
|
||||
|
||||
return response.json();
|
||||
if (isFailure(response)) {
|
||||
// resolveConfig merges in the defaults.
|
||||
return {};
|
||||
}
|
||||
|
||||
try {
|
||||
return await response.json();
|
||||
} catch (e) {
|
||||
logger.warn(`Config at ${fetchTarget} is not JSON, using defaults`, e);
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
+24
-24
@@ -7,30 +7,30 @@ Please see LICENSE in the repository root for full details.
|
||||
|
||||
import { type RelationType } from "matrix-js-sdk";
|
||||
|
||||
import catSoundOgg from "../sound/reactions/cat.ogg?url";
|
||||
import catSoundMp3 from "../sound/reactions/cat.mp3?url";
|
||||
import clapSoundOgg from "../sound/reactions/clap.ogg?url";
|
||||
import clapSoundMp3 from "../sound/reactions/clap.mp3?url";
|
||||
import cricketsSoundOgg from "../sound/reactions/crickets.ogg?url";
|
||||
import cricketsSoundMp3 from "../sound/reactions/crickets.mp3?url";
|
||||
import dogSoundOgg from "../sound/reactions/dog.ogg?url";
|
||||
import dogSoundMp3 from "../sound/reactions/dog.mp3?url";
|
||||
import genericSoundOgg from "../sound/reactions/generic.ogg?url";
|
||||
import genericSoundMp3 from "../sound/reactions/generic.mp3?url";
|
||||
import lightbulbSoundOgg from "../sound/reactions/lightbulb.ogg?url";
|
||||
import lightbulbSoundMp3 from "../sound/reactions/lightbulb.mp3?url";
|
||||
import partySoundOgg from "../sound/reactions/party.ogg?url";
|
||||
import partySoundMp3 from "../sound/reactions/party.mp3?url";
|
||||
import deerSoundOgg from "../sound/reactions/deer.ogg?url";
|
||||
import deerSoundMp3 from "../sound/reactions/deer.mp3?url";
|
||||
import rockSoundOgg from "../sound/reactions/rock.ogg?url";
|
||||
import rockSoundMp3 from "../sound/reactions/rock.mp3?url";
|
||||
import waveSoundOgg from "../sound/reactions/wave.ogg?url";
|
||||
import waveSoundMp3 from "../sound/reactions/wave.mp3?url";
|
||||
import baduntssSoundOgg from "../sound/reactions/baduntss.ogg?url";
|
||||
import baduntssSoundMp3 from "../sound/reactions/baduntss.mp3?url";
|
||||
import bellSoundOgg from "../sound/reactions/bell.ogg?url";
|
||||
import bellSoundMp3 from "../sound/reactions/bell.mp3?url";
|
||||
import catSoundOgg from "../sound/reactions/cat.ogg?url&no-inline";
|
||||
import catSoundMp3 from "../sound/reactions/cat.mp3?url&no-inline";
|
||||
import clapSoundOgg from "../sound/reactions/clap.ogg?url&no-inline";
|
||||
import clapSoundMp3 from "../sound/reactions/clap.mp3?url&no-inline";
|
||||
import cricketsSoundOgg from "../sound/reactions/crickets.ogg?url&no-inline";
|
||||
import cricketsSoundMp3 from "../sound/reactions/crickets.mp3?url&no-inline";
|
||||
import dogSoundOgg from "../sound/reactions/dog.ogg?url&no-inline";
|
||||
import dogSoundMp3 from "../sound/reactions/dog.mp3?url&no-inline";
|
||||
import genericSoundOgg from "../sound/reactions/generic.ogg?url&no-inline";
|
||||
import genericSoundMp3 from "../sound/reactions/generic.mp3?url&no-inline";
|
||||
import lightbulbSoundOgg from "../sound/reactions/lightbulb.ogg?url&no-inline";
|
||||
import lightbulbSoundMp3 from "../sound/reactions/lightbulb.mp3?url&no-inline";
|
||||
import partySoundOgg from "../sound/reactions/party.ogg?url&no-inline";
|
||||
import partySoundMp3 from "../sound/reactions/party.mp3?url&no-inline";
|
||||
import deerSoundOgg from "../sound/reactions/deer.ogg?url&no-inline";
|
||||
import deerSoundMp3 from "../sound/reactions/deer.mp3?url&no-inline";
|
||||
import rockSoundOgg from "../sound/reactions/rock.ogg?url&no-inline";
|
||||
import rockSoundMp3 from "../sound/reactions/rock.mp3?url&no-inline";
|
||||
import waveSoundOgg from "../sound/reactions/wave.ogg?url&no-inline";
|
||||
import waveSoundMp3 from "../sound/reactions/wave.mp3?url&no-inline";
|
||||
import baduntssSoundOgg from "../sound/reactions/baduntss.ogg?url&no-inline";
|
||||
import baduntssSoundMp3 from "../sound/reactions/baduntss.mp3?url&no-inline";
|
||||
import bellSoundOgg from "../sound/reactions/bell.ogg?url&no-inline";
|
||||
import bellSoundMp3 from "../sound/reactions/bell.mp3?url&no-inline";
|
||||
|
||||
export const ElementCallReactionEventType = "io.element.call.reaction";
|
||||
|
||||
|
||||
@@ -8,18 +8,18 @@ Please see LICENSE in the repository root for full details.
|
||||
import { type ReactNode, useEffect } from "react";
|
||||
|
||||
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import joinCallSoundMp3 from "../sound/join_call.mp3";
|
||||
import joinCallSoundOgg from "../sound/join_call.ogg";
|
||||
import leftCallSoundMp3 from "../sound/left_call.mp3";
|
||||
import leftCallSoundOgg from "../sound/left_call.ogg";
|
||||
import handSoundOgg from "../sound/raise_hand.ogg";
|
||||
import handSoundMp3 from "../sound/raise_hand.mp3";
|
||||
import screenShareStartedOgg from "../sound/screen_share_started.ogg";
|
||||
import screenShareStartedMp3 from "../sound/screen_share_started.mp3";
|
||||
import declineMp3 from "../sound/call_declined.mp3?url";
|
||||
import declineOgg from "../sound/call_declined.ogg?url";
|
||||
import timeoutMp3 from "../sound/call_timeout.mp3?url";
|
||||
import timeoutOgg from "../sound/call_timeout.ogg?url";
|
||||
import joinCallSoundMp3 from "../sound/join_call.mp3?url&no-inline";
|
||||
import joinCallSoundOgg from "../sound/join_call.ogg?url&no-inline";
|
||||
import leftCallSoundMp3 from "../sound/left_call.mp3?url&no-inline";
|
||||
import leftCallSoundOgg from "../sound/left_call.ogg?url&no-inline";
|
||||
import handSoundOgg from "../sound/raise_hand.ogg?url&no-inline";
|
||||
import handSoundMp3 from "../sound/raise_hand.mp3?url&no-inline";
|
||||
import screenShareStartedOgg from "../sound/screen_share_started.ogg?url&no-inline";
|
||||
import screenShareStartedMp3 from "../sound/screen_share_started.mp3?url&no-inline";
|
||||
import declineMp3 from "../sound/call_declined.mp3?url&no-inline";
|
||||
import declineOgg from "../sound/call_declined.ogg?url&no-inline";
|
||||
import timeoutMp3 from "../sound/call_timeout.mp3?url&no-inline";
|
||||
import timeoutOgg from "../sound/call_timeout.ogg?url&no-inline";
|
||||
import { useAudioContext } from "../useAudioContext";
|
||||
import { prefetchSounds } from "../soundUtils";
|
||||
import { useLatest } from "../useLatest";
|
||||
|
||||
@@ -75,7 +75,6 @@ import {
|
||||
type RTCSlotOptions,
|
||||
constant,
|
||||
} from "@element-hq/matrixrtc-sdk";
|
||||
import { map } from "rxjs";
|
||||
import {
|
||||
advancedCamera,
|
||||
advancedScreenShare,
|
||||
@@ -181,7 +180,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
|
||||
props.muteStates,
|
||||
mediaDevices,
|
||||
capture,
|
||||
backgroundEffectsState$.value.processor,
|
||||
backgroundEffectsState$.value,
|
||||
),
|
||||
});
|
||||
|
||||
@@ -195,9 +194,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
|
||||
...callViewModelOptionsFromParams(urlParams),
|
||||
encryptionSystem: props.e2eeSystem,
|
||||
capture,
|
||||
videoProcessor$: scope.behavior(
|
||||
backgroundEffectsState$.pipe(map((state) => state.processor)),
|
||||
),
|
||||
backgroundEffectsState$,
|
||||
hostBridge,
|
||||
autoLeaveWhenOthersLeft,
|
||||
waitForCallPickup: waitForCallPickup && sendNotificationType === "ring",
|
||||
@@ -241,6 +238,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
|
||||
vm,
|
||||
props.muteStates,
|
||||
mediaDevices,
|
||||
backgroundEffectsState$,
|
||||
`${props.client.getUserId()}:${props.client.getDeviceId()}`,
|
||||
{ showControls: urlParams.showControls, header: urlParams.header },
|
||||
);
|
||||
@@ -824,9 +822,7 @@ export function rtcParticipationOptions(
|
||||
const session = config.matrix_rtc_session;
|
||||
return {
|
||||
sendNotificationType: urlParams.sendNotificationType,
|
||||
applicationData: urlParams.callIntent
|
||||
? { "m.call.intent": urlParams.callIntent }
|
||||
: undefined,
|
||||
callIntent: urlParams.callIntent,
|
||||
timings: {
|
||||
syncDisconnectGracePeriodMs: config.sync_disconnect_grace_period_ms,
|
||||
networkErrorRetryMs: session.network_error_retry_ms,
|
||||
|
||||
+13
-5
@@ -16,7 +16,6 @@ import {
|
||||
type LocalMedia,
|
||||
type LocalVideoMediaTrack,
|
||||
} from "@element-hq/matrixrtc-sdk";
|
||||
import { map } from "rxjs";
|
||||
|
||||
import inCallStyles from "./InCallView.module.css";
|
||||
import styles from "./LobbyView.module.css";
|
||||
@@ -134,9 +133,7 @@ export const LobbyView: FC<Props> = ({
|
||||
muteStates,
|
||||
devices,
|
||||
captureSettings(),
|
||||
mediaScope.behavior(
|
||||
backgroundEffectsState$.pipe(map((state) => state.processor)),
|
||||
),
|
||||
backgroundEffectsState$,
|
||||
),
|
||||
);
|
||||
return (): void => {
|
||||
@@ -158,12 +155,15 @@ export const LobbyView: FC<Props> = ({
|
||||
null,
|
||||
);
|
||||
useEffect(() => {
|
||||
if (mediaVm === null) return;
|
||||
const footerScope = new ObservableScope();
|
||||
setFooterVm(
|
||||
createLobbyFooterViewModel(
|
||||
footerScope,
|
||||
muteStates,
|
||||
devices,
|
||||
backgroundEffectsState$,
|
||||
mediaVm.videoTrack$,
|
||||
openSettings,
|
||||
hangup,
|
||||
// Logo and header are connected: only show the logo in SPA with header.
|
||||
@@ -173,7 +173,15 @@ export const LobbyView: FC<Props> = ({
|
||||
return (): void => {
|
||||
footerScope.end();
|
||||
};
|
||||
}, [devices, hangup, hideHeader, muteStates, openSettings]);
|
||||
}, [
|
||||
devices,
|
||||
backgroundEffectsState$,
|
||||
hangup,
|
||||
hideHeader,
|
||||
mediaVm,
|
||||
muteStates,
|
||||
openSettings,
|
||||
]);
|
||||
|
||||
// TODO: Unify this component with InCallView, so we can get slick joining
|
||||
// animations and don't have to feel bad about reusing its CSS
|
||||
|
||||
@@ -8,6 +8,7 @@ Please see LICENSE in the repository root for full details.
|
||||
import { expect, type MockedFunction, test, vi } from "vitest";
|
||||
import { act, render } from "@testing-library/react";
|
||||
import { BehaviorSubject } from "rxjs";
|
||||
import { RTC_CALL_INTENT_AUDIO } from "matrix-js-sdk/lib/matrixrtc";
|
||||
|
||||
import { useAudioContext } from "../useAudioContext";
|
||||
import { createRingingMedia } from "../state/media/RingingMediaViewModel";
|
||||
@@ -39,7 +40,7 @@ test("ringtone plays on loop while ringing", () => {
|
||||
userId: alice.userId,
|
||||
displayName$: constant("Alice"),
|
||||
mxcAvatarUrl$: constant(undefined),
|
||||
intent: "audio",
|
||||
intent: RTC_CALL_INTENT_AUDIO,
|
||||
pickupState$,
|
||||
});
|
||||
|
||||
|
||||
@@ -12,8 +12,8 @@ import { type RingingMediaViewModel } from "../state/media/RingingMediaViewModel
|
||||
import { useBehavior } from "../useBehavior";
|
||||
import { useInitial } from "../useInitial";
|
||||
import { prefetchSounds } from "../soundUtils";
|
||||
import ringtoneMp3 from "../sound/ringtone.mp3?url";
|
||||
import ringtoneOgg from "../sound/ringtone.ogg?url";
|
||||
import ringtoneMp3 from "../sound/ringtone.mp3?url&no-inline";
|
||||
import ringtoneOgg from "../sound/ringtone.ogg?url&no-inline";
|
||||
import { type UseAudioContext, useAudioContext } from "../useAudioContext";
|
||||
import { useLatest } from "../useLatest";
|
||||
|
||||
|
||||
+7
-26
@@ -34,23 +34,6 @@ describe("prefetchSounds", () => {
|
||||
expect(bytesOf(sounds.beep)).toEqual([1, 2, 3]);
|
||||
});
|
||||
|
||||
it("decodes base64 data: URLs without fetching them", async () => {
|
||||
// "OggS" — the bytes 0x4f 0x67 0x67 0x53
|
||||
const sounds = await prefetchSounds({
|
||||
beep: { ogg: "data:audio/ogg;base64,T2dnUw==" },
|
||||
});
|
||||
expect(fetchSpy).not.toHaveBeenCalled();
|
||||
expect(bytesOf(sounds.beep)).toEqual([0x4f, 0x67, 0x67, 0x53]);
|
||||
});
|
||||
|
||||
it("leaves out a sound with a percent-encoded data: URL", async () => {
|
||||
const sounds = await prefetchSounds({
|
||||
beep: { ogg: "data:audio/ogg,Ogg%53" },
|
||||
});
|
||||
expect(fetchSpy).not.toHaveBeenCalled();
|
||||
expect(sounds).toEqual({});
|
||||
});
|
||||
|
||||
it("leaves out a sound whose response is not okay", async () => {
|
||||
fetchSpy.mockResolvedValue(new Response(null, { status: 404 }));
|
||||
const sounds = await prefetchSounds({
|
||||
@@ -60,18 +43,16 @@ describe("prefetchSounds", () => {
|
||||
});
|
||||
|
||||
it("leaves out a sound that fails to load, keeping the others", async () => {
|
||||
fetchSpy.mockRejectedValue(new TypeError("Failed to fetch"));
|
||||
// Fetched in the order given
|
||||
fetchSpy
|
||||
.mockRejectedValueOnce(new TypeError("Failed to fetch"))
|
||||
.mockResolvedValueOnce(
|
||||
new Response(new Uint8Array([1, 2, 3]), { status: 200 }),
|
||||
);
|
||||
const sounds = await prefetchSounds({
|
||||
beep: { ogg: "https://example.org/beep.ogg" },
|
||||
boop: { ogg: "data:audio/ogg;base64,T2dnUw==" },
|
||||
boop: { ogg: "https://example.org/boop.ogg" },
|
||||
});
|
||||
expect(Object.keys(sounds)).toEqual(["boop"]);
|
||||
});
|
||||
|
||||
it("leaves out a sound with a malformed data: URL", async () => {
|
||||
const sounds = await prefetchSounds({
|
||||
beep: { ogg: "data:audio/ogg;base64" },
|
||||
});
|
||||
expect(sounds).toEqual({});
|
||||
});
|
||||
});
|
||||
|
||||
+1
-27
@@ -32,38 +32,12 @@ function getPreferredAudioFormat(): "ogg" | "mp3" {
|
||||
|
||||
const preferredFormat = getPreferredAudioFormat();
|
||||
|
||||
/**
|
||||
* Decode a `data:` URL into the bytes it carries.
|
||||
*
|
||||
* The component build inlines the sound files as `data:` URLs. Those could be
|
||||
* `fetch`ed like any other URL, but that fetch is subject to the host page's
|
||||
* Content Security Policy, and `connect-src` lists rarely allow `data:` (a `*`
|
||||
* source does not cover it). Reading the bytes out of the URL directly needs no
|
||||
* network access, so no policy can stop it.
|
||||
* @param url The `data:` URL to decode.
|
||||
* @returns The decoded bytes.
|
||||
*/
|
||||
function decodeDataUrl(url: string): ArrayBuffer {
|
||||
const separator = url.indexOf(",");
|
||||
if (!url.startsWith("data:") || separator === -1)
|
||||
throw new Error("Not a valid data: URL");
|
||||
const header = url.slice("data:".length, separator);
|
||||
// Vite only percent-encodes SVGs; every other inlined asset is base64.
|
||||
if (!header.split(";").includes("base64"))
|
||||
throw new Error("Only base64 data: URLs are supported");
|
||||
// `decoded` is a binary string: one char per byte
|
||||
const decoded = atob(url.slice(separator + 1));
|
||||
const bytes = Uint8Array.from(decoded, (c) => c.charCodeAt(0));
|
||||
return bytes.buffer;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load the bytes of a single sound file.
|
||||
* @param url Where the sound is: an ordinary URL to fetch, or a `data:` URL.
|
||||
* @param url Where the sound is.
|
||||
* @returns The bytes, or null if the sound could not be loaded.
|
||||
*/
|
||||
async function loadSound(url: string): Promise<ArrayBuffer | null> {
|
||||
if (url.startsWith("data:")) return decodeDataUrl(url);
|
||||
const response = await fetch(url);
|
||||
if (isFailure(response)) return null;
|
||||
return await response.arrayBuffer();
|
||||
|
||||
@@ -7,7 +7,11 @@ Please see LICENSE in the repository root for full details.
|
||||
|
||||
import { it, vi, expect, beforeEach, afterEach, describe } from "vitest";
|
||||
import { firstValueFrom, of, Subject, take, toArray } from "rxjs";
|
||||
import { type RTCCallIntent } from "matrix-js-sdk/lib/matrixrtc";
|
||||
import {
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
RTC_CALL_INTENT_VIDEO,
|
||||
type RTCCallIntent,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
|
||||
import { AndroidControlledAudioOutput } from "./AndroidControlledAudioOutput.ts";
|
||||
import type { Controls, OutputDevice } from "../controls";
|
||||
@@ -117,7 +121,7 @@ describe("Default selection", () => {
|
||||
const controlledAudioOutput = new AndroidControlledAudioOutput(
|
||||
of(BASE_DEVICE_LIST),
|
||||
testScope,
|
||||
"video",
|
||||
RTC_CALL_INTENT_VIDEO,
|
||||
mockControls,
|
||||
);
|
||||
|
||||
@@ -142,7 +146,7 @@ describe("Default selection", () => {
|
||||
const controlledAudioOutput = new AndroidControlledAudioOutput(
|
||||
of(BASE_DEVICE_LIST),
|
||||
testScope,
|
||||
"audio",
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
mockControls,
|
||||
);
|
||||
|
||||
@@ -163,7 +167,7 @@ describe("Default selection", () => {
|
||||
});
|
||||
});
|
||||
|
||||
["audio", "video"].forEach((callIntent) => {
|
||||
[RTC_CALL_INTENT_AUDIO, RTC_CALL_INTENT_VIDEO].forEach((callIntent) => {
|
||||
it(`Default to BT headset for ${callIntent} calls if present`, async () => {
|
||||
const controlledAudioOutput = new AndroidControlledAudioOutput(
|
||||
of(BT_HEADSET_BASE_DEVICE_LIST),
|
||||
@@ -190,7 +194,7 @@ describe("Default selection", () => {
|
||||
});
|
||||
});
|
||||
|
||||
["audio", "video"].forEach((callIntent) => {
|
||||
[RTC_CALL_INTENT_AUDIO, RTC_CALL_INTENT_VIDEO].forEach((callIntent) => {
|
||||
it(`Default to wired headset for ${callIntent} calls if present`, async () => {
|
||||
const controlledAudioOutput = new AndroidControlledAudioOutput(
|
||||
of(WIRED_HEADSET_BASE_DEVICE_LIST),
|
||||
@@ -300,7 +304,7 @@ describe("Test select a device", () => {
|
||||
b: BT_HEADSET_BASE_DEVICE_LIST,
|
||||
}),
|
||||
testScope,
|
||||
"audio",
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
mockControls,
|
||||
);
|
||||
|
||||
@@ -340,7 +344,7 @@ describe("Test select a device", () => {
|
||||
c: BASE_DEVICE_LIST,
|
||||
}),
|
||||
testScope,
|
||||
"audio",
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
mockControls,
|
||||
);
|
||||
|
||||
@@ -393,7 +397,7 @@ describe("Available device changes", () => {
|
||||
});
|
||||
|
||||
it("When a BT headset is added, control should switch to use it", () => {
|
||||
createAudioControlledOutput("video");
|
||||
createAudioControlledOutput(RTC_CALL_INTENT_VIDEO);
|
||||
|
||||
// Emit the base device list, the speaker should be selected
|
||||
availableSource$.next(BASE_DEVICE_LIST);
|
||||
@@ -419,7 +423,9 @@ describe("Available device changes", () => {
|
||||
|
||||
// Android does not set `isExternalHeadset` to true for wired headphones, so we can't test this case.'
|
||||
it.skip("When a wired headset is added, control should switch to use it", async () => {
|
||||
const controlledAudioOutput = createAudioControlledOutput("video");
|
||||
const controlledAudioOutput = createAudioControlledOutput(
|
||||
RTC_CALL_INTENT_VIDEO,
|
||||
);
|
||||
|
||||
// Emit the base device list, the speaker should be selected
|
||||
availableSource$.next(BASE_DEVICE_LIST);
|
||||
@@ -446,7 +452,7 @@ describe("Available device changes", () => {
|
||||
});
|
||||
|
||||
it("When the active bt headset is removed on audio call, control should switch to earpiece", () => {
|
||||
createAudioControlledOutput("audio");
|
||||
createAudioControlledOutput(RTC_CALL_INTENT_AUDIO);
|
||||
|
||||
// Emit the BT headset device list, the BT headset should be selected
|
||||
availableSource$.next(BT_HEADSET_BASE_DEVICE_LIST);
|
||||
@@ -472,7 +478,7 @@ describe("Available device changes", () => {
|
||||
});
|
||||
|
||||
it("When the active bt headset is removed on video call, control should switch to speaker", () => {
|
||||
createAudioControlledOutput("video");
|
||||
createAudioControlledOutput(RTC_CALL_INTENT_VIDEO);
|
||||
|
||||
availableSource$.next(BT_HEADSET_BASE_DEVICE_LIST);
|
||||
|
||||
@@ -497,7 +503,7 @@ describe("Available device changes", () => {
|
||||
});
|
||||
|
||||
it("Do not repeatidly set the same device", () => {
|
||||
createAudioControlledOutput("video");
|
||||
createAudioControlledOutput(RTC_CALL_INTENT_VIDEO);
|
||||
|
||||
availableSource$.next(BT_HEADSET_BASE_DEVICE_LIST);
|
||||
availableSource$.next(BT_HEADSET_BASE_DEVICE_LIST);
|
||||
|
||||
@@ -22,7 +22,10 @@ import {
|
||||
type SelectedAudioOutputDevice,
|
||||
} from "./MediaDevices.ts";
|
||||
import type { ObservableScope } from "./ObservableScope.ts";
|
||||
import type { RTCCallIntent } from "matrix-js-sdk/lib/matrixrtc";
|
||||
import {
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
type RTCCallIntent,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
import { type Controls, type OutputDevice } from "../controls.ts";
|
||||
import { type Behavior } from "./Behavior.ts";
|
||||
|
||||
@@ -102,7 +105,7 @@ export class AndroidControlledAudioOutput implements MediaDevice<
|
||||
* @constructor
|
||||
* @param controlledDevices$ - The list of available output devices coming from the hosting application, ordered by preference order (most preferred first).
|
||||
* @param scope - The ObservableScope to create the Behaviors in.
|
||||
* @param initialIntent - The initial call intent (e.g. "audio" or "video") that can be used to determine the default audio routing (e.g. default to earpiece for audio calls and speaker for video calls).
|
||||
* @param initialIntent - The initial call intent (e.g. "m.audio" or "m.video") that can be used to determine the default audio routing (e.g. default to earpiece for audio calls and speaker for video calls).
|
||||
* @param controls - The controls provided by the hosting application to control the audio routing and notify of user actions.
|
||||
*/
|
||||
public constructor(
|
||||
@@ -316,7 +319,7 @@ export class AndroidControlledAudioOutput implements MediaDevice<
|
||||
this.logger.debug(
|
||||
`Android routing logic intent: ${this.initialIntent} finding best default...`,
|
||||
);
|
||||
if (this.initialIntent === "audio") {
|
||||
if (this.initialIntent === RTC_CALL_INTENT_AUDIO) {
|
||||
const systemProposed = available[0];
|
||||
// If no headset is connected, android will route to the speaker by default,
|
||||
// but for a voice call we want to route to the earpiece instead,
|
||||
|
||||
@@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { BehaviorSubject, distinctUntilChanged, map } from "rxjs";
|
||||
import { BehaviorSubject, distinctUntilChanged, map, of } from "rxjs";
|
||||
import { type BackgroundProcessorWrapper } from "@livekit/track-processors";
|
||||
|
||||
import {
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
type BackgroundEffectsOptions,
|
||||
type BackgroundEffectsState,
|
||||
} from "./BackgroundEffects";
|
||||
import { type AddedBackground } from "../tracks/backgroundImages";
|
||||
import { shippedBackgrounds } from "../tracks/backgroundEffectCatalog";
|
||||
import { flushPromises, testScope, withTestScheduler } from "../utils/test";
|
||||
|
||||
@@ -46,25 +47,41 @@ function fakePipeline(): {
|
||||
};
|
||||
}
|
||||
|
||||
/** One letter per state: idle, attached. */
|
||||
/** One letter per state: idle, preparing, waiting for a frame, attached. */
|
||||
function letter(state: BackgroundEffectsState): string {
|
||||
return state.processor === undefined ? "i" : "a";
|
||||
if (state.supported === false) return "x";
|
||||
if (state.wait === "preparing") return "p";
|
||||
if (state.processor === undefined) return "i";
|
||||
return state.wait === "settling" ? "w" : "a";
|
||||
}
|
||||
|
||||
describe("the pipeline's state", () => {
|
||||
function testState({
|
||||
effect,
|
||||
answer = "-y",
|
||||
firstFrame = "",
|
||||
expected,
|
||||
}: {
|
||||
effect: string;
|
||||
/** When the trial build answers, from being asked. */
|
||||
answer?: string;
|
||||
firstFrame?: string;
|
||||
expected: string;
|
||||
}): void {
|
||||
withTestScheduler(({ behavior, expectObservable }) => {
|
||||
withTestScheduler(({ behavior, cold, schedule, expectObservable }) => {
|
||||
const transformer = {
|
||||
onFirstFrame: undefined as (() => void) | undefined,
|
||||
};
|
||||
const effects = createBackgroundEffects(testScope(), {
|
||||
supported: true,
|
||||
effect$: behavior(effect, { n: "none", b: "blur" }),
|
||||
setEffect: vi.fn(),
|
||||
added$: new BehaviorSubject<AddedBackground[] | undefined>([]),
|
||||
canSegment: () => cold(answer, { y: true, n: false }),
|
||||
pipeline: fakePipeline().pipeline,
|
||||
transformer,
|
||||
});
|
||||
schedule(firstFrame, { f: () => transformer.onFirstFrame?.() });
|
||||
expectObservable(
|
||||
effects.state$.pipe(map(letter), distinctUntilChanged()),
|
||||
).toBe(expected);
|
||||
@@ -73,12 +90,27 @@ describe("the pipeline's state", () => {
|
||||
|
||||
it("defaults to no effect", () => testState({ effect: "n", expected: "i" }));
|
||||
|
||||
it("attaches only once it is known the pipeline builds", () =>
|
||||
testState({ effect: "nb", answer: "--y", expected: "ip-w" }));
|
||||
|
||||
it("offers none where the pipeline fails to build", () =>
|
||||
testState({ effect: "nb", answer: "-n", expected: "ipx" }));
|
||||
|
||||
it("waits from the first effect until a frame carries it", () =>
|
||||
testState({
|
||||
effect: " nb---n-b",
|
||||
firstFrame: "---f",
|
||||
expected: " ipwa",
|
||||
}));
|
||||
|
||||
it("attaches on first use and stays attached", () =>
|
||||
testState({ effect: "nbn", expected: "ia" }));
|
||||
testState({ effect: "nb--n", firstFrame: "---f", expected: "ipwa" }));
|
||||
});
|
||||
|
||||
describe("background effects", () => {
|
||||
let effect$: BehaviorSubject<string>;
|
||||
let added$: BehaviorSubject<AddedBackground[] | undefined>;
|
||||
let setEffect: (raw: string) => void;
|
||||
let fake: ReturnType<typeof fakePipeline>;
|
||||
|
||||
function build(
|
||||
@@ -87,7 +119,11 @@ describe("background effects", () => {
|
||||
return createBackgroundEffects(testScope(), {
|
||||
supported: true,
|
||||
effect$,
|
||||
setEffect,
|
||||
added$,
|
||||
canSegment: () => of(true),
|
||||
pipeline: fake.pipeline,
|
||||
transformer: { onFirstFrame: undefined },
|
||||
...options,
|
||||
});
|
||||
}
|
||||
@@ -95,14 +131,25 @@ describe("background effects", () => {
|
||||
effect$.next(raw);
|
||||
await flushPromises();
|
||||
};
|
||||
const blur = async (on: boolean): Promise<void> =>
|
||||
choose(on ? "blur" : "none");
|
||||
|
||||
beforeEach(() => {
|
||||
effect$ = new BehaviorSubject("none");
|
||||
added$ = new BehaviorSubject<AddedBackground[] | undefined>(undefined);
|
||||
setEffect = vi.fn((raw: string) => effect$.next(raw));
|
||||
fake = fakePipeline();
|
||||
});
|
||||
|
||||
it("offers none for the session where the pipeline fails to build", async () => {
|
||||
const effects = build({ canSegment: () => of(false) });
|
||||
await choose("blur");
|
||||
|
||||
expect(effects.state$.value.supported).toBe(false);
|
||||
expect(effects.state$.value.processor).toBeUndefined();
|
||||
expect(effects.state$.value.wait).toBeUndefined();
|
||||
// A browser update may fix it, so the next session asks again.
|
||||
expect(effect$.value).toBe("blur");
|
||||
});
|
||||
|
||||
it("puts a shipped background on as that picture", async () => {
|
||||
build();
|
||||
await choose(`image:${shippedBackgrounds[0].id}`);
|
||||
@@ -116,10 +163,10 @@ describe("background effects", () => {
|
||||
|
||||
it("switches in place rather than reattaching", async () => {
|
||||
const effects = build();
|
||||
await blur(true);
|
||||
await choose("blur");
|
||||
const pipeline = effects.state$.value.processor;
|
||||
await blur(false);
|
||||
await blur(true);
|
||||
await choose("none");
|
||||
await choose("blur");
|
||||
|
||||
expect(effects.state$.value.processor).toBe(pipeline);
|
||||
expect(fake.switches).toEqual([
|
||||
@@ -132,18 +179,18 @@ describe("background effects", () => {
|
||||
it("keeps switching after a switch fails", async () => {
|
||||
vi.mocked(fake.pipeline.switchTo).mockRejectedValueOnce(new Error("lost"));
|
||||
build();
|
||||
await blur(true);
|
||||
await blur(false);
|
||||
await choose("blur");
|
||||
await choose("none");
|
||||
expect(fake.switches).toEqual([{ mode: "disabled" }]);
|
||||
});
|
||||
|
||||
it("switches one at a time, skipping those overtaken", async () => {
|
||||
const finished = fake.holdNext();
|
||||
build();
|
||||
await blur(true);
|
||||
await blur(false);
|
||||
await blur(true);
|
||||
await blur(false);
|
||||
await choose("blur");
|
||||
await choose("none");
|
||||
await choose("blur");
|
||||
await choose("none");
|
||||
expect(fake.switches).toHaveLength(1);
|
||||
|
||||
finished();
|
||||
@@ -153,4 +200,63 @@ describe("background effects", () => {
|
||||
{ mode: "disabled" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("clears a remembered effect when it cannot be honoured", () => {
|
||||
effect$.next(`image:${shippedBackgrounds[0].id}`);
|
||||
const effects = build({ supported: false });
|
||||
expect(effect$.value).toBe("none");
|
||||
expect(effects.state$.value.processor).toBeUndefined();
|
||||
});
|
||||
|
||||
it("puts an added background on as its picture", async () => {
|
||||
added$.next([{ id: "mine", url: "blob:mine" }]);
|
||||
build();
|
||||
await choose("added:mine");
|
||||
expect(fake.switches).toEqual([
|
||||
{ mode: "virtual-background", imagePath: "blob:mine" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("switches between any two effects in place", async () => {
|
||||
added$.next([{ id: "mine", url: "blob:mine" }]);
|
||||
const shipped = shippedBackgrounds[0];
|
||||
const options: Record<string, unknown> = {
|
||||
none: { mode: "disabled" },
|
||||
blur: { mode: "background-blur", blurRadius: 15 },
|
||||
[`image:${shipped.id}`]: {
|
||||
mode: "virtual-background",
|
||||
imagePath: shipped.imagePath,
|
||||
},
|
||||
"added:mine": { mode: "virtual-background", imagePath: "blob:mine" },
|
||||
};
|
||||
// Each kind to each other kind, and back.
|
||||
const order = [
|
||||
"blur",
|
||||
`image:${shipped.id}`,
|
||||
"added:mine",
|
||||
"blur",
|
||||
"added:mine",
|
||||
`image:${shipped.id}`,
|
||||
"none",
|
||||
"added:mine",
|
||||
"none",
|
||||
"blur",
|
||||
];
|
||||
const effects = build();
|
||||
await choose(order[0]);
|
||||
const pipeline = effects.state$.value.processor;
|
||||
for (const raw of order.slice(1)) await choose(raw);
|
||||
|
||||
expect(effects.state$.value.processor).toBe(pipeline);
|
||||
expect(fake.switches).toEqual(order.map((raw) => options[raw]));
|
||||
});
|
||||
|
||||
it("forgets an added background the device no longer keeps", () => {
|
||||
effect$.next("added:gone");
|
||||
build();
|
||||
// Not before the device has said what it keeps.
|
||||
expect(effect$.value).toBe("added:gone");
|
||||
added$.next([]);
|
||||
expect(effect$.value).toBe("none");
|
||||
});
|
||||
});
|
||||
|
||||
+116
-19
@@ -5,7 +5,19 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { combineLatest, distinctUntilChanged, map, scan } from "rxjs";
|
||||
import {
|
||||
combineLatest,
|
||||
distinctUntilChanged,
|
||||
first,
|
||||
from,
|
||||
map,
|
||||
Observable,
|
||||
type ObservableInput,
|
||||
of,
|
||||
scan,
|
||||
startWith,
|
||||
switchMap,
|
||||
} from "rxjs";
|
||||
import {
|
||||
type BackgroundProcessorWrapper,
|
||||
type SwitchBackgroundProcessorOptions,
|
||||
@@ -21,15 +33,23 @@ import { type Behavior } from "./Behavior";
|
||||
import { type ObservableScope } from "./ObservableScope";
|
||||
import {
|
||||
type BackgroundEffect,
|
||||
type EffectId,
|
||||
imagePathFor,
|
||||
parseEffect,
|
||||
} from "../tracks/backgroundEffectCatalog";
|
||||
import { type AddedBackground } from "../tracks/backgroundImages";
|
||||
|
||||
const blurRadius = 15;
|
||||
|
||||
export interface BackgroundEffectsState {
|
||||
supported: boolean | undefined;
|
||||
processor: undefined | VideoProcessor;
|
||||
/**
|
||||
* The effect is not ready:
|
||||
* preparing, asking whether the pipeline can be built.
|
||||
* settling, once it is attached until a frame carrying the effect
|
||||
*/
|
||||
wait?: "preparing" | "settling";
|
||||
}
|
||||
|
||||
export interface BackgroundEffectsOptions {
|
||||
@@ -37,8 +57,16 @@ export interface BackgroundEffectsOptions {
|
||||
supported: boolean;
|
||||
/** The effect chosen, as the setting stores it. */
|
||||
effect$: Behavior<string>;
|
||||
/** Stores a choice, to forget one that can't be honoured. */
|
||||
setEffect: (id: EffectId) => void;
|
||||
/** The backgrounds the device keeps, undefined until it has said. */
|
||||
added$: Behavior<AddedBackground[] | undefined>;
|
||||
/** Builds a segmenter on trial, and answers whether it could. */
|
||||
canSegment: () => ObservableInput<boolean>;
|
||||
/** The background processor pipeline to be switched between effects. */
|
||||
pipeline: BackgroundProcessorWrapper;
|
||||
/** Tells, once, that a frame carrying an effect has been drawn. */
|
||||
transformer: { onFirstFrame: (() => void) | undefined };
|
||||
}
|
||||
|
||||
/** The background effect pipeline, as the camera tracks and the menus see it. */
|
||||
@@ -49,25 +77,81 @@ export interface BackgroundEffects {
|
||||
/** Switches the pipeline as the choice changes, for as long as the scope lasts. */
|
||||
export function createBackgroundEffects(
|
||||
scope: ObservableScope,
|
||||
{ supported, effect$, pipeline }: BackgroundEffectsOptions,
|
||||
{
|
||||
supported,
|
||||
effect$,
|
||||
setEffect,
|
||||
added$,
|
||||
canSegment,
|
||||
pipeline,
|
||||
transformer,
|
||||
}: BackgroundEffectsOptions,
|
||||
): BackgroundEffects {
|
||||
// Wrapped once, so the call and the preview see one processor throughout.
|
||||
const processor = fromLivekitProcessor(pipeline);
|
||||
const choice$ = effect$.pipe(map(parseEffect));
|
||||
const wanted$ = choice$.pipe(
|
||||
map((effect) => effect.kind !== "none"),
|
||||
distinctUntilChanged(),
|
||||
const wanted$ = scope.behavior(
|
||||
choice$.pipe(
|
||||
map((effect) => effect.kind !== "none"),
|
||||
distinctUntilChanged(),
|
||||
),
|
||||
);
|
||||
|
||||
combineLatest([choice$, added$])
|
||||
.pipe(scope.bind())
|
||||
.subscribe(([effect, added]) => {
|
||||
if (effect.kind === "none") return;
|
||||
if (
|
||||
!supported ||
|
||||
(effect.kind === "added" &&
|
||||
added !== undefined &&
|
||||
!added.some(({ id }) => id === effect.id))
|
||||
)
|
||||
setEffect("none");
|
||||
});
|
||||
|
||||
// Asked once an effect is first wanted. A browser update may fix what
|
||||
// fails, so the choice is kept for the next session rather than cleared.
|
||||
const buildable$ = scope.behavior<boolean | undefined>(
|
||||
supported
|
||||
? wanted$.pipe(
|
||||
first(Boolean),
|
||||
switchMap(() => from(canSegment())),
|
||||
startWith(undefined),
|
||||
)
|
||||
: of(undefined),
|
||||
);
|
||||
|
||||
const drewAFrame$ = scope.behavior(
|
||||
new Observable<boolean>((subscriber) => {
|
||||
subscriber.next(false);
|
||||
transformer.onFirstFrame = (): void => subscriber.next(true);
|
||||
return (): void => {
|
||||
transformer.onFirstFrame = undefined;
|
||||
};
|
||||
}),
|
||||
);
|
||||
|
||||
// In one step, so that nothing sees the pipeline neither preparing nor
|
||||
// attached.
|
||||
const state$ = scope.behavior(
|
||||
wanted$.pipe(
|
||||
scan<boolean, BackgroundEffectsState>(
|
||||
(previous, wanted) => {
|
||||
combineLatest([wanted$, buildable$, drewAFrame$]).pipe(
|
||||
scan<[boolean, boolean | undefined, boolean], BackgroundEffectsState>(
|
||||
(previous, [wanted, buildable, drewAFrame]) => {
|
||||
// Attached the first time an effect is wanted and never detached
|
||||
// after, so someone who never turns one on pays for none of it.
|
||||
const enable =
|
||||
previous.processor !== undefined || (supported && wanted);
|
||||
return { supported, processor: enable ? processor : undefined };
|
||||
previous.processor !== undefined || (wanted && buildable === true);
|
||||
const preparing = supported && wanted && buildable === undefined;
|
||||
return {
|
||||
supported: supported && buildable !== false,
|
||||
processor: enable ? processor : undefined,
|
||||
wait: preparing
|
||||
? "preparing"
|
||||
: enable && !drewAFrame
|
||||
? "settling"
|
||||
: undefined,
|
||||
};
|
||||
},
|
||||
{ supported, processor: undefined },
|
||||
),
|
||||
@@ -77,9 +161,16 @@ export function createBackgroundEffects(
|
||||
const switchOptions$ = scope.behavior<
|
||||
SwitchBackgroundProcessorOptions | undefined
|
||||
>(
|
||||
combineLatest([state$, choice$]).pipe(
|
||||
map(([{ processor }, effect]) =>
|
||||
processor === undefined ? undefined : switchOptionsFor(effect),
|
||||
combineLatest([state$, choice$, added$]).pipe(
|
||||
map(([{ processor }, effect, added]) =>
|
||||
processor === undefined
|
||||
? undefined
|
||||
: switchOptionsFor(
|
||||
effect,
|
||||
effect.kind === "added"
|
||||
? added?.find(({ id }) => id === effect.id)?.url
|
||||
: undefined,
|
||||
),
|
||||
),
|
||||
distinctUntilChanged(deepCompare),
|
||||
),
|
||||
@@ -97,18 +188,24 @@ export function createBackgroundEffects(
|
||||
return { state$ };
|
||||
}
|
||||
|
||||
/** `addedUrl` is the picture of the added background chosen, if still kept. */
|
||||
function switchOptionsFor(
|
||||
effect: BackgroundEffect,
|
||||
addedUrl: string | undefined,
|
||||
): SwitchBackgroundProcessorOptions {
|
||||
const withImage = (
|
||||
imagePath: string | undefined,
|
||||
): SwitchBackgroundProcessorOptions =>
|
||||
imagePath
|
||||
? { mode: "virtual-background", imagePath }
|
||||
: { mode: "disabled" };
|
||||
switch (effect.kind) {
|
||||
case "blur":
|
||||
return { mode: "background-blur", blurRadius };
|
||||
case "shipped": {
|
||||
const imagePath = imagePathFor(effect.id);
|
||||
return imagePath
|
||||
? { mode: "virtual-background", imagePath }
|
||||
: { mode: "disabled" };
|
||||
}
|
||||
case "shipped":
|
||||
return withImage(imagePathFor(effect.id));
|
||||
case "added":
|
||||
return withImage(addedUrl);
|
||||
default:
|
||||
return { mode: "disabled" };
|
||||
}
|
||||
|
||||
@@ -12,7 +12,11 @@ import {
|
||||
MatrixEvent,
|
||||
type Room,
|
||||
} from "matrix-js-sdk";
|
||||
import { type RTCCallIntent } from "matrix-js-sdk/lib/matrixrtc";
|
||||
import {
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
RTC_CALL_INTENT_VIDEO,
|
||||
type RTCCallIntent,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
import { map, mergeMap, NEVER, type Observable, startWith } from "rxjs";
|
||||
|
||||
import { withTestScheduler } from "../../utils/test";
|
||||
@@ -33,10 +37,12 @@ function mockRingEvent(
|
||||
eventId: string,
|
||||
lifetimeMs: number | undefined,
|
||||
sender = local.userId,
|
||||
legacyIntent?: string,
|
||||
): CallNotificationWrapper {
|
||||
return {
|
||||
event_id: eventId,
|
||||
...(lifetimeMs === undefined ? {} : { lifetime: lifetimeMs }),
|
||||
...(legacyIntent === undefined ? {} : { "m.call.intent": legacyIntent }),
|
||||
notification_type: "ring",
|
||||
sender,
|
||||
} as unknown as CallNotificationWrapper;
|
||||
@@ -128,7 +134,25 @@ test("ring attempt times out after nobody joins", () => {
|
||||
});
|
||||
|
||||
expectObservable(summarizeRingAttempts$(ringAttempts$)).toBe("-a 29ms A", {
|
||||
a: { intent: "audio", recipient: alice.userId },
|
||||
a: { intent: RTC_CALL_INTENT_AUDIO, recipient: alice.userId },
|
||||
A: { outcome: "timeout" },
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
test("ring attempt maps the legacy intent on the notification to its stable value", () => {
|
||||
withTestScheduler(({ scope, expectObservable, hot }) => {
|
||||
const { ringAttempts$ } = createCallNotificationLifecycle$({
|
||||
scope,
|
||||
...defaultProps,
|
||||
memberUserIds$: constant<string[]>([]),
|
||||
sentCallNotification$: hot("-a", {
|
||||
a: mockRingEvent("$notif1", 30, local.userId, "video"),
|
||||
}),
|
||||
});
|
||||
|
||||
expectObservable(summarizeRingAttempts$(ringAttempts$)).toBe("-a 29ms A", {
|
||||
a: { intent: RTC_CALL_INTENT_VIDEO, recipient: alice.userId },
|
||||
A: { outcome: "timeout" },
|
||||
});
|
||||
});
|
||||
@@ -148,7 +172,7 @@ test("ring attempt is accepted once recipient joins", () => {
|
||||
});
|
||||
|
||||
expectObservable(summarizeRingAttempts$(ringAttempts$)).toBe("-aA", {
|
||||
a: { intent: "audio", recipient: alice.userId },
|
||||
a: { intent: RTC_CALL_INTENT_AUDIO, recipient: alice.userId },
|
||||
A: { outcome: "accept" },
|
||||
});
|
||||
});
|
||||
@@ -166,7 +190,7 @@ test("ring attempt is immediately accepted if recipient is already joined", () =
|
||||
});
|
||||
|
||||
expectObservable(summarizeRingAttempts$(ringAttempts$)).toBe("-(aA)", {
|
||||
a: { intent: "audio", recipient: alice.userId },
|
||||
a: { intent: RTC_CALL_INTENT_AUDIO, recipient: alice.userId },
|
||||
A: { outcome: "accept" },
|
||||
});
|
||||
});
|
||||
@@ -201,7 +225,7 @@ test("ring attempt can be declined", () => {
|
||||
});
|
||||
|
||||
expectObservable(summarizeRingAttempts$(ringAttempts$)).toBe("-aA", {
|
||||
a: { intent: "audio", recipient: alice.userId },
|
||||
a: { intent: RTC_CALL_INTENT_AUDIO, recipient: alice.userId },
|
||||
A: { outcome: "decline" },
|
||||
});
|
||||
});
|
||||
@@ -236,7 +260,7 @@ test("ring attempt times out if recipient declines too late", () => {
|
||||
});
|
||||
|
||||
expectObservable(summarizeRingAttempts$(ringAttempts$)).toBe("-a 29ms A", {
|
||||
a: { intent: "audio", recipient: alice.userId },
|
||||
a: { intent: RTC_CALL_INTENT_AUDIO, recipient: alice.userId },
|
||||
A: { outcome: "timeout" },
|
||||
});
|
||||
});
|
||||
@@ -271,7 +295,7 @@ test("decline event relating to wrong event is ignored (times out)", () => {
|
||||
});
|
||||
|
||||
expectObservable(summarizeRingAttempts$(ringAttempts$)).toBe("-a 29ms A", {
|
||||
a: { intent: "audio", recipient: alice.userId },
|
||||
a: { intent: RTC_CALL_INTENT_AUDIO, recipient: alice.userId },
|
||||
A: { outcome: "timeout" },
|
||||
});
|
||||
});
|
||||
@@ -306,7 +330,7 @@ test("decline event from wrong sender is ignored (times out)", () => {
|
||||
});
|
||||
|
||||
expectObservable(summarizeRingAttempts$(ringAttempts$)).toBe("-a 29ms A", {
|
||||
a: { intent: "audio", recipient: alice.userId },
|
||||
a: { intent: RTC_CALL_INTENT_AUDIO, recipient: alice.userId },
|
||||
A: { outcome: "timeout" },
|
||||
});
|
||||
});
|
||||
|
||||
@@ -9,7 +9,9 @@ import {
|
||||
type IRTCNotificationContent,
|
||||
type MatrixRTCSession,
|
||||
MatrixRTCSessionEvent,
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
type RTCCallIntent,
|
||||
toStableCallIntent,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
import {
|
||||
filter,
|
||||
@@ -155,7 +157,10 @@ export function createCallNotificationLifecycle$({
|
||||
);
|
||||
|
||||
return of({
|
||||
intent: notificationEvent["m.call.intent"] ?? "audio",
|
||||
// The notification still carries the legacy intent values
|
||||
intent:
|
||||
toStableCallIntent(notificationEvent["m.call.intent"]) ??
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
recipient,
|
||||
outcome$: race(timeout$, accept$, decline$).pipe(
|
||||
take(1),
|
||||
|
||||
@@ -21,6 +21,7 @@ import {
|
||||
type CallMembership,
|
||||
type IRTCNotificationContent,
|
||||
MatrixRTCSessionEvent,
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
import { deepCompare } from "matrix-js-sdk/lib/utils";
|
||||
|
||||
@@ -70,9 +71,11 @@ import {
|
||||
} from "./CallViewModel.ts";
|
||||
import { type Handler, MuteState } from "../MuteStates.ts";
|
||||
import {
|
||||
type LocalVideoMediaTrack,
|
||||
type RTCParticipation,
|
||||
type VideoProcessor,
|
||||
} from "@element-hq/matrixrtc-sdk";
|
||||
import { type BackgroundEffectsState } from "../BackgroundEffects";
|
||||
|
||||
initializeWidget();
|
||||
|
||||
@@ -1557,7 +1560,7 @@ describe.each(modes)("CallViewModel (%s mode)", (mode) => {
|
||||
b: expect.objectContaining({
|
||||
type: "ringing",
|
||||
userId: alice.userId,
|
||||
intent: "audio",
|
||||
intent: RTC_CALL_INTENT_AUDIO,
|
||||
}),
|
||||
});
|
||||
// Layout should show placeholder media for the participant we're
|
||||
@@ -1603,7 +1606,7 @@ describe.each(modes)("CallViewModel (%s mode)", (mode) => {
|
||||
b: expect.objectContaining({
|
||||
type: "ringing",
|
||||
userId: alice.userId,
|
||||
intent: "audio",
|
||||
intent: RTC_CALL_INTENT_AUDIO,
|
||||
}),
|
||||
});
|
||||
// Layout should show placeholder media for the participant we're
|
||||
@@ -1692,28 +1695,113 @@ describe("a camera with a video processor", () => {
|
||||
muteStates,
|
||||
mockMediaDevices({}),
|
||||
{},
|
||||
processor,
|
||||
{ supported: true, processor },
|
||||
).find((request) => request.source === "camera"),
|
||||
).toMatchObject({ processor });
|
||||
});
|
||||
|
||||
it("leaves the camera out of the join while the processor is prepared", () => {
|
||||
const muteStates = mockMuteStates();
|
||||
vi.spyOn(muteStates.video.enabled$, "value", "get").mockReturnValue(true);
|
||||
const sources = (wait?: "preparing"): string[] =>
|
||||
initialPublishRequests(
|
||||
muteStates,
|
||||
mockMediaDevices({}),
|
||||
{},
|
||||
{
|
||||
supported: true,
|
||||
processor: undefined,
|
||||
wait,
|
||||
},
|
||||
).map((request) => request.source);
|
||||
expect(sources("preparing")).not.toContain("camera");
|
||||
expect(sources()).toContain("camera");
|
||||
});
|
||||
|
||||
it("waits to make a camera track until it is known the processor builds", async () => {
|
||||
const setHandler = vi.spyOn(MuteState.prototype, "setHandler");
|
||||
onTestFinished(() => setHandler.mockRestore());
|
||||
const effects$ = new BehaviorSubject<BackgroundEffectsState>({
|
||||
supported: true,
|
||||
processor: undefined,
|
||||
wait: "preparing",
|
||||
});
|
||||
let participation: RTCParticipation | undefined;
|
||||
withCallViewModel(
|
||||
{},
|
||||
(_vm, _session, { rtcParticipation }) => {
|
||||
participation = rtcParticipation;
|
||||
},
|
||||
{ backgroundEffectsState$: effects$ },
|
||||
);
|
||||
const camera = participation!.localMember$.value?.tracks$.value?.find(
|
||||
(track) => track.source === "camera",
|
||||
) as LocalVideoMediaTrack;
|
||||
const turnedOn = cameraHandler(setHandler)(true);
|
||||
await flushPromises();
|
||||
expect(camera.setEnabled).not.toHaveBeenCalledWith(true);
|
||||
|
||||
effects$.next({ supported: true, processor });
|
||||
await turnedOn;
|
||||
expect(camera.setEnabled).toHaveBeenCalledWith(true);
|
||||
expect(participation!.publish).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("turns the camera on only once its processor is on the track", async () => {
|
||||
const setHandler = vi.spyOn(MuteState.prototype, "setHandler");
|
||||
onTestFinished(() => setHandler.mockRestore());
|
||||
const effects$ = new BehaviorSubject<BackgroundEffectsState>({
|
||||
supported: true,
|
||||
processor: undefined,
|
||||
wait: "preparing",
|
||||
});
|
||||
let participation: RTCParticipation | undefined;
|
||||
withCallViewModel(
|
||||
{},
|
||||
(_vm, _session, { rtcParticipation }) => {
|
||||
participation = rtcParticipation;
|
||||
},
|
||||
{ backgroundEffectsState$: effects$ },
|
||||
);
|
||||
const camera = participation!.localMember$.value?.tracks$.value?.find(
|
||||
(track) => track.source === "camera",
|
||||
) as LocalVideoMediaTrack;
|
||||
// Attaching takes its time, as the pipeline is built on the track
|
||||
let attached!: () => void;
|
||||
vi.mocked(camera.setProcessor).mockReturnValueOnce(
|
||||
new Promise<void>((resolve) => {
|
||||
attached = resolve;
|
||||
}),
|
||||
);
|
||||
const turnedOn = cameraHandler(setHandler)(true);
|
||||
effects$.next({ supported: true, processor });
|
||||
await flushPromises();
|
||||
expect(camera.setProcessor).toHaveBeenCalledWith(processor);
|
||||
expect(camera.setEnabled).not.toHaveBeenCalledWith(true);
|
||||
|
||||
attached();
|
||||
await turnedOn;
|
||||
expect(camera.setEnabled).toHaveBeenCalledWith(true);
|
||||
});
|
||||
|
||||
// The SDK builds the processor on the camera's restart, before the room
|
||||
// sees a frame, so the track is unmuted rather than published anew
|
||||
it("turns on a camera turned off before the processor was chosen", async () => {
|
||||
const setHandler = vi.spyOn(MuteState.prototype, "setHandler");
|
||||
onTestFinished(() => setHandler.mockRestore());
|
||||
const videoProcessor$ = new BehaviorSubject<VideoProcessor | undefined>(
|
||||
undefined,
|
||||
);
|
||||
const effects$ = new BehaviorSubject<BackgroundEffectsState>({
|
||||
supported: true,
|
||||
processor: undefined,
|
||||
});
|
||||
let participation: RTCParticipation | undefined;
|
||||
withCallViewModel(
|
||||
{},
|
||||
(_vm, _session, { rtcParticipation }) => {
|
||||
participation = rtcParticipation;
|
||||
},
|
||||
{ videoProcessor$ },
|
||||
{ backgroundEffectsState$: effects$ },
|
||||
);
|
||||
videoProcessor$.next(processor);
|
||||
effects$.next({ supported: true, processor });
|
||||
await cameraHandler(setHandler)(true);
|
||||
await flushPromises();
|
||||
|
||||
|
||||
@@ -37,6 +37,7 @@ import {
|
||||
combineLatest,
|
||||
distinctUntilChanged,
|
||||
filter,
|
||||
firstValueFrom,
|
||||
fromEvent,
|
||||
map,
|
||||
merge,
|
||||
@@ -77,6 +78,7 @@ import {
|
||||
type ReactionInfo,
|
||||
type ReactionOption,
|
||||
} from "../../reactions";
|
||||
import { type BackgroundEffectsState } from "../BackgroundEffects";
|
||||
import { shallowEquals as arrayShallowEquals } from "../../utils/array";
|
||||
import { type MediaDevices } from "../MediaDevices";
|
||||
import { type MuteStates } from "../MuteStates";
|
||||
@@ -170,13 +172,16 @@ export function initialPublishRequests(
|
||||
muteStates: MuteStates,
|
||||
mediaDevices: MediaDevices,
|
||||
capture: CaptureSettings,
|
||||
processor?: VideoProcessor,
|
||||
effects?: BackgroundEffectsState,
|
||||
): PublishRequest[] {
|
||||
const requests: PublishRequest[] = [];
|
||||
if (muteStates.audio.enabled$.value)
|
||||
requests.push(publishRequest("microphone", mediaDevices, capture));
|
||||
if (muteStates.video.enabled$.value)
|
||||
requests.push(publishRequest("camera", mediaDevices, capture, processor));
|
||||
// Left out while the processor is prepared; the call publishes it after
|
||||
if (muteStates.video.enabled$.value && effects?.wait !== "preparing")
|
||||
requests.push(
|
||||
publishRequest("camera", mediaDevices, capture, effects?.processor),
|
||||
);
|
||||
return requests;
|
||||
}
|
||||
|
||||
@@ -187,8 +192,8 @@ export interface CallViewModelOptions {
|
||||
* defaults otherwise.
|
||||
*/
|
||||
capture?: CaptureSettings;
|
||||
/** Background blur and the like, applied to the camera as it changes. */
|
||||
videoProcessor$?: Behavior<VideoProcessor | undefined>;
|
||||
/** The background effects state (preparing, supported...) including the processor itself. */
|
||||
backgroundEffectsState$?: Behavior<BackgroundEffectsState>;
|
||||
/**
|
||||
* The application hosting Element Call, which can ask it to hang up and wants
|
||||
* to know when the user joins or leaves. Defaults to no host.
|
||||
@@ -363,6 +368,8 @@ export interface CallViewModel {
|
||||
keyRotationSuppressed$: Behavior<boolean>;
|
||||
/** The local member, once our membership has been seen in the room. */
|
||||
localMember$: Behavior<LocalRTCMember | null>;
|
||||
/** Our own camera track, while one is published. */
|
||||
camera$: Behavior<LocalVideoMediaTrack | undefined>;
|
||||
remoteMembers$: Behavior<RemoteRTCMember[]>;
|
||||
/** The transports the client holds a connection to, for the developer panel. */
|
||||
connectedTransports$: Behavior<TransportMetadata[]>;
|
||||
@@ -488,7 +495,21 @@ export function createCallViewModel$(
|
||||
sentCallNotification$ = of(null),
|
||||
} = options;
|
||||
|
||||
const { capture = {}, videoProcessor$ = constant(undefined) } = options;
|
||||
const {
|
||||
capture = {},
|
||||
backgroundEffectsState$ = constant<BackgroundEffectsState>({
|
||||
supported: false,
|
||||
processor: undefined,
|
||||
}),
|
||||
} = options;
|
||||
const videoProcessor$ = scope.behavior(
|
||||
backgroundEffectsState$.pipe(map(({ processor }) => processor)),
|
||||
);
|
||||
// Resolves once it is known whether the processor builds
|
||||
const processorKnown = async (): Promise<unknown> =>
|
||||
firstValueFrom(
|
||||
backgroundEffectsState$.pipe(filter(({ wait }) => wait !== "preparing")),
|
||||
);
|
||||
const localTracks$ = scope.behavior<
|
||||
(LocalAudioMediaTrack | LocalVideoMediaTrack)[]
|
||||
>(
|
||||
@@ -512,6 +533,19 @@ export function createCallViewModel$(
|
||||
"screenShare",
|
||||
) as Behavior<LocalVideoMediaTrack | undefined>;
|
||||
|
||||
const processorAttached = followProcessor(
|
||||
scope,
|
||||
camera$,
|
||||
videoProcessor$,
|
||||
logger,
|
||||
);
|
||||
|
||||
// A camera turned on waits for its processor to be known, then to be on
|
||||
// the track: LiveKit restarts the track with whatever processor it has then.
|
||||
const cameraReady = async (): Promise<void> => {
|
||||
await processorKnown();
|
||||
await processorAttached();
|
||||
};
|
||||
const supply = (source: "microphone" | "camera"): TrackSupply => ({
|
||||
publish: async () =>
|
||||
rtcParticipation.publish(
|
||||
@@ -522,6 +556,7 @@ export function createCallViewModel$(
|
||||
source === "camera" ? videoProcessor$.value : undefined,
|
||||
),
|
||||
),
|
||||
ready: source === "camera" ? cameraReady : undefined,
|
||||
});
|
||||
followMuteState(
|
||||
scope,
|
||||
@@ -548,6 +583,29 @@ export function createCallViewModel$(
|
||||
);
|
||||
followDevice(scope, camera$, mediaDevices.videoInput, "camera", logger);
|
||||
|
||||
// A camera wanted at the join was left out of it while the processor was
|
||||
// prepared (see initialPublishRequests), and is published once that is known
|
||||
if (
|
||||
backgroundEffectsState$.value.wait === "preparing" &&
|
||||
muteStates.video.enabled$.value
|
||||
)
|
||||
processorKnown()
|
||||
.then(async () => {
|
||||
if (camera$.value !== undefined || !muteStates.video.enabled$.value)
|
||||
return;
|
||||
await rtcParticipation.publish(
|
||||
publishRequest(
|
||||
"camera",
|
||||
mediaDevices,
|
||||
capture,
|
||||
videoProcessor$.value,
|
||||
),
|
||||
);
|
||||
})
|
||||
.catch((e) => {
|
||||
logger.error("Failed to publish the camera", e);
|
||||
});
|
||||
|
||||
// The output device follows the selection onto the client the same way. A
|
||||
// host that routes audio itself picks the output outside the browser, so the
|
||||
// client is left alone
|
||||
@@ -561,8 +619,6 @@ export function createCallViewModel$(
|
||||
});
|
||||
});
|
||||
|
||||
followProcessor(scope, camera$, videoProcessor$, logger);
|
||||
|
||||
const members$ = scope.behavior<(LocalRTCMember | RemoteRTCMember)[]>(
|
||||
combineLatest(
|
||||
[rtcParticipation.localMember$, rtcParticipation.remoteMembers$],
|
||||
@@ -1508,6 +1564,7 @@ export function createCallViewModel$(
|
||||
leave,
|
||||
toggleScreenSharing: toggleScreenSharing,
|
||||
sharingScreen$: sharingScreen$,
|
||||
camera$,
|
||||
|
||||
tapScreen: (): void => screenTap$.next(),
|
||||
tapControls: (): void => controlsTap$.next(),
|
||||
|
||||
@@ -21,6 +21,10 @@ import {
|
||||
EARPIECE_CONFIG_ID,
|
||||
IOSControlledAudioOutput,
|
||||
} from "./IOSControlledAudioOutput";
|
||||
import {
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
RTC_CALL_INTENT_VIDEO,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
|
||||
// `vi.mock` calls are hoisted above all imports, so the static imports below
|
||||
// already see these mocks. Force the iOS platform so that the virtual earpiece
|
||||
@@ -81,7 +85,7 @@ describe("Default selection", () => {
|
||||
const output = new IOSControlledAudioOutput(
|
||||
constant(false),
|
||||
testScope,
|
||||
"audio",
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
);
|
||||
const selected = latestSelection(output);
|
||||
|
||||
@@ -100,7 +104,7 @@ describe("Default selection", () => {
|
||||
const output = new IOSControlledAudioOutput(
|
||||
constant(false),
|
||||
testScope,
|
||||
"video",
|
||||
RTC_CALL_INTENT_VIDEO,
|
||||
);
|
||||
const selected = latestSelection(output);
|
||||
|
||||
@@ -113,7 +117,7 @@ describe("Default selection", () => {
|
||||
const output = new IOSControlledAudioOutput(
|
||||
constant(false),
|
||||
testScope,
|
||||
"audio",
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
);
|
||||
const selected = latestSelection(output);
|
||||
|
||||
@@ -130,7 +134,7 @@ describe("Explicit selection", () => {
|
||||
const output = new IOSControlledAudioOutput(
|
||||
constant(false),
|
||||
testScope,
|
||||
"audio",
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
);
|
||||
const selected = latestSelection(output);
|
||||
|
||||
@@ -150,7 +154,7 @@ describe("Explicit selection", () => {
|
||||
const output = new IOSControlledAudioOutput(
|
||||
constant(false),
|
||||
testScope,
|
||||
"audio",
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
);
|
||||
const selected = latestSelection(output);
|
||||
|
||||
|
||||
@@ -8,7 +8,10 @@ Please see LICENSE in the repository root for full details.
|
||||
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
|
||||
import { combineLatest, merge, startWith, Subject, tap } from "rxjs";
|
||||
|
||||
import type { RTCCallIntent } from "matrix-js-sdk/lib/matrixrtc";
|
||||
import {
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
type RTCCallIntent,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
import {
|
||||
availableOutputDevices$ as controlledAvailableOutputDevices$,
|
||||
outputDevice$ as controlledOutputSelection$,
|
||||
@@ -111,7 +114,7 @@ export class IOSControlledAudioOutput implements MediaDevice<
|
||||
* Chooses the default output device when no explicit selection (from the user
|
||||
* or the hosting application) has been made yet.
|
||||
*
|
||||
* For voice calls (`initialIntent === "audio"`) we want to start on the
|
||||
* For voice calls (`initialIntent === "m.audio"`) we want to start on the
|
||||
* earpiece rather than the speaker, like a regular phone call. We only
|
||||
* override when the device that would otherwise be the default is the
|
||||
* speaker: if the host already routed to a headset (e.g. Bluetooth) — which
|
||||
@@ -122,7 +125,7 @@ export class IOSControlledAudioOutput implements MediaDevice<
|
||||
available: Map<string, AudioOutputDeviceLabel>,
|
||||
): string | undefined {
|
||||
const firstId = available.keys().next().value;
|
||||
if (this.initialIntent === "audio") {
|
||||
if (this.initialIntent === RTC_CALL_INTENT_AUDIO) {
|
||||
const firstLabel =
|
||||
firstId !== undefined ? available.get(firstId) : undefined;
|
||||
if (firstLabel?.type === "speaker") {
|
||||
|
||||
@@ -21,6 +21,7 @@ import { type MediaDevices } from "./MediaDevices";
|
||||
import { MuteStates } from "./MuteStates";
|
||||
import { nullHostBridge } from "../HostBridge";
|
||||
import { flushPromises, mockMediaDevices, testScope } from "../utils/test";
|
||||
import { type BackgroundEffectsState } from "./BackgroundEffects";
|
||||
|
||||
describe("LobbyMediaViewModel", () => {
|
||||
it("opens what starts enabled in one capture and follows the devices", async () => {
|
||||
@@ -135,7 +136,7 @@ function setup(
|
||||
muteStates,
|
||||
mediaDevices,
|
||||
{},
|
||||
constant(processor),
|
||||
constant<BackgroundEffectsState>({ supported: true, processor }),
|
||||
);
|
||||
return { media, vm, muteStates, mediaDevices };
|
||||
}
|
||||
|
||||
@@ -13,7 +13,6 @@ import {
|
||||
type ObservableScope,
|
||||
type PublishRequest,
|
||||
trackBySource$,
|
||||
type VideoProcessor,
|
||||
} from "@element-hq/matrixrtc-sdk";
|
||||
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
|
||||
import { combineLatest, debounceTime, map } from "rxjs";
|
||||
@@ -25,6 +24,7 @@ import {
|
||||
import { type MediaDevices } from "./MediaDevices";
|
||||
import { type MuteStates } from "./MuteStates";
|
||||
import { followDevice, followProcessor } from "./localMediaControls";
|
||||
import { type BackgroundEffectsState } from "./BackgroundEffects";
|
||||
|
||||
export interface LobbyMediaViewModel {
|
||||
/** The camera, once open, for the preview. */
|
||||
@@ -42,9 +42,13 @@ export function createLobbyMediaViewModel(
|
||||
muteStates: MuteStates,
|
||||
mediaDevices: MediaDevices,
|
||||
capture: CaptureSettings,
|
||||
videoProcessor$: Behavior<VideoProcessor | undefined>,
|
||||
/** No camera is opened while its pipeline is prepared: it would show the room without the processor. */
|
||||
backgroundEffectsState$: Behavior<BackgroundEffectsState>,
|
||||
): LobbyMediaViewModel {
|
||||
const logger = rootLogger.getChild("[LobbyMedia]");
|
||||
const videoProcessor$ = scope.behavior(
|
||||
backgroundEffectsState$.pipe(map(({ processor }) => processor)),
|
||||
);
|
||||
const microphone$ = trackBySource$(
|
||||
scope,
|
||||
media.tracks$,
|
||||
@@ -64,13 +68,19 @@ export function createLobbyMediaViewModel(
|
||||
muteStates.video.enabled$,
|
||||
microphone$,
|
||||
camera$,
|
||||
backgroundEffectsState$,
|
||||
])
|
||||
.pipe(debounceTime(0), scope.bind())
|
||||
.subscribe(([audio, video, microphone, camera]) => {
|
||||
.subscribe(([audio, video, microphone, camera, { wait }]) => {
|
||||
const open: PublishRequest[] = [];
|
||||
if (audio && microphone === undefined && !opening.has("microphone"))
|
||||
open.push(publishRequest("microphone", mediaDevices, capture));
|
||||
if (video && camera === undefined && !opening.has("camera"))
|
||||
if (
|
||||
video &&
|
||||
camera === undefined &&
|
||||
!opening.has("camera") &&
|
||||
wait !== "preparing"
|
||||
)
|
||||
open.push(
|
||||
publishRequest(
|
||||
"camera",
|
||||
|
||||
@@ -35,6 +35,10 @@ import { AudioOutput, MediaDevices } from "./MediaDevices";
|
||||
import { AndroidControlledAudioOutput } from "./AndroidControlledAudioOutput";
|
||||
import { IOSControlledAudioOutput } from "./IOSControlledAudioOutput";
|
||||
import { ObservableScope } from "./ObservableScope";
|
||||
import {
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
RTC_CALL_INTENT_VIDEO,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
|
||||
// Which audio output implementation is used is decided by what the app hosting
|
||||
// Element Call told it, rather than being discovered from the environment.
|
||||
@@ -52,7 +56,7 @@ describe("MediaDevices audio output", () => {
|
||||
|
||||
const devices = new MediaDevices(new ObservableScope(), {
|
||||
controlledAudioDevices: true,
|
||||
callIntent: "audio",
|
||||
callIntent: RTC_CALL_INTENT_AUDIO,
|
||||
});
|
||||
|
||||
expect(devices.audioOutput).toBeInstanceOf(AndroidControlledAudioOutput);
|
||||
@@ -63,7 +67,7 @@ describe("MediaDevices audio output", () => {
|
||||
|
||||
const devices = new MediaDevices(new ObservableScope(), {
|
||||
controlledAudioDevices: true,
|
||||
callIntent: "video",
|
||||
callIntent: RTC_CALL_INTENT_VIDEO,
|
||||
});
|
||||
|
||||
expect(devices.audioOutput).toBeInstanceOf(IOSControlledAudioOutput);
|
||||
|
||||
@@ -6,7 +6,11 @@ Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { test, expect } from "vitest";
|
||||
import { type RTCCallIntent } from "matrix-js-sdk/lib/matrixrtc";
|
||||
import {
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
RTC_CALL_INTENT_VIDEO,
|
||||
type RTCCallIntent,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
|
||||
import { calculateInitialMuteState } from "./initialMuteState";
|
||||
|
||||
@@ -14,10 +18,10 @@ test.each<{
|
||||
callIntent: RTCCallIntent;
|
||||
allowJoinUnmutedViaIntent: boolean;
|
||||
}>([
|
||||
{ callIntent: "audio", allowJoinUnmutedViaIntent: false },
|
||||
{ callIntent: "audio", allowJoinUnmutedViaIntent: true },
|
||||
{ callIntent: "video", allowJoinUnmutedViaIntent: false },
|
||||
{ callIntent: "video", allowJoinUnmutedViaIntent: true },
|
||||
{ callIntent: RTC_CALL_INTENT_AUDIO, allowJoinUnmutedViaIntent: false },
|
||||
{ callIntent: RTC_CALL_INTENT_AUDIO, allowJoinUnmutedViaIntent: true },
|
||||
{ callIntent: RTC_CALL_INTENT_VIDEO, allowJoinUnmutedViaIntent: false },
|
||||
{ callIntent: RTC_CALL_INTENT_VIDEO, allowJoinUnmutedViaIntent: true },
|
||||
{ callIntent: "unknown", allowJoinUnmutedViaIntent: false },
|
||||
{ callIntent: "unknown", allowJoinUnmutedViaIntent: true },
|
||||
])(
|
||||
@@ -29,15 +33,15 @@ test.each<{
|
||||
allowJoinUnmutedViaIntent,
|
||||
);
|
||||
expect(audioEnabled).toBe(true);
|
||||
expect(videoEnabled).toBe(callIntent !== "audio");
|
||||
expect(videoEnabled).toBe(callIntent !== RTC_CALL_INTENT_AUDIO);
|
||||
},
|
||||
);
|
||||
|
||||
test.each<{
|
||||
callIntent: RTCCallIntent;
|
||||
}>([
|
||||
{ callIntent: "audio" },
|
||||
{ callIntent: "video" },
|
||||
{ callIntent: RTC_CALL_INTENT_AUDIO },
|
||||
{ callIntent: RTC_CALL_INTENT_VIDEO },
|
||||
{ callIntent: "unknown" },
|
||||
])(
|
||||
"Should always mute on start if skipping lobby and the host does not vouch for the intent (callIntent: $callIntent)",
|
||||
@@ -55,8 +59,8 @@ test.each<{
|
||||
test.each<{
|
||||
callIntent: RTCCallIntent;
|
||||
}>([
|
||||
{ callIntent: "audio" },
|
||||
{ callIntent: "video" },
|
||||
{ callIntent: RTC_CALL_INTENT_AUDIO },
|
||||
{ callIntent: RTC_CALL_INTENT_VIDEO },
|
||||
{ callIntent: "unknown" },
|
||||
])(
|
||||
"Can start unmuted if skipping lobby and the host vouches for the intent (callIntent: $callIntent)",
|
||||
@@ -67,6 +71,6 @@ test.each<{
|
||||
true,
|
||||
);
|
||||
expect(audioEnabled).toBe(true);
|
||||
expect(videoEnabled).toBe(callIntent !== "audio");
|
||||
expect(videoEnabled).toBe(callIntent !== RTC_CALL_INTENT_AUDIO);
|
||||
},
|
||||
);
|
||||
|
||||
@@ -6,7 +6,10 @@ Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
import { type RTCCallIntent } from "matrix-js-sdk/lib/matrixrtc";
|
||||
import {
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
type RTCCallIntent,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
|
||||
/**
|
||||
* Calculates the initial mute state for media devices based on configuration.
|
||||
@@ -40,10 +43,10 @@ export function calculateInitialMuteState(
|
||||
|
||||
// A host that vouches for the intent is a trusted environment, so it allows
|
||||
// unmuted by default. Same for when showing a lobby, as users can adjust
|
||||
// their settings there. Additionally, if the call intent is "audio", we
|
||||
// their settings there. Additionally, if the call intent is "m.audio", we
|
||||
// disable video by default.
|
||||
return {
|
||||
audioEnabled: true,
|
||||
videoEnabled: callIntent != "audio",
|
||||
videoEnabled: callIntent != RTC_CALL_INTENT_AUDIO,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -21,6 +21,8 @@ import { type Handler } from "./MuteStates";
|
||||
/** How a mute switch gets a track where there is none. */
|
||||
export interface TrackSupply {
|
||||
publish: () => Promise<unknown>;
|
||||
/** Awaited before a switch turned on acts: a camera waits for its processor to be known. */
|
||||
ready?: () => Promise<unknown>;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -45,6 +47,7 @@ export function followMuteState(
|
||||
let desired = muteState.enabled$.value;
|
||||
muteState.setHandler(async (enabled) => {
|
||||
desired = enabled;
|
||||
if (enabled) await supply.ready?.();
|
||||
const track = track$.value;
|
||||
if (track !== undefined) {
|
||||
desired = await track.setEnabled(enabled);
|
||||
@@ -94,18 +97,23 @@ export function followDevice(
|
||||
/**
|
||||
* Attached once the camera track exists rather than carried in the publish
|
||||
* request, so that a new track and a change of processor take the same path.
|
||||
* Answers with a wait for the latest attachment to have gone through: a
|
||||
* camera turned on before then would restart without its processor.
|
||||
*/
|
||||
export function followProcessor(
|
||||
scope: ObservableScope,
|
||||
camera$: Behavior<LocalVideoMediaTrack | undefined>,
|
||||
videoProcessor$: Behavior<VideoProcessor | undefined>,
|
||||
logger: Logger,
|
||||
): void {
|
||||
): () => Promise<void> {
|
||||
let attached = Promise.resolve();
|
||||
combineLatest([camera$, videoProcessor$])
|
||||
.pipe(scope.bind())
|
||||
.subscribe(([camera, processor]) => {
|
||||
camera?.setProcessor(processor).catch((e) => {
|
||||
logger.error("Failed to set the video processor", e);
|
||||
});
|
||||
attached =
|
||||
camera?.setProcessor(processor).catch((e) => {
|
||||
logger.error("Failed to set the video processor", e);
|
||||
}) ?? Promise.resolve();
|
||||
});
|
||||
return async () => attached;
|
||||
}
|
||||
|
||||
@@ -8,7 +8,10 @@ Please see LICENSE in the repository root for full details.
|
||||
import { test, expect } from "vitest";
|
||||
import { act, render, screen } from "@testing-library/react";
|
||||
import { axe } from "vitest-axe";
|
||||
import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
|
||||
import {
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
type MatrixRTCSession,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
import { BehaviorSubject } from "rxjs";
|
||||
|
||||
import { GridTile } from "./GridTile";
|
||||
@@ -122,7 +125,7 @@ test("GridTile displays ringing media", async () => {
|
||||
const vm = createRingingMedia({
|
||||
pickupState$,
|
||||
id: "test",
|
||||
intent: "audio",
|
||||
intent: RTC_CALL_INTENT_AUDIO,
|
||||
userId: "@alice:example.org",
|
||||
displayName$: constant("Alice"),
|
||||
mxcAvatarUrl$: constant(undefined),
|
||||
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
EndCallIcon,
|
||||
} from "@vector-im/compound-design-tokens/assets/web/icons";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { RTC_CALL_INTENT_VIDEO } from "matrix-js-sdk/lib/matrixrtc";
|
||||
|
||||
import { type RingingMediaViewModel } from "../state/media/RingingMediaViewModel";
|
||||
import { useBehavior } from "../useBehavior";
|
||||
@@ -25,7 +26,7 @@ export const RingingStatus: FC<Props> = ({ vm }) => {
|
||||
const pickupState = useBehavior(vm.pickupState$);
|
||||
const Icon =
|
||||
pickupState === "ringing"
|
||||
? vm.intent === "video"
|
||||
? vm.intent === RTC_CALL_INTENT_VIDEO
|
||||
? VideoCallSolidIcon
|
||||
: VoiceCallSolidIcon
|
||||
: EndCallIcon;
|
||||
|
||||
@@ -11,6 +11,7 @@ import { axe } from "vitest-axe";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
import { TooltipProvider } from "@vector-im/compound-web";
|
||||
import { BehaviorSubject } from "rxjs";
|
||||
import { RTC_CALL_INTENT_AUDIO } from "matrix-js-sdk/lib/matrixrtc";
|
||||
|
||||
import { SpotlightTile } from "./SpotlightTile";
|
||||
import {
|
||||
@@ -176,7 +177,7 @@ test("SpotlightTile displays ringing media", async () => {
|
||||
const vm = createRingingMedia({
|
||||
pickupState$,
|
||||
id: "test",
|
||||
intent: "audio",
|
||||
intent: RTC_CALL_INTENT_AUDIO,
|
||||
userId: "@alice:example.org",
|
||||
displayName$: constant("Alice"),
|
||||
mxcAvatarUrl$: constant(undefined),
|
||||
|
||||
@@ -13,10 +13,20 @@ import {
|
||||
} from "@livekit/track-processors";
|
||||
import { ImageSegmenter } from "@mediapipe/tasks-vision";
|
||||
|
||||
import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer";
|
||||
import {
|
||||
BackgroundEffectTransformer,
|
||||
canSegment,
|
||||
} from "./BackgroundEffectTransformer";
|
||||
|
||||
const frame = {} as VideoFrame;
|
||||
const controller = {} as TransformStreamDefaultController<VideoFrame>;
|
||||
const stats = { processingTimeMs: 1, segmentationTimeMs: 1, filterTimeMs: 0 };
|
||||
|
||||
describe("BackgroundEffectTransformer", () => {
|
||||
afterEach(() => vi.restoreAllMocks());
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it("loads a background chosen before it was built", async () => {
|
||||
vi.spyOn(VideoTransformer.prototype, "init").mockResolvedValue();
|
||||
@@ -38,6 +48,82 @@ describe("BackgroundEffectTransformer", () => {
|
||||
expect(load).toHaveBeenCalledWith("/background.jpg");
|
||||
});
|
||||
|
||||
it("runs MediaPipe from its own bundle", async () => {
|
||||
vi.spyOn(VideoTransformer.prototype, "init").mockResolvedValue();
|
||||
const create = vi
|
||||
.spyOn(ImageSegmenter, "createFromOptions")
|
||||
.mockResolvedValue({} as ImageSegmenter);
|
||||
|
||||
await new BackgroundEffectTransformer({ backgroundDisabled: true }).init(
|
||||
{} as VideoTransformerInitOptions,
|
||||
);
|
||||
|
||||
const [fileset, options] = create.mock.calls[0];
|
||||
expect(fileset.wasmLoaderPath).toMatch(/vision_wasm_internal\.js/);
|
||||
expect(fileset.wasmBinaryPath).toMatch(/vision_wasm_internal\.wasm/);
|
||||
expect(options.baseOptions?.modelAssetPath).toMatch(
|
||||
/selfie_segmenter\.tflite/,
|
||||
);
|
||||
});
|
||||
|
||||
it("reports the first frame carrying an effect, once", async () => {
|
||||
// The library reports every frame it draws.
|
||||
vi.spyOn(BackgroundTransformer.prototype, "transform").mockImplementation(
|
||||
async function (this: BackgroundTransformer) {
|
||||
this.options.onFrameProcessed?.(stats);
|
||||
return Promise.resolve();
|
||||
},
|
||||
);
|
||||
const transformer = new BackgroundEffectTransformer({});
|
||||
transformer.onFirstFrame = vi.fn();
|
||||
|
||||
await transformer.transform(frame, controller);
|
||||
await transformer.transform(frame, controller);
|
||||
expect(transformer.onFirstFrame).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not report a frame passed through untouched", async () => {
|
||||
vi.spyOn(BackgroundTransformer.prototype, "transform").mockResolvedValue();
|
||||
const transformer = new BackgroundEffectTransformer({
|
||||
backgroundDisabled: true,
|
||||
});
|
||||
transformer.onFirstFrame = vi.fn();
|
||||
|
||||
await transformer.transform(frame, controller);
|
||||
expect(transformer.onFirstFrame).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps reporting drawn frames to whoever asked", async () => {
|
||||
vi.spyOn(BackgroundTransformer.prototype, "transform").mockImplementation(
|
||||
async function (this: BackgroundTransformer) {
|
||||
this.options.onFrameProcessed?.(stats);
|
||||
return Promise.resolve();
|
||||
},
|
||||
);
|
||||
const onFrameProcessed = vi.fn();
|
||||
const transformer = new BackgroundEffectTransformer({ onFrameProcessed });
|
||||
|
||||
await transformer.transform(frame, controller);
|
||||
expect(onFrameProcessed).toHaveBeenCalledWith(stats);
|
||||
});
|
||||
|
||||
it("never sends the first frame on untouched", async () => {
|
||||
// The library copies its first frame through untouched, ahead of the
|
||||
// drawn one, unless told the first frame is past
|
||||
const seen: boolean[] = [];
|
||||
vi.spyOn(BackgroundTransformer.prototype, "transform").mockImplementation(
|
||||
async function (this: BackgroundTransformer) {
|
||||
seen.push(this.isFirstFrame);
|
||||
return Promise.resolve();
|
||||
},
|
||||
);
|
||||
const transformer = new BackgroundEffectTransformer({});
|
||||
expect(transformer.isFirstFrame).toBe(true);
|
||||
|
||||
await transformer.transform(frame, controller);
|
||||
expect(seen).toEqual([false]);
|
||||
});
|
||||
|
||||
it("passes frames on untouched once disabled, even after blurring", async () => {
|
||||
const transformer = new BackgroundEffectTransformer({});
|
||||
await transformer.update({ blurRadius: 15, backgroundDisabled: false });
|
||||
@@ -49,4 +135,33 @@ describe("BackgroundEffectTransformer", () => {
|
||||
expect(transformer.options.blurRadius).toBeUndefined();
|
||||
expect(transformer.options.imagePath).toBeUndefined();
|
||||
});
|
||||
|
||||
it("segments on the CPU where the GPU refuses", async () => {
|
||||
vi.spyOn(VideoTransformer.prototype, "init").mockResolvedValue();
|
||||
const build = vi
|
||||
.spyOn(ImageSegmenter, "createFromOptions")
|
||||
.mockRejectedValueOnce(new Error("blocklisted"))
|
||||
.mockResolvedValue({} as ImageSegmenter);
|
||||
|
||||
await new BackgroundEffectTransformer({}).init(
|
||||
{} as VideoTransformerInitOptions,
|
||||
);
|
||||
expect(build.mock.calls.map(([, o]) => o.baseOptions?.delegate)).toEqual([
|
||||
"GPU",
|
||||
"CPU",
|
||||
]);
|
||||
});
|
||||
|
||||
it("says whether a segmenter can be built at all", async () => {
|
||||
vi.stubGlobal("OffscreenCanvas", class {});
|
||||
const close = vi.fn();
|
||||
const build = vi
|
||||
.spyOn(ImageSegmenter, "createFromOptions")
|
||||
.mockResolvedValue({ close } as unknown as ImageSegmenter);
|
||||
expect(await canSegment()).toBe(true);
|
||||
expect(close).toHaveBeenCalled();
|
||||
|
||||
build.mockRejectedValue(new Error("no model"));
|
||||
expect(await canSegment()).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -11,11 +11,18 @@ import {
|
||||
VideoTransformer,
|
||||
type VideoTransformerInitOptions,
|
||||
type BackgroundOptions,
|
||||
type SegmenterOptions,
|
||||
} from "@livekit/track-processors";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
import { ImageSegmenter } from "@mediapipe/tasks-vision";
|
||||
|
||||
import modelAssetPath from "../mediapipe/imageSegmenter/selfie_segmenter.tflite?url";
|
||||
// Only the SIMD build of MediaPipe's runtime: every browser Element Call
|
||||
// supports has WebAssembly SIMD. As files rather than data URLs, so that a
|
||||
// host's bundler serves them as assets of its own, and a browser fetches them
|
||||
// only when an effect is first switched on.
|
||||
import wasmLoaderPath from "@mediapipe/tasks-vision/vision_wasm_internal.js?url&no-inline";
|
||||
import wasmBinaryPath from "@mediapipe/tasks-vision/vision_wasm_internal.wasm?url&no-inline";
|
||||
import modelAssetPath from "../mediapipe/imageSegmenter/selfie_segmenter.tflite?url&no-inline";
|
||||
|
||||
interface WasmFileset {
|
||||
/** The path to the Wasm loader script. */
|
||||
@@ -24,26 +31,7 @@ interface WasmFileset {
|
||||
wasmBinaryPath: string;
|
||||
}
|
||||
|
||||
// The MediaPipe package, by default, ships some alternative versions of the
|
||||
// WASM files which avoid SIMD for compatibility with older browsers. But SIMD
|
||||
// in WASM is actually fine by our support policy, so we include just the SIMD
|
||||
// versions.
|
||||
// It's really not ideal that we have to reference these internal files from
|
||||
// MediaPipe and depend on node_modules having this specific structure. It's
|
||||
// easy to see this breaking if our dependencies changed and MediaPipe were
|
||||
// no longer hoisted, or if we switched to another dependency loader such as
|
||||
// yarn PnP.
|
||||
// https://github.com/google-ai-edge/mediapipe/issues/5961
|
||||
const wasmFileset: WasmFileset = {
|
||||
wasmLoaderPath: new URL(
|
||||
"../../node_modules/@mediapipe/tasks-vision/wasm/vision_wasm_internal.js",
|
||||
import.meta.url,
|
||||
).href,
|
||||
wasmBinaryPath: new URL(
|
||||
"../../node_modules/@mediapipe/tasks-vision/wasm/vision_wasm_internal.wasm",
|
||||
import.meta.url,
|
||||
).href,
|
||||
};
|
||||
const wasmFileset: WasmFileset = { wasmLoaderPath, wasmBinaryPath };
|
||||
|
||||
/**
|
||||
* Track processor that applies effects such as blurring to a user's background.
|
||||
@@ -53,6 +41,43 @@ const wasmFileset: WasmFileset = {
|
||||
* resource fetched from the public internet.
|
||||
*/
|
||||
export class BackgroundEffectTransformer extends BackgroundTransformer {
|
||||
/**
|
||||
* Called once, when the first frame carrying an effect has been drawn.
|
||||
* This is used for drewAFrame$ to show the correct loading state.
|
||||
*/
|
||||
public onFirstFrame: (() => void) | undefined;
|
||||
private drewAFrame = false;
|
||||
|
||||
public constructor(opts: BackgroundOptions) {
|
||||
super(opts);
|
||||
// Called by the library for a frame it drew, never for one it passed on
|
||||
// untouched.
|
||||
this.options.onFrameProcessed = (stats): void => {
|
||||
opts.onFrameProcessed?.(stats);
|
||||
if (this.drewAFrame) return;
|
||||
this.drewAFrame = true;
|
||||
this.onFirstFrame?.();
|
||||
};
|
||||
}
|
||||
|
||||
public override async transform(
|
||||
frame: VideoFrame,
|
||||
controller: TransformStreamDefaultController<VideoFrame>,
|
||||
): Promise<void> {
|
||||
// From the LK docs:
|
||||
// > Render a copy of the first frame is rendered to the screen as soon as possible to act
|
||||
// > as a less jarring initial state than a solid color while the synchronous work below
|
||||
// > (segmentation + frame rendering) occurs.
|
||||
// We do not want this copy (it leaks unblurred media).
|
||||
// So we always set `isFirstFrame` to false -> never show an unprocessed frame.
|
||||
// It does not have any impact other than setting the unprocessed frame while loading the processor.
|
||||
//
|
||||
// To not show the blank frame, we have our own mechanism for hiding the loading overlay:
|
||||
// `onFirstFrame` is what the EC UI will listen to and only then show the video feed.
|
||||
this.isFirstFrame = false;
|
||||
await super.transform(frame, controller);
|
||||
}
|
||||
|
||||
/**
|
||||
* As the library's, except that disabling also drops the blur radius. The
|
||||
* library skips a frame on `isDisabled ?? backgroundDisabled`, and
|
||||
@@ -78,17 +103,10 @@ export class BackgroundEffectTransformer extends BackgroundTransformer {
|
||||
inputElement: inputVideo,
|
||||
});
|
||||
|
||||
this.imageSegmenter = await ImageSegmenter.createFromOptions(wasmFileset, {
|
||||
baseOptions: {
|
||||
modelAssetPath,
|
||||
delegate: "GPU",
|
||||
...this.options.segmenterOptions,
|
||||
},
|
||||
canvas: this.canvas,
|
||||
runningMode: "VIDEO",
|
||||
outputCategoryMask: true,
|
||||
outputConfidenceMasks: false,
|
||||
});
|
||||
this.imageSegmenter = await createSegmenter(
|
||||
this.canvas,
|
||||
this.options.segmenterOptions,
|
||||
);
|
||||
|
||||
// BackgroundTransformer's own init applies these, and this one replaces it.
|
||||
if (this.options.imagePath) {
|
||||
@@ -102,3 +120,38 @@ export class BackgroundEffectTransformer extends BackgroundTransformer {
|
||||
this.gl?.setBackgroundDisabled(this.options.backgroundDisabled ?? false);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a segmenter can be built here, asked once before any camera depends
|
||||
* on one: a pipeline that fails to build takes the camera down with it.
|
||||
*/
|
||||
export async function canSegment(): Promise<boolean> {
|
||||
try {
|
||||
(await createSegmenter(new OffscreenCanvas(1, 1))).close();
|
||||
return true;
|
||||
} catch (e) {
|
||||
logger.warn("Background effects cannot run here", e);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** On the GPU, or on the CPU where the GPU refuses, as a blocklisted one does. */
|
||||
async function createSegmenter(
|
||||
canvas: HTMLCanvasElement | OffscreenCanvas | undefined,
|
||||
segmenterOptions?: SegmenterOptions,
|
||||
): Promise<ImageSegmenter> {
|
||||
const build = async (delegate: "GPU" | "CPU"): Promise<ImageSegmenter> =>
|
||||
ImageSegmenter.createFromOptions(wasmFileset, {
|
||||
baseOptions: { modelAssetPath, delegate, ...segmenterOptions },
|
||||
canvas,
|
||||
runningMode: "VIDEO",
|
||||
outputCategoryMask: true,
|
||||
outputConfidenceMasks: false,
|
||||
});
|
||||
try {
|
||||
return await build("GPU");
|
||||
} catch (e) {
|
||||
logger.warn("Segmenting on the CPU, as the GPU refused", e);
|
||||
return build("CPU");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,8 +19,10 @@ import {
|
||||
type BackgroundEffectsState,
|
||||
} from "../state/BackgroundEffects";
|
||||
import { backgroundEffect } from "../settings/settings";
|
||||
import { type AddedBackground } from "./backgroundImages";
|
||||
import { flushPromises, testScope } from "../utils/test";
|
||||
|
||||
const sdkSupportMock = vi.hoisted(() => vi.fn(() => true));
|
||||
const pipelines = vi.hoisted(() => ({
|
||||
built: 0,
|
||||
destroyed: 0,
|
||||
@@ -41,12 +43,21 @@ vi.mock("@livekit/track-processors", () => ({
|
||||
return Promise.resolve();
|
||||
});
|
||||
}),
|
||||
supportsBackgroundProcessors: (): boolean => true,
|
||||
supportsBackgroundProcessors: (): boolean => sdkSupportMock(),
|
||||
}));
|
||||
const platformMock = vi.hoisted(() => ({ platform: "desktop" }));
|
||||
vi.mock("../Platform", () => platformMock);
|
||||
vi.mock("./backgroundImages", async () => {
|
||||
const { BehaviorSubject } = await import("rxjs");
|
||||
return {
|
||||
addedBackgrounds: {
|
||||
added$: new BehaviorSubject<AddedBackground[] | undefined>(undefined),
|
||||
},
|
||||
};
|
||||
});
|
||||
vi.mock("./BackgroundEffectTransformer", () => ({
|
||||
BackgroundEffectTransformer: vi.fn(),
|
||||
BackgroundEffectTransformer: vi.fn(function () {}),
|
||||
canSegment: async (): Promise<boolean> => Promise.resolve(true),
|
||||
}));
|
||||
|
||||
describe("BackgroundEffectsProvider", () => {
|
||||
@@ -77,6 +88,7 @@ describe("BackgroundEffectsProvider", () => {
|
||||
pipelines.destroyed = 0;
|
||||
pipelines.switches = [];
|
||||
platformMock.platform = "desktop";
|
||||
sdkSupportMock.mockReturnValue(true);
|
||||
backgroundEffect.setValue("none");
|
||||
});
|
||||
afterEach(() => backgroundEffect.setValue("none"));
|
||||
@@ -93,13 +105,13 @@ describe("BackgroundEffectsProvider", () => {
|
||||
expect(latest().processor).toBeDefined();
|
||||
});
|
||||
|
||||
it("does not run on a phone, even where its browser could", async () => {
|
||||
it("runs on a phone whose browser can run it", async () => {
|
||||
platformMock.platform = "ios";
|
||||
render(surface());
|
||||
await blur(true);
|
||||
|
||||
expect(latest().supported).toBe(false);
|
||||
expect(latest().processor).toBeUndefined();
|
||||
expect(latest().supported).toBe(true);
|
||||
expect(latest().processor).toBeDefined();
|
||||
});
|
||||
|
||||
it("switches to an effect chosen while the camera is off", async () => {
|
||||
|
||||
@@ -5,12 +5,18 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { BackgroundProcessorWrapper } from "@livekit/track-processors";
|
||||
import {
|
||||
BackgroundProcessorWrapper,
|
||||
supportsBackgroundProcessors,
|
||||
} from "@livekit/track-processors";
|
||||
import { createContext, type FC, type JSX, use } from "react";
|
||||
|
||||
import { backgroundEffect as backgroundEffectSetting } from "../settings/settings";
|
||||
import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer";
|
||||
import { supportsBackgroundProcessors } from "./backgroundProcessing";
|
||||
import {
|
||||
BackgroundEffectTransformer,
|
||||
canSegment,
|
||||
} from "./BackgroundEffectTransformer";
|
||||
import { addedBackgrounds } from "./backgroundImages";
|
||||
import { type Behavior } from "../state/Behavior";
|
||||
import { type ObservableScope } from "../state/ObservableScope";
|
||||
import {
|
||||
@@ -45,13 +51,17 @@ export function useBackgroundEffectsState$(): Behavior<BackgroundEffectsState> {
|
||||
export function createAppBackgroundEffects(
|
||||
scope: ObservableScope,
|
||||
): BackgroundEffects {
|
||||
const transformer = new BackgroundEffectTransformer({
|
||||
backgroundDisabled: true,
|
||||
});
|
||||
return createBackgroundEffects(scope, {
|
||||
supported: supportsBackgroundProcessors(),
|
||||
effect$: backgroundEffectSetting.value$,
|
||||
pipeline: new BackgroundProcessorWrapper(
|
||||
new BackgroundEffectTransformer({ backgroundDisabled: true }),
|
||||
"background-effect",
|
||||
),
|
||||
setEffect: backgroundEffectSetting.setValue,
|
||||
added$: addedBackgrounds.added$,
|
||||
canSegment,
|
||||
pipeline: new BackgroundProcessorWrapper(transformer, "background-effect"),
|
||||
transformer,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
serializeEffect,
|
||||
shippedBackgrounds,
|
||||
} from "./backgroundEffectCatalog";
|
||||
import { maxAddedBackgrounds } from "./backgroundImages";
|
||||
|
||||
describe("the chosen background effect", () => {
|
||||
test("round-trips through its stored form", () => {
|
||||
@@ -22,6 +23,7 @@ describe("the chosen background effect", () => {
|
||||
{ kind: "none" },
|
||||
{ kind: "blur" },
|
||||
...shippedBackgrounds.map(({ id }) => ({ kind: "shipped" as const, id })),
|
||||
{ kind: "added", id: "d3b07384" },
|
||||
];
|
||||
for (const effect of effects)
|
||||
expect(parseEffect(serializeEffect(effect))).toEqual(effect);
|
||||
@@ -30,6 +32,7 @@ describe("the chosen background effect", () => {
|
||||
test("falls back to no effect when the stored form names nothing we ship", () => {
|
||||
expect(parseEffect("image:gone")).toEqual({ kind: "none" });
|
||||
expect(parseEffect("")).toEqual({ kind: "none" });
|
||||
expect(parseEffect("added:")).toEqual({ kind: "none" });
|
||||
});
|
||||
|
||||
test("reads a stored value that is not a string as no effect", () => {
|
||||
@@ -37,6 +40,14 @@ describe("the chosen background effect", () => {
|
||||
expect(parseEffect(raw)).toEqual({ kind: "none" });
|
||||
});
|
||||
|
||||
test("fills the grid at the limit and one short of it", () => {
|
||||
// No effect, blur and the shipped ones, then the added ones.
|
||||
const fixed = 2 + shippedBackgrounds.length;
|
||||
expect((fixed + maxAddedBackgrounds) % 3).toBe(0);
|
||||
// One short, the add tile takes the last place.
|
||||
expect((fixed + maxAddedBackgrounds - 1 + 1) % 3).toBe(0);
|
||||
});
|
||||
|
||||
test("gives every shipped background an image to draw", () => {
|
||||
for (const { id } of shippedBackgrounds)
|
||||
expect(imagePathFor(id)).toBeTruthy();
|
||||
|
||||
@@ -5,8 +5,8 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import arc from "../graphics/background-arc.jpg?url";
|
||||
import glow from "../graphics/background-glow.jpg?url";
|
||||
import arc from "../graphics/background-arc.jpg?url&no-inline";
|
||||
import glow from "../graphics/background-glow.jpg?url&no-inline";
|
||||
|
||||
export interface ShippedBackground {
|
||||
/** Stored in the user's settings when chosen, so it must not change. */
|
||||
@@ -22,16 +22,18 @@ export const shippedBackgrounds: ShippedBackground[] = [
|
||||
export type BackgroundEffect =
|
||||
| { kind: "none" }
|
||||
| { kind: "blur" }
|
||||
| { kind: "shipped"; id: string };
|
||||
| { kind: "shipped"; id: string }
|
||||
| { kind: "added"; id: string };
|
||||
|
||||
/** An effect as the setting stores it, and as the menu names its option. */
|
||||
export type EffectId = "none" | "blur" | `image:${string}`;
|
||||
export type EffectId = "none" | "blur" | `image:${string}` | `added:${string}`;
|
||||
|
||||
export const noEffect: BackgroundEffect = { kind: "none" };
|
||||
|
||||
/**
|
||||
* Reads a stored effect; one we no longer ship reads as no effect. Stored by
|
||||
* hand or by another build, the value need not even be a string.
|
||||
* Reads a stored effect; one we no longer ship reads as no effect. Whether an
|
||||
* added one is still kept only the device knows. Stored by hand or by another
|
||||
* build, the value need not even be a string.
|
||||
*/
|
||||
export function parseEffect(raw: unknown): BackgroundEffect {
|
||||
if (typeof raw !== "string") return noEffect;
|
||||
@@ -41,6 +43,10 @@ export function parseEffect(raw: unknown): BackgroundEffect {
|
||||
if (shippedBackgrounds.some((b) => b.id === id))
|
||||
return { kind: "shipped", id };
|
||||
}
|
||||
if (raw.startsWith("added:")) {
|
||||
const id = raw.slice("added:".length);
|
||||
if (id) return { kind: "added", id };
|
||||
}
|
||||
return noEffect;
|
||||
}
|
||||
|
||||
@@ -50,6 +56,8 @@ export function serializeEffect(effect: BackgroundEffect): EffectId {
|
||||
return "blur";
|
||||
case "shipped":
|
||||
return `image:${effect.id}`;
|
||||
case "added":
|
||||
return `added:${effect.id}`;
|
||||
default:
|
||||
return "none";
|
||||
}
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { IDBFactory } from "fake-indexeddb";
|
||||
|
||||
import {
|
||||
AddedBackgrounds,
|
||||
type BackgroundImageStorage,
|
||||
IndexedDBImageStorage,
|
||||
type KeptImage,
|
||||
maxAddedBackgrounds,
|
||||
prepareImage,
|
||||
UnusableImage,
|
||||
} from "./backgroundImages";
|
||||
import { flushPromises } from "../utils/test";
|
||||
|
||||
describe("AddedBackgrounds", () => {
|
||||
beforeEach(() => {
|
||||
let n = 0;
|
||||
vi.spyOn(URL, "createObjectURL").mockImplementation(() => `blob:${++n}`);
|
||||
});
|
||||
|
||||
it("offers what the device keeps, oldest first", async () => {
|
||||
const image = new Blob(["x"], { type: "image/png" });
|
||||
const store = new AddedBackgrounds(
|
||||
memoryStorage([
|
||||
{ id: "later", image, addedAt: 2 },
|
||||
{ id: "earlier", image, addedAt: 1 },
|
||||
]),
|
||||
asGiven,
|
||||
);
|
||||
expect(store.added$.value).toBeUndefined();
|
||||
await flushPromises();
|
||||
expect(store.added$.value?.map(({ id }) => id)).toEqual([
|
||||
"earlier",
|
||||
"later",
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps an added image and offers it after the others", async () => {
|
||||
const storage = memoryStorage();
|
||||
const store = new AddedBackgrounds(storage, asGiven);
|
||||
await flushPromises();
|
||||
const id = await store.add(new Blob(["x"], { type: "image/png" }));
|
||||
expect(store.added$.value?.map((a) => a.id)).toEqual([id]);
|
||||
expect((await storage.list()).map((k) => k.id)).toEqual([id]);
|
||||
});
|
||||
|
||||
it("keeps the prepared image, not the file", async () => {
|
||||
const storage = memoryStorage();
|
||||
const prepared = new Blob(["y"], { type: "image/webp" });
|
||||
const store = new AddedBackgrounds(storage, async () =>
|
||||
Promise.resolve(prepared),
|
||||
);
|
||||
await store.add(new Blob(["x"], { type: "image/png" }));
|
||||
expect((await storage.list())[0].image).toBe(prepared);
|
||||
});
|
||||
|
||||
it("keeps nothing of a file it refuses", async () => {
|
||||
const storage = memoryStorage();
|
||||
const store = new AddedBackgrounds(storage, async () =>
|
||||
Promise.reject(new UnusableImage("animated")),
|
||||
);
|
||||
await flushPromises();
|
||||
await expect(store.add(new Blob(["x"]))).rejects.toThrow(UnusableImage);
|
||||
expect(await storage.list()).toEqual([]);
|
||||
expect(store.added$.value).toEqual([]);
|
||||
});
|
||||
|
||||
it("keeps no more than the grid holds", async () => {
|
||||
const storage = memoryStorage();
|
||||
const store = new AddedBackgrounds(storage, asGiven);
|
||||
for (let i = 0; i < maxAddedBackgrounds; i++)
|
||||
await store.add(new Blob([`${i}`]));
|
||||
await expect(store.add(new Blob(["more"]))).rejects.toThrow(RangeError);
|
||||
expect(await storage.list()).toHaveLength(maxAddedBackgrounds);
|
||||
});
|
||||
|
||||
it("forgets a removed image, and lets its picture go", async () => {
|
||||
const revoke = vi
|
||||
.spyOn(URL, "revokeObjectURL")
|
||||
.mockImplementation(() => {});
|
||||
const storage = memoryStorage();
|
||||
const store = new AddedBackgrounds(storage, asGiven);
|
||||
await flushPromises();
|
||||
const first = await store.add(new Blob(["a"]));
|
||||
const second = await store.add(new Blob(["b"]));
|
||||
const url = store.added$.value![0].url;
|
||||
|
||||
await store.remove(first);
|
||||
expect(store.added$.value?.map((a) => a.id)).toEqual([second]);
|
||||
expect((await storage.list()).map((k) => k.id)).toEqual([second]);
|
||||
expect(revoke).toHaveBeenCalledWith(url);
|
||||
});
|
||||
|
||||
it("offers nothing where the browser keeps nothing", async () => {
|
||||
const store = new AddedBackgrounds(null);
|
||||
await flushPromises();
|
||||
expect(store.added$.value).toEqual([]);
|
||||
await expect(store.add(new Blob(["x"]))).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("offers none, rather than waiting, where the device can't be read", async () => {
|
||||
const unreadable: BackgroundImageStorage = {
|
||||
...memoryStorage(),
|
||||
list: async () => Promise.reject(new Error("storage cleared")),
|
||||
};
|
||||
const store = new AddedBackgrounds(unreadable);
|
||||
await flushPromises();
|
||||
expect(store.added$.value).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("prepareImage", () => {
|
||||
// Refused before anything is decoded, which only a real browser could do.
|
||||
it("refuses a file that isn't an image", async () => {
|
||||
await expect(
|
||||
prepareImage(new Blob(["x"], { type: "text/plain" })),
|
||||
).rejects.toEqual(new UnusableImage("not-an-image"));
|
||||
});
|
||||
|
||||
it("refuses an animated type where the browser can't count frames", async () => {
|
||||
expect(globalThis.ImageDecoder).toBeUndefined();
|
||||
await expect(
|
||||
prepareImage(new Blob(["GIF89a"], { type: "image/gif" })),
|
||||
).rejects.toEqual(new UnusableImage("animated"));
|
||||
});
|
||||
});
|
||||
|
||||
describe("IndexedDBImageStorage", () => {
|
||||
it("keeps images across sessions until they are deleted", async () => {
|
||||
const indexedDB = new IDBFactory();
|
||||
const image = new Blob(["x"], { type: "image/webp" });
|
||||
const first = new IndexedDBImageStorage(indexedDB);
|
||||
await first.put({ id: "a", image, addedAt: 1 });
|
||||
await first.put({ id: "b", image, addedAt: 2 });
|
||||
|
||||
// A session of its own, as after a reload.
|
||||
const second = new IndexedDBImageStorage(indexedDB);
|
||||
expect(
|
||||
(await second.list()).map(({ id, addedAt }) => [id, addedAt]),
|
||||
).toEqual([
|
||||
["a", 1],
|
||||
["b", 2],
|
||||
]);
|
||||
await second.delete("a");
|
||||
expect((await first.list()).map(({ id }) => id)).toEqual(["b"]);
|
||||
});
|
||||
});
|
||||
|
||||
// Decoding needs a real browser; the end-to-end checks cover it.
|
||||
const asGiven = async (file: Blob): Promise<Blob> => Promise.resolve(file);
|
||||
|
||||
function memoryStorage(kept: KeptImage[] = []): BackgroundImageStorage {
|
||||
return {
|
||||
list: async (): Promise<KeptImage[]> => Promise.resolve([...kept]),
|
||||
put: async (image): Promise<void> => {
|
||||
kept.push(image);
|
||||
return Promise.resolve();
|
||||
},
|
||||
delete: async (id): Promise<void> => {
|
||||
const index = kept.findIndex((k) => k.id === id);
|
||||
if (index !== -1) kept.splice(index, 1);
|
||||
return Promise.resolve();
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { BehaviorSubject } from "rxjs";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
|
||||
import { type Behavior } from "../state/Behavior";
|
||||
|
||||
export const maxAddedBackgrounds = 5;
|
||||
|
||||
/** The longest edge kept; the pipeline scales every image to the camera. */
|
||||
export const maxStoredEdge = 1920;
|
||||
|
||||
/** Why a file can't be used as a background. */
|
||||
export type UnusableReason = "not-an-image" | "animated" | "undecodable";
|
||||
|
||||
export class UnusableImage extends Error {
|
||||
public constructor(public readonly reason: UnusableReason) {
|
||||
super(reason);
|
||||
this.name = "UnusableImage";
|
||||
}
|
||||
}
|
||||
|
||||
/** A background the user added, as the menu and the pipeline use it. */
|
||||
export interface AddedBackground {
|
||||
id: string;
|
||||
url: string;
|
||||
}
|
||||
|
||||
/** A background the user added, as it is kept on the device. */
|
||||
export interface KeptImage {
|
||||
id: string;
|
||||
image: Blob;
|
||||
addedAt: number;
|
||||
}
|
||||
|
||||
/** Where added backgrounds are kept. */
|
||||
export interface BackgroundImageStorage {
|
||||
list(): Promise<KeptImage[]>;
|
||||
put(image: KeptImage): Promise<void>;
|
||||
delete(id: string): Promise<void>;
|
||||
}
|
||||
|
||||
/** The backgrounds this device keeps, which never leave it. */
|
||||
export class AddedBackgrounds {
|
||||
private readonly subject$ = new BehaviorSubject<
|
||||
AddedBackground[] | undefined
|
||||
>(undefined);
|
||||
/** Undefined until what the device keeps has been read. */
|
||||
public readonly added$: Behavior<AddedBackground[] | undefined> =
|
||||
this.subject$;
|
||||
|
||||
public constructor(
|
||||
private readonly storage: BackgroundImageStorage | null,
|
||||
private readonly prepare: (file: Blob) => Promise<Blob> = prepareImage,
|
||||
) {
|
||||
this.read().catch((e) => {
|
||||
logger.warn("Could not read added backgrounds", e);
|
||||
this.subject$.next([]);
|
||||
});
|
||||
}
|
||||
|
||||
private async read(): Promise<void> {
|
||||
const kept = (await this.storage?.list()) ?? [];
|
||||
kept.sort((a, b) => a.addedAt - b.addedAt);
|
||||
this.subject$.next(
|
||||
kept.map(({ id, image }) => ({ id, url: URL.createObjectURL(image) })),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Keeps a file as a background, and answers with its id. Throws
|
||||
* {@link UnusableImage} for a file that can't be used, and a RangeError
|
||||
* once maxAddedBackgrounds are kept.
|
||||
*/
|
||||
public async add(file: Blob): Promise<string> {
|
||||
if (this.storage === null)
|
||||
throw new Error("This browser keeps no backgrounds");
|
||||
const image = await this.prepare(file);
|
||||
if ((await this.storage.list()).length >= maxAddedBackgrounds)
|
||||
throw new RangeError(
|
||||
`${maxAddedBackgrounds} backgrounds are kept already`,
|
||||
);
|
||||
const kept: KeptImage = {
|
||||
id: crypto.randomUUID(),
|
||||
image,
|
||||
addedAt: Date.now(),
|
||||
};
|
||||
await this.storage.put(kept);
|
||||
this.subject$.next([
|
||||
...(this.subject$.value ?? []),
|
||||
{ id: kept.id, url: URL.createObjectURL(image) },
|
||||
]);
|
||||
return kept.id;
|
||||
}
|
||||
|
||||
public async remove(id: string): Promise<void> {
|
||||
await this.storage?.delete(id);
|
||||
const current = this.subject$.value ?? [];
|
||||
const gone = current.find((a) => a.id === id);
|
||||
if (gone) URL.revokeObjectURL(gone.url);
|
||||
this.subject$.next(current.filter((a) => a.id !== id));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Refuses what can't be used, reduces what is larger than is kept, and lays
|
||||
* every image on an opaque ground: a background has to cover what is behind
|
||||
* it, so a transparent pixel would be a hole in it.
|
||||
*/
|
||||
export async function prepareImage(file: Blob): Promise<Blob> {
|
||||
if (!file.type.startsWith("image/")) throw new UnusableImage("not-an-image");
|
||||
if (await isAnimated(file)) throw new UnusableImage("animated");
|
||||
|
||||
let bitmap: ImageBitmap;
|
||||
try {
|
||||
bitmap = await createImageBitmap(file);
|
||||
} catch (e) {
|
||||
logger.debug("Could not decode the chosen file", e);
|
||||
throw new UnusableImage("undecodable");
|
||||
}
|
||||
try {
|
||||
const scale = Math.min(
|
||||
1,
|
||||
maxStoredEdge / Math.max(bitmap.width, bitmap.height),
|
||||
);
|
||||
const canvas = new OffscreenCanvas(
|
||||
Math.round(bitmap.width * scale),
|
||||
Math.round(bitmap.height * scale),
|
||||
);
|
||||
const context = canvas.getContext("2d");
|
||||
if (!context) throw new UnusableImage("undecodable");
|
||||
context.fillStyle = "black";
|
||||
context.fillRect(0, 0, canvas.width, canvas.height);
|
||||
context.drawImage(bitmap, 0, 0, canvas.width, canvas.height);
|
||||
return await canvas.convertToBlob({ type: "image/webp", quality: 0.9 });
|
||||
} finally {
|
||||
bitmap.close();
|
||||
}
|
||||
}
|
||||
|
||||
// Read from the file, not its type: a still and an animated WebP share one.
|
||||
async function isAnimated(file: Blob): Promise<boolean> {
|
||||
if (typeof ImageDecoder === "undefined") return /gif|apng/.test(file.type);
|
||||
try {
|
||||
const decoder = new ImageDecoder({
|
||||
data: await file.arrayBuffer(),
|
||||
type: file.type,
|
||||
});
|
||||
// Some browsers have no selected track until the tracks are ready.
|
||||
await decoder.tracks.ready;
|
||||
await decoder.completed;
|
||||
return (decoder.tracks.selectedTrack?.frameCount ?? 1) > 1;
|
||||
} catch (e) {
|
||||
logger.debug("Could not read the frame count, judging by type", e);
|
||||
return /gif|apng/.test(file.type);
|
||||
}
|
||||
}
|
||||
|
||||
const DB_NAME = "element-call-background-images";
|
||||
const STORE = "backgrounds";
|
||||
|
||||
export class IndexedDBImageStorage implements BackgroundImageStorage {
|
||||
private db?: Promise<IDBDatabase>;
|
||||
|
||||
public constructor(private readonly indexedDB: IDBFactory) {}
|
||||
|
||||
private async open(): Promise<IDBDatabase> {
|
||||
return (this.db ??= new Promise((resolve, reject) => {
|
||||
const request = this.indexedDB.open(DB_NAME, 1);
|
||||
request.onupgradeneeded = (): void => {
|
||||
request.result.createObjectStore(STORE, { keyPath: "id" });
|
||||
};
|
||||
request.onsuccess = (): void => resolve(request.result);
|
||||
request.onerror = (): void => reject(request.error);
|
||||
}));
|
||||
}
|
||||
|
||||
private async run<T>(
|
||||
mode: IDBTransactionMode,
|
||||
body: (store: IDBObjectStore) => IDBRequest<T>,
|
||||
): Promise<T> {
|
||||
const db = await this.open();
|
||||
return new Promise((resolve, reject) => {
|
||||
const request = body(db.transaction(STORE, mode).objectStore(STORE));
|
||||
request.onsuccess = (): void => resolve(request.result);
|
||||
request.onerror = (): void => reject(request.error);
|
||||
});
|
||||
}
|
||||
|
||||
public async list(): Promise<KeptImage[]> {
|
||||
return this.run<KeptImage[]>("readonly", (s) => s.getAll());
|
||||
}
|
||||
|
||||
public async put(image: KeptImage): Promise<void> {
|
||||
await this.run("readwrite", (s) => s.put(image));
|
||||
}
|
||||
|
||||
public async delete(id: string): Promise<void> {
|
||||
await this.run("readwrite", (s) => s.delete(id));
|
||||
}
|
||||
}
|
||||
|
||||
function openStorage(): BackgroundImageStorage | null {
|
||||
// Merely reading indexedDB throws where it is disabled.
|
||||
try {
|
||||
return globalThis.indexedDB
|
||||
? new IndexedDBImageStorage(globalThis.indexedDB)
|
||||
: null;
|
||||
} catch (e) {
|
||||
logger.warn("Could not reach IndexedDB", e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export const addedBackgrounds = new AddedBackgrounds(openStorage());
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user