mirror of
https://github.com/vector-im/element-call.git
synced 2026-11-13 01:12:29 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b74e63c1c0 |
@@ -1,35 +0,0 @@
|
||||
---
|
||||
name: handoff
|
||||
description: Implement the first cut of a task and stop for a direction check before the quality pass.
|
||||
argument-hint: <the task>
|
||||
disable-model-invocation: true
|
||||
---
|
||||
|
||||
Task: $ARGUMENTS
|
||||
|
||||
Stop after the first implementation and hand it back before the quality pass.
|
||||
|
||||
## Before the hand-off
|
||||
|
||||
- Run the narrowest check that rules out an obviously broken hand-off: the test
|
||||
file for the touched code, `tsc` on its own, a single story. Not the full
|
||||
`pnpm lint`, not the whole suite.
|
||||
- If a wider check is needed before feedback can be useful, keep it to the touched
|
||||
code and say why it was needed.
|
||||
- Leave wider refactors, extra tests and documentation polish out. They are
|
||||
follow-up work, not part of the first hand-off.
|
||||
|
||||
## The hand-off
|
||||
|
||||
Stop and ask whether the direction is right. The message says:
|
||||
|
||||
- what was built and where, in a few lines,
|
||||
- which check ran and what it returned, verbatim if it failed,
|
||||
- what was left out on purpose,
|
||||
- the decisions that could have gone another way.
|
||||
|
||||
Do not continue past this point on your own.
|
||||
|
||||
## After the user confirms
|
||||
|
||||
The skill ends here. Continue with the quality pass as usual.
|
||||
@@ -14,7 +14,7 @@ permissions:
|
||||
jobs:
|
||||
prevent-blocked:
|
||||
name: Prevent blocked
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
pull-requests: read
|
||||
steps:
|
||||
|
||||
@@ -17,7 +17,7 @@ env:
|
||||
jobs:
|
||||
build_and_deploy:
|
||||
name: Build & publish docker
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write # required to upload release asset
|
||||
packages: write # needed for publishing packages to GHCR
|
||||
|
||||
@@ -29,7 +29,7 @@ on:
|
||||
jobs:
|
||||
build:
|
||||
name: Build Element Call
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
# Deploy main branch to call.element.dev after build completes
|
||||
if: github.ref == 'refs/heads/main'
|
||||
needs: build_full_element_call
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Deploy to call.element.dev
|
||||
uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410 # v6
|
||||
@@ -92,7 +92,7 @@ jobs:
|
||||
build_storybook:
|
||||
name: Build Storybook
|
||||
if: contains(github.event.pull_request.labels.*.name, 'storybook build')
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
|
||||
@@ -13,7 +13,7 @@ permissions:
|
||||
|
||||
jobs:
|
||||
pr-changelog-label:
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: yogevbd/enforce-label-action@a3c219da6b8fa73f6ba62b68ff09c469b3a1c024 # 2.2.2
|
||||
with:
|
||||
|
||||
@@ -13,7 +13,7 @@ jobs:
|
||||
name: Review
|
||||
# Only run on labelling and skip forks.
|
||||
if: github.event.label.name == 'Claude-Review' && github.event.pull_request.head.repo.full_name == github.repository
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
||||
|
||||
@@ -32,7 +32,7 @@ on:
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
deployments: write
|
||||
environment: Netlify
|
||||
|
||||
@@ -10,7 +10,7 @@ on:
|
||||
jobs:
|
||||
lint:
|
||||
name: Lint, format & type check
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
|
||||
@@ -17,7 +17,7 @@ jobs:
|
||||
github.event.workflow_run.conclusion == 'success' &&
|
||||
github.event.workflow_run.event == 'pull_request' &&
|
||||
github.event.workflow_run.head_repository.full_name == github.repository
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
pr_number: ${{ steps.prdetails.outputs.pr_id }}
|
||||
pr_data_json: ${{ steps.prdetails.outputs.data }}
|
||||
|
||||
@@ -17,7 +17,7 @@ permissions: {}
|
||||
jobs:
|
||||
versioning:
|
||||
name: Versioning
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
DRY_RUN: ${{ steps.dry_run.outputs.DRY_RUN }}
|
||||
PREFIXED_VERSION: ${{ steps.prefixed_version.outputs.PREFIXED_VERSION }}
|
||||
@@ -59,7 +59,7 @@ jobs:
|
||||
publish_npm:
|
||||
needs: versioning
|
||||
name: Publish NPM
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
ARTIFACT_VERSION: ${{ steps.artifact_version.outputs.ARTIFACT_VERSION }}
|
||||
permissions:
|
||||
@@ -111,7 +111,7 @@ jobs:
|
||||
needs: [versioning, publish_npm]
|
||||
if: ${{ needs.versioning.outputs.DRY_RUN == 'false' }}
|
||||
name: Update release notes
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-latest
|
||||
# Shared with publish-embedded-packages.yaml, which appends to the same release: two
|
||||
# appends at once would each write back the body without the other's section
|
||||
concurrency:
|
||||
|
||||
@@ -14,7 +14,7 @@ on:
|
||||
jobs:
|
||||
versioning:
|
||||
name: Versioning
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
DRY_RUN: ${{ steps.dry_run.outputs.DRY_RUN }}
|
||||
PREFIXED_VERSION: ${{ steps.prefixed_version.outputs.PREFIXED_VERSION }}
|
||||
@@ -76,7 +76,7 @@ jobs:
|
||||
needs: [build_element_call, versioning]
|
||||
if: always()
|
||||
name: Publish tarball
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write # required to upload release asset
|
||||
steps:
|
||||
@@ -107,7 +107,7 @@ jobs:
|
||||
needs: [build_element_call, versioning]
|
||||
if: always()
|
||||
name: Publish NPM
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
ARTIFACT_VERSION: ${{ steps.artifact_version.outputs.ARTIFACT_VERSION }}
|
||||
permissions:
|
||||
@@ -152,7 +152,7 @@ jobs:
|
||||
needs: [build_element_call, versioning]
|
||||
if: always()
|
||||
name: Publish Android AAR
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
ARTIFACT_VERSION: ${{ steps.artifact_version.outputs.ARTIFACT_VERSION }}
|
||||
permissions:
|
||||
@@ -212,7 +212,7 @@ jobs:
|
||||
needs: [build_element_call, versioning]
|
||||
if: always()
|
||||
name: Publish SwiftPM Library
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
ARTIFACT_VERSION: ${{ steps.artifact_version.outputs.ARTIFACT_VERSION }}
|
||||
permissions:
|
||||
@@ -282,7 +282,7 @@ jobs:
|
||||
needs: [versioning, publish_npm, publish_android, publish_ios]
|
||||
if: always()
|
||||
name: Update release notes
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-latest
|
||||
# Shared with publish-component.yaml, which appends to the same release: two appends
|
||||
# at once would each write back the body without the other's section
|
||||
concurrency:
|
||||
|
||||
@@ -23,7 +23,7 @@ jobs:
|
||||
needs: build_element_call
|
||||
if: always()
|
||||
name: Publish tarball
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write # required to upload release asset
|
||||
packages: write
|
||||
@@ -68,7 +68,7 @@ jobs:
|
||||
add_docker_release_note:
|
||||
needs: publish_docker
|
||||
name: Add docker release note
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Add release note
|
||||
uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2
|
||||
|
||||
@@ -25,7 +25,7 @@ on:
|
||||
jobs:
|
||||
vitest:
|
||||
name: Run unit tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
@@ -71,27 +71,10 @@ jobs:
|
||||
flags: unittests
|
||||
fail_ci_if_error: true
|
||||
playwright:
|
||||
name: Run end-to-end tests (${{ matrix.part }})
|
||||
name: Run end-to-end tests
|
||||
if: ${{ !inputs.disable_e2e_tests }}
|
||||
timeout-minutes: 60
|
||||
runs-on: ubuntu-24.04
|
||||
# We manually shard the tests across runners. Playwright's own shards cut
|
||||
# the list of tests into equal counts in file order, which would leave the
|
||||
# widget suite, the slowest tests by far, together at the end.
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- part: chromium-1
|
||||
args: --project=chromium --shard=1/2
|
||||
- part: chromium-2
|
||||
args: --project=chromium --shard=2/2
|
||||
- part: widget-1
|
||||
args: --project=widget --shard=1/2
|
||||
- part: widget-2
|
||||
args: --project=widget --shard=2/2
|
||||
- part: firefox-mobile
|
||||
args: --project=firefox --project=mobile
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
with:
|
||||
@@ -124,64 +107,17 @@ jobs:
|
||||
run: pnpm playwright install --with-deps
|
||||
- name: Run backend components
|
||||
run: |
|
||||
# Make sure the directory for the embedded build exists as Docker might
|
||||
# create it with wrong access rights otherwise.
|
||||
mkdir -p embedded/web/dist
|
||||
docker compose -f docker-compose-dev.yml -f docker-compose-playwright.yml pull
|
||||
docker compose -f docker-compose-dev.yml -f docker-compose-playwright.yml up -d
|
||||
docker ps
|
||||
- name: Run Playwright tests
|
||||
env:
|
||||
USE_DOCKER: 1
|
||||
ARGS: ${{ matrix.args }}
|
||||
# Unquoted on purpose: the part's arguments are several words
|
||||
run: pnpm exec playwright test $ARGS
|
||||
run: pnpm exec playwright test
|
||||
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
if: ${{ !cancelled() }}
|
||||
with:
|
||||
name: blob-report-${{ matrix.part }}
|
||||
path: blob-report
|
||||
if-no-files-found: error
|
||||
# Only read by the merge below.
|
||||
retention-days: 1
|
||||
|
||||
e2e_report:
|
||||
# Combines the blobs into a single HTML report and blocks PRs on failure.
|
||||
name: Run end-to-end tests
|
||||
needs: playwright
|
||||
if: ${{ always() && !inputs.disable_e2e_tests }}
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
|
||||
with:
|
||||
# Inside a called workflow, github.repository is the caller so we need to make sure to still check out the EC repo here.
|
||||
repository: ${{ inputs.matrix-js-sdk-sha && 'element-hq/element-call' || github.repository }}
|
||||
persist-credentials: false
|
||||
- name: Enable Corepack
|
||||
run: corepack enable
|
||||
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
|
||||
with:
|
||||
cache: "pnpm"
|
||||
node-version-file: ".node-version"
|
||||
- name: Install dependencies
|
||||
run: scripts/layered.sh --ignore-pnpmfile
|
||||
env:
|
||||
PR_HEAD_REPO_OWNER: ${{ github.event.pull_request.head.repo.owner.login }}
|
||||
JS_SDK_GITHUB_BASE_REF: ${{ inputs.matrix-js-sdk-sha }}
|
||||
- name: Download the parts' reports
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
|
||||
with:
|
||||
pattern: blob-report-*
|
||||
path: all-blob-reports
|
||||
merge-multiple: true
|
||||
- name: Merge into one HTML report
|
||||
run: pnpm exec playwright merge-reports --reporter html ./all-blob-reports
|
||||
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: html-report
|
||||
path: playwright-report
|
||||
if-no-files-found: error
|
||||
retention-days: 4
|
||||
- name: Fail on a failed part
|
||||
if: ${{ needs.playwright.result != 'success' }}
|
||||
run: exit 1
|
||||
|
||||
@@ -7,7 +7,7 @@ on:
|
||||
|
||||
jobs:
|
||||
download:
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ on:
|
||||
|
||||
jobs:
|
||||
upload:
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ permissions: {}
|
||||
jobs:
|
||||
zizmor:
|
||||
name: Run zizmor 🌈
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
security-events: write # Required for upload-sarif (used by zizmor-action) to upload SARIF files.
|
||||
steps:
|
||||
|
||||
@@ -44,7 +44,3 @@ storybook-static
|
||||
|
||||
# Nested checkout made by scripts/layered.sh
|
||||
/matrix-js-sdk/
|
||||
|
||||
# Claude
|
||||
/.claude/*
|
||||
!/.claude/skills/
|
||||
|
||||
@@ -16,35 +16,19 @@ host's page. It is the MatrixRTC reference implementation.
|
||||
`@media`, global `i18next` or the `widget` global — take it from a provider.
|
||||
- The change works standalone, as a widget and as a component. Say what you checked.
|
||||
- Every gate below is green before you push.
|
||||
- Hand off after the first implementation, before the quality pass. Then commit on
|
||||
the user's word where the setup allows it, or hand them the message.
|
||||
|
||||
## Read before you
|
||||
|
||||
| … | … |
|
||||
| -------------------------------- | ---------------------------------------------------------- |
|
||||
| start a task of any size | [docs/agents/workflow.md](docs/agents/workflow.md) |
|
||||
| touch call logic or a view model | [docs/agents/architecture.md](docs/agents/architecture.md) |
|
||||
| write any code | [docs/agents/code-style.md](docs/agents/code-style.md) |
|
||||
| write a test, story or e2e spec | [docs/agents/testing.md](docs/agents/testing.md) |
|
||||
| open a PR | [CONTRIBUTING.md](CONTRIBUTING.md) |
|
||||
|
||||
## Scratch files live in `agent-workspace/`
|
||||
|
||||
Git-ignored. One kebab-case subfolder per task, matching the branch topic:
|
||||
`agent-workspace/<slug>/`.
|
||||
|
||||
| File | Purpose |
|
||||
| --------------------------- | -------------------------------------------------------------------------------- |
|
||||
| `plan.md` | The concrete approach — files, symbols, edit order — once direction is confirmed |
|
||||
| `commit-msg.txt` | `git commit -F agent-workspace/<slug>/commit-msg.txt` |
|
||||
| `pr-body.md` | `gh pr create --body-file agent-workspace/<slug>/pr-body.md` |
|
||||
| `implementation-summary.md` | What was built, decisions, trade-offs |
|
||||
| `NN-prompt.md` | Raw prompt text, numbered, when worth keeping |
|
||||
|
||||
- Nothing here is durable. The folder is ignored, so anything worth keeping is
|
||||
promoted into the PR body or a doc before the task ends.
|
||||
- Never write scratch files to the repo root. Stray `load_test_summary.md` and
|
||||
`config.json_` files are what this folder prevents.
|
||||
- Start `pr-body.md` from `.github/PULL_REQUEST_TEMPLATE.md` and fill every
|
||||
section.
|
||||
| implement from a feature spec | [FEATURES_SPEC/AGENTS.md](FEATURES_SPEC/AGENTS.md) |
|
||||
|
||||
## Gates
|
||||
|
||||
@@ -58,11 +42,6 @@ pnpm i18n:check
|
||||
`knip` fails on dead code; a file deliberately inert ahead of its consumer goes in
|
||||
`knip.ts` `ignoreFiles` with a reason.
|
||||
|
||||
Ready means every gate is green and the change is covered at the layers
|
||||
[testing.md](docs/agents/testing.md) asks for. Read the diff against
|
||||
[code-style.md](docs/agents/code-style.md) first. Re-run the whole checklist after
|
||||
any fix.
|
||||
|
||||
## A good PR
|
||||
|
||||
- One slice, under ~400 changed lines, green on its own, linked to its issue.
|
||||
|
||||
@@ -221,68 +221,119 @@ See also:
|
||||
|
||||
- [Developing with linked packages](./docs/linking.md)
|
||||
|
||||
#### Element Call as a component
|
||||
#### Element Call as a component (experimental)
|
||||
|
||||
Element Call can be rendered inside another React app instead of an iframe.
|
||||
The host supplies its own Matrix client, and the call stays inside the element
|
||||
it is mounted in.
|
||||
Element Call can also be embedded directly into another React application
|
||||
rather than being loaded in an iframe as a widget. `pnpm build:component`
|
||||
builds it as a library into `component/dist` (the bundle, its stylesheet and
|
||||
type declarations), and
|
||||
|
||||
##### Install
|
||||
```sh
|
||||
pnpm dev:component
|
||||
```
|
||||
|
||||
serves a harness on port 3001 that stands in for such an application: it signs
|
||||
in twice against the development backend and shows two calls side by side, in
|
||||
resizable boxes, with page furniture of its own around them. Use it to see how
|
||||
Element Call behaves when it does not own the page — the size it is given,
|
||||
whether it stays inside its container, and what it says to its host, which is
|
||||
logged along the bottom. The harness is served with the same development
|
||||
certificate as the app, so unless the development CA is trusted, the browser
|
||||
needs a certificate exception for `https://localhost:3001` as well (see the
|
||||
note under [Backend](#backend)). It reads the same `public/config.json` as
|
||||
`pnpm dev` if one exists, and runs with Element Call's defaults otherwise.
|
||||
|
||||
The call lays itself out for the size of the element it is mounted in, not the
|
||||
window: a host that shrinks the container to a corner of its page gets the
|
||||
picture-in-picture layout, just as a host that shrank the whole iframe used to.
|
||||
The breakpoints in the stylesheets the component uses are
|
||||
`@container element-call` queries against its root element for the same reason;
|
||||
for the standalone app the root is the page, so they mean what the media queries
|
||||
they replaced did. (The standalone-only views, such as the home and login pages,
|
||||
still use plain media queries, since the component never shows them.)
|
||||
|
||||
The component's stylesheet is confined to the element it is mounted in: the
|
||||
build rewrites every selector so that it matches only Element Call's root or
|
||||
what is inside it, with `html`, `body` and `:root` standing for that root (see
|
||||
`component/build/scopeStylesToRoot.ts`). A host's own page keeps its styles,
|
||||
and Element Call brings its own fonts and design tokens along.
|
||||
|
||||
The component speaks every language the app does. English is bundled in; the
|
||||
other locales are split into chunks the host's bundler loads the first time
|
||||
they are needed. It starts in the browser's language, and follows the host's
|
||||
own language setting through the `language` prop (`supportedLanguages` lists
|
||||
the tags it accepts, and anything else falls back to its base language or to
|
||||
English). The `theme` prop works the same way and takes the same values as the
|
||||
widget's `theme` URL parameter: `light`, `dark`, `light-high-contrast` or
|
||||
`dark-high-contrast`. Both can change while a call is running without
|
||||
disturbing it.
|
||||
|
||||
A host must call and await `initializeElementCall(config)` once before
|
||||
rendering the component: it loads the `Intl` polyfills, applies the
|
||||
deployment-wide `config.json`-style configuration and sets up translations.
|
||||
The component itself takes the host's `client` and the `roomId` to call in, an
|
||||
`intent` saying what the user asked for (which decides whether to show the
|
||||
lobby, ring, and so on), an optional `config` overriding what the intent
|
||||
implies, and an optional `hostBridge` through which Element Call tells the host
|
||||
that the user has joined or hung up, that it wants to stay on screen, and so
|
||||
on. The host makes its own requests (`join`, `hangUp`, `setDeviceMute`) through
|
||||
the handle exposed on `ref`. The full API is documented in the type declarations
|
||||
(`component/index.tsx` and `component/host.ts`).
|
||||
|
||||
A few things differ from the widget on purpose: the component draws a solid
|
||||
background rather than a gradient unless told otherwise, never offers to edit
|
||||
the user's profile (the account is the host's), scopes its keyboard shortcuts to
|
||||
its own root element so that several instances can share a page, and only shows
|
||||
its own post-call and error screens when the host has not supplied a `close()`
|
||||
callback; with one, it asks the host to unmount it instead. The
|
||||
[global JS controls](./docs/controls.md) on `window` are unchanged and remain
|
||||
page-wide, so with several instances on one page they apply to all of them.
|
||||
|
||||
Each release publishes the package to npm, with the same version as the release:
|
||||
|
||||
```sh
|
||||
npm install @element-hq/element-call-component
|
||||
```
|
||||
|
||||
Releases publish the package with the same version as Element Call. For an
|
||||
unreleased commit, depend on this repository directly:
|
||||
To use an unreleased commit, a host can instead install it as a git dependency
|
||||
on the `component` directory of this repository,
|
||||
|
||||
```json
|
||||
"@element-hq/element-call-component": "github:element-hq/element-call#main&path:/component"
|
||||
```
|
||||
|
||||
This builds on install, so the host needs pnpm (via Corepack) and must allow
|
||||
the build (`allowBuilds` in its `pnpm-workspace.yaml`).
|
||||
whose `prepare` script runs the build on install. That build needs pnpm (via
|
||||
Corepack) on the host's machine, runs a full `pnpm install` of this repository,
|
||||
since it inherits the `--max-old-space-size` setting of
|
||||
the app build; the host's pnpm also has to allow it to run at all
|
||||
(`allowBuilds` in its `pnpm-workspace.yaml`). Note that `component/` is a pnpm
|
||||
project of its own for this reason, so pnpm commands run from inside that
|
||||
directory target it rather than the repository; run them from the repository
|
||||
root. The host imports the component from
|
||||
`@element-hq/element-call-component` and the stylesheet from
|
||||
`@element-hq/element-call-component/style.css`, and has to provide `react`,
|
||||
`react-dom`, `matrix-js-sdk`, `@vector-im/compound-web` and
|
||||
`@vector-im/compound-design-tokens` itself, since the bundle leaves them
|
||||
external.
|
||||
Compound's stylesheet are the host's to load too: the component shares
|
||||
the host's copy of Compound, (required for example for tooltips)
|
||||
|
||||
##### Use
|
||||
The component is large, and a host will usually load it lazily, only once a
|
||||
call is shown. Everything a host needs in order to talk about a call before
|
||||
then is also exported
|
||||
from `@element-hq/element-call-component/api` (a small entry point that does not
|
||||
load the component).
|
||||
|
||||
1. Provide the externals yourself: `react`, `react-dom`, `matrix-js-sdk`,
|
||||
`@vector-im/compound-web`, `@vector-im/compound-design-tokens`, and
|
||||
Compound's stylesheet.
|
||||
2. Import `@element-hq/element-call-component/style.css`.
|
||||
3. Await `initializeElementCall(config)` once before rendering.
|
||||
4. Render the component with `client`, `roomId` and an `intent`; optionally
|
||||
pass `config`, `hostBridge`, `language` and `theme` (both can change
|
||||
mid-call). Control it through the `ref` handle (`join`, `hangUp`,
|
||||
`setDeviceMute`).
|
||||
|
||||
Load the component lazily. Types, enums and `configurationForIntent` are
|
||||
available without it from `@element-hq/element-call-component/api`. The full
|
||||
API is in `component/index.tsx`, `component/api.ts` and `component/host.ts`.
|
||||
|
||||
##### How it differs from the widget
|
||||
|
||||
- It lays out for its container, not the window: a small container gets the
|
||||
picture-in-picture layout.
|
||||
- Its styles and keyboard shortcuts are scoped to its root, so the host page is
|
||||
untouched and several instances can share a page. The
|
||||
[global JS controls](./docs/controls.md) stay page-wide.
|
||||
- It draws a solid background by default and never offers profile editing.
|
||||
- If the host bridge implements `close()`, it asks to be unmounted instead of
|
||||
showing its own post-call and error screens.
|
||||
- Sounds, images and the MediaPipe runtime are separate files the host's
|
||||
bundler copies (webpack 5, Vite, Rollup); MediaPipe is fetched only when a
|
||||
background effect is first used.
|
||||
|
||||
##### Develop
|
||||
|
||||
- `pnpm build:component` builds the library into `component/dist`.
|
||||
- `pnpm dev:component` serves a test host on `https://localhost:3001`: two
|
||||
calls side by side in resizable boxes, with host messages logged at the
|
||||
bottom. It uses the development backend, `public/config.json` if present,
|
||||
and needs the same certificate exception as `pnpm dev` (see
|
||||
[Backend](#backend)).
|
||||
- Run pnpm commands from the repository root; `component/` is its own pnpm
|
||||
project.
|
||||
- props and the types they are made of
|
||||
- the handle
|
||||
- the host bridge
|
||||
- `UserIntent`
|
||||
- `HeaderStyle`
|
||||
- `BackgroundStyle` enums
|
||||
- `configurationForIntent`, the defaults each intent implies
|
||||
Import from it wherever a value such as
|
||||
`BackgroundStyle.Solid` is needed on a path that must stay light; the main
|
||||
entry point re-exports all of it too.
|
||||
|
||||
### Backend
|
||||
|
||||
|
||||
@@ -26,7 +26,8 @@
|
||||
"features": {
|
||||
"feature_element_call_video_rooms": true,
|
||||
"feature_video_rooms": true,
|
||||
"feature_group_calls": true
|
||||
"feature_group_calls": true,
|
||||
"feature_release_announcement": false
|
||||
},
|
||||
"default_federate": true,
|
||||
"default_theme": "light",
|
||||
@@ -39,10 +40,7 @@
|
||||
},
|
||||
"setting_defaults": {
|
||||
"breadcrumbs": true,
|
||||
"feature_group_calls": true,
|
||||
"releaseAnnouncementData": {
|
||||
"room_list_section": true
|
||||
}
|
||||
"feature_group_calls": true
|
||||
},
|
||||
"jitsi": {
|
||||
"preferred_domain": "meet.element.io"
|
||||
|
||||
@@ -26,7 +26,8 @@
|
||||
"features": {
|
||||
"feature_element_call_video_rooms": true,
|
||||
"feature_video_rooms": true,
|
||||
"feature_group_calls": true
|
||||
"feature_group_calls": true,
|
||||
"feature_release_announcement": false
|
||||
},
|
||||
"default_federate": true,
|
||||
"default_theme": "light",
|
||||
@@ -39,10 +40,7 @@
|
||||
},
|
||||
"setting_defaults": {
|
||||
"breadcrumbs": true,
|
||||
"feature_group_calls": true,
|
||||
"releaseAnnouncementData": {
|
||||
"room_list_section": true
|
||||
}
|
||||
"feature_group_calls": true
|
||||
},
|
||||
"jitsi": {
|
||||
"preferred_domain": "meet.element.io"
|
||||
|
||||
@@ -19,12 +19,7 @@ services:
|
||||
- ./backend/playwright_homeserver-othersite.yaml:/data/cfg/homeserver.yaml:Z
|
||||
element-web:
|
||||
# Pin to a SHA so that upstream cannot break our tests. Renovate handles regular updates.
|
||||
image: ghcr.io/element-hq/element-web:develop@sha256:72791f0b2541208f8f7edd9c6b5a7eaf88accd8ed1f0c4db1bc62bfe18eb90fe
|
||||
image: ghcr.io/element-hq/element-web:develop@sha256:e063472b6bdf6d0bfa1d43dfa4ef7effee1efd2c9684daee665d1bedfe746a47
|
||||
element-web-1:
|
||||
# Pin to a SHA so that upstream cannot break our tests. Renovate handles regular updates.
|
||||
image: ghcr.io/element-hq/element-web:develop@sha256:72791f0b2541208f8f7edd9c6b5a7eaf88accd8ed1f0c4db1bc62bfe18eb90fe
|
||||
volumes:
|
||||
# The other site opens the embedded package under test in place of the
|
||||
# copy its image bundles (the first site opens the dev server's app.
|
||||
# Built by the Playwright run (see build.setup.ts).
|
||||
- ./embedded/web/dist:/app/widgets/element-call:ro,Z
|
||||
image: ghcr.io/element-hq/element-web:develop@sha256:e063472b6bdf6d0bfa1d43dfa4ef7effee1efd2c9684daee665d1bedfe746a47
|
||||
|
||||
+1
-5
@@ -14,11 +14,7 @@ This folder contains documentation for setup, usage, and development of Element
|
||||
|
||||
Entry point: [AGENTS.md](../AGENTS.md).
|
||||
|
||||
- [Agent workflow](./agents/workflow.md)
|
||||
- [Architecture](./agents/architecture.md)
|
||||
- [Code style](./agents/code-style.md)
|
||||
- [Testing](./agents/testing.md)
|
||||
|
||||
Opt-in skills in [`.claude/skills/`](../.claude/skills/):
|
||||
|
||||
- `/handoff <task>` — get the first implementation back for a direction check
|
||||
before lint, the full suite and polish.
|
||||
|
||||
@@ -34,10 +34,6 @@ translations. The browser is Playwright's, so a fresh clone needs
|
||||
|
||||
## End-to-end
|
||||
|
||||
The Element Web instance `HOST1` opens the app the dev server serves. The instance
|
||||
`HOST2` (the other site) opens the embedded package as a host ships it. This package
|
||||
is built by the `build-embedded` setup project and mounted in `docker-compose-playwright.yml`.
|
||||
|
||||
- `playwright/*.spec.ts` — standalone, on Chromium and Firefox.
|
||||
- `playwright/widget/` — widget mode, where cross-context bugs surface.
|
||||
- `playwright/component/` — the component in a host page, via the harness on port
|
||||
@@ -52,8 +48,6 @@ Test what a user observes: the peer sees the change, it survives a reconnect, it
|
||||
right after a reload. Reuse `playwright/spa-helpers.ts`,
|
||||
`playwright/widget/test-helpers.ts`, `playwright/fixtures/`.
|
||||
|
||||
To mark new release announcements as viewed, add them in `backend/ew.test*.config.json`.
|
||||
|
||||
```sh
|
||||
pnpm test # unit + storybook
|
||||
pnpm backend # Synapse + LiveKit, required for e2e
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
# Agent workflow
|
||||
|
||||
## Scratch files live in `agent-workspace/`
|
||||
|
||||
Git-ignored. One kebab-case subfolder per task, matching the branch topic:
|
||||
`agent-workspace/<slug>/`.
|
||||
|
||||
| File | Purpose |
|
||||
| --------------------------- | -------------------------------------------------------------------------------- |
|
||||
| `plan.md` | The concrete approach — files, symbols, edit order — once direction is confirmed |
|
||||
| `commit-msg.txt` | `git commit -F agent-workspace/<slug>/commit-msg.txt` |
|
||||
| `pr-body.md` | `gh pr create --body-file agent-workspace/<slug>/pr-body.md` |
|
||||
| `implementation-summary.md` | What was built, decisions, trade-offs |
|
||||
| `NN-prompt.md` | Raw prompt text, numbered, when worth keeping |
|
||||
|
||||
- Nothing here is durable. The folder is ignored, so anything worth keeping is
|
||||
promoted into the PR body, a doc, or a feature spec before the task ends.
|
||||
- Never write scratch files to the repo root. Stray `load_test_summary.md` and
|
||||
`config.json_` files are what this folder prevents.
|
||||
- Start `pr-body.md` from `.github/PULL_REQUEST_TEMPLATE.md` and fill every
|
||||
section.
|
||||
- Commit subjects are plain imperative English. No conventional-commits prefixes.
|
||||
- A `plan.md` and a feature spec sit at different altitudes; neither replaces the
|
||||
other. A spec in `FEATURES_SPEC/` is durable and deliberately abstract —
|
||||
behaviour, decisions, acceptance criteria — so the feature can be rebuilt against
|
||||
a `main` nobody has seen yet. A plan is one slice of it landing on today's
|
||||
`main`: the paths, symbols and edit order the spec's decisions and criteria must
|
||||
not name. `FEATURES_SPEC/AGENTS.md` wins wherever the two genuinely overlap.
|
||||
|
||||
## Hand off before the quality pass
|
||||
|
||||
- Implement the change, run the narrowest check that rules out an obviously broken
|
||||
handoff, then stop and ask whether the direction is right.
|
||||
- Full `pnpm lint`, the whole suite, coverage and benchmarks come after the
|
||||
direction is confirmed.
|
||||
- Wider refactors, extra tests and documentation polish are follow-up work, not
|
||||
part of the first handoff.
|
||||
- If a check is needed before feedback, keep it to the touched code and say why.
|
||||
|
||||
## Commit and PR readiness
|
||||
|
||||
- Commit once the user confirms direction, or asks for one. Not before. Where the
|
||||
setup has no git identity, a sandbox included, write the message to
|
||||
`commit-msg.txt` and hand it over instead.
|
||||
- Ready means every gate in [AGENTS.md](../../AGENTS.md#gates) is green and the
|
||||
change is covered at the layers [testing.md](./testing.md) asks for. Read the diff
|
||||
against [code-style.md](./code-style.md) first.
|
||||
- Re-run the whole checklist after any fix. Only on green, either way.
|
||||
- Once a human has started reviewing, fix forward — never force-push a
|
||||
regeneration over a review in progress.
|
||||
+1
-1
@@ -107,7 +107,7 @@
|
||||
"livekit-client": "^2.18.1",
|
||||
"lodash-es": "^4.17.21",
|
||||
"loglevel": "^1.9.1",
|
||||
"matrix-js-sdk": "github:matrix-org/matrix-js-sdk#dc8610e796c92553c658300a4882dde961a0678a",
|
||||
"matrix-js-sdk": "github:matrix-org/matrix-js-sdk#a9ef09fbf2f1644c8663d0aeaf6a921cd16e7b01",
|
||||
"matrix-widget-api": "^1.18.0",
|
||||
"node-stdlib-browser": "^1.3.1",
|
||||
"normalize.css": "^8.0.1",
|
||||
|
||||
+21
-33
@@ -26,19 +26,6 @@ process.env.NODE_EXTRA_CA_CERTS = join(
|
||||
"backend/dev_tls_local-ca.crt",
|
||||
);
|
||||
|
||||
const desktopChrome = {
|
||||
...devices["Desktop Chrome"],
|
||||
permissions: ["clipboard-write", "clipboard-read", "microphone", "camera"],
|
||||
ignoreHTTPSErrors: true,
|
||||
launchOptions: {
|
||||
args: [
|
||||
"--use-fake-ui-for-media-stream",
|
||||
"--use-fake-device-for-media-stream",
|
||||
"--mute-audio",
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* See https://playwright.dev/docs/test-configuration.
|
||||
*/
|
||||
@@ -56,11 +43,11 @@ export default defineConfig({
|
||||
* Reporter to use. See https://playwright.dev/docs/test-reporters
|
||||
*
|
||||
* On CI, `list` prints a line per test as it finishes, which keeps the job
|
||||
* log moving; `github` annotates failures on the PR; the `blob` format is
|
||||
* uploaded by sharded CI runs and combined into a HTML report by the workflow.
|
||||
* log moving; `github` annotates failures on the PR; `html` is uploaded as
|
||||
* an artifact and must never try to open a browser there.
|
||||
*/
|
||||
reporter: process.env.CI
|
||||
? [["list"], ["github"], ["blob"]]
|
||||
? [["list"], ["github"], ["html", { open: "never" }]]
|
||||
: [["list"], ["html"]],
|
||||
/* Stop a broken run early on CI instead of waiting out every retry. */
|
||||
maxFailures: process.env.CI ? 5 : undefined,
|
||||
@@ -76,28 +63,29 @@ export default defineConfig({
|
||||
},
|
||||
/* Configure projects for major browsers */
|
||||
projects: [
|
||||
{
|
||||
// Builds the embedded package into the directory the other site's
|
||||
// Element Web container serves it from (see docker-compose-playwright.yml).
|
||||
name: "build-embedded",
|
||||
testMatch: "**/widget/embedded/build.setup.ts",
|
||||
},
|
||||
{
|
||||
name: "chromium",
|
||||
testIgnore: ["**/mobile/**", "**/widget/**"],
|
||||
use: desktopChrome,
|
||||
},
|
||||
{
|
||||
// Element Call as a widget in Element Web, on Chromium. This uses its
|
||||
// own project so that it can be sharded on CI.
|
||||
name: "widget",
|
||||
dependencies: ["build-embedded"],
|
||||
testMatch: "**/widget/**/*.@(spec|test).ts",
|
||||
use: desktopChrome,
|
||||
testIgnore: "**/mobile/**",
|
||||
use: {
|
||||
...devices["Desktop Chrome"],
|
||||
permissions: [
|
||||
"clipboard-write",
|
||||
"clipboard-read",
|
||||
"microphone",
|
||||
"camera",
|
||||
],
|
||||
ignoreHTTPSErrors: true,
|
||||
launchOptions: {
|
||||
args: [
|
||||
"--use-fake-ui-for-media-stream",
|
||||
"--use-fake-device-for-media-stream",
|
||||
"--mute-audio",
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "firefox",
|
||||
dependencies: ["build-embedded"],
|
||||
testIgnore: "**/mobile/**",
|
||||
use: {
|
||||
...devices["Desktop Firefox"],
|
||||
|
||||
@@ -211,46 +211,6 @@ test.describe("background effects section", () => {
|
||||
});
|
||||
});
|
||||
|
||||
test.describe("a pipeline that fails to build", () => {
|
||||
test("offers no effects for the session, and keeps the choice", async ({
|
||||
page,
|
||||
}) => {
|
||||
test.setTimeout(120_000);
|
||||
await page.goto("/");
|
||||
await SpaHelpers.createCall(page, "Failing user", "Pipeline fails");
|
||||
const preview = page.locator("video").first();
|
||||
const camera = await cameraColour(preview);
|
||||
await page.route("**/*.tflite*", async (route) => route.abort());
|
||||
|
||||
await page.getByRole("button", { name: "Camera", exact: true }).click();
|
||||
const tile = page.getByRole("menuitemradio", { name: "Background 1" });
|
||||
await tile.click();
|
||||
await expect(
|
||||
page.getByText("Background effects are not supported on this platform."),
|
||||
).toBeVisible({ timeout: 30_000 });
|
||||
await expect(tile).toBeDisabled();
|
||||
await page.keyboard.press("Escape");
|
||||
// The camera and microphone carry on without it.
|
||||
expect(distance(await cameraColour(preview), camera)).toBeLessThan(30);
|
||||
await expect(page.getByTestId("incall_mute")).toHaveAttribute(
|
||||
"aria-checked",
|
||||
"true",
|
||||
);
|
||||
|
||||
await page.unroute("**/*.tflite*");
|
||||
await page.reload();
|
||||
await expect(page.getByTestId("lobby_joinCall")).toBeVisible();
|
||||
await page.getByRole("button", { name: "Camera", exact: true }).click();
|
||||
await expect(tile).toHaveAttribute("aria-checked", "true");
|
||||
const picture = await averageColour(tile.locator("img"));
|
||||
await expect
|
||||
.poll(async () => distance(await averageColour(preview), picture), {
|
||||
timeout: 60_000,
|
||||
})
|
||||
.toBeLessThan(distance(camera, picture) / 2);
|
||||
});
|
||||
});
|
||||
|
||||
test.describe("the chosen effect", () => {
|
||||
test("chosen effect survives leaving and rejoining", async ({ page }) => {
|
||||
test.setTimeout(120_000);
|
||||
|
||||
@@ -29,10 +29,47 @@ export interface MyFixtures {
|
||||
addUser: (username: string, host: string) => Promise<UserBaseFixture>;
|
||||
}
|
||||
|
||||
// Minimal config.json for the local element-web instance
|
||||
const CONFIG_JSON = {
|
||||
default_server_config: {
|
||||
"m.homeserver": {
|
||||
base_url: "https://synapse.m.localhost",
|
||||
server_name: "synapse.m.localhost",
|
||||
},
|
||||
},
|
||||
|
||||
element_call: {
|
||||
participant_limit: 8,
|
||||
brand: "Element Call",
|
||||
},
|
||||
|
||||
// The default language is set here for test consistency
|
||||
setting_defaults: {
|
||||
language: "en-GB",
|
||||
feature_group_calls: true,
|
||||
},
|
||||
|
||||
// the location tests want a map style url.
|
||||
map_style_url:
|
||||
"https://api.maptiler.com/maps/streets/style.json?key=fU3vlMsMn4Jb6dnEIFsx",
|
||||
|
||||
features: {
|
||||
// We don't want to go through the feature announcement during the e2e test
|
||||
feature_release_announcement: false,
|
||||
feature_element_call_video_rooms: true,
|
||||
feature_video_rooms: true,
|
||||
feature_group_calls: true,
|
||||
},
|
||||
};
|
||||
|
||||
export const widgetTest = test.extend<MyFixtures>({
|
||||
// allow per-test override: `widgetTest.use({ callType: "dm" })`
|
||||
callType: ["room", { option: true }],
|
||||
asWidget: async ({ browser, callType }, pUse) => {
|
||||
asWidget: async ({ browser, context, callType }, pUse) => {
|
||||
await context.route(`http://localhost:8081/config.json*`, async (route) => {
|
||||
await route.fulfill({ json: CONFIG_JSON });
|
||||
});
|
||||
|
||||
const brooksDisplayName = `brooks_${Date.now()}`;
|
||||
const whistlerDisplayName = `whistler_${Date.now()}`;
|
||||
|
||||
@@ -127,14 +164,13 @@ export const widgetTest = test.extend<MyFixtures>({
|
||||
* Provide a way to add additional users within a test.
|
||||
* The returned user will be registered on the default homeserver, the name will be made unique by appending a timestamp.
|
||||
*/
|
||||
addUser: async ({ browser }, use, testInfo) => {
|
||||
addUser: async ({ browser }, use) => {
|
||||
await use(
|
||||
async (
|
||||
username: string,
|
||||
host: string = HOST1,
|
||||
): Promise<UserBaseFixture> => {
|
||||
// Tests in parallel workers can reach this in the same millisecond.
|
||||
const uniqueSuffix = `${Date.now()}_${testInfo.parallelIndex}`;
|
||||
const uniqueSuffix = Date.now();
|
||||
const { page, clientHandle, mxId } = await TestHelpers.registerUser(
|
||||
browser,
|
||||
`${username.toLowerCase()}_${uniqueSuffix}`,
|
||||
|
||||
@@ -1,20 +0,0 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { test as setup } from "@playwright/test";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { mkdirSync } from "node:fs";
|
||||
|
||||
const EMBEDDED_DIST = "embedded/web/dist";
|
||||
|
||||
setup("build the embedded package", () => {
|
||||
setup.setTimeout(10 * 60 * 1000);
|
||||
mkdirSync(EMBEDDED_DIST, { recursive: true });
|
||||
execFileSync("pnpm", ["build:embedded", "--outDir", EMBEDDED_DIST], {
|
||||
stdio: "inherit",
|
||||
});
|
||||
});
|
||||
@@ -1,56 +0,0 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { expect, type Route } from "@playwright/test";
|
||||
|
||||
import { widgetTest } from "../../fixtures/widget-user.ts";
|
||||
import { HOST2, TestHelpers } from "../test-helpers.ts";
|
||||
|
||||
/** Where the other site's Element Web serves the embedded package from */
|
||||
const CONFIG_JSON = "**/widgets/element-call/config.json";
|
||||
|
||||
const HOST_ANSWERS: Record<string, (route: Route) => Promise<void>> = {
|
||||
"a 404": async (route) => route.fulfill({ status: 404 }),
|
||||
"a failed fetch": async (route) => route.abort("failed"),
|
||||
"an HTML page": async (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: "text/html",
|
||||
body: "<!doctype html><title>Element</title>",
|
||||
}),
|
||||
};
|
||||
|
||||
widgetTest.skip(
|
||||
({ browserName }) => browserName === "firefox",
|
||||
"The widget suite does not run on Firefox, see the other specs here",
|
||||
);
|
||||
|
||||
for (const [answer, respond] of Object.entries(HOST_ANSWERS)) {
|
||||
widgetTest(
|
||||
`reaches the lobby when config.json gets ${answer}`,
|
||||
async ({ addUser }) => {
|
||||
const { page } = await addUser("Brooks", HOST2);
|
||||
await TestHelpers.createRoom("Config Room", page);
|
||||
|
||||
let answered = 0;
|
||||
await page.route(CONFIG_JSON, async (route) => {
|
||||
answered++;
|
||||
await respond(route);
|
||||
});
|
||||
|
||||
await TestHelpers.startCallInCurrentRoom(page);
|
||||
|
||||
await expect(
|
||||
page
|
||||
.locator('iframe[title="Element Call"]')
|
||||
.contentFrame()
|
||||
.getByTestId("lobby_joinCall"),
|
||||
).toBeVisible({ timeout: 15000 });
|
||||
expect(answered).toBeGreaterThan(0);
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -144,11 +144,7 @@ export class TestHelpers {
|
||||
|
||||
await this.dismissStartupToasts(page);
|
||||
|
||||
// The other site's Element Web opens the embedded package it serves itself
|
||||
// (see docker-compose-playwright.yml), not the dev server's app.
|
||||
if (host !== HOST2) {
|
||||
await TestHelpers.setDevToolElementCallDevUrl(page);
|
||||
}
|
||||
await TestHelpers.setDevToolElementCallDevUrl(page);
|
||||
|
||||
const clientHandle = await page.evaluateHandle(() =>
|
||||
window.mxMatrixClientPeg.get(),
|
||||
@@ -157,10 +153,9 @@ export class TestHelpers {
|
||||
return { page, clientHandle, mxId };
|
||||
}
|
||||
|
||||
// Dismisses the toasts a fresh device gets after sign-in.
|
||||
//
|
||||
// Toasts stack and only the top one can be dismissed, so look
|
||||
// at what is on top each time.
|
||||
// Dismisses any toasts that appear on startup, such as "Failed to load service worker" or "Back up your chats".
|
||||
// Toast can be stacked, and only the top one can be dismiss, so just look at what is on top and
|
||||
// dismiss (if part of expected toats)
|
||||
public static async dismissStartupToasts(page: Page): Promise<void> {
|
||||
const expectedToasts: {
|
||||
title: string;
|
||||
@@ -168,7 +163,6 @@ export class TestHelpers {
|
||||
confirm?: string;
|
||||
}[] = [
|
||||
{ title: "Failed to load service worker", button: "OK" },
|
||||
{ title: "Verify this device", button: "Later" },
|
||||
{ title: "Back up your chats", button: "Not now" },
|
||||
{
|
||||
title: "Turn on key storage",
|
||||
@@ -219,6 +213,30 @@ export class TestHelpers {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clears what Element Web stacks over the room list after sign-in. The
|
||||
* release announcement is held back while any toast is up, so it is closed
|
||||
* after the toasts, not before.
|
||||
*/
|
||||
public static async clearRoomListOverlays(page: Page): Promise<void> {
|
||||
await TestHelpers.dismissStartupToasts(page);
|
||||
await TestHelpers.closeReleaseAnnouncement(page, "Introducing Sections");
|
||||
}
|
||||
|
||||
public static async closeReleaseAnnouncement(
|
||||
page: Page,
|
||||
name: string,
|
||||
): Promise<void> {
|
||||
try {
|
||||
await page
|
||||
.getByRole("dialog", { name })
|
||||
.getByRole("button", { name: "OK" })
|
||||
.click({ timeout: 2000 });
|
||||
} catch {
|
||||
// Announcement not shown; nothing to do
|
||||
}
|
||||
}
|
||||
|
||||
public static async expandAllSections(page: Page): Promise<void> {
|
||||
try {
|
||||
await page
|
||||
@@ -264,14 +282,15 @@ export class TestHelpers {
|
||||
/**
|
||||
* Opens the room list's "New" menu and picks the given item.
|
||||
*
|
||||
* Driven by keyboard, as a toast arriving late sits over the button and
|
||||
* swallows clicks.
|
||||
* While the sections announcement is anchored to the "New" button, the
|
||||
* button has no menu at all, so the overlays go first. Driven by keyboard,
|
||||
* as a toast arriving afterwards sits over the button and swallows clicks.
|
||||
*/
|
||||
public static async chooseFromRoomListNewMenu(
|
||||
page: Page,
|
||||
itemName: string,
|
||||
): Promise<void> {
|
||||
await TestHelpers.dismissStartupToasts(page);
|
||||
await TestHelpers.clearRoomListOverlays(page);
|
||||
await page
|
||||
.getByRole("navigation", { name: "Room list" })
|
||||
.getByRole("button", { name: "New", exact: true })
|
||||
@@ -408,8 +427,9 @@ export class TestHelpers {
|
||||
name: "Toggle Invites section",
|
||||
});
|
||||
await expect(invitesSection).toBeVisible({ timeout: 10000 });
|
||||
// Toasts sit over the top of the room list, where this header is.
|
||||
await TestHelpers.dismissStartupToasts(page);
|
||||
// Toasts and the announcement sit over the top of the room list, where
|
||||
// this header is
|
||||
await TestHelpers.clearRoomListOverlays(page);
|
||||
if ((await invitesSection.getAttribute("aria-expanded")) !== "true") {
|
||||
await invitesSection.click();
|
||||
}
|
||||
|
||||
Generated
+26
-39
@@ -8,7 +8,7 @@ overrides:
|
||||
'@livekit/components-core>rxjs': ^7.8.1
|
||||
'@livekit/track-processors>@mediapipe/tasks-vision': ^0.10.18
|
||||
tar: ^7.5.11
|
||||
minimatch: ^11.0.0
|
||||
minimatch: ^10.2.3
|
||||
glob: ^10.5.0
|
||||
qs: ^6.15.2
|
||||
js-yaml: ^4.2.0
|
||||
@@ -191,8 +191,8 @@ importers:
|
||||
specifier: ^1.9.1
|
||||
version: 1.9.2
|
||||
matrix-js-sdk:
|
||||
specifier: github:matrix-org/matrix-js-sdk#dc8610e796c92553c658300a4882dde961a0678a
|
||||
version: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/dc8610e796c92553c658300a4882dde961a0678a
|
||||
specifier: github:matrix-org/matrix-js-sdk#a9ef09fbf2f1644c8663d0aeaf6a921cd16e7b01
|
||||
version: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/a9ef09fbf2f1644c8663d0aeaf6a921cd16e7b01
|
||||
matrix-widget-api:
|
||||
specifier: ^1.18.0
|
||||
version: 1.20.0
|
||||
@@ -1145,9 +1145,6 @@ packages:
|
||||
resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==}
|
||||
engines: {node: '>=12'}
|
||||
|
||||
'@isaacs/nonrecursive-glob-match@1.0.3':
|
||||
resolution: {integrity: sha512-SI7kyKHq4XXDRozw+r9GlRZkia1ZkFrpLW98r2IC2Bpr8YkFGE3fvjzmxCG0x78A++69HmDNrDvSMYBZmHGpKA==}
|
||||
|
||||
'@joshwooding/vite-plugin-react-docgen-typescript@0.7.0':
|
||||
resolution: {integrity: sha512-qvsTEwEFefhdirGOPnu9Wp6ChfIwy2dBCRuETU3uE+4cC+PFoxMSiiEhxk4lOluA34eARHA0OxqsEUYDqRMgeQ==}
|
||||
peerDependencies:
|
||||
@@ -3533,8 +3530,8 @@ packages:
|
||||
boolbase@1.0.0:
|
||||
resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==}
|
||||
|
||||
brace-expansion@5.0.12:
|
||||
resolution: {integrity: sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ==}
|
||||
brace-expansion@5.0.9:
|
||||
resolution: {integrity: sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==}
|
||||
engines: {node: 20 || >=22}
|
||||
|
||||
braces@3.0.3:
|
||||
@@ -4734,8 +4731,8 @@ packages:
|
||||
matrix-events-sdk@0.0.1:
|
||||
resolution: {integrity: sha512-1QEOsXO+bhyCroIe2/A5OwaxHvBm7EsSQ46DEDn8RBIfQwN5HWBpFvyWWR4QY0KHPPnnJdI99wgRiAl7Ad5qaA==}
|
||||
|
||||
matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/dc8610e796c92553c658300a4882dde961a0678a:
|
||||
resolution: {gitHosted: true, integrity: sha512-Ldt/VkCzBtre6m1cn0FqgrzRV0GrgWIoARgQUQO58ef2Edu2ERFCINPzp/psFS5RwoN3RIy9Rm4oPVc1ATkEWA==, tarball: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/dc8610e796c92553c658300a4882dde961a0678a}
|
||||
matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/a9ef09fbf2f1644c8663d0aeaf6a921cd16e7b01:
|
||||
resolution: {gitHosted: true, integrity: sha512-ZAhn/zB44MgrXqUzi9ffSFcmZY3+2nx7bEgfEJR7chYDFyT/XmlmfCJx7Ar4rOTAYZJRVnMxrEwiEHGDapUi0Q==, tarball: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/a9ef09fbf2f1644c8663d0aeaf6a921cd16e7b01}
|
||||
version: 45.0.0
|
||||
engines: {node: '>=22.0.0'}
|
||||
|
||||
@@ -4771,9 +4768,9 @@ packages:
|
||||
minimalistic-crypto-utils@1.0.1:
|
||||
resolution: {integrity: sha512-JIYlbt6g8i5jKfJ3xz7rF0LXmv2TkDxBLUkiBeZ7bAx4GnnNMr8xFpGnOxn6GhTEHx3SjRrZEoU+j04prX1ktg==}
|
||||
|
||||
minimatch@11.0.4:
|
||||
resolution: {integrity: sha512-/UJ1dEoVQTjPNDh6SGlUo+ueLMmoHTas4q+8yaAR/5t6nLAIiBDJtoYUh20IYleix7ehiQkgczb84PE3uzHkxg==}
|
||||
engines: {node: 20 || >=22}
|
||||
minimatch@10.2.6:
|
||||
resolution: {integrity: sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==}
|
||||
engines: {node: 18 || 20 || >=22}
|
||||
|
||||
minimist@1.2.8:
|
||||
resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==}
|
||||
@@ -4946,8 +4943,8 @@ packages:
|
||||
resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==}
|
||||
engines: {node: '>=10'}
|
||||
|
||||
p-retry@9.0.0:
|
||||
resolution: {integrity: sha512-HuEats8gYn/UUYa5uyveyQGv75mQmwdFtuMgnZlS12dj/7bdnJSO6keaTjgcZUhvkANTj5EO3lACUmV3xmgj0Q==}
|
||||
p-retry@8.0.1:
|
||||
resolution: {integrity: sha512-NAigyu8hfvJe7MsS18mnc4is0MZtau3QMdBklaJKK23oBQb6occO3UPM3vHmTckW8KhfyjMZaUOqdTFEYliHgg==}
|
||||
engines: {node: '>=22'}
|
||||
|
||||
p-try@2.2.0:
|
||||
@@ -5318,10 +5315,6 @@ packages:
|
||||
randomfill@1.0.4:
|
||||
resolution: {integrity: sha512-87lcbR8+MhcWcUiQ+9e+Rwx8MyR2P7qnt15ynUlbm3TU/fjbgz4GsvfSUDTemtCCtVCqb4ZcEFlyPNTh9bBTLw==}
|
||||
|
||||
re2js@2.9.1:
|
||||
resolution: {integrity: sha512-JIT3bEEUZTOR2YS1walNEIfH4cF/cQeZcBUFq7ldqO6lR6g0X5CBq/S8rsa9omzZcsNf8qc36jCkWpKKtncrZg==}
|
||||
engines: {node: '>=20.0.0'}
|
||||
|
||||
react-docgen-typescript@2.4.0:
|
||||
resolution: {integrity: sha512-ZtAp5XTO5HRzQctjPU0ybY0RRCQO19X/8fxn3w7y2VVTUbGHDKULPTL4ky3vB05euSgG5NpALhEhDPvQ56wvXg==}
|
||||
peerDependencies:
|
||||
@@ -6909,7 +6902,7 @@ snapshots:
|
||||
ignore: 5.3.2
|
||||
import-fresh: 3.3.1
|
||||
js-yaml: 4.3.1
|
||||
minimatch: 11.0.4
|
||||
minimatch: 10.2.6
|
||||
strip-json-comments: 3.1.1
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
@@ -6988,7 +6981,7 @@ snapshots:
|
||||
dependencies:
|
||||
'@humanwhocodes/object-schema': 2.0.3
|
||||
debug: 4.4.3(supports-color@7.2.0)
|
||||
minimatch: 11.0.4
|
||||
minimatch: 10.2.6
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
@@ -7124,8 +7117,6 @@ snapshots:
|
||||
wrap-ansi: 8.1.0
|
||||
wrap-ansi-cjs: wrap-ansi@7.0.0
|
||||
|
||||
'@isaacs/nonrecursive-glob-match@1.0.3': {}
|
||||
|
||||
'@joshwooding/vite-plugin-react-docgen-typescript@0.7.0(typescript@5.9.3)(vite@8.2.4(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.103.1)(terser@5.46.1)(yaml@2.9.0))':
|
||||
dependencies:
|
||||
glob: 10.5.0
|
||||
@@ -8785,7 +8776,7 @@ snapshots:
|
||||
'@typescript-eslint/types': 8.69.0
|
||||
'@typescript-eslint/visitor-keys': 8.69.0
|
||||
debug: 4.4.3(supports-color@7.2.0)
|
||||
minimatch: 11.0.4
|
||||
minimatch: 10.2.6
|
||||
semver: 7.8.5
|
||||
tinyglobby: 0.2.17
|
||||
ts-api-utils: 2.5.0(typescript@5.9.3)
|
||||
@@ -8800,7 +8791,7 @@ snapshots:
|
||||
'@typescript-eslint/types': 8.72.0
|
||||
'@typescript-eslint/visitor-keys': 8.72.0
|
||||
debug: 4.4.3(supports-color@7.2.0)
|
||||
minimatch: 11.0.4
|
||||
minimatch: 10.2.6
|
||||
semver: 7.8.5
|
||||
tinyglobby: 0.2.17
|
||||
ts-api-utils: 2.5.0(typescript@5.9.3)
|
||||
@@ -9125,7 +9116,7 @@ snapshots:
|
||||
|
||||
boolbase@1.0.0: {}
|
||||
|
||||
brace-expansion@5.0.12:
|
||||
brace-expansion@5.0.9:
|
||||
dependencies:
|
||||
balanced-match: 4.0.4
|
||||
|
||||
@@ -9670,7 +9661,7 @@ snapshots:
|
||||
json-stable-stringify-without-jsonify: 1.0.1
|
||||
levn: 0.4.1
|
||||
lodash.merge: 4.6.2
|
||||
minimatch: 11.0.4
|
||||
minimatch: 10.2.6
|
||||
natural-compare: 1.4.0
|
||||
optionator: 0.9.4
|
||||
strip-ansi: 6.0.1
|
||||
@@ -9790,7 +9781,7 @@ snapshots:
|
||||
|
||||
filelist@1.0.6:
|
||||
dependencies:
|
||||
minimatch: 11.0.4
|
||||
minimatch: 10.2.6
|
||||
|
||||
fill-range@7.1.1:
|
||||
dependencies:
|
||||
@@ -9890,7 +9881,7 @@ snapshots:
|
||||
dependencies:
|
||||
foreground-child: 3.3.1
|
||||
jackspeak: 3.4.3
|
||||
minimatch: 11.0.4
|
||||
minimatch: 10.2.6
|
||||
minipass: 7.1.3
|
||||
package-json-from-dist: 1.0.1
|
||||
path-scurry: 1.11.1
|
||||
@@ -10015,7 +10006,7 @@ snapshots:
|
||||
jiti: 2.7.0
|
||||
jsonc-parser: 3.3.1
|
||||
magic-string: 1.2.3
|
||||
minimatch: 11.0.4
|
||||
minimatch: 10.2.6
|
||||
ora: 9.4.1
|
||||
react: 19.2.8
|
||||
react-i18next: 17.0.13(i18next@26.4.1(typescript@5.9.3))(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(typescript@5.9.3)
|
||||
@@ -10413,7 +10404,7 @@ snapshots:
|
||||
|
||||
matrix-events-sdk@0.0.1: {}
|
||||
|
||||
matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/dc8610e796c92553c658300a4882dde961a0678a:
|
||||
matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/a9ef09fbf2f1644c8663d0aeaf6a921cd16e7b01:
|
||||
dependencies:
|
||||
'@babel/runtime': 8.0.8
|
||||
'@matrix-org/matrix-sdk-crypto-wasm': 18.9.0
|
||||
@@ -10423,8 +10414,7 @@ snapshots:
|
||||
loglevel: 1.9.2
|
||||
matrix-events-sdk: 0.0.1
|
||||
matrix-widget-api: 1.20.0
|
||||
p-retry: 9.0.0
|
||||
re2js: 2.9.1
|
||||
p-retry: 8.0.1
|
||||
sdp-transform: 3.0.0
|
||||
unhomoglyph: 1.0.6
|
||||
|
||||
@@ -10459,10 +10449,9 @@ snapshots:
|
||||
|
||||
minimalistic-crypto-utils@1.0.1: {}
|
||||
|
||||
minimatch@11.0.4:
|
||||
minimatch@10.2.6:
|
||||
dependencies:
|
||||
'@isaacs/nonrecursive-glob-match': 1.0.3
|
||||
brace-expansion: 5.0.12
|
||||
brace-expansion: 5.0.9
|
||||
|
||||
minimist@1.2.8: {}
|
||||
|
||||
@@ -10746,7 +10735,7 @@ snapshots:
|
||||
dependencies:
|
||||
p-limit: 3.1.0
|
||||
|
||||
p-retry@9.0.0:
|
||||
p-retry@8.0.1:
|
||||
dependencies:
|
||||
is-network-error: 1.3.2
|
||||
|
||||
@@ -11170,8 +11159,6 @@ snapshots:
|
||||
randombytes: 2.1.0
|
||||
safe-buffer: 5.2.1
|
||||
|
||||
re2js@2.9.1: {}
|
||||
|
||||
react-docgen-typescript@2.4.0(typescript@5.9.3):
|
||||
dependencies:
|
||||
typescript: 5.9.3
|
||||
|
||||
+1
-1
@@ -27,7 +27,7 @@ overrides:
|
||||
# Security fixes:
|
||||
# - https://github.com/advisories/GHSA-7r86-cg39-jmmj
|
||||
# - https://github.com/advisories/GHSA-23c5-xmqv-rm74
|
||||
"minimatch": "^11.0.0"
|
||||
"minimatch": "^10.2.3"
|
||||
# Security fix: https://github.com/element-hq/element-call/security/dependabot/109
|
||||
"glob": "^10.5.0"
|
||||
# Security fixes:
|
||||
|
||||
@@ -162,7 +162,6 @@ describe("createLocalMedia", () => {
|
||||
media.release("camera");
|
||||
expect(trackBehind(camera).stop).toHaveBeenCalled();
|
||||
expect(media.tracks$.value).toEqual([microphone]);
|
||||
expect(media.requests.map((r) => r.source)).toEqual(["microphone"]);
|
||||
|
||||
media.stop();
|
||||
expect(trackBehind(microphone).stop).toHaveBeenCalled();
|
||||
|
||||
@@ -107,7 +107,6 @@ export function createLocalMedia(
|
||||
if (source !== "microphone" && source !== "camera") return;
|
||||
capturedTracks.get(source)?.stop();
|
||||
capturedTracks.delete(source);
|
||||
requests.delete(source);
|
||||
publishCapturedTracks();
|
||||
};
|
||||
|
||||
|
||||
@@ -184,7 +184,6 @@ export function createRTCParticipation(
|
||||
encryptMedia: encryptionSystem.kind !== E2eeType.NONE,
|
||||
matrixRTCMode,
|
||||
sendNotificationType: options.sendNotificationType,
|
||||
callIntent: options.callIntent,
|
||||
applicationData: options.applicationData,
|
||||
timings,
|
||||
delayedLeave,
|
||||
|
||||
+4
-8
@@ -12,7 +12,6 @@ Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import {
|
||||
type RTCCallIntent,
|
||||
type RTCNotificationType,
|
||||
type Transport,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
@@ -104,14 +103,11 @@ export interface RTCParticipationOptions {
|
||||
* to a notification (ringing, timeouts, declines) is the application's job.
|
||||
*/
|
||||
sendNotificationType?: RTCNotificationType;
|
||||
/**
|
||||
* The kind of call the local member intends to take part in: `"m.audio"`
|
||||
* or `"m.video"`. Carried by the membership as a field of its own.
|
||||
*/
|
||||
callIntent?: RTCCallIntent;
|
||||
/**
|
||||
* Whatever the application wants to say about itself in the membership,
|
||||
* under namespaced keys. Opaque to the SDK.
|
||||
* under namespaced keys. Opaque to the SDK, apart from `m.call.intent`
|
||||
* (`"audio"` or `"video"`), which the membership format carries as a
|
||||
* field of its own.
|
||||
*/
|
||||
applicationData?: Record<string, unknown>;
|
||||
/** Timings the host has configured; the defaults otherwise. */
|
||||
@@ -291,7 +287,7 @@ export interface RTCMember {
|
||||
memberId: string;
|
||||
/** Which transport this member is on; undefined when the membership has none. */
|
||||
transport$: Behavior<TransportMetadata | undefined>;
|
||||
/** What the member's application says about itself, e.g. `intent`. */
|
||||
/** What the member's application says about itself, e.g. `m.call.intent`. */
|
||||
applicationData$: Behavior<Record<string, unknown>>;
|
||||
|
||||
displayName$: Behavior<string>;
|
||||
|
||||
@@ -60,7 +60,7 @@ describe("joinJsSdkSession", () => {
|
||||
expect(s.join).toHaveBeenCalledWith(ownMemberMock, [transportWithAlias], {
|
||||
notificationType: "ring",
|
||||
callIntent: undefined,
|
||||
applicationData: undefined,
|
||||
applicationData: {},
|
||||
manageMediaKeys: true,
|
||||
delayedLeaveEventRestartMs: 300_000,
|
||||
delayedLeaveEventDelayMs: 3_600_000,
|
||||
@@ -95,14 +95,13 @@ describe("joinJsSdkSession", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("passes the call intent and application data through and sends sticky events in Matrix 2.0 mode", () => {
|
||||
it("lifts the call intent out of the application data and sends sticky events in Matrix 2.0 mode", () => {
|
||||
const s = session();
|
||||
|
||||
joinJsSdkSession("!room:example.org", s, ownMemberMock, transport, {
|
||||
encryptMedia: true,
|
||||
matrixRTCMode: MatrixRTCMode.Matrix_2_0,
|
||||
callIntent: "m.audio",
|
||||
applicationData: { custom: 1 },
|
||||
applicationData: { "m.call.intent": "audio", custom: 1 },
|
||||
timings: defaultSessionTimings,
|
||||
delayedLeave: defaultSessionTimings.delayedLeave,
|
||||
});
|
||||
@@ -111,7 +110,7 @@ describe("joinJsSdkSession", () => {
|
||||
ownMemberMock,
|
||||
[transportWithAlias],
|
||||
expect.objectContaining({
|
||||
callIntent: "m.audio",
|
||||
callIntent: "audio",
|
||||
applicationData: { custom: 1 },
|
||||
unstableSendStickyEvents: true,
|
||||
}),
|
||||
@@ -125,8 +124,7 @@ describe("joinJsSdkSession", () => {
|
||||
joinJsSdkSession("!room:example.org", s, ownMemberMock, transport, {
|
||||
encryptMedia: true,
|
||||
matrixRTCMode: MatrixRTCMode.Matrix_2_0,
|
||||
callIntent: "m.audio",
|
||||
applicationData: { custom: 1 },
|
||||
applicationData: { "m.call.intent": "audio", custom: 1 },
|
||||
timings: defaultSessionTimings,
|
||||
delayedLeave: defaultSessionTimings.delayedLeave,
|
||||
});
|
||||
@@ -135,7 +133,7 @@ describe("joinJsSdkSession", () => {
|
||||
ownMemberMock,
|
||||
[transport],
|
||||
expect.objectContaining({
|
||||
callIntent: "m.audio",
|
||||
callIntent: "audio",
|
||||
applicationData: { custom: 1 },
|
||||
unstableSendStickyEvents: true,
|
||||
}),
|
||||
|
||||
@@ -23,7 +23,6 @@ interface Options {
|
||||
encryptMedia: boolean;
|
||||
matrixRTCMode: MatrixRTCMode;
|
||||
sendNotificationType?: RTCNotificationType;
|
||||
callIntent?: RTCCallIntent;
|
||||
applicationData?: Record<string, unknown>;
|
||||
timings: SessionTimings;
|
||||
/** The delayed leave timings to use: delegated to the SFU or not. */
|
||||
@@ -44,12 +43,15 @@ export function joinJsSdkSession(
|
||||
encryptMedia,
|
||||
matrixRTCMode,
|
||||
sendNotificationType,
|
||||
callIntent,
|
||||
applicationData,
|
||||
timings,
|
||||
delayedLeave,
|
||||
}: Options,
|
||||
): void {
|
||||
// The membership format carries the intent as a field of its own
|
||||
const { "m.call.intent": intent, ...rest } = applicationData ?? {};
|
||||
const callIntent =
|
||||
typeof intent === "string" ? (intent as RTCCallIntent) : undefined;
|
||||
// Give up on the network as soon as either the sync has been down for the
|
||||
// grace period or the delayed leave has probably been sent, whichever is
|
||||
// sooner
|
||||
@@ -76,7 +78,7 @@ export function joinJsSdkSession(
|
||||
{
|
||||
notificationType: sendNotificationType,
|
||||
callIntent,
|
||||
applicationData,
|
||||
applicationData: rest,
|
||||
manageMediaKeys: encryptMedia,
|
||||
delayedLeaveEventRestartMs: delayedLeave.restart_ms,
|
||||
delayedLeaveEventDelayMs: delayedLeave.delay_ms,
|
||||
|
||||
@@ -225,51 +225,6 @@ describe("createLivekitMediaTrack", () => {
|
||||
);
|
||||
});
|
||||
|
||||
// LiveKit's media stream track is the processed one while a processor runs,
|
||||
// and that one names no device; the camera behind it does
|
||||
it.each([
|
||||
{ deviceId: "cam-1", switches: false },
|
||||
{ deviceId: "cam-2", switches: true },
|
||||
])(
|
||||
"leaves a processed camera alone unless its device changes: $deviceId",
|
||||
async ({ deviceId, switches }) => {
|
||||
vi.stubGlobal("MediaStreamTrack", class {});
|
||||
onTestFinished(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
const camera = Object.assign(Object.create(LocalVideoTrack.prototype), {
|
||||
source: Track.Source.Camera,
|
||||
_mediaStreamTrack: {
|
||||
readyState: "live",
|
||||
label: "",
|
||||
getSettings: () => ({ deviceId: "cam-1" }),
|
||||
},
|
||||
// What LiveKit shows as the track, with no device of its own
|
||||
processor: {
|
||||
processedTrack: { label: "", getSettings: () => ({}) },
|
||||
},
|
||||
}) as LocalVideoTrack;
|
||||
const switchActiveDevice = vi.fn().mockResolvedValue(true);
|
||||
const { participant, publication, room } = fakes({
|
||||
track: camera as unknown as FakeTrack,
|
||||
});
|
||||
const track = createLocalLivekitMediaTrack(
|
||||
scope,
|
||||
Object.assign(participant, {
|
||||
isLocal: true,
|
||||
}) as unknown as LocalParticipant,
|
||||
publication as unknown as TrackPublication,
|
||||
Object.assign(room, { switchActiveDevice }),
|
||||
vi.fn(),
|
||||
);
|
||||
|
||||
await track.setDevice(deviceId);
|
||||
expect(switchActiveDevice.mock.calls).toEqual(
|
||||
switches ? [["videoinput", deviceId, false]] : [],
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
// LiveKit stops a camera that is turned off; the processor is built when it
|
||||
// is turned on again
|
||||
it.each([
|
||||
|
||||
@@ -199,13 +199,11 @@ export function createLocalLivekitMediaTrack(
|
||||
throw new MatrixRTCError(`A ${base.source} track has no device`);
|
||||
// LiveKit decides whether to restart by the constraints it keeps, which
|
||||
// a browser that does not echo the device id back (Firefox) leaves
|
||||
// without one; a track already on the device is left alone here. Asked
|
||||
// of the camera track itself: with a processor on, LiveKit's media
|
||||
// stream track is the processed one, which names no device
|
||||
// without one; a track already on the device is left alone here
|
||||
const current = publication.track;
|
||||
if (
|
||||
current instanceof LocalTrack &&
|
||||
(await current.getDeviceId(false)) === deviceId
|
||||
current.mediaStreamTrack.getSettings().deviceId === deviceId
|
||||
)
|
||||
return;
|
||||
await room.switchActiveDevice(
|
||||
|
||||
+8
-10
@@ -18,8 +18,6 @@ Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import {
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
RTC_CALL_INTENT_VIDEO,
|
||||
type RTCCallIntent,
|
||||
type RTCNotificationType,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
@@ -181,40 +179,40 @@ export function configurationForIntent(intent: UserIntent): UrlConfiguration {
|
||||
switch (intent) {
|
||||
case UserIntent.StartNewCall:
|
||||
preset.skipLobby = false;
|
||||
preset.callIntent = RTC_CALL_INTENT_VIDEO;
|
||||
preset.callIntent = "video";
|
||||
break;
|
||||
case UserIntent.JoinExistingCall:
|
||||
// On desktop this will be overridden based on which button was used to join the call
|
||||
preset.skipLobby = false;
|
||||
preset.callIntent = RTC_CALL_INTENT_VIDEO;
|
||||
preset.callIntent = "video";
|
||||
break;
|
||||
case UserIntent.StartNewCallVoice:
|
||||
preset.skipLobby = false;
|
||||
preset.callIntent = RTC_CALL_INTENT_AUDIO;
|
||||
preset.callIntent = "audio";
|
||||
break;
|
||||
case UserIntent.JoinExistingCallVoice:
|
||||
// On desktop this will be overridden based on which button was used to join the call
|
||||
preset.skipLobby = false;
|
||||
preset.callIntent = RTC_CALL_INTENT_AUDIO;
|
||||
preset.callIntent = "audio";
|
||||
break;
|
||||
case UserIntent.StartNewCallDMVoice:
|
||||
preset.callIntent = RTC_CALL_INTENT_AUDIO;
|
||||
preset.callIntent = "audio";
|
||||
// Fall through
|
||||
case UserIntent.StartNewCallDM:
|
||||
preset.skipLobby = true;
|
||||
preset.sendNotificationType = "ring";
|
||||
preset.autoLeaveWhenOthersLeft = true;
|
||||
preset.waitForCallPickup = true;
|
||||
preset.callIntent = preset.callIntent ?? RTC_CALL_INTENT_VIDEO;
|
||||
preset.callIntent = preset.callIntent ?? "video";
|
||||
break;
|
||||
case UserIntent.JoinExistingCallDMVoice:
|
||||
preset.callIntent = RTC_CALL_INTENT_AUDIO;
|
||||
preset.callIntent = "audio";
|
||||
// Fall through
|
||||
case UserIntent.JoinExistingCallDM:
|
||||
// On desktop this will be overridden based on which button was used to join the call
|
||||
preset.skipLobby = true;
|
||||
preset.autoLeaveWhenOthersLeft = true;
|
||||
preset.callIntent = preset.callIntent ?? RTC_CALL_INTENT_VIDEO;
|
||||
preset.callIntent = preset.callIntent ?? "video";
|
||||
break;
|
||||
// Non widget usecase defaults
|
||||
default:
|
||||
|
||||
+6
-10
@@ -21,10 +21,6 @@ import {
|
||||
UserIntent,
|
||||
} from "../src/UrlParams";
|
||||
import { mockConfig } from "./utils/test";
|
||||
import {
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
RTC_CALL_INTENT_VIDEO,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
|
||||
const ROOM_NAME = "roomNameHere";
|
||||
const ROOM_ID = "!d45f138fsd";
|
||||
@@ -279,7 +275,7 @@ describe("UrlParams", () => {
|
||||
).toMatchObject({
|
||||
...startNewCallDefaults("desktop"),
|
||||
skipLobby: false,
|
||||
callIntent: RTC_CALL_INTENT_VIDEO,
|
||||
callIntent: "video",
|
||||
});
|
||||
});
|
||||
|
||||
@@ -326,7 +322,7 @@ describe("UrlParams", () => {
|
||||
).toMatchObject({
|
||||
...startNewCallDefaults("desktop"),
|
||||
skipLobby: false,
|
||||
callIntent: RTC_CALL_INTENT_AUDIO,
|
||||
callIntent: "audio",
|
||||
});
|
||||
});
|
||||
|
||||
@@ -337,7 +333,7 @@ describe("UrlParams", () => {
|
||||
),
|
||||
).toMatchObject({
|
||||
...joinExistingCallDefaults("desktop"),
|
||||
callIntent: RTC_CALL_INTENT_AUDIO,
|
||||
callIntent: "audio",
|
||||
});
|
||||
});
|
||||
|
||||
@@ -352,7 +348,7 @@ describe("UrlParams", () => {
|
||||
sendNotificationType: "ring",
|
||||
autoLeaveWhenOthersLeft: true,
|
||||
waitForCallPickup: true,
|
||||
callIntent: RTC_CALL_INTENT_AUDIO,
|
||||
callIntent: "audio",
|
||||
});
|
||||
});
|
||||
|
||||
@@ -367,7 +363,7 @@ describe("UrlParams", () => {
|
||||
skipLobby: true,
|
||||
autoLeaveWhenOthersLeft: true,
|
||||
waitForCallPickup: false,
|
||||
callIntent: RTC_CALL_INTENT_VIDEO,
|
||||
callIntent: "video",
|
||||
});
|
||||
});
|
||||
|
||||
@@ -381,7 +377,7 @@ describe("UrlParams", () => {
|
||||
skipLobby: true,
|
||||
autoLeaveWhenOthersLeft: true,
|
||||
waitForCallPickup: false,
|
||||
callIntent: RTC_CALL_INTENT_AUDIO,
|
||||
callIntent: "audio",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -35,7 +35,7 @@ import { LayoutSwitch } from "../room/LayoutSwitch";
|
||||
import { type BackgroundEffectGridItem } from "./BackgroundEffectGrid";
|
||||
import { SelfPreview } from "./SelfPreview";
|
||||
import { type UnusableReason } from "../tracks/backgroundImages";
|
||||
import { type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk";
|
||||
import { type ShownCamera } from "../tracks/cameraTrack";
|
||||
|
||||
export interface BackgroundImageRefusal {
|
||||
reason: UnusableReason | "not-kept";
|
||||
@@ -95,7 +95,7 @@ export interface FooterState {
|
||||
/** Why the last file offered couldn't be kept, if it couldn't. */
|
||||
backgroundImageRefusal: BackgroundImageRefusal | undefined;
|
||||
/** The camera to preview, with the effect in force; null while it is off. */
|
||||
cameraTrack: LocalVideoMediaTrack | null;
|
||||
cameraTrack: ShownCamera | null;
|
||||
showFooter: boolean;
|
||||
|
||||
/* This is needed for WindowMode = "flat" */
|
||||
|
||||
@@ -7,17 +7,14 @@ Please see LICENSE in the repository root for full details.
|
||||
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { BehaviorSubject } from "rxjs";
|
||||
import { type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk";
|
||||
|
||||
import {
|
||||
flushPromises,
|
||||
testScope,
|
||||
mockMuteStates,
|
||||
mockMediaDevices,
|
||||
mockVideoTrack,
|
||||
} from "../utils/test";
|
||||
import { type Behavior, constant } from "../state/Behavior";
|
||||
import { type BackgroundEffectsState } from "../state/BackgroundEffects";
|
||||
import { constant } from "../state/Behavior";
|
||||
import type { CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import type { Layout } from "../state/layout-types";
|
||||
import type { SpotlightTileViewModel } from "../state/TileViewModel";
|
||||
@@ -27,6 +24,9 @@ import {
|
||||
createLobbyFooterViewModel,
|
||||
} from "./CallFooterViewModel";
|
||||
import { HeaderStyle } from "../UrlParams";
|
||||
import { type BackgroundEffectsState } from "../state/BackgroundEffects";
|
||||
import { type ShownCamera, SyncedCameraTrack } from "../tracks/cameraTrack";
|
||||
import { type MuteStates } from "../state/MuteStates";
|
||||
import { backgroundEffect as backgroundEffectSetting } from "../settings/settings";
|
||||
import { shippedBackgrounds } from "../tracks/backgroundEffectCatalog";
|
||||
import type * as BackgroundImages from "../tracks/backgroundImages";
|
||||
@@ -81,7 +81,6 @@ vi.mock("livekit-client", () => ({
|
||||
function buildMinimalCallViewModel(layout: Layout): CallViewModel {
|
||||
return {
|
||||
layout$: constant(layout),
|
||||
camera$: constant(undefined),
|
||||
edgeToEdge$: constant(false),
|
||||
showHeader$: constant(false),
|
||||
hangup: (): void => {},
|
||||
@@ -118,11 +117,6 @@ const noPipeline = constant<BackgroundEffectsState>({
|
||||
processor: undefined,
|
||||
});
|
||||
|
||||
/** The provider's verdict, which the SDK's check stands for here. */
|
||||
function pipelineAsTheSdkSays(): Behavior<BackgroundEffectsState> {
|
||||
return constant({ supported: sdkSupportMock(), processor: undefined });
|
||||
}
|
||||
|
||||
const twoMicsAndOneCamMediaDevices = mockMediaDevices({
|
||||
audioInput: {
|
||||
available$: constant(
|
||||
@@ -285,8 +279,7 @@ describe("createCallFooterViewModel", () => {
|
||||
testScope(),
|
||||
mockMuteStates(),
|
||||
twoMicsAndOneCamMediaDevices,
|
||||
pipelineAsTheSdkSays(),
|
||||
/* camera$ */ constant(null),
|
||||
noPipeline,
|
||||
/* openSettings */ undefined,
|
||||
/* hangup */ undefined,
|
||||
/* showLogo */ false,
|
||||
@@ -374,49 +367,22 @@ describe("createCallFooterViewModel", () => {
|
||||
const pipeline$ = new BehaviorSubject<BackgroundEffectsState>({
|
||||
supported: true,
|
||||
processor: undefined,
|
||||
wait: "settling",
|
||||
settling: true,
|
||||
});
|
||||
const vm = createLobbyFooterViewModel(
|
||||
testScope(),
|
||||
mockMuteStates(),
|
||||
twoMicsAndOneCamMediaDevices,
|
||||
pipeline$,
|
||||
/* camera$ */ constant(null),
|
||||
/* openSettings */ undefined,
|
||||
/* hangup */ undefined,
|
||||
/* showLogo */ false,
|
||||
);
|
||||
expect(vm.backgroundEffectSettling$.value).toBe(true);
|
||||
pipeline$.next({ ...pipeline$.value, wait: undefined });
|
||||
pipeline$.next({ ...pipeline$.value, settling: false });
|
||||
expect(vm.backgroundEffectSettling$.value).toBe(false);
|
||||
});
|
||||
|
||||
it("offers nothing once the pipeline fails to build, keeping the choice", () => {
|
||||
platformMock.mockReturnValue("desktop");
|
||||
backgroundEffectSetting.setValue("blur");
|
||||
const pipeline$ = new BehaviorSubject<BackgroundEffectsState>({
|
||||
supported: true,
|
||||
processor: undefined,
|
||||
});
|
||||
const vm = createLobbyFooterViewModel(
|
||||
testScope(),
|
||||
mockMuteStates(),
|
||||
twoMicsAndOneCamMediaDevices,
|
||||
pipeline$,
|
||||
/* camera$ */ constant(null),
|
||||
/* openSettings */ undefined,
|
||||
/* hangup */ undefined,
|
||||
/* showLogo */ false,
|
||||
);
|
||||
expect(vm.backgroundEffect$.value).toBe("blur");
|
||||
|
||||
pipeline$.next({ supported: false, processor: undefined });
|
||||
expect(vm.selectBackgroundEffect$.value).toBeUndefined();
|
||||
expect(vm.addBackgroundImage$.value).toBeUndefined();
|
||||
expect(vm.backgroundEffect$.value).toBe("none");
|
||||
expect(backgroundEffectSetting.getValue()).toBe("blur");
|
||||
});
|
||||
|
||||
it("adding is unavailable at the limit", () => {
|
||||
sdkSupportMock.mockReturnValue(true);
|
||||
const added$ = addedBackgrounds.added$ as BehaviorSubject<
|
||||
@@ -473,44 +439,33 @@ describe("createCallFooterViewModel", () => {
|
||||
expect(store.remove).toHaveBeenCalledWith("spare");
|
||||
});
|
||||
|
||||
it("previews the lobby's camera while there is one", () => {
|
||||
const track = mockVideoTrack({});
|
||||
const camera$ = new BehaviorSubject<LocalVideoMediaTrack | null>(track);
|
||||
it("previews the camera only while it is on", () => {
|
||||
const camera = new SyncedCameraTrack();
|
||||
const track = {} as ShownCamera;
|
||||
camera.report(track);
|
||||
const videoEnabled$ = new BehaviorSubject(true);
|
||||
const muteStates = mockMuteStates();
|
||||
const vm = createLobbyFooterViewModel(
|
||||
testScope(),
|
||||
mockMuteStates(),
|
||||
{
|
||||
...muteStates,
|
||||
video: { ...muteStates.video, enabled$: videoEnabled$ },
|
||||
} as unknown as MuteStates,
|
||||
twoMicsAndOneCamMediaDevices,
|
||||
noPipeline,
|
||||
camera$,
|
||||
constant({
|
||||
supported: true,
|
||||
processor: undefined,
|
||||
cameraTrack: camera,
|
||||
}),
|
||||
/* openSettings */ undefined,
|
||||
/* hangup */ undefined,
|
||||
/* showLogo */ false,
|
||||
);
|
||||
expect(vm.cameraTrack$.value).toBe(track);
|
||||
// The lobby releases the camera when it is turned off
|
||||
camera$.next(null);
|
||||
videoEnabled$.next(false);
|
||||
expect(vm.cameraTrack$.value).toBeNull();
|
||||
});
|
||||
|
||||
it("previews the call's camera only while it is not muted", () => {
|
||||
const muted$ = new BehaviorSubject(false);
|
||||
const track = mockVideoTrack({ muted$ });
|
||||
const vm = createCallFooterViewModel(
|
||||
testScope(),
|
||||
{ ...buildMinimalCallViewModel(gridLayout), camera$: constant(track) },
|
||||
mockMuteStates(),
|
||||
twoMicsAndOneCamMediaDevices,
|
||||
noPipeline,
|
||||
/* reactionIdentifier */ undefined,
|
||||
{ showControls: true, header: HeaderStyle.Standard },
|
||||
);
|
||||
expect(vm.cameraTrack$.value).toBe(track);
|
||||
muted$.next(true);
|
||||
expect(vm.cameraTrack$.value).toBeNull();
|
||||
muted$.next(false);
|
||||
expect(vm.cameraTrack$.value).toBe(track);
|
||||
});
|
||||
|
||||
it("availability is the same before and during a call", () => {
|
||||
for (const supported of [true, false]) {
|
||||
sdkSupportMock.mockReturnValue(supported);
|
||||
@@ -520,7 +475,7 @@ describe("createCallFooterViewModel", () => {
|
||||
buildMinimalCallViewModel(gridLayout),
|
||||
mockMuteStates(),
|
||||
twoMicsAndOneCamMediaDevices,
|
||||
pipelineAsTheSdkSays(),
|
||||
noPipeline,
|
||||
/* reactionIdentifier */ undefined,
|
||||
{ showControls: true, header: HeaderStyle.Standard },
|
||||
);
|
||||
|
||||
@@ -15,8 +15,10 @@ import {
|
||||
switchMap,
|
||||
} from "rxjs";
|
||||
import { supportsAudioOutputSelection } from "livekit-client";
|
||||
import { type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk";
|
||||
import { supportsModernBackgroundProcessors } from "@livekit/track-processors";
|
||||
import {
|
||||
supportsBackgroundProcessors,
|
||||
supportsModernBackgroundProcessors,
|
||||
} from "@livekit/track-processors";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
|
||||
import {
|
||||
@@ -24,6 +26,7 @@ import {
|
||||
serializeEffect,
|
||||
shippedBackgrounds,
|
||||
} from "../tracks/backgroundEffectCatalog";
|
||||
import { type BackgroundEffectsState } from "../state/BackgroundEffects";
|
||||
import {
|
||||
type AddedBackground,
|
||||
addedBackgrounds,
|
||||
@@ -32,7 +35,6 @@ import {
|
||||
} from "../tracks/backgroundImages";
|
||||
|
||||
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import { type BackgroundEffectsState } from "../state/BackgroundEffects";
|
||||
import { type MenuOptions } from "./MediaMuteAndSwitchButton";
|
||||
import { type MediaDevices } from "../state/MediaDevices";
|
||||
import {
|
||||
@@ -88,8 +90,7 @@ function buildDeviceBehaviors(
|
||||
/** return empty arrays for audioOptions and videoOptions*/
|
||||
disableSwitcher$: Behavior<boolean>,
|
||||
backgroundEffectsState$: Observable<BackgroundEffectsState>,
|
||||
/** The camera as the pipeline has it, for the menu's preview. */
|
||||
camera$: Observable<LocalVideoMediaTrack | null | undefined>,
|
||||
videoEnabled$: Behavior<boolean>,
|
||||
): Pick<
|
||||
ViewModel<FooterSnapshot>,
|
||||
| "audioOptions$"
|
||||
@@ -126,14 +127,10 @@ function buildDeviceBehaviors(
|
||||
),
|
||||
);
|
||||
|
||||
// Also false once the pipeline has failed to build this session.
|
||||
const supported$ = backgroundEffectsState$.pipe(
|
||||
map(({ supported }) => supported === true),
|
||||
distinctUntilChanged(),
|
||||
);
|
||||
const supported = supportsBackgroundProcessors();
|
||||
const slow = !supportsModernBackgroundProcessors();
|
||||
const offered$ = combineLatest([disableSwitcher$, supported$]).pipe(
|
||||
map(([switcherDisabled, supported]) => !switcherDisabled && supported),
|
||||
const offered$ = disableSwitcher$.pipe(
|
||||
map((switcherDisabled) => !switcherDisabled && supported),
|
||||
);
|
||||
// A new object for each refusal, so the same one twice is shown twice.
|
||||
const refusal$ = new BehaviorSubject<BackgroundImageRefusal | undefined>(
|
||||
@@ -164,10 +161,8 @@ function buildDeviceBehaviors(
|
||||
),
|
||||
selectVideoButtonOption$: constant(mediaDevices.videoInput.select),
|
||||
backgroundEffect$: scope.behavior(
|
||||
combineLatest([backgroundEffectSetting.value$, supported$]).pipe(
|
||||
map(([raw, supported]) =>
|
||||
supported ? serializeEffect(parseEffect(raw)) : "none",
|
||||
),
|
||||
backgroundEffectSetting.value$.pipe(
|
||||
map((raw) => (supported ? serializeEffect(parseEffect(raw)) : "none")),
|
||||
),
|
||||
),
|
||||
selectBackgroundEffect$: scope.behavior(
|
||||
@@ -196,7 +191,7 @@ function buildDeviceBehaviors(
|
||||
),
|
||||
backgroundEffectSettling$: scope.behavior(
|
||||
backgroundEffectsState$.pipe(
|
||||
map(({ wait }) => wait !== undefined),
|
||||
map(({ settling }) => settling ?? false),
|
||||
distinctUntilChanged(),
|
||||
),
|
||||
false,
|
||||
@@ -220,15 +215,13 @@ function buildDeviceBehaviors(
|
||||
),
|
||||
),
|
||||
backgroundImageRefusal$: refusal$,
|
||||
// None while muted: the call mutes its track rather than releasing it.
|
||||
cameraTrack$: scope.behavior(
|
||||
camera$.pipe(
|
||||
switchMap((camera) =>
|
||||
camera
|
||||
? camera.muted$.pipe(map((muted) => (muted ? null : camera)))
|
||||
: of(null),
|
||||
combineLatest([
|
||||
backgroundEffectsState$.pipe(
|
||||
switchMap(({ cameraTrack }) => cameraTrack?.track$ ?? of(null)),
|
||||
),
|
||||
),
|
||||
videoEnabled$,
|
||||
]).pipe(map(([track, videoEnabled]) => (videoEnabled ? track : null))),
|
||||
null,
|
||||
),
|
||||
// Never the one in force: taking it away would leave nothing chosen.
|
||||
@@ -260,8 +253,8 @@ function buildDeviceBehaviors(
|
||||
* @param callModel - The root CallViewModel; provides layout, grid mode, reactions, etc.
|
||||
* @param muteStates - Audio and video mute state + toggles.
|
||||
* @param mediaDevices - Available and selected input devices.
|
||||
* @param backgroundEffectsState$ - The background pipeline's state: whether
|
||||
* effects run here, and the wait for the first.
|
||||
* @param backgroundEffectsState$ - The background pipeline's state: whether effects
|
||||
* run here, the wait for the first, and the camera it is synced to.
|
||||
* @param reactionIdentifier - The local user's reaction identifier string, or
|
||||
* undefined when reactions are not supported (hides the reaction button).
|
||||
* @param options - `showControls`: whether the call controls should be shown.
|
||||
@@ -292,7 +285,7 @@ export function createCallFooterViewModel(
|
||||
mediaDevices,
|
||||
disableDeviceSwitcher$,
|
||||
backgroundEffectsState$,
|
||||
callModel.camera$,
|
||||
muteStates.video.enabled$,
|
||||
),
|
||||
// candidat to move into the FooterViewModel
|
||||
showFooter$: callModel.showFooter$,
|
||||
@@ -355,9 +348,8 @@ export function createCallFooterViewModel(
|
||||
* @param scope - ObservableScope that bounds the lifetime of derived behaviors.
|
||||
* @param muteStates - Audio and video mute state + toggles.
|
||||
* @param mediaDevices - Available and selected input devices.
|
||||
* @param backgroundEffectsState$ - The background pipeline's state: whether
|
||||
* effects run here, and the wait for the first.
|
||||
* @param camera$ - The lobby's camera, for the camera menu's preview.
|
||||
* @param backgroundEffectsState$ - The background pipeline's state: whether effects
|
||||
* run here, the wait for the first, and the camera it is synced to.
|
||||
* @param openSettings - Callback to open the settings modal, or undefined.
|
||||
* @param hangup - Callback to leave/cancel, or undefined (hides the button).
|
||||
* @param showLogo - Whether to show the Element Call logo.
|
||||
@@ -367,7 +359,6 @@ export function createLobbyFooterViewModel(
|
||||
muteStates: MuteStates,
|
||||
mediaDevices: MediaDevices,
|
||||
backgroundEffectsState$: Observable<BackgroundEffectsState>,
|
||||
camera$: Behavior<LocalVideoMediaTrack | null>,
|
||||
openSettings: (() => void) | undefined,
|
||||
hangup: (() => void) | undefined,
|
||||
showLogo: boolean,
|
||||
@@ -416,7 +407,7 @@ export function createLobbyFooterViewModel(
|
||||
mediaDevices,
|
||||
constant(false),
|
||||
backgroundEffectsState$,
|
||||
camera$,
|
||||
muteStates.video.enabled$,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -114,9 +114,6 @@ const MENU_WIDTH = 296;
|
||||
/** Space kept between the menu and the call area's sides. */
|
||||
const MENU_MARGIN = 16;
|
||||
|
||||
/** Below the preview, as its stylesheet draws it. */
|
||||
const PREVIEW_GAP = 12;
|
||||
|
||||
export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
|
||||
enabled,
|
||||
busy,
|
||||
@@ -218,6 +215,12 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
|
||||
const [previewBlock, setPreviewBlock] = useState(0);
|
||||
useEffect(() => {
|
||||
if (!menuOpen) return;
|
||||
// From the tokens the stylesheets size the preview with, so that it is
|
||||
// placed once rather than measured and moved.
|
||||
const tokens = getComputedStyle(rootElement);
|
||||
const px = (token: string): number =>
|
||||
parseFloat(tokens.getPropertyValue(token)) || 0;
|
||||
const below = px("--cpd-space-2x") + px("--cpd-space-1x");
|
||||
// Followed, since a host can resize the call while the menu is open.
|
||||
const subscription = observeElementSize$(rootElement)
|
||||
.pipe(
|
||||
@@ -231,7 +234,7 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
|
||||
)
|
||||
.subscribe(({ height, width }) => {
|
||||
setListShare(height);
|
||||
setPreviewBlock(Math.round((width * 9) / 16) + PREVIEW_GAP);
|
||||
setPreviewBlock(Math.round((width * 9) / 16) + below);
|
||||
});
|
||||
return (): void => subscription.unsubscribe();
|
||||
}, [menuOpen, rootElement]);
|
||||
|
||||
@@ -1,47 +0,0 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { act, render } from "@testing-library/react";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { BehaviorSubject } from "rxjs";
|
||||
|
||||
import { SelfPreview } from "./SelfPreview";
|
||||
import styles from "./SelfPreview.module.css";
|
||||
import { mockVideoTrack } from "../utils/test";
|
||||
|
||||
describe("SelfPreview", () => {
|
||||
it("detaches only its own element from a camera shown twice", () => {
|
||||
const track = mockVideoTrack({});
|
||||
const first = render(<SelfPreview track={track} />);
|
||||
const second = render(<SelfPreview track={track} />);
|
||||
const [a, b] = [first, second].map((r) =>
|
||||
r.container.querySelector("video"),
|
||||
);
|
||||
expect(track.attach).toHaveBeenCalledWith(a);
|
||||
expect(track.attach).toHaveBeenCalledWith(b);
|
||||
|
||||
first.unmount();
|
||||
expect(track.detach).toHaveBeenCalledExactlyOnceWith(a);
|
||||
});
|
||||
|
||||
it("mirrors the camera while it faces the user, and only then", () => {
|
||||
const facingMode$ = new BehaviorSubject<"user" | "environment">("user");
|
||||
const { container } = render(
|
||||
<SelfPreview track={mockVideoTrack({ facingMode$ })} />,
|
||||
);
|
||||
const video = (): HTMLVideoElement => container.querySelector("video")!;
|
||||
expect(video().classList.contains(styles.mirrored)).toBe(true);
|
||||
act(() => facingMode$.next("environment"));
|
||||
expect(video().classList.contains(styles.mirrored)).toBe(false);
|
||||
});
|
||||
|
||||
it("says the camera is off while there is none", () => {
|
||||
const { container } = render(<SelfPreview track={null} />);
|
||||
expect(container.querySelector("video")).toBeNull();
|
||||
expect(container.querySelector("svg")).not.toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -5,18 +5,16 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { type FC, useEffect, useRef } from "react";
|
||||
import { constant, type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk";
|
||||
import { type FC } from "react";
|
||||
import { VideoCallOffIcon } from "@vector-im/compound-design-tokens/assets/web/icons";
|
||||
|
||||
import styles from "./SelfPreview.module.css";
|
||||
import { useBehavior } from "../useBehavior";
|
||||
|
||||
const noFacingMode = constant(undefined);
|
||||
import { useAttachedTrack } from "../useAttachedTrack";
|
||||
import { type ShownCamera } from "../tracks/cameraTrack";
|
||||
|
||||
interface Props {
|
||||
/** The camera as the pipeline has it, or none while the camera is off. */
|
||||
track: LocalVideoMediaTrack | null;
|
||||
track: ShownCamera | null;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -24,17 +22,7 @@ interface Props {
|
||||
* shown a second time rather than captured again.
|
||||
*/
|
||||
export const SelfPreview: FC<Props> = ({ track }) => {
|
||||
const videoRef = useRef<HTMLVideoElement | null>(null);
|
||||
useEffect(() => {
|
||||
const element = videoRef.current;
|
||||
if (!track || !element) return;
|
||||
track.attach(element);
|
||||
// This element only: the pre-join preview may be showing the same camera
|
||||
return (): void => {
|
||||
track.detach(element);
|
||||
};
|
||||
}, [track]);
|
||||
const facingMode = useBehavior(track?.facingMode$ ?? noFacingMode);
|
||||
const { videoRef, mirrored } = useAttachedTrack(track);
|
||||
|
||||
if (!track) return <VideoCallOffIcon width={32} height={32} />;
|
||||
return (
|
||||
@@ -42,7 +30,7 @@ export const SelfPreview: FC<Props> = ({ track }) => {
|
||||
ref={videoRef}
|
||||
muted
|
||||
playsInline
|
||||
className={facingMode === "user" ? styles.mirrored : undefined}
|
||||
className={mirrored ? styles.mirrored : undefined}
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
||||
@@ -108,22 +108,6 @@ describe("Config.initWith", () => {
|
||||
expect(Config.get().media_quality).toEqual(DEFAULT_CONFIG.media_quality);
|
||||
});
|
||||
|
||||
it("falls back to the defaults when config.json is not JSON", async () => {
|
||||
vi.spyOn(globalThis, "fetch").mockResolvedValue(
|
||||
new Response("<!doctype html><title>Element</title>", {
|
||||
status: 200,
|
||||
headers: { "content-type": "text/html" },
|
||||
}),
|
||||
);
|
||||
const warnSpy = vi.spyOn(logger, "warn").mockImplementation(() => {});
|
||||
Reflect.set(Config, "internalInstance", undefined);
|
||||
|
||||
await expect(Config.init()).resolves.toBeUndefined();
|
||||
|
||||
expect(Config.get().media_quality).toEqual(DEFAULT_CONFIG.media_quality);
|
||||
expect(warnSpy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("replaces a config initialized earlier", () => {
|
||||
Config.initWith({ ssla: "https://first.invalid/ssla" });
|
||||
Config.initWith({ ssla: "https://second.invalid/ssla" });
|
||||
|
||||
@@ -156,10 +156,5 @@ async function downloadConfig(fetchTarget: string): Promise<ConfigOptions> {
|
||||
return {};
|
||||
}
|
||||
|
||||
try {
|
||||
return await response.json();
|
||||
} catch (e) {
|
||||
logger.warn(`Config at ${fetchTarget} is not JSON, using defaults`, e);
|
||||
return {};
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
+24
-24
@@ -7,30 +7,30 @@ Please see LICENSE in the repository root for full details.
|
||||
|
||||
import { type RelationType } from "matrix-js-sdk";
|
||||
|
||||
import catSoundOgg from "../sound/reactions/cat.ogg?url&no-inline";
|
||||
import catSoundMp3 from "../sound/reactions/cat.mp3?url&no-inline";
|
||||
import clapSoundOgg from "../sound/reactions/clap.ogg?url&no-inline";
|
||||
import clapSoundMp3 from "../sound/reactions/clap.mp3?url&no-inline";
|
||||
import cricketsSoundOgg from "../sound/reactions/crickets.ogg?url&no-inline";
|
||||
import cricketsSoundMp3 from "../sound/reactions/crickets.mp3?url&no-inline";
|
||||
import dogSoundOgg from "../sound/reactions/dog.ogg?url&no-inline";
|
||||
import dogSoundMp3 from "../sound/reactions/dog.mp3?url&no-inline";
|
||||
import genericSoundOgg from "../sound/reactions/generic.ogg?url&no-inline";
|
||||
import genericSoundMp3 from "../sound/reactions/generic.mp3?url&no-inline";
|
||||
import lightbulbSoundOgg from "../sound/reactions/lightbulb.ogg?url&no-inline";
|
||||
import lightbulbSoundMp3 from "../sound/reactions/lightbulb.mp3?url&no-inline";
|
||||
import partySoundOgg from "../sound/reactions/party.ogg?url&no-inline";
|
||||
import partySoundMp3 from "../sound/reactions/party.mp3?url&no-inline";
|
||||
import deerSoundOgg from "../sound/reactions/deer.ogg?url&no-inline";
|
||||
import deerSoundMp3 from "../sound/reactions/deer.mp3?url&no-inline";
|
||||
import rockSoundOgg from "../sound/reactions/rock.ogg?url&no-inline";
|
||||
import rockSoundMp3 from "../sound/reactions/rock.mp3?url&no-inline";
|
||||
import waveSoundOgg from "../sound/reactions/wave.ogg?url&no-inline";
|
||||
import waveSoundMp3 from "../sound/reactions/wave.mp3?url&no-inline";
|
||||
import baduntssSoundOgg from "../sound/reactions/baduntss.ogg?url&no-inline";
|
||||
import baduntssSoundMp3 from "../sound/reactions/baduntss.mp3?url&no-inline";
|
||||
import bellSoundOgg from "../sound/reactions/bell.ogg?url&no-inline";
|
||||
import bellSoundMp3 from "../sound/reactions/bell.mp3?url&no-inline";
|
||||
import catSoundOgg from "../sound/reactions/cat.ogg?url";
|
||||
import catSoundMp3 from "../sound/reactions/cat.mp3?url";
|
||||
import clapSoundOgg from "../sound/reactions/clap.ogg?url";
|
||||
import clapSoundMp3 from "../sound/reactions/clap.mp3?url";
|
||||
import cricketsSoundOgg from "../sound/reactions/crickets.ogg?url";
|
||||
import cricketsSoundMp3 from "../sound/reactions/crickets.mp3?url";
|
||||
import dogSoundOgg from "../sound/reactions/dog.ogg?url";
|
||||
import dogSoundMp3 from "../sound/reactions/dog.mp3?url";
|
||||
import genericSoundOgg from "../sound/reactions/generic.ogg?url";
|
||||
import genericSoundMp3 from "../sound/reactions/generic.mp3?url";
|
||||
import lightbulbSoundOgg from "../sound/reactions/lightbulb.ogg?url";
|
||||
import lightbulbSoundMp3 from "../sound/reactions/lightbulb.mp3?url";
|
||||
import partySoundOgg from "../sound/reactions/party.ogg?url";
|
||||
import partySoundMp3 from "../sound/reactions/party.mp3?url";
|
||||
import deerSoundOgg from "../sound/reactions/deer.ogg?url";
|
||||
import deerSoundMp3 from "../sound/reactions/deer.mp3?url";
|
||||
import rockSoundOgg from "../sound/reactions/rock.ogg?url";
|
||||
import rockSoundMp3 from "../sound/reactions/rock.mp3?url";
|
||||
import waveSoundOgg from "../sound/reactions/wave.ogg?url";
|
||||
import waveSoundMp3 from "../sound/reactions/wave.mp3?url";
|
||||
import baduntssSoundOgg from "../sound/reactions/baduntss.ogg?url";
|
||||
import baduntssSoundMp3 from "../sound/reactions/baduntss.mp3?url";
|
||||
import bellSoundOgg from "../sound/reactions/bell.ogg?url";
|
||||
import bellSoundMp3 from "../sound/reactions/bell.mp3?url";
|
||||
|
||||
export const ElementCallReactionEventType = "io.element.call.reaction";
|
||||
|
||||
|
||||
@@ -8,18 +8,18 @@ Please see LICENSE in the repository root for full details.
|
||||
import { type ReactNode, useEffect } from "react";
|
||||
|
||||
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import joinCallSoundMp3 from "../sound/join_call.mp3?url&no-inline";
|
||||
import joinCallSoundOgg from "../sound/join_call.ogg?url&no-inline";
|
||||
import leftCallSoundMp3 from "../sound/left_call.mp3?url&no-inline";
|
||||
import leftCallSoundOgg from "../sound/left_call.ogg?url&no-inline";
|
||||
import handSoundOgg from "../sound/raise_hand.ogg?url&no-inline";
|
||||
import handSoundMp3 from "../sound/raise_hand.mp3?url&no-inline";
|
||||
import screenShareStartedOgg from "../sound/screen_share_started.ogg?url&no-inline";
|
||||
import screenShareStartedMp3 from "../sound/screen_share_started.mp3?url&no-inline";
|
||||
import declineMp3 from "../sound/call_declined.mp3?url&no-inline";
|
||||
import declineOgg from "../sound/call_declined.ogg?url&no-inline";
|
||||
import timeoutMp3 from "../sound/call_timeout.mp3?url&no-inline";
|
||||
import timeoutOgg from "../sound/call_timeout.ogg?url&no-inline";
|
||||
import joinCallSoundMp3 from "../sound/join_call.mp3";
|
||||
import joinCallSoundOgg from "../sound/join_call.ogg";
|
||||
import leftCallSoundMp3 from "../sound/left_call.mp3";
|
||||
import leftCallSoundOgg from "../sound/left_call.ogg";
|
||||
import handSoundOgg from "../sound/raise_hand.ogg";
|
||||
import handSoundMp3 from "../sound/raise_hand.mp3";
|
||||
import screenShareStartedOgg from "../sound/screen_share_started.ogg";
|
||||
import screenShareStartedMp3 from "../sound/screen_share_started.mp3";
|
||||
import declineMp3 from "../sound/call_declined.mp3?url";
|
||||
import declineOgg from "../sound/call_declined.ogg?url";
|
||||
import timeoutMp3 from "../sound/call_timeout.mp3?url";
|
||||
import timeoutOgg from "../sound/call_timeout.ogg?url";
|
||||
import { useAudioContext } from "../useAudioContext";
|
||||
import { prefetchSounds } from "../soundUtils";
|
||||
import { useLatest } from "../useLatest";
|
||||
|
||||
+19
-4
@@ -75,6 +75,7 @@ import {
|
||||
type RTCSlotOptions,
|
||||
constant,
|
||||
} from "@element-hq/matrixrtc-sdk";
|
||||
import { map } from "rxjs";
|
||||
import {
|
||||
advancedCamera,
|
||||
advancedScreenShare,
|
||||
@@ -109,7 +110,11 @@ import { useBehavior } from "../useBehavior.ts";
|
||||
import { useValueBehavior } from "../useValueBehavior.ts";
|
||||
import { Toast } from "../Toast.tsx";
|
||||
import overlayStyles from "../Overlay.module.css";
|
||||
import { useBackgroundEffectsState$ } from "../tracks/BackgroundEffectsContext";
|
||||
import {
|
||||
reportCamera,
|
||||
useBackgroundEffectsState$,
|
||||
} from "../tracks/BackgroundEffectsContext.tsx";
|
||||
import { sdkCamera } from "../tracks/cameraTrack.ts";
|
||||
import { type Layout } from "../state/layout-types.ts";
|
||||
import { ObservableScope } from "../state/ObservableScope.ts";
|
||||
import { CallFooter, type FooterSnapshot } from "../components/CallFooter.tsx";
|
||||
@@ -180,7 +185,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
|
||||
props.muteStates,
|
||||
mediaDevices,
|
||||
capture,
|
||||
backgroundEffectsState$.value,
|
||||
backgroundEffectsState$.value.processor,
|
||||
),
|
||||
});
|
||||
|
||||
@@ -194,7 +199,9 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
|
||||
...callViewModelOptionsFromParams(urlParams),
|
||||
encryptionSystem: props.e2eeSystem,
|
||||
capture,
|
||||
backgroundEffectsState$,
|
||||
videoProcessor$: scope.behavior(
|
||||
backgroundEffectsState$.pipe(map((state) => state.processor)),
|
||||
),
|
||||
hostBridge,
|
||||
autoLeaveWhenOthersLeft,
|
||||
waitForCallPickup: waitForCallPickup && sendNotificationType === "ring",
|
||||
@@ -208,6 +215,12 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
|
||||
reactionsReader.reactions$,
|
||||
);
|
||||
setVm(vm);
|
||||
// The call's camera is the one the camera menu shows from here on.
|
||||
reportCamera(
|
||||
scope,
|
||||
vm.camera$.pipe(map((camera) => (camera ? sdkCamera(camera) : null))),
|
||||
backgroundEffectsState$,
|
||||
);
|
||||
|
||||
vm.leave$.pipe(scope.bind()).subscribe(props.onLeft);
|
||||
|
||||
@@ -822,7 +835,9 @@ export function rtcParticipationOptions(
|
||||
const session = config.matrix_rtc_session;
|
||||
return {
|
||||
sendNotificationType: urlParams.sendNotificationType,
|
||||
callIntent: urlParams.callIntent,
|
||||
applicationData: urlParams.callIntent
|
||||
? { "m.call.intent": urlParams.callIntent }
|
||||
: undefined,
|
||||
timings: {
|
||||
syncDisconnectGracePeriodMs: config.sync_disconnect_grace_period_ms,
|
||||
networkErrorRetryMs: session.network_error_retry_ms,
|
||||
|
||||
@@ -28,6 +28,7 @@ const stable = vi.hoisted(() => ({
|
||||
media: undefined as unknown,
|
||||
}));
|
||||
vi.mock("../tracks/BackgroundEffectsContext", () => ({
|
||||
reportCamera: (): void => {},
|
||||
useBackgroundEffectsState$: (): Behavior<BackgroundEffectsState> =>
|
||||
(stable.processorState$ ??= constant({
|
||||
supported: false,
|
||||
|
||||
@@ -45,6 +45,7 @@ const leaveToHome = vi.fn();
|
||||
// One instance: the view keys an effect on it
|
||||
const processorState$ = vi.hoisted(() => ({ current: undefined as unknown }));
|
||||
vi.mock("../tracks/BackgroundEffectsContext", () => ({
|
||||
reportCamera: (): void => {},
|
||||
useBackgroundEffectsState$: (): Behavior<BackgroundEffectsState> =>
|
||||
(processorState$.current ??= constant({
|
||||
supported: false,
|
||||
|
||||
+21
-12
@@ -16,6 +16,7 @@ import {
|
||||
type LocalMedia,
|
||||
type LocalVideoMediaTrack,
|
||||
} from "@element-hq/matrixrtc-sdk";
|
||||
import { map } from "rxjs";
|
||||
|
||||
import inCallStyles from "./InCallView.module.css";
|
||||
import styles from "./LobbyView.module.css";
|
||||
@@ -30,7 +31,11 @@ import { LeaveToHomeLink } from "../button/LeaveToHomeLink";
|
||||
import { useLeaveToHome } from "../LeaveToHomeContext";
|
||||
import { useMediaDevices } from "../MediaDevicesContext";
|
||||
import { ObservableScope } from "../state/ObservableScope";
|
||||
import { useBackgroundEffectsState$ } from "../tracks/BackgroundEffectsContext";
|
||||
import {
|
||||
reportCamera,
|
||||
useBackgroundEffectsState$,
|
||||
} from "../tracks/BackgroundEffectsContext";
|
||||
import { sdkCamera } from "../tracks/cameraTrack";
|
||||
import { useBehavior } from "../useBehavior";
|
||||
import { CallFooter, type FooterSnapshot } from "../components/CallFooter";
|
||||
import { useCallViewKeyboardShortcuts } from "../useCallViewKeyboardShortcuts";
|
||||
@@ -126,16 +131,23 @@ export const LobbyView: FC<Props> = ({
|
||||
const [mediaVm, setMediaVm] = useState<LobbyMediaViewModel | null>(null);
|
||||
useEffect(() => {
|
||||
const mediaScope = new ObservableScope();
|
||||
setMediaVm(
|
||||
createLobbyMediaViewModel(
|
||||
mediaScope,
|
||||
media,
|
||||
muteStates,
|
||||
devices,
|
||||
captureSettings(),
|
||||
backgroundEffectsState$,
|
||||
const vm = createLobbyMediaViewModel(
|
||||
mediaScope,
|
||||
media,
|
||||
muteStates,
|
||||
devices,
|
||||
captureSettings(),
|
||||
mediaScope.behavior(
|
||||
backgroundEffectsState$.pipe(map((state) => state.processor)),
|
||||
),
|
||||
);
|
||||
// The lobby's camera is the one the camera menu shows until the call's
|
||||
reportCamera(
|
||||
mediaScope,
|
||||
vm.videoTrack$.pipe(map((camera) => camera && sdkCamera(camera))),
|
||||
backgroundEffectsState$,
|
||||
);
|
||||
setMediaVm(vm);
|
||||
return (): void => {
|
||||
mediaScope.end();
|
||||
};
|
||||
@@ -155,7 +167,6 @@ export const LobbyView: FC<Props> = ({
|
||||
null,
|
||||
);
|
||||
useEffect(() => {
|
||||
if (mediaVm === null) return;
|
||||
const footerScope = new ObservableScope();
|
||||
setFooterVm(
|
||||
createLobbyFooterViewModel(
|
||||
@@ -163,7 +174,6 @@ export const LobbyView: FC<Props> = ({
|
||||
muteStates,
|
||||
devices,
|
||||
backgroundEffectsState$,
|
||||
mediaVm.videoTrack$,
|
||||
openSettings,
|
||||
hangup,
|
||||
// Logo and header are connected: only show the logo in SPA with header.
|
||||
@@ -178,7 +188,6 @@ export const LobbyView: FC<Props> = ({
|
||||
backgroundEffectsState$,
|
||||
hangup,
|
||||
hideHeader,
|
||||
mediaVm,
|
||||
muteStates,
|
||||
openSettings,
|
||||
]);
|
||||
|
||||
@@ -8,7 +8,6 @@ Please see LICENSE in the repository root for full details.
|
||||
import { expect, type MockedFunction, test, vi } from "vitest";
|
||||
import { act, render } from "@testing-library/react";
|
||||
import { BehaviorSubject } from "rxjs";
|
||||
import { RTC_CALL_INTENT_AUDIO } from "matrix-js-sdk/lib/matrixrtc";
|
||||
|
||||
import { useAudioContext } from "../useAudioContext";
|
||||
import { createRingingMedia } from "../state/media/RingingMediaViewModel";
|
||||
@@ -40,7 +39,7 @@ test("ringtone plays on loop while ringing", () => {
|
||||
userId: alice.userId,
|
||||
displayName$: constant("Alice"),
|
||||
mxcAvatarUrl$: constant(undefined),
|
||||
intent: RTC_CALL_INTENT_AUDIO,
|
||||
intent: "audio",
|
||||
pickupState$,
|
||||
});
|
||||
|
||||
|
||||
@@ -12,8 +12,8 @@ import { type RingingMediaViewModel } from "../state/media/RingingMediaViewModel
|
||||
import { useBehavior } from "../useBehavior";
|
||||
import { useInitial } from "../useInitial";
|
||||
import { prefetchSounds } from "../soundUtils";
|
||||
import ringtoneMp3 from "../sound/ringtone.mp3?url&no-inline";
|
||||
import ringtoneOgg from "../sound/ringtone.ogg?url&no-inline";
|
||||
import ringtoneMp3 from "../sound/ringtone.mp3?url";
|
||||
import ringtoneOgg from "../sound/ringtone.ogg?url";
|
||||
import { type UseAudioContext, useAudioContext } from "../useAudioContext";
|
||||
import { useLatest } from "../useLatest";
|
||||
|
||||
|
||||
@@ -5,9 +5,9 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { useEffect, useMemo, useRef, type FC, type ReactNode } from "react";
|
||||
import { useMemo, type FC, type ReactNode } from "react";
|
||||
import useMeasure from "react-use-measure";
|
||||
import { constant, type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk";
|
||||
import { type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk";
|
||||
import classNames from "classnames";
|
||||
import { useTranslation } from "react-i18next";
|
||||
|
||||
@@ -15,9 +15,8 @@ import { TileAvatar } from "../tile/TileAvatar";
|
||||
import styles from "./VideoPreview.module.css";
|
||||
import { type EncryptionSystem } from "../e2ee/sharedKeyManagement";
|
||||
import videoPlaceholder from "../graphics/video-placeholder.gif";
|
||||
import { useBehavior } from "../useBehavior";
|
||||
|
||||
const noFacingMode = constant(undefined);
|
||||
import { useAttachedTrack } from "../useAttachedTrack";
|
||||
import { sdkCamera } from "../tracks/cameraTrack";
|
||||
|
||||
export type MatrixInfo = {
|
||||
userId: string;
|
||||
@@ -46,15 +45,9 @@ export const VideoPreview: FC<Props> = ({
|
||||
const { t } = useTranslation();
|
||||
const [previewRef, previewBounds] = useMeasure();
|
||||
|
||||
const videoEl = useRef<HTMLVideoElement | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
const element = videoEl.current;
|
||||
if (!element || !videoTrack) return;
|
||||
videoTrack.attach(element);
|
||||
return (): void => videoTrack.detach(element);
|
||||
}, [videoTrack]);
|
||||
const facingMode = useBehavior(videoTrack?.facingMode$ ?? noFacingMode);
|
||||
const { videoRef, mirrored } = useAttachedTrack(
|
||||
videoTrack && sdkCamera(videoTrack),
|
||||
);
|
||||
|
||||
const cameraIsStarting = useMemo(
|
||||
() => videoEnabled && !videoTrack,
|
||||
@@ -64,8 +57,8 @@ export const VideoPreview: FC<Props> = ({
|
||||
return (
|
||||
<div className={classNames(styles.preview)} ref={previewRef}>
|
||||
<video
|
||||
className={facingMode === "user" ? styles.mirror : undefined}
|
||||
ref={videoEl}
|
||||
className={mirrored ? styles.mirror : undefined}
|
||||
ref={videoRef}
|
||||
muted
|
||||
playsInline
|
||||
// There's no reason for this to be focusable
|
||||
|
||||
+26
-7
@@ -34,6 +34,23 @@ describe("prefetchSounds", () => {
|
||||
expect(bytesOf(sounds.beep)).toEqual([1, 2, 3]);
|
||||
});
|
||||
|
||||
it("decodes base64 data: URLs without fetching them", async () => {
|
||||
// "OggS" — the bytes 0x4f 0x67 0x67 0x53
|
||||
const sounds = await prefetchSounds({
|
||||
beep: { ogg: "data:audio/ogg;base64,T2dnUw==" },
|
||||
});
|
||||
expect(fetchSpy).not.toHaveBeenCalled();
|
||||
expect(bytesOf(sounds.beep)).toEqual([0x4f, 0x67, 0x67, 0x53]);
|
||||
});
|
||||
|
||||
it("leaves out a sound with a percent-encoded data: URL", async () => {
|
||||
const sounds = await prefetchSounds({
|
||||
beep: { ogg: "data:audio/ogg,Ogg%53" },
|
||||
});
|
||||
expect(fetchSpy).not.toHaveBeenCalled();
|
||||
expect(sounds).toEqual({});
|
||||
});
|
||||
|
||||
it("leaves out a sound whose response is not okay", async () => {
|
||||
fetchSpy.mockResolvedValue(new Response(null, { status: 404 }));
|
||||
const sounds = await prefetchSounds({
|
||||
@@ -43,16 +60,18 @@ describe("prefetchSounds", () => {
|
||||
});
|
||||
|
||||
it("leaves out a sound that fails to load, keeping the others", async () => {
|
||||
// Fetched in the order given
|
||||
fetchSpy
|
||||
.mockRejectedValueOnce(new TypeError("Failed to fetch"))
|
||||
.mockResolvedValueOnce(
|
||||
new Response(new Uint8Array([1, 2, 3]), { status: 200 }),
|
||||
);
|
||||
fetchSpy.mockRejectedValue(new TypeError("Failed to fetch"));
|
||||
const sounds = await prefetchSounds({
|
||||
beep: { ogg: "https://example.org/beep.ogg" },
|
||||
boop: { ogg: "https://example.org/boop.ogg" },
|
||||
boop: { ogg: "data:audio/ogg;base64,T2dnUw==" },
|
||||
});
|
||||
expect(Object.keys(sounds)).toEqual(["boop"]);
|
||||
});
|
||||
|
||||
it("leaves out a sound with a malformed data: URL", async () => {
|
||||
const sounds = await prefetchSounds({
|
||||
beep: { ogg: "data:audio/ogg;base64" },
|
||||
});
|
||||
expect(sounds).toEqual({});
|
||||
});
|
||||
});
|
||||
|
||||
+27
-1
@@ -32,12 +32,38 @@ function getPreferredAudioFormat(): "ogg" | "mp3" {
|
||||
|
||||
const preferredFormat = getPreferredAudioFormat();
|
||||
|
||||
/**
|
||||
* Decode a `data:` URL into the bytes it carries.
|
||||
*
|
||||
* The component build inlines the sound files as `data:` URLs. Those could be
|
||||
* `fetch`ed like any other URL, but that fetch is subject to the host page's
|
||||
* Content Security Policy, and `connect-src` lists rarely allow `data:` (a `*`
|
||||
* source does not cover it). Reading the bytes out of the URL directly needs no
|
||||
* network access, so no policy can stop it.
|
||||
* @param url The `data:` URL to decode.
|
||||
* @returns The decoded bytes.
|
||||
*/
|
||||
function decodeDataUrl(url: string): ArrayBuffer {
|
||||
const separator = url.indexOf(",");
|
||||
if (!url.startsWith("data:") || separator === -1)
|
||||
throw new Error("Not a valid data: URL");
|
||||
const header = url.slice("data:".length, separator);
|
||||
// Vite only percent-encodes SVGs; every other inlined asset is base64.
|
||||
if (!header.split(";").includes("base64"))
|
||||
throw new Error("Only base64 data: URLs are supported");
|
||||
// `decoded` is a binary string: one char per byte
|
||||
const decoded = atob(url.slice(separator + 1));
|
||||
const bytes = Uint8Array.from(decoded, (c) => c.charCodeAt(0));
|
||||
return bytes.buffer;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load the bytes of a single sound file.
|
||||
* @param url Where the sound is.
|
||||
* @param url Where the sound is: an ordinary URL to fetch, or a `data:` URL.
|
||||
* @returns The bytes, or null if the sound could not be loaded.
|
||||
*/
|
||||
async function loadSound(url: string): Promise<ArrayBuffer | null> {
|
||||
if (url.startsWith("data:")) return decodeDataUrl(url);
|
||||
const response = await fetch(url);
|
||||
if (isFailure(response)) return null;
|
||||
return await response.arrayBuffer();
|
||||
|
||||
@@ -7,11 +7,7 @@ Please see LICENSE in the repository root for full details.
|
||||
|
||||
import { it, vi, expect, beforeEach, afterEach, describe } from "vitest";
|
||||
import { firstValueFrom, of, Subject, take, toArray } from "rxjs";
|
||||
import {
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
RTC_CALL_INTENT_VIDEO,
|
||||
type RTCCallIntent,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
import { type RTCCallIntent } from "matrix-js-sdk/lib/matrixrtc";
|
||||
|
||||
import { AndroidControlledAudioOutput } from "./AndroidControlledAudioOutput.ts";
|
||||
import type { Controls, OutputDevice } from "../controls";
|
||||
@@ -121,7 +117,7 @@ describe("Default selection", () => {
|
||||
const controlledAudioOutput = new AndroidControlledAudioOutput(
|
||||
of(BASE_DEVICE_LIST),
|
||||
testScope,
|
||||
RTC_CALL_INTENT_VIDEO,
|
||||
"video",
|
||||
mockControls,
|
||||
);
|
||||
|
||||
@@ -146,7 +142,7 @@ describe("Default selection", () => {
|
||||
const controlledAudioOutput = new AndroidControlledAudioOutput(
|
||||
of(BASE_DEVICE_LIST),
|
||||
testScope,
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
"audio",
|
||||
mockControls,
|
||||
);
|
||||
|
||||
@@ -167,7 +163,7 @@ describe("Default selection", () => {
|
||||
});
|
||||
});
|
||||
|
||||
[RTC_CALL_INTENT_AUDIO, RTC_CALL_INTENT_VIDEO].forEach((callIntent) => {
|
||||
["audio", "video"].forEach((callIntent) => {
|
||||
it(`Default to BT headset for ${callIntent} calls if present`, async () => {
|
||||
const controlledAudioOutput = new AndroidControlledAudioOutput(
|
||||
of(BT_HEADSET_BASE_DEVICE_LIST),
|
||||
@@ -194,7 +190,7 @@ describe("Default selection", () => {
|
||||
});
|
||||
});
|
||||
|
||||
[RTC_CALL_INTENT_AUDIO, RTC_CALL_INTENT_VIDEO].forEach((callIntent) => {
|
||||
["audio", "video"].forEach((callIntent) => {
|
||||
it(`Default to wired headset for ${callIntent} calls if present`, async () => {
|
||||
const controlledAudioOutput = new AndroidControlledAudioOutput(
|
||||
of(WIRED_HEADSET_BASE_DEVICE_LIST),
|
||||
@@ -304,7 +300,7 @@ describe("Test select a device", () => {
|
||||
b: BT_HEADSET_BASE_DEVICE_LIST,
|
||||
}),
|
||||
testScope,
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
"audio",
|
||||
mockControls,
|
||||
);
|
||||
|
||||
@@ -344,7 +340,7 @@ describe("Test select a device", () => {
|
||||
c: BASE_DEVICE_LIST,
|
||||
}),
|
||||
testScope,
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
"audio",
|
||||
mockControls,
|
||||
);
|
||||
|
||||
@@ -397,7 +393,7 @@ describe("Available device changes", () => {
|
||||
});
|
||||
|
||||
it("When a BT headset is added, control should switch to use it", () => {
|
||||
createAudioControlledOutput(RTC_CALL_INTENT_VIDEO);
|
||||
createAudioControlledOutput("video");
|
||||
|
||||
// Emit the base device list, the speaker should be selected
|
||||
availableSource$.next(BASE_DEVICE_LIST);
|
||||
@@ -423,9 +419,7 @@ describe("Available device changes", () => {
|
||||
|
||||
// Android does not set `isExternalHeadset` to true for wired headphones, so we can't test this case.'
|
||||
it.skip("When a wired headset is added, control should switch to use it", async () => {
|
||||
const controlledAudioOutput = createAudioControlledOutput(
|
||||
RTC_CALL_INTENT_VIDEO,
|
||||
);
|
||||
const controlledAudioOutput = createAudioControlledOutput("video");
|
||||
|
||||
// Emit the base device list, the speaker should be selected
|
||||
availableSource$.next(BASE_DEVICE_LIST);
|
||||
@@ -452,7 +446,7 @@ describe("Available device changes", () => {
|
||||
});
|
||||
|
||||
it("When the active bt headset is removed on audio call, control should switch to earpiece", () => {
|
||||
createAudioControlledOutput(RTC_CALL_INTENT_AUDIO);
|
||||
createAudioControlledOutput("audio");
|
||||
|
||||
// Emit the BT headset device list, the BT headset should be selected
|
||||
availableSource$.next(BT_HEADSET_BASE_DEVICE_LIST);
|
||||
@@ -478,7 +472,7 @@ describe("Available device changes", () => {
|
||||
});
|
||||
|
||||
it("When the active bt headset is removed on video call, control should switch to speaker", () => {
|
||||
createAudioControlledOutput(RTC_CALL_INTENT_VIDEO);
|
||||
createAudioControlledOutput("video");
|
||||
|
||||
availableSource$.next(BT_HEADSET_BASE_DEVICE_LIST);
|
||||
|
||||
@@ -503,7 +497,7 @@ describe("Available device changes", () => {
|
||||
});
|
||||
|
||||
it("Do not repeatidly set the same device", () => {
|
||||
createAudioControlledOutput(RTC_CALL_INTENT_VIDEO);
|
||||
createAudioControlledOutput("video");
|
||||
|
||||
availableSource$.next(BT_HEADSET_BASE_DEVICE_LIST);
|
||||
availableSource$.next(BT_HEADSET_BASE_DEVICE_LIST);
|
||||
|
||||
@@ -22,10 +22,7 @@ import {
|
||||
type SelectedAudioOutputDevice,
|
||||
} from "./MediaDevices.ts";
|
||||
import type { ObservableScope } from "./ObservableScope.ts";
|
||||
import {
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
type RTCCallIntent,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
import type { RTCCallIntent } from "matrix-js-sdk/lib/matrixrtc";
|
||||
import { type Controls, type OutputDevice } from "../controls.ts";
|
||||
import { type Behavior } from "./Behavior.ts";
|
||||
|
||||
@@ -105,7 +102,7 @@ export class AndroidControlledAudioOutput implements MediaDevice<
|
||||
* @constructor
|
||||
* @param controlledDevices$ - The list of available output devices coming from the hosting application, ordered by preference order (most preferred first).
|
||||
* @param scope - The ObservableScope to create the Behaviors in.
|
||||
* @param initialIntent - The initial call intent (e.g. "m.audio" or "m.video") that can be used to determine the default audio routing (e.g. default to earpiece for audio calls and speaker for video calls).
|
||||
* @param initialIntent - The initial call intent (e.g. "audio" or "video") that can be used to determine the default audio routing (e.g. default to earpiece for audio calls and speaker for video calls).
|
||||
* @param controls - The controls provided by the hosting application to control the audio routing and notify of user actions.
|
||||
*/
|
||||
public constructor(
|
||||
@@ -319,7 +316,7 @@ export class AndroidControlledAudioOutput implements MediaDevice<
|
||||
this.logger.debug(
|
||||
`Android routing logic intent: ${this.initialIntent} finding best default...`,
|
||||
);
|
||||
if (this.initialIntent === RTC_CALL_INTENT_AUDIO) {
|
||||
if (this.initialIntent === "audio") {
|
||||
const systemProposed = available[0];
|
||||
// If no headset is connected, android will route to the speaker by default,
|
||||
// but for a voice call we want to route to the earpiece instead,
|
||||
|
||||
@@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { BehaviorSubject, distinctUntilChanged, map, of } from "rxjs";
|
||||
import { BehaviorSubject, distinctUntilChanged, map } from "rxjs";
|
||||
import { type BackgroundProcessorWrapper } from "@livekit/track-processors";
|
||||
|
||||
import {
|
||||
@@ -47,28 +47,23 @@ function fakePipeline(): {
|
||||
};
|
||||
}
|
||||
|
||||
/** One letter per state: idle, preparing, waiting for a frame, attached. */
|
||||
/** One letter per state: idle, waiting for a frame, attached. */
|
||||
function letter(state: BackgroundEffectsState): string {
|
||||
if (state.supported === false) return "x";
|
||||
if (state.wait === "preparing") return "p";
|
||||
if (state.processor === undefined) return "i";
|
||||
return state.wait === "settling" ? "w" : "a";
|
||||
return state.settling ? "w" : "a";
|
||||
}
|
||||
|
||||
describe("the pipeline's state", () => {
|
||||
function testState({
|
||||
effect,
|
||||
answer = "-y",
|
||||
firstFrame = "",
|
||||
expected,
|
||||
}: {
|
||||
effect: string;
|
||||
/** When the trial build answers, from being asked. */
|
||||
answer?: string;
|
||||
firstFrame?: string;
|
||||
expected: string;
|
||||
}): void {
|
||||
withTestScheduler(({ behavior, cold, schedule, expectObservable }) => {
|
||||
withTestScheduler(({ behavior, schedule, expectObservable }) => {
|
||||
const transformer = {
|
||||
onFirstFrame: undefined as (() => void) | undefined,
|
||||
};
|
||||
@@ -77,7 +72,6 @@ describe("the pipeline's state", () => {
|
||||
effect$: behavior(effect, { n: "none", b: "blur" }),
|
||||
setEffect: vi.fn(),
|
||||
added$: new BehaviorSubject<AddedBackground[] | undefined>([]),
|
||||
canSegment: () => cold(answer, { y: true, n: false }),
|
||||
pipeline: fakePipeline().pipeline,
|
||||
transformer,
|
||||
});
|
||||
@@ -90,21 +84,15 @@ describe("the pipeline's state", () => {
|
||||
|
||||
it("defaults to no effect", () => testState({ effect: "n", expected: "i" }));
|
||||
|
||||
it("attaches only once it is known the pipeline builds", () =>
|
||||
testState({ effect: "nb", answer: "--y", expected: "ip-w" }));
|
||||
|
||||
it("offers none where the pipeline fails to build", () =>
|
||||
testState({ effect: "nb", answer: "-n", expected: "ipx" }));
|
||||
|
||||
it("waits from the first effect until a frame carries it", () =>
|
||||
testState({
|
||||
effect: " nb---n-b",
|
||||
firstFrame: "---f",
|
||||
expected: " ipwa",
|
||||
effect: " nb-n-b",
|
||||
firstFrame: " --f",
|
||||
expected: " iwa",
|
||||
}));
|
||||
|
||||
it("attaches on first use and stays attached", () =>
|
||||
testState({ effect: "nb--n", firstFrame: "---f", expected: "ipwa" }));
|
||||
testState({ effect: "nb-n", firstFrame: "--f", expected: "iwa" }));
|
||||
});
|
||||
|
||||
describe("background effects", () => {
|
||||
@@ -121,7 +109,6 @@ describe("background effects", () => {
|
||||
effect$,
|
||||
setEffect,
|
||||
added$,
|
||||
canSegment: () => of(true),
|
||||
pipeline: fake.pipeline,
|
||||
transformer: { onFirstFrame: undefined },
|
||||
...options,
|
||||
@@ -131,6 +118,8 @@ describe("background effects", () => {
|
||||
effect$.next(raw);
|
||||
await flushPromises();
|
||||
};
|
||||
const blur = async (on: boolean): Promise<void> =>
|
||||
choose(on ? "blur" : "none");
|
||||
|
||||
beforeEach(() => {
|
||||
effect$ = new BehaviorSubject("none");
|
||||
@@ -139,17 +128,6 @@ describe("background effects", () => {
|
||||
fake = fakePipeline();
|
||||
});
|
||||
|
||||
it("offers none for the session where the pipeline fails to build", async () => {
|
||||
const effects = build({ canSegment: () => of(false) });
|
||||
await choose("blur");
|
||||
|
||||
expect(effects.state$.value.supported).toBe(false);
|
||||
expect(effects.state$.value.processor).toBeUndefined();
|
||||
expect(effects.state$.value.wait).toBeUndefined();
|
||||
// A browser update may fix it, so the next session asks again.
|
||||
expect(effect$.value).toBe("blur");
|
||||
});
|
||||
|
||||
it("puts a shipped background on as that picture", async () => {
|
||||
build();
|
||||
await choose(`image:${shippedBackgrounds[0].id}`);
|
||||
@@ -163,10 +141,10 @@ describe("background effects", () => {
|
||||
|
||||
it("switches in place rather than reattaching", async () => {
|
||||
const effects = build();
|
||||
await choose("blur");
|
||||
await blur(true);
|
||||
const pipeline = effects.state$.value.processor;
|
||||
await choose("none");
|
||||
await choose("blur");
|
||||
await blur(false);
|
||||
await blur(true);
|
||||
|
||||
expect(effects.state$.value.processor).toBe(pipeline);
|
||||
expect(fake.switches).toEqual([
|
||||
@@ -179,18 +157,18 @@ describe("background effects", () => {
|
||||
it("keeps switching after a switch fails", async () => {
|
||||
vi.mocked(fake.pipeline.switchTo).mockRejectedValueOnce(new Error("lost"));
|
||||
build();
|
||||
await choose("blur");
|
||||
await choose("none");
|
||||
await blur(true);
|
||||
await blur(false);
|
||||
expect(fake.switches).toEqual([{ mode: "disabled" }]);
|
||||
});
|
||||
|
||||
it("switches one at a time, skipping those overtaken", async () => {
|
||||
const finished = fake.holdNext();
|
||||
build();
|
||||
await choose("blur");
|
||||
await choose("none");
|
||||
await choose("blur");
|
||||
await choose("none");
|
||||
await blur(true);
|
||||
await blur(false);
|
||||
await blur(true);
|
||||
await blur(false);
|
||||
expect(fake.switches).toHaveLength(1);
|
||||
|
||||
finished();
|
||||
|
||||
@@ -8,15 +8,9 @@ Please see LICENSE in the repository root for full details.
|
||||
import {
|
||||
combineLatest,
|
||||
distinctUntilChanged,
|
||||
first,
|
||||
from,
|
||||
map,
|
||||
Observable,
|
||||
type ObservableInput,
|
||||
of,
|
||||
scan,
|
||||
startWith,
|
||||
switchMap,
|
||||
} from "rxjs";
|
||||
import {
|
||||
type BackgroundProcessorWrapper,
|
||||
@@ -38,18 +32,17 @@ import {
|
||||
parseEffect,
|
||||
} from "../tracks/backgroundEffectCatalog";
|
||||
import { type AddedBackground } from "../tracks/backgroundImages";
|
||||
import { SyncedCameraTrack } from "../tracks/cameraTrack";
|
||||
|
||||
const blurRadius = 15;
|
||||
|
||||
export interface BackgroundEffectsState {
|
||||
supported: boolean | undefined;
|
||||
processor: undefined | VideoProcessor;
|
||||
/**
|
||||
* The effect is not ready:
|
||||
* preparing, asking whether the pipeline can be built.
|
||||
* settling, once it is attached until a frame carrying the effect
|
||||
*/
|
||||
wait?: "preparing" | "settling";
|
||||
/** From the first effect chosen until a frame carrying it is drawn. */
|
||||
settling?: boolean;
|
||||
/** The camera this provider's pipeline is synced to. */
|
||||
cameraTrack?: SyncedCameraTrack;
|
||||
}
|
||||
|
||||
export interface BackgroundEffectsOptions {
|
||||
@@ -61,8 +54,6 @@ export interface BackgroundEffectsOptions {
|
||||
setEffect: (id: EffectId) => void;
|
||||
/** The backgrounds the device keeps, undefined until it has said. */
|
||||
added$: Behavior<AddedBackground[] | undefined>;
|
||||
/** Builds a segmenter on trial, and answers whether it could. */
|
||||
canSegment: () => ObservableInput<boolean>;
|
||||
/** The background processor pipeline to be switched between effects. */
|
||||
pipeline: BackgroundProcessorWrapper;
|
||||
/** Tells, once, that a frame carrying an effect has been drawn. */
|
||||
@@ -82,19 +73,18 @@ export function createBackgroundEffects(
|
||||
effect$,
|
||||
setEffect,
|
||||
added$,
|
||||
canSegment,
|
||||
pipeline,
|
||||
transformer,
|
||||
}: BackgroundEffectsOptions,
|
||||
): BackgroundEffects {
|
||||
// Wrapped once, so the call and the preview see one processor throughout.
|
||||
const processor = fromLivekitProcessor(pipeline);
|
||||
// The camera the pipeline is synced to.
|
||||
const cameraTrack = new SyncedCameraTrack();
|
||||
const choice$ = effect$.pipe(map(parseEffect));
|
||||
const wanted$ = scope.behavior(
|
||||
choice$.pipe(
|
||||
map((effect) => effect.kind !== "none"),
|
||||
distinctUntilChanged(),
|
||||
),
|
||||
const wanted$ = choice$.pipe(
|
||||
map((effect) => effect.kind !== "none"),
|
||||
distinctUntilChanged(),
|
||||
);
|
||||
|
||||
combineLatest([choice$, added$])
|
||||
@@ -110,18 +100,6 @@ export function createBackgroundEffects(
|
||||
setEffect("none");
|
||||
});
|
||||
|
||||
// Asked once an effect is first wanted. A browser update may fix what
|
||||
// fails, so the choice is kept for the next session rather than cleared.
|
||||
const buildable$ = scope.behavior<boolean | undefined>(
|
||||
supported
|
||||
? wanted$.pipe(
|
||||
first(Boolean),
|
||||
switchMap(() => from(canSegment())),
|
||||
startWith(undefined),
|
||||
)
|
||||
: of(undefined),
|
||||
);
|
||||
|
||||
const drewAFrame$ = scope.behavior(
|
||||
new Observable<boolean>((subscriber) => {
|
||||
subscriber.next(false);
|
||||
@@ -132,25 +110,19 @@ export function createBackgroundEffects(
|
||||
}),
|
||||
);
|
||||
|
||||
// In one step, so that nothing sees the pipeline neither preparing nor
|
||||
// attached.
|
||||
const state$ = scope.behavior(
|
||||
combineLatest([wanted$, buildable$, drewAFrame$]).pipe(
|
||||
scan<[boolean, boolean | undefined, boolean], BackgroundEffectsState>(
|
||||
(previous, [wanted, buildable, drewAFrame]) => {
|
||||
combineLatest([wanted$, drewAFrame$]).pipe(
|
||||
scan<[boolean, boolean], BackgroundEffectsState>(
|
||||
(previous, [wanted, drewAFrame]) => {
|
||||
// Attached the first time an effect is wanted and never detached
|
||||
// after, so someone who never turns one on pays for none of it.
|
||||
const enable =
|
||||
previous.processor !== undefined || (wanted && buildable === true);
|
||||
const preparing = supported && wanted && buildable === undefined;
|
||||
previous.processor !== undefined || (supported && wanted);
|
||||
return {
|
||||
supported: supported && buildable !== false,
|
||||
supported,
|
||||
processor: enable ? processor : undefined,
|
||||
wait: preparing
|
||||
? "preparing"
|
||||
: enable && !drewAFrame
|
||||
? "settling"
|
||||
: undefined,
|
||||
settling: enable && !drewAFrame,
|
||||
cameraTrack,
|
||||
};
|
||||
},
|
||||
{ supported, processor: undefined },
|
||||
|
||||
@@ -12,11 +12,7 @@ import {
|
||||
MatrixEvent,
|
||||
type Room,
|
||||
} from "matrix-js-sdk";
|
||||
import {
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
RTC_CALL_INTENT_VIDEO,
|
||||
type RTCCallIntent,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
import { type RTCCallIntent } from "matrix-js-sdk/lib/matrixrtc";
|
||||
import { map, mergeMap, NEVER, type Observable, startWith } from "rxjs";
|
||||
|
||||
import { withTestScheduler } from "../../utils/test";
|
||||
@@ -37,12 +33,10 @@ function mockRingEvent(
|
||||
eventId: string,
|
||||
lifetimeMs: number | undefined,
|
||||
sender = local.userId,
|
||||
legacyIntent?: string,
|
||||
): CallNotificationWrapper {
|
||||
return {
|
||||
event_id: eventId,
|
||||
...(lifetimeMs === undefined ? {} : { lifetime: lifetimeMs }),
|
||||
...(legacyIntent === undefined ? {} : { "m.call.intent": legacyIntent }),
|
||||
notification_type: "ring",
|
||||
sender,
|
||||
} as unknown as CallNotificationWrapper;
|
||||
@@ -134,25 +128,7 @@ test("ring attempt times out after nobody joins", () => {
|
||||
});
|
||||
|
||||
expectObservable(summarizeRingAttempts$(ringAttempts$)).toBe("-a 29ms A", {
|
||||
a: { intent: RTC_CALL_INTENT_AUDIO, recipient: alice.userId },
|
||||
A: { outcome: "timeout" },
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
test("ring attempt maps the legacy intent on the notification to its stable value", () => {
|
||||
withTestScheduler(({ scope, expectObservable, hot }) => {
|
||||
const { ringAttempts$ } = createCallNotificationLifecycle$({
|
||||
scope,
|
||||
...defaultProps,
|
||||
memberUserIds$: constant<string[]>([]),
|
||||
sentCallNotification$: hot("-a", {
|
||||
a: mockRingEvent("$notif1", 30, local.userId, "video"),
|
||||
}),
|
||||
});
|
||||
|
||||
expectObservable(summarizeRingAttempts$(ringAttempts$)).toBe("-a 29ms A", {
|
||||
a: { intent: RTC_CALL_INTENT_VIDEO, recipient: alice.userId },
|
||||
a: { intent: "audio", recipient: alice.userId },
|
||||
A: { outcome: "timeout" },
|
||||
});
|
||||
});
|
||||
@@ -172,7 +148,7 @@ test("ring attempt is accepted once recipient joins", () => {
|
||||
});
|
||||
|
||||
expectObservable(summarizeRingAttempts$(ringAttempts$)).toBe("-aA", {
|
||||
a: { intent: RTC_CALL_INTENT_AUDIO, recipient: alice.userId },
|
||||
a: { intent: "audio", recipient: alice.userId },
|
||||
A: { outcome: "accept" },
|
||||
});
|
||||
});
|
||||
@@ -190,7 +166,7 @@ test("ring attempt is immediately accepted if recipient is already joined", () =
|
||||
});
|
||||
|
||||
expectObservable(summarizeRingAttempts$(ringAttempts$)).toBe("-(aA)", {
|
||||
a: { intent: RTC_CALL_INTENT_AUDIO, recipient: alice.userId },
|
||||
a: { intent: "audio", recipient: alice.userId },
|
||||
A: { outcome: "accept" },
|
||||
});
|
||||
});
|
||||
@@ -225,7 +201,7 @@ test("ring attempt can be declined", () => {
|
||||
});
|
||||
|
||||
expectObservable(summarizeRingAttempts$(ringAttempts$)).toBe("-aA", {
|
||||
a: { intent: RTC_CALL_INTENT_AUDIO, recipient: alice.userId },
|
||||
a: { intent: "audio", recipient: alice.userId },
|
||||
A: { outcome: "decline" },
|
||||
});
|
||||
});
|
||||
@@ -260,7 +236,7 @@ test("ring attempt times out if recipient declines too late", () => {
|
||||
});
|
||||
|
||||
expectObservable(summarizeRingAttempts$(ringAttempts$)).toBe("-a 29ms A", {
|
||||
a: { intent: RTC_CALL_INTENT_AUDIO, recipient: alice.userId },
|
||||
a: { intent: "audio", recipient: alice.userId },
|
||||
A: { outcome: "timeout" },
|
||||
});
|
||||
});
|
||||
@@ -295,7 +271,7 @@ test("decline event relating to wrong event is ignored (times out)", () => {
|
||||
});
|
||||
|
||||
expectObservable(summarizeRingAttempts$(ringAttempts$)).toBe("-a 29ms A", {
|
||||
a: { intent: RTC_CALL_INTENT_AUDIO, recipient: alice.userId },
|
||||
a: { intent: "audio", recipient: alice.userId },
|
||||
A: { outcome: "timeout" },
|
||||
});
|
||||
});
|
||||
@@ -330,7 +306,7 @@ test("decline event from wrong sender is ignored (times out)", () => {
|
||||
});
|
||||
|
||||
expectObservable(summarizeRingAttempts$(ringAttempts$)).toBe("-a 29ms A", {
|
||||
a: { intent: RTC_CALL_INTENT_AUDIO, recipient: alice.userId },
|
||||
a: { intent: "audio", recipient: alice.userId },
|
||||
A: { outcome: "timeout" },
|
||||
});
|
||||
});
|
||||
|
||||
@@ -9,9 +9,7 @@ import {
|
||||
type IRTCNotificationContent,
|
||||
type MatrixRTCSession,
|
||||
MatrixRTCSessionEvent,
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
type RTCCallIntent,
|
||||
toStableCallIntent,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
import {
|
||||
filter,
|
||||
@@ -157,10 +155,7 @@ export function createCallNotificationLifecycle$({
|
||||
);
|
||||
|
||||
return of({
|
||||
// The notification still carries the legacy intent values
|
||||
intent:
|
||||
toStableCallIntent(notificationEvent["m.call.intent"]) ??
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
intent: notificationEvent["m.call.intent"] ?? "audio",
|
||||
recipient,
|
||||
outcome$: race(timeout$, accept$, decline$).pipe(
|
||||
take(1),
|
||||
|
||||
@@ -21,7 +21,6 @@ import {
|
||||
type CallMembership,
|
||||
type IRTCNotificationContent,
|
||||
MatrixRTCSessionEvent,
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
import { deepCompare } from "matrix-js-sdk/lib/utils";
|
||||
|
||||
@@ -71,11 +70,9 @@ import {
|
||||
} from "./CallViewModel.ts";
|
||||
import { type Handler, MuteState } from "../MuteStates.ts";
|
||||
import {
|
||||
type LocalVideoMediaTrack,
|
||||
type RTCParticipation,
|
||||
type VideoProcessor,
|
||||
} from "@element-hq/matrixrtc-sdk";
|
||||
import { type BackgroundEffectsState } from "../BackgroundEffects";
|
||||
|
||||
initializeWidget();
|
||||
|
||||
@@ -1560,7 +1557,7 @@ describe.each(modes)("CallViewModel (%s mode)", (mode) => {
|
||||
b: expect.objectContaining({
|
||||
type: "ringing",
|
||||
userId: alice.userId,
|
||||
intent: RTC_CALL_INTENT_AUDIO,
|
||||
intent: "audio",
|
||||
}),
|
||||
});
|
||||
// Layout should show placeholder media for the participant we're
|
||||
@@ -1606,7 +1603,7 @@ describe.each(modes)("CallViewModel (%s mode)", (mode) => {
|
||||
b: expect.objectContaining({
|
||||
type: "ringing",
|
||||
userId: alice.userId,
|
||||
intent: RTC_CALL_INTENT_AUDIO,
|
||||
intent: "audio",
|
||||
}),
|
||||
});
|
||||
// Layout should show placeholder media for the participant we're
|
||||
@@ -1695,113 +1692,28 @@ describe("a camera with a video processor", () => {
|
||||
muteStates,
|
||||
mockMediaDevices({}),
|
||||
{},
|
||||
{ supported: true, processor },
|
||||
processor,
|
||||
).find((request) => request.source === "camera"),
|
||||
).toMatchObject({ processor });
|
||||
});
|
||||
|
||||
it("leaves the camera out of the join while the processor is prepared", () => {
|
||||
const muteStates = mockMuteStates();
|
||||
vi.spyOn(muteStates.video.enabled$, "value", "get").mockReturnValue(true);
|
||||
const sources = (wait?: "preparing"): string[] =>
|
||||
initialPublishRequests(
|
||||
muteStates,
|
||||
mockMediaDevices({}),
|
||||
{},
|
||||
{
|
||||
supported: true,
|
||||
processor: undefined,
|
||||
wait,
|
||||
},
|
||||
).map((request) => request.source);
|
||||
expect(sources("preparing")).not.toContain("camera");
|
||||
expect(sources()).toContain("camera");
|
||||
});
|
||||
|
||||
it("waits to make a camera track until it is known the processor builds", async () => {
|
||||
const setHandler = vi.spyOn(MuteState.prototype, "setHandler");
|
||||
onTestFinished(() => setHandler.mockRestore());
|
||||
const effects$ = new BehaviorSubject<BackgroundEffectsState>({
|
||||
supported: true,
|
||||
processor: undefined,
|
||||
wait: "preparing",
|
||||
});
|
||||
let participation: RTCParticipation | undefined;
|
||||
withCallViewModel(
|
||||
{},
|
||||
(_vm, _session, { rtcParticipation }) => {
|
||||
participation = rtcParticipation;
|
||||
},
|
||||
{ backgroundEffectsState$: effects$ },
|
||||
);
|
||||
const camera = participation!.localMember$.value?.tracks$.value?.find(
|
||||
(track) => track.source === "camera",
|
||||
) as LocalVideoMediaTrack;
|
||||
const turnedOn = cameraHandler(setHandler)(true);
|
||||
await flushPromises();
|
||||
expect(camera.setEnabled).not.toHaveBeenCalledWith(true);
|
||||
|
||||
effects$.next({ supported: true, processor });
|
||||
await turnedOn;
|
||||
expect(camera.setEnabled).toHaveBeenCalledWith(true);
|
||||
expect(participation!.publish).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("turns the camera on only once its processor is on the track", async () => {
|
||||
const setHandler = vi.spyOn(MuteState.prototype, "setHandler");
|
||||
onTestFinished(() => setHandler.mockRestore());
|
||||
const effects$ = new BehaviorSubject<BackgroundEffectsState>({
|
||||
supported: true,
|
||||
processor: undefined,
|
||||
wait: "preparing",
|
||||
});
|
||||
let participation: RTCParticipation | undefined;
|
||||
withCallViewModel(
|
||||
{},
|
||||
(_vm, _session, { rtcParticipation }) => {
|
||||
participation = rtcParticipation;
|
||||
},
|
||||
{ backgroundEffectsState$: effects$ },
|
||||
);
|
||||
const camera = participation!.localMember$.value?.tracks$.value?.find(
|
||||
(track) => track.source === "camera",
|
||||
) as LocalVideoMediaTrack;
|
||||
// Attaching takes its time, as the pipeline is built on the track
|
||||
let attached!: () => void;
|
||||
vi.mocked(camera.setProcessor).mockReturnValueOnce(
|
||||
new Promise<void>((resolve) => {
|
||||
attached = resolve;
|
||||
}),
|
||||
);
|
||||
const turnedOn = cameraHandler(setHandler)(true);
|
||||
effects$.next({ supported: true, processor });
|
||||
await flushPromises();
|
||||
expect(camera.setProcessor).toHaveBeenCalledWith(processor);
|
||||
expect(camera.setEnabled).not.toHaveBeenCalledWith(true);
|
||||
|
||||
attached();
|
||||
await turnedOn;
|
||||
expect(camera.setEnabled).toHaveBeenCalledWith(true);
|
||||
});
|
||||
|
||||
// The SDK builds the processor on the camera's restart, before the room
|
||||
// sees a frame, so the track is unmuted rather than published anew
|
||||
it("turns on a camera turned off before the processor was chosen", async () => {
|
||||
const setHandler = vi.spyOn(MuteState.prototype, "setHandler");
|
||||
onTestFinished(() => setHandler.mockRestore());
|
||||
const effects$ = new BehaviorSubject<BackgroundEffectsState>({
|
||||
supported: true,
|
||||
processor: undefined,
|
||||
});
|
||||
const videoProcessor$ = new BehaviorSubject<VideoProcessor | undefined>(
|
||||
undefined,
|
||||
);
|
||||
let participation: RTCParticipation | undefined;
|
||||
withCallViewModel(
|
||||
{},
|
||||
(_vm, _session, { rtcParticipation }) => {
|
||||
participation = rtcParticipation;
|
||||
},
|
||||
{ backgroundEffectsState$: effects$ },
|
||||
{ videoProcessor$ },
|
||||
);
|
||||
effects$.next({ supported: true, processor });
|
||||
videoProcessor$.next(processor);
|
||||
await cameraHandler(setHandler)(true);
|
||||
await flushPromises();
|
||||
|
||||
|
||||
@@ -37,7 +37,6 @@ import {
|
||||
combineLatest,
|
||||
distinctUntilChanged,
|
||||
filter,
|
||||
firstValueFrom,
|
||||
fromEvent,
|
||||
map,
|
||||
merge,
|
||||
@@ -78,7 +77,6 @@ import {
|
||||
type ReactionInfo,
|
||||
type ReactionOption,
|
||||
} from "../../reactions";
|
||||
import { type BackgroundEffectsState } from "../BackgroundEffects";
|
||||
import { shallowEquals as arrayShallowEquals } from "../../utils/array";
|
||||
import { type MediaDevices } from "../MediaDevices";
|
||||
import { type MuteStates } from "../MuteStates";
|
||||
@@ -172,16 +170,13 @@ export function initialPublishRequests(
|
||||
muteStates: MuteStates,
|
||||
mediaDevices: MediaDevices,
|
||||
capture: CaptureSettings,
|
||||
effects?: BackgroundEffectsState,
|
||||
processor?: VideoProcessor,
|
||||
): PublishRequest[] {
|
||||
const requests: PublishRequest[] = [];
|
||||
if (muteStates.audio.enabled$.value)
|
||||
requests.push(publishRequest("microphone", mediaDevices, capture));
|
||||
// Left out while the processor is prepared; the call publishes it after
|
||||
if (muteStates.video.enabled$.value && effects?.wait !== "preparing")
|
||||
requests.push(
|
||||
publishRequest("camera", mediaDevices, capture, effects?.processor),
|
||||
);
|
||||
if (muteStates.video.enabled$.value)
|
||||
requests.push(publishRequest("camera", mediaDevices, capture, processor));
|
||||
return requests;
|
||||
}
|
||||
|
||||
@@ -192,8 +187,8 @@ export interface CallViewModelOptions {
|
||||
* defaults otherwise.
|
||||
*/
|
||||
capture?: CaptureSettings;
|
||||
/** The background effects state (preparing, supported...) including the processor itself. */
|
||||
backgroundEffectsState$?: Behavior<BackgroundEffectsState>;
|
||||
/** Background blur and the like, applied to the camera as it changes. */
|
||||
videoProcessor$?: Behavior<VideoProcessor | undefined>;
|
||||
/**
|
||||
* The application hosting Element Call, which can ask it to hang up and wants
|
||||
* to know when the user joins or leaves. Defaults to no host.
|
||||
@@ -495,21 +490,7 @@ export function createCallViewModel$(
|
||||
sentCallNotification$ = of(null),
|
||||
} = options;
|
||||
|
||||
const {
|
||||
capture = {},
|
||||
backgroundEffectsState$ = constant<BackgroundEffectsState>({
|
||||
supported: false,
|
||||
processor: undefined,
|
||||
}),
|
||||
} = options;
|
||||
const videoProcessor$ = scope.behavior(
|
||||
backgroundEffectsState$.pipe(map(({ processor }) => processor)),
|
||||
);
|
||||
// Resolves once it is known whether the processor builds
|
||||
const processorKnown = async (): Promise<unknown> =>
|
||||
firstValueFrom(
|
||||
backgroundEffectsState$.pipe(filter(({ wait }) => wait !== "preparing")),
|
||||
);
|
||||
const { capture = {}, videoProcessor$ = constant(undefined) } = options;
|
||||
const localTracks$ = scope.behavior<
|
||||
(LocalAudioMediaTrack | LocalVideoMediaTrack)[]
|
||||
>(
|
||||
@@ -533,19 +514,6 @@ export function createCallViewModel$(
|
||||
"screenShare",
|
||||
) as Behavior<LocalVideoMediaTrack | undefined>;
|
||||
|
||||
const processorAttached = followProcessor(
|
||||
scope,
|
||||
camera$,
|
||||
videoProcessor$,
|
||||
logger,
|
||||
);
|
||||
|
||||
// A camera turned on waits for its processor to be known, then to be on
|
||||
// the track: LiveKit restarts the track with whatever processor it has then.
|
||||
const cameraReady = async (): Promise<void> => {
|
||||
await processorKnown();
|
||||
await processorAttached();
|
||||
};
|
||||
const supply = (source: "microphone" | "camera"): TrackSupply => ({
|
||||
publish: async () =>
|
||||
rtcParticipation.publish(
|
||||
@@ -556,7 +524,6 @@ export function createCallViewModel$(
|
||||
source === "camera" ? videoProcessor$.value : undefined,
|
||||
),
|
||||
),
|
||||
ready: source === "camera" ? cameraReady : undefined,
|
||||
});
|
||||
followMuteState(
|
||||
scope,
|
||||
@@ -583,29 +550,6 @@ export function createCallViewModel$(
|
||||
);
|
||||
followDevice(scope, camera$, mediaDevices.videoInput, "camera", logger);
|
||||
|
||||
// A camera wanted at the join was left out of it while the processor was
|
||||
// prepared (see initialPublishRequests), and is published once that is known
|
||||
if (
|
||||
backgroundEffectsState$.value.wait === "preparing" &&
|
||||
muteStates.video.enabled$.value
|
||||
)
|
||||
processorKnown()
|
||||
.then(async () => {
|
||||
if (camera$.value !== undefined || !muteStates.video.enabled$.value)
|
||||
return;
|
||||
await rtcParticipation.publish(
|
||||
publishRequest(
|
||||
"camera",
|
||||
mediaDevices,
|
||||
capture,
|
||||
videoProcessor$.value,
|
||||
),
|
||||
);
|
||||
})
|
||||
.catch((e) => {
|
||||
logger.error("Failed to publish the camera", e);
|
||||
});
|
||||
|
||||
// The output device follows the selection onto the client the same way. A
|
||||
// host that routes audio itself picks the output outside the browser, so the
|
||||
// client is left alone
|
||||
@@ -619,6 +563,8 @@ export function createCallViewModel$(
|
||||
});
|
||||
});
|
||||
|
||||
followProcessor(scope, camera$, videoProcessor$, logger);
|
||||
|
||||
const members$ = scope.behavior<(LocalRTCMember | RemoteRTCMember)[]>(
|
||||
combineLatest(
|
||||
[rtcParticipation.localMember$, rtcParticipation.remoteMembers$],
|
||||
|
||||
@@ -21,10 +21,6 @@ import {
|
||||
EARPIECE_CONFIG_ID,
|
||||
IOSControlledAudioOutput,
|
||||
} from "./IOSControlledAudioOutput";
|
||||
import {
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
RTC_CALL_INTENT_VIDEO,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
|
||||
// `vi.mock` calls are hoisted above all imports, so the static imports below
|
||||
// already see these mocks. Force the iOS platform so that the virtual earpiece
|
||||
@@ -85,7 +81,7 @@ describe("Default selection", () => {
|
||||
const output = new IOSControlledAudioOutput(
|
||||
constant(false),
|
||||
testScope,
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
"audio",
|
||||
);
|
||||
const selected = latestSelection(output);
|
||||
|
||||
@@ -104,7 +100,7 @@ describe("Default selection", () => {
|
||||
const output = new IOSControlledAudioOutput(
|
||||
constant(false),
|
||||
testScope,
|
||||
RTC_CALL_INTENT_VIDEO,
|
||||
"video",
|
||||
);
|
||||
const selected = latestSelection(output);
|
||||
|
||||
@@ -117,7 +113,7 @@ describe("Default selection", () => {
|
||||
const output = new IOSControlledAudioOutput(
|
||||
constant(false),
|
||||
testScope,
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
"audio",
|
||||
);
|
||||
const selected = latestSelection(output);
|
||||
|
||||
@@ -134,7 +130,7 @@ describe("Explicit selection", () => {
|
||||
const output = new IOSControlledAudioOutput(
|
||||
constant(false),
|
||||
testScope,
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
"audio",
|
||||
);
|
||||
const selected = latestSelection(output);
|
||||
|
||||
@@ -154,7 +150,7 @@ describe("Explicit selection", () => {
|
||||
const output = new IOSControlledAudioOutput(
|
||||
constant(false),
|
||||
testScope,
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
"audio",
|
||||
);
|
||||
const selected = latestSelection(output);
|
||||
|
||||
|
||||
@@ -8,10 +8,7 @@ Please see LICENSE in the repository root for full details.
|
||||
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
|
||||
import { combineLatest, merge, startWith, Subject, tap } from "rxjs";
|
||||
|
||||
import {
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
type RTCCallIntent,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
import type { RTCCallIntent } from "matrix-js-sdk/lib/matrixrtc";
|
||||
import {
|
||||
availableOutputDevices$ as controlledAvailableOutputDevices$,
|
||||
outputDevice$ as controlledOutputSelection$,
|
||||
@@ -114,7 +111,7 @@ export class IOSControlledAudioOutput implements MediaDevice<
|
||||
* Chooses the default output device when no explicit selection (from the user
|
||||
* or the hosting application) has been made yet.
|
||||
*
|
||||
* For voice calls (`initialIntent === "m.audio"`) we want to start on the
|
||||
* For voice calls (`initialIntent === "audio"`) we want to start on the
|
||||
* earpiece rather than the speaker, like a regular phone call. We only
|
||||
* override when the device that would otherwise be the default is the
|
||||
* speaker: if the host already routed to a headset (e.g. Bluetooth) — which
|
||||
@@ -125,7 +122,7 @@ export class IOSControlledAudioOutput implements MediaDevice<
|
||||
available: Map<string, AudioOutputDeviceLabel>,
|
||||
): string | undefined {
|
||||
const firstId = available.keys().next().value;
|
||||
if (this.initialIntent === RTC_CALL_INTENT_AUDIO) {
|
||||
if (this.initialIntent === "audio") {
|
||||
const firstLabel =
|
||||
firstId !== undefined ? available.get(firstId) : undefined;
|
||||
if (firstLabel?.type === "speaker") {
|
||||
|
||||
@@ -21,7 +21,6 @@ import { type MediaDevices } from "./MediaDevices";
|
||||
import { MuteStates } from "./MuteStates";
|
||||
import { nullHostBridge } from "../HostBridge";
|
||||
import { flushPromises, mockMediaDevices, testScope } from "../utils/test";
|
||||
import { type BackgroundEffectsState } from "./BackgroundEffects";
|
||||
|
||||
describe("LobbyMediaViewModel", () => {
|
||||
it("opens what starts enabled in one capture and follows the devices", async () => {
|
||||
@@ -136,7 +135,7 @@ function setup(
|
||||
muteStates,
|
||||
mediaDevices,
|
||||
{},
|
||||
constant<BackgroundEffectsState>({ supported: true, processor }),
|
||||
constant(processor),
|
||||
);
|
||||
return { media, vm, muteStates, mediaDevices };
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ import {
|
||||
type ObservableScope,
|
||||
type PublishRequest,
|
||||
trackBySource$,
|
||||
type VideoProcessor,
|
||||
} from "@element-hq/matrixrtc-sdk";
|
||||
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
|
||||
import { combineLatest, debounceTime, map } from "rxjs";
|
||||
@@ -24,7 +25,6 @@ import {
|
||||
import { type MediaDevices } from "./MediaDevices";
|
||||
import { type MuteStates } from "./MuteStates";
|
||||
import { followDevice, followProcessor } from "./localMediaControls";
|
||||
import { type BackgroundEffectsState } from "./BackgroundEffects";
|
||||
|
||||
export interface LobbyMediaViewModel {
|
||||
/** The camera, once open, for the preview. */
|
||||
@@ -42,13 +42,9 @@ export function createLobbyMediaViewModel(
|
||||
muteStates: MuteStates,
|
||||
mediaDevices: MediaDevices,
|
||||
capture: CaptureSettings,
|
||||
/** No camera is opened while its pipeline is prepared: it would show the room without the processor. */
|
||||
backgroundEffectsState$: Behavior<BackgroundEffectsState>,
|
||||
videoProcessor$: Behavior<VideoProcessor | undefined>,
|
||||
): LobbyMediaViewModel {
|
||||
const logger = rootLogger.getChild("[LobbyMedia]");
|
||||
const videoProcessor$ = scope.behavior(
|
||||
backgroundEffectsState$.pipe(map(({ processor }) => processor)),
|
||||
);
|
||||
const microphone$ = trackBySource$(
|
||||
scope,
|
||||
media.tracks$,
|
||||
@@ -68,19 +64,13 @@ export function createLobbyMediaViewModel(
|
||||
muteStates.video.enabled$,
|
||||
microphone$,
|
||||
camera$,
|
||||
backgroundEffectsState$,
|
||||
])
|
||||
.pipe(debounceTime(0), scope.bind())
|
||||
.subscribe(([audio, video, microphone, camera, { wait }]) => {
|
||||
.subscribe(([audio, video, microphone, camera]) => {
|
||||
const open: PublishRequest[] = [];
|
||||
if (audio && microphone === undefined && !opening.has("microphone"))
|
||||
open.push(publishRequest("microphone", mediaDevices, capture));
|
||||
if (
|
||||
video &&
|
||||
camera === undefined &&
|
||||
!opening.has("camera") &&
|
||||
wait !== "preparing"
|
||||
)
|
||||
if (video && camera === undefined && !opening.has("camera"))
|
||||
open.push(
|
||||
publishRequest(
|
||||
"camera",
|
||||
|
||||
@@ -35,10 +35,6 @@ import { AudioOutput, MediaDevices } from "./MediaDevices";
|
||||
import { AndroidControlledAudioOutput } from "./AndroidControlledAudioOutput";
|
||||
import { IOSControlledAudioOutput } from "./IOSControlledAudioOutput";
|
||||
import { ObservableScope } from "./ObservableScope";
|
||||
import {
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
RTC_CALL_INTENT_VIDEO,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
|
||||
// Which audio output implementation is used is decided by what the app hosting
|
||||
// Element Call told it, rather than being discovered from the environment.
|
||||
@@ -56,7 +52,7 @@ describe("MediaDevices audio output", () => {
|
||||
|
||||
const devices = new MediaDevices(new ObservableScope(), {
|
||||
controlledAudioDevices: true,
|
||||
callIntent: RTC_CALL_INTENT_AUDIO,
|
||||
callIntent: "audio",
|
||||
});
|
||||
|
||||
expect(devices.audioOutput).toBeInstanceOf(AndroidControlledAudioOutput);
|
||||
@@ -67,7 +63,7 @@ describe("MediaDevices audio output", () => {
|
||||
|
||||
const devices = new MediaDevices(new ObservableScope(), {
|
||||
controlledAudioDevices: true,
|
||||
callIntent: RTC_CALL_INTENT_VIDEO,
|
||||
callIntent: "video",
|
||||
});
|
||||
|
||||
expect(devices.audioOutput).toBeInstanceOf(IOSControlledAudioOutput);
|
||||
|
||||
@@ -6,11 +6,7 @@ Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { test, expect } from "vitest";
|
||||
import {
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
RTC_CALL_INTENT_VIDEO,
|
||||
type RTCCallIntent,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
import { type RTCCallIntent } from "matrix-js-sdk/lib/matrixrtc";
|
||||
|
||||
import { calculateInitialMuteState } from "./initialMuteState";
|
||||
|
||||
@@ -18,10 +14,10 @@ test.each<{
|
||||
callIntent: RTCCallIntent;
|
||||
allowJoinUnmutedViaIntent: boolean;
|
||||
}>([
|
||||
{ callIntent: RTC_CALL_INTENT_AUDIO, allowJoinUnmutedViaIntent: false },
|
||||
{ callIntent: RTC_CALL_INTENT_AUDIO, allowJoinUnmutedViaIntent: true },
|
||||
{ callIntent: RTC_CALL_INTENT_VIDEO, allowJoinUnmutedViaIntent: false },
|
||||
{ callIntent: RTC_CALL_INTENT_VIDEO, allowJoinUnmutedViaIntent: true },
|
||||
{ callIntent: "audio", allowJoinUnmutedViaIntent: false },
|
||||
{ callIntent: "audio", allowJoinUnmutedViaIntent: true },
|
||||
{ callIntent: "video", allowJoinUnmutedViaIntent: false },
|
||||
{ callIntent: "video", allowJoinUnmutedViaIntent: true },
|
||||
{ callIntent: "unknown", allowJoinUnmutedViaIntent: false },
|
||||
{ callIntent: "unknown", allowJoinUnmutedViaIntent: true },
|
||||
])(
|
||||
@@ -33,15 +29,15 @@ test.each<{
|
||||
allowJoinUnmutedViaIntent,
|
||||
);
|
||||
expect(audioEnabled).toBe(true);
|
||||
expect(videoEnabled).toBe(callIntent !== RTC_CALL_INTENT_AUDIO);
|
||||
expect(videoEnabled).toBe(callIntent !== "audio");
|
||||
},
|
||||
);
|
||||
|
||||
test.each<{
|
||||
callIntent: RTCCallIntent;
|
||||
}>([
|
||||
{ callIntent: RTC_CALL_INTENT_AUDIO },
|
||||
{ callIntent: RTC_CALL_INTENT_VIDEO },
|
||||
{ callIntent: "audio" },
|
||||
{ callIntent: "video" },
|
||||
{ callIntent: "unknown" },
|
||||
])(
|
||||
"Should always mute on start if skipping lobby and the host does not vouch for the intent (callIntent: $callIntent)",
|
||||
@@ -59,8 +55,8 @@ test.each<{
|
||||
test.each<{
|
||||
callIntent: RTCCallIntent;
|
||||
}>([
|
||||
{ callIntent: RTC_CALL_INTENT_AUDIO },
|
||||
{ callIntent: RTC_CALL_INTENT_VIDEO },
|
||||
{ callIntent: "audio" },
|
||||
{ callIntent: "video" },
|
||||
{ callIntent: "unknown" },
|
||||
])(
|
||||
"Can start unmuted if skipping lobby and the host vouches for the intent (callIntent: $callIntent)",
|
||||
@@ -71,6 +67,6 @@ test.each<{
|
||||
true,
|
||||
);
|
||||
expect(audioEnabled).toBe(true);
|
||||
expect(videoEnabled).toBe(callIntent !== RTC_CALL_INTENT_AUDIO);
|
||||
expect(videoEnabled).toBe(callIntent !== "audio");
|
||||
},
|
||||
);
|
||||
|
||||
@@ -6,10 +6,7 @@ Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
import {
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
type RTCCallIntent,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
import { type RTCCallIntent } from "matrix-js-sdk/lib/matrixrtc";
|
||||
|
||||
/**
|
||||
* Calculates the initial mute state for media devices based on configuration.
|
||||
@@ -43,10 +40,10 @@ export function calculateInitialMuteState(
|
||||
|
||||
// A host that vouches for the intent is a trusted environment, so it allows
|
||||
// unmuted by default. Same for when showing a lobby, as users can adjust
|
||||
// their settings there. Additionally, if the call intent is "m.audio", we
|
||||
// their settings there. Additionally, if the call intent is "audio", we
|
||||
// disable video by default.
|
||||
return {
|
||||
audioEnabled: true,
|
||||
videoEnabled: callIntent != RTC_CALL_INTENT_AUDIO,
|
||||
videoEnabled: callIntent != "audio",
|
||||
};
|
||||
}
|
||||
|
||||
@@ -21,8 +21,6 @@ import { type Handler } from "./MuteStates";
|
||||
/** How a mute switch gets a track where there is none. */
|
||||
export interface TrackSupply {
|
||||
publish: () => Promise<unknown>;
|
||||
/** Awaited before a switch turned on acts: a camera waits for its processor to be known. */
|
||||
ready?: () => Promise<unknown>;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -47,7 +45,6 @@ export function followMuteState(
|
||||
let desired = muteState.enabled$.value;
|
||||
muteState.setHandler(async (enabled) => {
|
||||
desired = enabled;
|
||||
if (enabled) await supply.ready?.();
|
||||
const track = track$.value;
|
||||
if (track !== undefined) {
|
||||
desired = await track.setEnabled(enabled);
|
||||
@@ -97,23 +94,18 @@ export function followDevice(
|
||||
/**
|
||||
* Attached once the camera track exists rather than carried in the publish
|
||||
* request, so that a new track and a change of processor take the same path.
|
||||
* Answers with a wait for the latest attachment to have gone through: a
|
||||
* camera turned on before then would restart without its processor.
|
||||
*/
|
||||
export function followProcessor(
|
||||
scope: ObservableScope,
|
||||
camera$: Behavior<LocalVideoMediaTrack | undefined>,
|
||||
videoProcessor$: Behavior<VideoProcessor | undefined>,
|
||||
logger: Logger,
|
||||
): () => Promise<void> {
|
||||
let attached = Promise.resolve();
|
||||
): void {
|
||||
combineLatest([camera$, videoProcessor$])
|
||||
.pipe(scope.bind())
|
||||
.subscribe(([camera, processor]) => {
|
||||
attached =
|
||||
camera?.setProcessor(processor).catch((e) => {
|
||||
logger.error("Failed to set the video processor", e);
|
||||
}) ?? Promise.resolve();
|
||||
camera?.setProcessor(processor).catch((e) => {
|
||||
logger.error("Failed to set the video processor", e);
|
||||
});
|
||||
});
|
||||
return async () => attached;
|
||||
}
|
||||
|
||||
@@ -8,10 +8,7 @@ Please see LICENSE in the repository root for full details.
|
||||
import { test, expect } from "vitest";
|
||||
import { act, render, screen } from "@testing-library/react";
|
||||
import { axe } from "vitest-axe";
|
||||
import {
|
||||
RTC_CALL_INTENT_AUDIO,
|
||||
type MatrixRTCSession,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
|
||||
import { BehaviorSubject } from "rxjs";
|
||||
|
||||
import { GridTile } from "./GridTile";
|
||||
@@ -125,7 +122,7 @@ test("GridTile displays ringing media", async () => {
|
||||
const vm = createRingingMedia({
|
||||
pickupState$,
|
||||
id: "test",
|
||||
intent: RTC_CALL_INTENT_AUDIO,
|
||||
intent: "audio",
|
||||
userId: "@alice:example.org",
|
||||
displayName$: constant("Alice"),
|
||||
mxcAvatarUrl$: constant(undefined),
|
||||
|
||||
@@ -12,7 +12,6 @@ import {
|
||||
EndCallIcon,
|
||||
} from "@vector-im/compound-design-tokens/assets/web/icons";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { RTC_CALL_INTENT_VIDEO } from "matrix-js-sdk/lib/matrixrtc";
|
||||
|
||||
import { type RingingMediaViewModel } from "../state/media/RingingMediaViewModel";
|
||||
import { useBehavior } from "../useBehavior";
|
||||
@@ -26,7 +25,7 @@ export const RingingStatus: FC<Props> = ({ vm }) => {
|
||||
const pickupState = useBehavior(vm.pickupState$);
|
||||
const Icon =
|
||||
pickupState === "ringing"
|
||||
? vm.intent === RTC_CALL_INTENT_VIDEO
|
||||
? vm.intent === "video"
|
||||
? VideoCallSolidIcon
|
||||
: VoiceCallSolidIcon
|
||||
: EndCallIcon;
|
||||
|
||||
@@ -11,7 +11,6 @@ import { axe } from "vitest-axe";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
import { TooltipProvider } from "@vector-im/compound-web";
|
||||
import { BehaviorSubject } from "rxjs";
|
||||
import { RTC_CALL_INTENT_AUDIO } from "matrix-js-sdk/lib/matrixrtc";
|
||||
|
||||
import { SpotlightTile } from "./SpotlightTile";
|
||||
import {
|
||||
@@ -177,7 +176,7 @@ test("SpotlightTile displays ringing media", async () => {
|
||||
const vm = createRingingMedia({
|
||||
pickupState$,
|
||||
id: "test",
|
||||
intent: RTC_CALL_INTENT_AUDIO,
|
||||
intent: "audio",
|
||||
userId: "@alice:example.org",
|
||||
displayName$: constant("Alice"),
|
||||
mxcAvatarUrl$: constant(undefined),
|
||||
|
||||
@@ -13,20 +13,13 @@ import {
|
||||
} from "@livekit/track-processors";
|
||||
import { ImageSegmenter } from "@mediapipe/tasks-vision";
|
||||
|
||||
import {
|
||||
BackgroundEffectTransformer,
|
||||
canSegment,
|
||||
} from "./BackgroundEffectTransformer";
|
||||
import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer";
|
||||
|
||||
const frame = {} as VideoFrame;
|
||||
const controller = {} as TransformStreamDefaultController<VideoFrame>;
|
||||
const stats = { processingTimeMs: 1, segmentationTimeMs: 1, filterTimeMs: 0 };
|
||||
|
||||
describe("BackgroundEffectTransformer", () => {
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
afterEach(() => vi.restoreAllMocks());
|
||||
|
||||
it("loads a background chosen before it was built", async () => {
|
||||
vi.spyOn(VideoTransformer.prototype, "init").mockResolvedValue();
|
||||
@@ -48,29 +41,10 @@ describe("BackgroundEffectTransformer", () => {
|
||||
expect(load).toHaveBeenCalledWith("/background.jpg");
|
||||
});
|
||||
|
||||
it("runs MediaPipe from its own bundle", async () => {
|
||||
vi.spyOn(VideoTransformer.prototype, "init").mockResolvedValue();
|
||||
const create = vi
|
||||
.spyOn(ImageSegmenter, "createFromOptions")
|
||||
.mockResolvedValue({} as ImageSegmenter);
|
||||
|
||||
await new BackgroundEffectTransformer({ backgroundDisabled: true }).init(
|
||||
{} as VideoTransformerInitOptions,
|
||||
);
|
||||
|
||||
const [fileset, options] = create.mock.calls[0];
|
||||
expect(fileset.wasmLoaderPath).toMatch(/vision_wasm_internal\.js/);
|
||||
expect(fileset.wasmBinaryPath).toMatch(/vision_wasm_internal\.wasm/);
|
||||
expect(options.baseOptions?.modelAssetPath).toMatch(
|
||||
/selfie_segmenter\.tflite/,
|
||||
);
|
||||
});
|
||||
|
||||
it("reports the first frame carrying an effect, once", async () => {
|
||||
// The library reports every frame it draws.
|
||||
vi.spyOn(BackgroundTransformer.prototype, "transform").mockImplementation(
|
||||
async function (this: BackgroundTransformer) {
|
||||
this.options.onFrameProcessed?.(stats);
|
||||
this.isFirstFrame = false;
|
||||
return Promise.resolve();
|
||||
},
|
||||
);
|
||||
@@ -93,37 +67,6 @@ describe("BackgroundEffectTransformer", () => {
|
||||
expect(transformer.onFirstFrame).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps reporting drawn frames to whoever asked", async () => {
|
||||
vi.spyOn(BackgroundTransformer.prototype, "transform").mockImplementation(
|
||||
async function (this: BackgroundTransformer) {
|
||||
this.options.onFrameProcessed?.(stats);
|
||||
return Promise.resolve();
|
||||
},
|
||||
);
|
||||
const onFrameProcessed = vi.fn();
|
||||
const transformer = new BackgroundEffectTransformer({ onFrameProcessed });
|
||||
|
||||
await transformer.transform(frame, controller);
|
||||
expect(onFrameProcessed).toHaveBeenCalledWith(stats);
|
||||
});
|
||||
|
||||
it("never sends the first frame on untouched", async () => {
|
||||
// The library copies its first frame through untouched, ahead of the
|
||||
// drawn one, unless told the first frame is past
|
||||
const seen: boolean[] = [];
|
||||
vi.spyOn(BackgroundTransformer.prototype, "transform").mockImplementation(
|
||||
async function (this: BackgroundTransformer) {
|
||||
seen.push(this.isFirstFrame);
|
||||
return Promise.resolve();
|
||||
},
|
||||
);
|
||||
const transformer = new BackgroundEffectTransformer({});
|
||||
expect(transformer.isFirstFrame).toBe(true);
|
||||
|
||||
await transformer.transform(frame, controller);
|
||||
expect(seen).toEqual([false]);
|
||||
});
|
||||
|
||||
it("passes frames on untouched once disabled, even after blurring", async () => {
|
||||
const transformer = new BackgroundEffectTransformer({});
|
||||
await transformer.update({ blurRadius: 15, backgroundDisabled: false });
|
||||
@@ -135,33 +78,4 @@ describe("BackgroundEffectTransformer", () => {
|
||||
expect(transformer.options.blurRadius).toBeUndefined();
|
||||
expect(transformer.options.imagePath).toBeUndefined();
|
||||
});
|
||||
|
||||
it("segments on the CPU where the GPU refuses", async () => {
|
||||
vi.spyOn(VideoTransformer.prototype, "init").mockResolvedValue();
|
||||
const build = vi
|
||||
.spyOn(ImageSegmenter, "createFromOptions")
|
||||
.mockRejectedValueOnce(new Error("blocklisted"))
|
||||
.mockResolvedValue({} as ImageSegmenter);
|
||||
|
||||
await new BackgroundEffectTransformer({}).init(
|
||||
{} as VideoTransformerInitOptions,
|
||||
);
|
||||
expect(build.mock.calls.map(([, o]) => o.baseOptions?.delegate)).toEqual([
|
||||
"GPU",
|
||||
"CPU",
|
||||
]);
|
||||
});
|
||||
|
||||
it("says whether a segmenter can be built at all", async () => {
|
||||
vi.stubGlobal("OffscreenCanvas", class {});
|
||||
const close = vi.fn();
|
||||
const build = vi
|
||||
.spyOn(ImageSegmenter, "createFromOptions")
|
||||
.mockResolvedValue({ close } as unknown as ImageSegmenter);
|
||||
expect(await canSegment()).toBe(true);
|
||||
expect(close).toHaveBeenCalled();
|
||||
|
||||
build.mockRejectedValue(new Error("no model"));
|
||||
expect(await canSegment()).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -11,18 +11,11 @@ import {
|
||||
VideoTransformer,
|
||||
type VideoTransformerInitOptions,
|
||||
type BackgroundOptions,
|
||||
type SegmenterOptions,
|
||||
} from "@livekit/track-processors";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
import { ImageSegmenter } from "@mediapipe/tasks-vision";
|
||||
|
||||
// Only the SIMD build of MediaPipe's runtime: every browser Element Call
|
||||
// supports has WebAssembly SIMD. As files rather than data URLs, so that a
|
||||
// host's bundler serves them as assets of its own, and a browser fetches them
|
||||
// only when an effect is first switched on.
|
||||
import wasmLoaderPath from "@mediapipe/tasks-vision/vision_wasm_internal.js?url&no-inline";
|
||||
import wasmBinaryPath from "@mediapipe/tasks-vision/vision_wasm_internal.wasm?url&no-inline";
|
||||
import modelAssetPath from "../mediapipe/imageSegmenter/selfie_segmenter.tflite?url&no-inline";
|
||||
import modelAssetPath from "../mediapipe/imageSegmenter/selfie_segmenter.tflite?url";
|
||||
|
||||
interface WasmFileset {
|
||||
/** The path to the Wasm loader script. */
|
||||
@@ -31,7 +24,26 @@ interface WasmFileset {
|
||||
wasmBinaryPath: string;
|
||||
}
|
||||
|
||||
const wasmFileset: WasmFileset = { wasmLoaderPath, wasmBinaryPath };
|
||||
// The MediaPipe package, by default, ships some alternative versions of the
|
||||
// WASM files which avoid SIMD for compatibility with older browsers. But SIMD
|
||||
// in WASM is actually fine by our support policy, so we include just the SIMD
|
||||
// versions.
|
||||
// It's really not ideal that we have to reference these internal files from
|
||||
// MediaPipe and depend on node_modules having this specific structure. It's
|
||||
// easy to see this breaking if our dependencies changed and MediaPipe were
|
||||
// no longer hoisted, or if we switched to another dependency loader such as
|
||||
// yarn PnP.
|
||||
// https://github.com/google-ai-edge/mediapipe/issues/5961
|
||||
const wasmFileset: WasmFileset = {
|
||||
wasmLoaderPath: new URL(
|
||||
"../../node_modules/@mediapipe/tasks-vision/wasm/vision_wasm_internal.js",
|
||||
import.meta.url,
|
||||
).href,
|
||||
wasmBinaryPath: new URL(
|
||||
"../../node_modules/@mediapipe/tasks-vision/wasm/vision_wasm_internal.wasm",
|
||||
import.meta.url,
|
||||
).href,
|
||||
};
|
||||
|
||||
/**
|
||||
* Track processor that applies effects such as blurring to a user's background.
|
||||
@@ -41,41 +53,18 @@ const wasmFileset: WasmFileset = { wasmLoaderPath, wasmBinaryPath };
|
||||
* resource fetched from the public internet.
|
||||
*/
|
||||
export class BackgroundEffectTransformer extends BackgroundTransformer {
|
||||
/**
|
||||
* Called once, when the first frame carrying an effect has been drawn.
|
||||
* This is used for drewAFrame$ to show the correct loading state.
|
||||
*/
|
||||
/** Called once, when the first frame carrying an effect has been drawn. */
|
||||
public onFirstFrame: (() => void) | undefined;
|
||||
private drewAFrame = false;
|
||||
|
||||
public constructor(opts: BackgroundOptions) {
|
||||
super(opts);
|
||||
// Called by the library for a frame it drew, never for one it passed on
|
||||
// untouched.
|
||||
this.options.onFrameProcessed = (stats): void => {
|
||||
opts.onFrameProcessed?.(stats);
|
||||
if (this.drewAFrame) return;
|
||||
this.drewAFrame = true;
|
||||
this.onFirstFrame?.();
|
||||
};
|
||||
}
|
||||
|
||||
public override async transform(
|
||||
frame: VideoFrame,
|
||||
controller: TransformStreamDefaultController<VideoFrame>,
|
||||
): Promise<void> {
|
||||
// From the LK docs:
|
||||
// > Render a copy of the first frame is rendered to the screen as soon as possible to act
|
||||
// > as a less jarring initial state than a solid color while the synchronous work below
|
||||
// > (segmentation + frame rendering) occurs.
|
||||
// We do not want this copy (it leaks unblurred media).
|
||||
// So we always set `isFirstFrame` to false -> never show an unprocessed frame.
|
||||
// It does not have any impact other than setting the unprocessed frame while loading the processor.
|
||||
//
|
||||
// To not show the blank frame, we have our own mechanism for hiding the loading overlay:
|
||||
// `onFirstFrame` is what the EC UI will listen to and only then show the video feed.
|
||||
this.isFirstFrame = false;
|
||||
const priming = this.isFirstFrame;
|
||||
await super.transform(frame, controller);
|
||||
// Cleared only once a frame has been through the segmenter; a frame
|
||||
// passed through untouched leaves it set.
|
||||
if (priming && !this.isFirstFrame) this.onFirstFrame?.();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -103,10 +92,17 @@ export class BackgroundEffectTransformer extends BackgroundTransformer {
|
||||
inputElement: inputVideo,
|
||||
});
|
||||
|
||||
this.imageSegmenter = await createSegmenter(
|
||||
this.canvas,
|
||||
this.options.segmenterOptions,
|
||||
);
|
||||
this.imageSegmenter = await ImageSegmenter.createFromOptions(wasmFileset, {
|
||||
baseOptions: {
|
||||
modelAssetPath,
|
||||
delegate: "GPU",
|
||||
...this.options.segmenterOptions,
|
||||
},
|
||||
canvas: this.canvas,
|
||||
runningMode: "VIDEO",
|
||||
outputCategoryMask: true,
|
||||
outputConfidenceMasks: false,
|
||||
});
|
||||
|
||||
// BackgroundTransformer's own init applies these, and this one replaces it.
|
||||
if (this.options.imagePath) {
|
||||
@@ -120,38 +116,3 @@ export class BackgroundEffectTransformer extends BackgroundTransformer {
|
||||
this.gl?.setBackgroundDisabled(this.options.backgroundDisabled ?? false);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a segmenter can be built here, asked once before any camera depends
|
||||
* on one: a pipeline that fails to build takes the camera down with it.
|
||||
*/
|
||||
export async function canSegment(): Promise<boolean> {
|
||||
try {
|
||||
(await createSegmenter(new OffscreenCanvas(1, 1))).close();
|
||||
return true;
|
||||
} catch (e) {
|
||||
logger.warn("Background effects cannot run here", e);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** On the GPU, or on the CPU where the GPU refuses, as a blocklisted one does. */
|
||||
async function createSegmenter(
|
||||
canvas: HTMLCanvasElement | OffscreenCanvas | undefined,
|
||||
segmenterOptions?: SegmenterOptions,
|
||||
): Promise<ImageSegmenter> {
|
||||
const build = async (delegate: "GPU" | "CPU"): Promise<ImageSegmenter> =>
|
||||
ImageSegmenter.createFromOptions(wasmFileset, {
|
||||
baseOptions: { modelAssetPath, delegate, ...segmenterOptions },
|
||||
canvas,
|
||||
runningMode: "VIDEO",
|
||||
outputCategoryMask: true,
|
||||
outputConfidenceMasks: false,
|
||||
});
|
||||
try {
|
||||
return await build("GPU");
|
||||
} catch (e) {
|
||||
logger.warn("Segmenting on the CPU, as the GPU refused", e);
|
||||
return build("CPU");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,9 +8,11 @@ Please see LICENSE in the repository root for full details.
|
||||
import { act, createElement, type FC } from "react";
|
||||
import { render } from "@testing-library/react";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { BehaviorSubject } from "rxjs";
|
||||
|
||||
import {
|
||||
BackgroundEffectsProvider,
|
||||
reportCamera,
|
||||
useBackgroundEffects,
|
||||
createAppBackgroundEffects,
|
||||
} from "./BackgroundEffectsContext";
|
||||
@@ -19,8 +21,10 @@ import {
|
||||
type BackgroundEffectsState,
|
||||
} from "../state/BackgroundEffects";
|
||||
import { backgroundEffect } from "../settings/settings";
|
||||
import { sdkCamera, type ShownCamera, SyncedCameraTrack } from "./cameraTrack";
|
||||
import { type AddedBackground } from "./backgroundImages";
|
||||
import { flushPromises, testScope } from "../utils/test";
|
||||
import { constant } from "../state/Behavior";
|
||||
import { flushPromises, mockVideoTrack, testScope } from "../utils/test";
|
||||
|
||||
const sdkSupportMock = vi.hoisted(() => vi.fn(() => true));
|
||||
const pipelines = vi.hoisted(() => ({
|
||||
@@ -56,10 +60,86 @@ vi.mock("./backgroundImages", async () => {
|
||||
};
|
||||
});
|
||||
vi.mock("./BackgroundEffectTransformer", () => ({
|
||||
BackgroundEffectTransformer: vi.fn(function () {}),
|
||||
canSegment: async (): Promise<boolean> => Promise.resolve(true),
|
||||
BackgroundEffectTransformer: vi.fn(),
|
||||
}));
|
||||
|
||||
describe("the synced camera", () => {
|
||||
it("is the track last synced, and cleared only while still its own", () => {
|
||||
const synced = new SyncedCameraTrack();
|
||||
const pipeline = constant<BackgroundEffectsState>({
|
||||
supported: false,
|
||||
processor: undefined,
|
||||
cameraTrack: synced,
|
||||
});
|
||||
const lobby = sdkCamera(mockVideoTrack({ id: "lobby" }));
|
||||
const call = sdkCamera(mockVideoTrack({ id: "call" }));
|
||||
const lobbyCamera$ = new BehaviorSubject<ShownCamera | null>(lobby);
|
||||
reportCamera(testScope(), lobbyCamera$, pipeline);
|
||||
expect(synced.track$.value).toBe(lobby);
|
||||
|
||||
const callScope = testScope();
|
||||
reportCamera(callScope, constant(call), pipeline);
|
||||
expect(synced.track$.value).toBe(call);
|
||||
|
||||
lobbyCamera$.next(null);
|
||||
expect(synced.track$.value).toBe(call);
|
||||
callScope.end();
|
||||
expect(synced.track$.value).toBeNull();
|
||||
});
|
||||
|
||||
it("reports the camera once, however often the pipeline's state changes", () => {
|
||||
const synced = new SyncedCameraTrack();
|
||||
const report = vi.spyOn(synced, "report");
|
||||
const state$ = new BehaviorSubject<BackgroundEffectsState>({
|
||||
supported: true,
|
||||
processor: undefined,
|
||||
cameraTrack: synced,
|
||||
});
|
||||
reportCamera(testScope(), constant(sdkCamera(mockVideoTrack({}))), state$);
|
||||
state$.next({ ...state$.value, settling: true });
|
||||
state$.next({ ...state$.value, settling: false });
|
||||
expect(report).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("is the SDK's camera once the call reports it", () => {
|
||||
const synced = new SyncedCameraTrack();
|
||||
const state$ = constant<BackgroundEffectsState>({
|
||||
supported: true,
|
||||
processor: undefined,
|
||||
cameraTrack: synced,
|
||||
});
|
||||
const callCamera = mockVideoTrack({ facingMode$: constant("user") });
|
||||
const callCamera$ = new BehaviorSubject<ShownCamera | null>(
|
||||
sdkCamera(callCamera),
|
||||
);
|
||||
reportCamera(testScope(), callCamera$, state$);
|
||||
expect(synced.track$.value).toBe(sdkCamera(callCamera));
|
||||
expect(synced.track$.value?.facesUser()).toBe(true);
|
||||
|
||||
const element = document.createElement("video");
|
||||
synced.track$.value!.attach(element);
|
||||
expect(callCamera.attach).toHaveBeenCalledWith(element);
|
||||
|
||||
callCamera$.next(null);
|
||||
expect(synced.track$.value).toBeNull();
|
||||
});
|
||||
|
||||
it("keeps each call's camera to its own", () => {
|
||||
const mine = createAppBackgroundEffects(testScope());
|
||||
const theirs = createAppBackgroundEffects(testScope());
|
||||
const first = sdkCamera(mockVideoTrack({ id: "first" }));
|
||||
const second = sdkCamera(mockVideoTrack({ id: "second" }));
|
||||
reportCamera(testScope(), constant(first), mine.state$);
|
||||
reportCamera(testScope(), constant(second), theirs.state$);
|
||||
|
||||
expect(mine.state$.value.cameraTrack).not.toBe(
|
||||
theirs.state$.value.cameraTrack,
|
||||
);
|
||||
expect(mine.state$.value.cameraTrack?.track$.value).toBe(first);
|
||||
expect(theirs.state$.value.cameraTrack?.track$.value).toBe(second);
|
||||
});
|
||||
});
|
||||
|
||||
describe("BackgroundEffectsProvider", () => {
|
||||
let seen: BackgroundEffectsState[];
|
||||
const Surface: FC = () => {
|
||||
|
||||
@@ -5,6 +5,14 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import {
|
||||
combineLatest,
|
||||
distinctUntilChanged,
|
||||
EMPTY,
|
||||
map,
|
||||
Observable,
|
||||
switchMap,
|
||||
} from "rxjs";
|
||||
import {
|
||||
BackgroundProcessorWrapper,
|
||||
supportsBackgroundProcessors,
|
||||
@@ -12,10 +20,8 @@ import {
|
||||
import { createContext, type FC, type JSX, use } from "react";
|
||||
|
||||
import { backgroundEffect as backgroundEffectSetting } from "../settings/settings";
|
||||
import {
|
||||
BackgroundEffectTransformer,
|
||||
canSegment,
|
||||
} from "./BackgroundEffectTransformer";
|
||||
import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer";
|
||||
import { type ShownCamera } from "./cameraTrack";
|
||||
import { addedBackgrounds } from "./backgroundImages";
|
||||
import { type Behavior } from "../state/Behavior";
|
||||
import { type ObservableScope } from "../state/ObservableScope";
|
||||
@@ -59,7 +65,6 @@ export function createAppBackgroundEffects(
|
||||
effect$: backgroundEffectSetting.value$,
|
||||
setEffect: backgroundEffectSetting.setValue,
|
||||
added$: addedBackgrounds.added$,
|
||||
canSegment,
|
||||
pipeline: new BackgroundProcessorWrapper(transformer, "background-effect"),
|
||||
transformer,
|
||||
});
|
||||
@@ -75,3 +80,32 @@ export const BackgroundEffectsProvider: FC<Props> = ({ effects, children }) => (
|
||||
{children}
|
||||
</BackgroundEffectsContext>
|
||||
);
|
||||
|
||||
/**
|
||||
* Reports the camera the pipeline is synced to while the scope lasts: the
|
||||
* lobby's before joining, or the call's, each syncing its own.
|
||||
*/
|
||||
export function reportCamera(
|
||||
scope: ObservableScope,
|
||||
camera$: Observable<ShownCamera | null>,
|
||||
backgroundEffectsState$: Behavior<BackgroundEffectsState>,
|
||||
): void {
|
||||
// Keyed on the camera alone: reporting again for every change of state would
|
||||
// blank the preview each time.
|
||||
combineLatest([
|
||||
camera$,
|
||||
backgroundEffectsState$.pipe(
|
||||
map(({ cameraTrack }) => cameraTrack),
|
||||
distinctUntilChanged(),
|
||||
),
|
||||
])
|
||||
.pipe(
|
||||
switchMap(([camera, cameraTrack]) =>
|
||||
camera === null || cameraTrack === undefined
|
||||
? EMPTY
|
||||
: new Observable<never>(() => cameraTrack.report(camera)),
|
||||
),
|
||||
scope.bind(),
|
||||
)
|
||||
.subscribe();
|
||||
}
|
||||
|
||||
@@ -5,8 +5,8 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import arc from "../graphics/background-arc.jpg?url&no-inline";
|
||||
import glow from "../graphics/background-glow.jpg?url&no-inline";
|
||||
import arc from "../graphics/background-arc.jpg?url";
|
||||
import glow from "../graphics/background-glow.jpg?url";
|
||||
|
||||
export interface ShippedBackground {
|
||||
/** Stored in the user's settings when chosen, so it must not change. */
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { BehaviorSubject } from "rxjs";
|
||||
import { type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk";
|
||||
|
||||
import { type Behavior } from "../state/Behavior";
|
||||
|
||||
/**
|
||||
* A camera that can be shown in a video element: the MatrixRTC SDK's track,
|
||||
* before joining and in the call alike.
|
||||
*/
|
||||
export interface ShownCamera {
|
||||
attach(element: HTMLVideoElement): void;
|
||||
/** Detaches this element only: another may be showing the same camera. */
|
||||
detach(element: HTMLVideoElement): void;
|
||||
/** Whether it faces the user, so that it is shown mirrored. */
|
||||
facesUser(): boolean;
|
||||
}
|
||||
|
||||
// One per track, so that a view showing it isn't attached again on every render.
|
||||
const shown = new WeakMap<object, ShownCamera>();
|
||||
|
||||
function shownOnce(track: object, make: () => ShownCamera): ShownCamera {
|
||||
let camera = shown.get(track);
|
||||
if (camera === undefined) {
|
||||
camera = make();
|
||||
shown.set(track, camera);
|
||||
}
|
||||
return camera;
|
||||
}
|
||||
|
||||
/** A camera as the MatrixRTC SDK has it. */
|
||||
export function sdkCamera(track: LocalVideoMediaTrack): ShownCamera {
|
||||
return shownOnce(track, () => ({
|
||||
attach: (element): void => track.attach(element),
|
||||
detach: (element): void => track.detach(element),
|
||||
facesUser: (): boolean => track.facingMode$.value === "user",
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* The camera track a background pipeline is synced to: the lobby's before
|
||||
* joining, the call's after, none while the camera is off.
|
||||
*/
|
||||
export class SyncedCameraTrack {
|
||||
private readonly subject$ = new BehaviorSubject<ShownCamera | null>(null);
|
||||
public readonly track$: Behavior<ShownCamera | null> = this.subject$;
|
||||
|
||||
public report(track: ShownCamera): () => void {
|
||||
this.subject$.next(track);
|
||||
// Only if still this one, so a lobby leaving doesn't blank a call starting.
|
||||
return (): void => {
|
||||
if (this.subject$.value === track) this.subject$.next(null);
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { type FC } from "react";
|
||||
import { render } from "@testing-library/react";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { useAttachedTrack } from "./useAttachedTrack";
|
||||
import { type ShownCamera } from "./tracks/cameraTrack";
|
||||
|
||||
function cameraTrack(facing: string): ShownCamera {
|
||||
return {
|
||||
attach: vi.fn(),
|
||||
detach: vi.fn(),
|
||||
facesUser: (): boolean => facing === "user",
|
||||
};
|
||||
}
|
||||
|
||||
const Video: FC<{ track: ShownCamera | null }> = ({ track }) => {
|
||||
const { videoRef, mirrored } = useAttachedTrack(track);
|
||||
return <video ref={videoRef} data-mirrored={mirrored} />;
|
||||
};
|
||||
|
||||
describe("useAttachedTrack", () => {
|
||||
it("detaches only its own element from a track shown twice", () => {
|
||||
const track = cameraTrack("user");
|
||||
const first = render(<Video track={track} />);
|
||||
const second = render(<Video track={track} />);
|
||||
const [a, b] = [first, second].map((r) =>
|
||||
r.container.querySelector("video"),
|
||||
);
|
||||
expect(track.attach).toHaveBeenCalledWith(a);
|
||||
expect(track.attach).toHaveBeenCalledWith(b);
|
||||
|
||||
first.unmount();
|
||||
expect(track.detach).toHaveBeenCalledExactlyOnceWith(a);
|
||||
});
|
||||
|
||||
it("mirrors a camera facing the user, and only that", () => {
|
||||
const { container, rerender } = render(
|
||||
<Video track={cameraTrack("user")} />,
|
||||
);
|
||||
expect(container.querySelector("video")?.dataset.mirrored).toBe("true");
|
||||
rerender(<Video track={cameraTrack("environment")} />);
|
||||
expect(container.querySelector("video")?.dataset.mirrored).toBe("false");
|
||||
rerender(<Video track={null} />);
|
||||
expect(container.querySelector("video")?.dataset.mirrored).toBe("false");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,31 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { type RefObject, useEffect, useRef } from "react";
|
||||
|
||||
import { type ShownCamera } from "./tracks/cameraTrack";
|
||||
|
||||
/**
|
||||
* Shows a camera in the video element given the returned ref, and says
|
||||
* whether to mirror it, as a camera facing the user is. Only that element is
|
||||
* detached again: another may be showing the same camera.
|
||||
*/
|
||||
export function useAttachedTrack(camera: ShownCamera | null): {
|
||||
videoRef: RefObject<HTMLVideoElement | null>;
|
||||
mirrored: boolean;
|
||||
} {
|
||||
const videoRef = useRef<HTMLVideoElement | null>(null);
|
||||
useEffect(() => {
|
||||
const element = videoRef.current;
|
||||
if (!camera || !element) return;
|
||||
camera.attach(element);
|
||||
return (): void => {
|
||||
camera.detach(element);
|
||||
};
|
||||
}, [camera]);
|
||||
return { videoRef, mirrored: camera?.facesUser() ?? false };
|
||||
}
|
||||
@@ -6,7 +6,6 @@ Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { defineConfig } from "vite";
|
||||
import { posix } from "node:path";
|
||||
|
||||
import { vitePluginsConfig } from "./vite.config.ts";
|
||||
import { scopeStylesToRoot } from "./component/build/scopeStylesToRoot.ts";
|
||||
@@ -35,25 +34,6 @@ export default defineConfig(({ mode }) => {
|
||||
// whatever is in `public` — including the developer's own config.json, which
|
||||
// is not in the repository — into the output we would publish.
|
||||
publicDir: false,
|
||||
// A library does not know where it will be served from, so every file it
|
||||
// emits beside itself — the sounds, the pictures, MediaPipe's runtime — is
|
||||
// referenced relative to the module's own URL. (A library build inlines
|
||||
// each asset as a data URL unless its import says `no-inline`, so the
|
||||
// imports do.)
|
||||
base: "./",
|
||||
experimental: {
|
||||
// Relative to the chunk that references it, wherever that chunk lands,
|
||||
// and spelled `./assets/…` rather than `assets/…`: a host's bundler reads
|
||||
// a bare name as a package to look up.
|
||||
renderBuiltUrl: (filename, { hostId, hostType }) => {
|
||||
if (hostType !== "js") return { relative: true };
|
||||
const relative = posix.relative(posix.dirname(hostId), filename);
|
||||
const specifier = relative.startsWith(".") ? relative : `./${relative}`;
|
||||
return {
|
||||
runtime: `new URL(${JSON.stringify(specifier)}, import.meta.url).href`,
|
||||
};
|
||||
},
|
||||
},
|
||||
// A host's document is not ours to style: everything in the stylesheet is
|
||||
// confined to the element Element Call is mounted in
|
||||
css: { postcss: { plugins: [scopeStylesToRoot()] } },
|
||||
@@ -84,15 +64,6 @@ export default defineConfig(({ mode }) => {
|
||||
cssFileName: "element-call",
|
||||
},
|
||||
rollupOptions: {
|
||||
output: {
|
||||
// Hashed, as the app's are, so that a file a host serves as it is
|
||||
// stays correct across versions of the package. Not the stylesheet,
|
||||
// which the package exports by name (see `cssFileName`).
|
||||
assetFileNames: ({ names }) =>
|
||||
names.some((name) => name.endsWith(".css"))
|
||||
? "[name][extname]"
|
||||
: "assets/[name]-[hash][extname]",
|
||||
},
|
||||
// The host already has these, and a second copy of any of them does not
|
||||
// merely bloat the bundle: React would hold two sets of hooks, the
|
||||
// Matrix client would run two sync loops, and a second Compound would
|
||||
|
||||
Reference in New Issue
Block a user