Compare commits

..
Author SHA1 Message Date
Timo K. 9b86a9c8f4 Update main.ts 2026-11-13 18:03:23 +01:00
Timo K. e2e504d307 Data channel lk support 2026-11-06 10:07:00 +01:00
Timo K. de16cb4e37 refactor media sdk api 2026-11-05 16:57:39 +01:00
160 changed files with 2783 additions and 5831 deletions
-35
View File
@@ -1,35 +0,0 @@
---
name: handoff
description: Implement the first cut of a task and stop for a direction check before the quality pass.
argument-hint: <the task>
disable-model-invocation: true
---
Task: $ARGUMENTS
Stop after the first implementation and hand it back before the quality pass.
## Before the hand-off
- Run the narrowest check that rules out an obviously broken hand-off: the test
file for the touched code, `tsc` on its own, a single story. Not the full
`pnpm lint`, not the whole suite.
- If a wider check is needed before feedback can be useful, keep it to the touched
code and say why it was needed.
- Leave wider refactors, extra tests and documentation polish out. They are
follow-up work, not part of the first hand-off.
## The hand-off
Stop and ask whether the direction is right. The message says:
- what was built and where, in a few lines,
- which check ran and what it returned, verbatim if it failed,
- what was left out on purpose,
- the decisions that could have gone another way.
Do not continue past this point on your own.
## After the user confirms
The skill ends here. Continue with the quality pass as usual.
+1 -1
View File
@@ -14,7 +14,7 @@ permissions:
jobs:
prevent-blocked:
name: Prevent blocked
runs-on: ubuntu-24.04
runs-on: ubuntu-latest
permissions:
pull-requests: read
steps:
@@ -17,7 +17,7 @@ env:
jobs:
build_and_deploy:
name: Build & publish docker
runs-on: ubuntu-24.04
runs-on: ubuntu-latest
permissions:
contents: write # required to upload release asset
packages: write # needed for publishing packages to GHCR
+2 -2
View File
@@ -7,7 +7,7 @@ on:
type: string
package:
type: string # This would ideally be a `choice` type, but that isn't supported yet
description: The package type to be built. Must be one of 'full', 'embedded', or 'widget-rtc-sdk'
description: The package type to be built. Must be one of 'full', 'embedded', or 'sdk'
required: true
build_mode:
type: string # This would ideally be a `choice` type, but that isn't supported yet
@@ -29,7 +29,7 @@ on:
jobs:
build:
name: Build Element Call
runs-on: ubuntu-24.04
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+8 -6
View File
@@ -25,7 +25,7 @@ jobs:
# Deploy main branch to call.element.dev after build completes
if: github.ref == 'refs/heads/main'
needs: build_full_element_call
runs-on: ubuntu-24.04
runs-on: ubuntu-latest
steps:
- name: Deploy to call.element.dev
uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410 # v6
@@ -72,12 +72,14 @@ jobs:
SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
build_widget_rtc_sdk:
# The widget wrapper around the sdk/ package (widget-rtc-sdk/), not the
# package itself. Deployed by pr-deploy.yaml as the pr<N>-widget-rtc-sdk preview.
build_sdk_element_call:
# The 'sdk' package is the CallViewModel bundle that predates the sdk/
# package (build:sdk:production, sdk-target-based-on-call-view-model/), not
# the package itself. The name is shared with pr-deploy.yaml, which always
# runs from main, so it stays until the bundle's consumers have moved over.
uses: ./.github/workflows/build-element-call.yaml
with:
package: widget-rtc-sdk
package: sdk
vite_app_version: ${{ github.event.release.tag_name || github.sha }}
build_mode: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'development build') && 'development' || 'production' }}
secrets:
@@ -90,7 +92,7 @@ jobs:
build_storybook:
name: Build Storybook
if: contains(github.event.pull_request.labels.*.name, 'storybook build')
runs-on: ubuntu-24.04
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+1 -1
View File
@@ -13,7 +13,7 @@ permissions:
jobs:
pr-changelog-label:
runs-on: ubuntu-24.04
runs-on: ubuntu-latest
steps:
- uses: yogevbd/enforce-label-action@a3c219da6b8fa73f6ba62b68ff09c469b3a1c024 # 2.2.2
with:
+1 -1
View File
@@ -13,7 +13,7 @@ jobs:
name: Review
# Only run on labelling and skip forks.
if: github.event.label.name == 'Claude-Review' && github.event.pull_request.head.repo.full_name == github.repository
runs-on: ubuntu-24.04
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+12 -14
View File
@@ -17,7 +17,7 @@ on:
package:
required: true
type: string
description: Which package to deploy - 'full', 'embedded', 'widget-rtc-sdk', or 'storybook'
description: Which package to deploy - 'full', 'embedded', 'sdk', or 'storybook'
artifact_run_id:
required: false
type: string
@@ -32,7 +32,7 @@ on:
jobs:
deploy:
runs-on: ubuntu-24.04
runs-on: ubuntu-latest
permissions:
deployments: write
environment: Netlify
@@ -64,27 +64,25 @@ jobs:
run: curl -s https://raw.githubusercontent.com/element-hq/element-call/main/config/netlify_redirects > webapp/_redirects
- name: Add config file
# Only the app reads a config.json beside it. A widget gets everything
# about the homeserver from its host.
if: inputs.package == 'full'
run: curl -s "https://raw.githubusercontent.com/${INPUTS_PR_HEAD_FULL_NAME}/${INPUTS_PR_HEAD_REF}/config/config_netlify_preview.json" > webapp/config.json
# Not needed for storybook deployments
if: inputs.package != 'storybook'
run: |
if [ "${INPUTS_PACKAGE}" = "full" ]; then
curl -s "https://raw.githubusercontent.com/${INPUTS_PR_HEAD_FULL_NAME}/${INPUTS_PR_HEAD_REF}/config/config_netlify_preview.json" > webapp/config.json
else
curl -s "https://raw.githubusercontent.com/${INPUTS_PR_HEAD_FULL_NAME}/${INPUTS_PR_HEAD_REF}/config/config_netlify_preview_sdk.json" > webapp/config.json
fi
env:
INPUTS_PACKAGE: ${{ inputs.package }}
INPUTS_PR_HEAD_FULL_NAME: ${{ inputs.pr_head_full_name }}
INPUTS_PR_HEAD_REF: ${{ inputs.pr_head_ref }}
- name: Add headers file
# A widget page on another origin imports the widget RTC SDK bundle as a
# module, which the browser only allows with CORS
if: inputs.package == 'widget-rtc-sdk'
run: |
printf '/*\n Access-Control-Allow-Origin: *\n' > webapp/_headers
- name: ☁️ Deploy to Netlify
id: netlify
uses: nwtgck/actions-netlify@4cbaf4c08f1a7bfa537d6113472ef4424e4eb654 # v3.0
with:
publish-dir: webapp
deploy-message: "Deploy from GitHub Actions"
alias: ${{ inputs.package == 'widget-rtc-sdk' && format('pr{0}-widget-rtc-sdk', inputs.pr_number) || inputs.package == 'storybook' && format('pr{0}-storybook', inputs.pr_number) || format('pr{0}', inputs.pr_number) }}
alias: ${{ inputs.package == 'sdk' && format('pr{0}-sdk', inputs.pr_number) || inputs.package == 'storybook' && format('pr{0}-storybook', inputs.pr_number) || format('pr{0}', inputs.pr_number) }}
env:
NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}
+1 -1
View File
@@ -10,7 +10,7 @@ on:
jobs:
lint:
name: Lint, format & type check
runs-on: ubuntu-24.04
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+3 -3
View File
@@ -17,7 +17,7 @@ jobs:
github.event.workflow_run.conclusion == 'success' &&
github.event.workflow_run.event == 'pull_request' &&
github.event.workflow_run.head_repository.full_name == github.repository
runs-on: ubuntu-24.04
runs-on: ubuntu-latest
outputs:
pr_number: ${{ steps.prdetails.outputs.pr_id }}
pr_data_json: ${{ steps.prdetails.outputs.data }}
@@ -46,7 +46,7 @@ jobs:
NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}
netlify-widget-rtc-sdk:
netlify-sdk:
needs: prdetails
permissions:
deployments: write
@@ -57,7 +57,7 @@ jobs:
pr_head_full_name: ${{ github.event.workflow_run.head_repository.full_name }}
pr_head_ref: ${{ needs.prdetails.outputs.pr_data_json && fromJSON(needs.prdetails.outputs.pr_data_json).head.ref }}
deployment_ref: ${{ needs.prdetails.outputs.pr_data_json && fromJSON(needs.prdetails.outputs.pr_data_json).head.sha || github.ref || github.head_ref }}
package: widget-rtc-sdk
package: sdk
secrets:
ELEMENT_BOT_TOKEN: ${{ secrets.ELEMENT_BOT_TOKEN }}
NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
+3 -3
View File
@@ -17,7 +17,7 @@ permissions: {}
jobs:
versioning:
name: Versioning
runs-on: ubuntu-24.04
runs-on: ubuntu-latest
outputs:
DRY_RUN: ${{ steps.dry_run.outputs.DRY_RUN }}
PREFIXED_VERSION: ${{ steps.prefixed_version.outputs.PREFIXED_VERSION }}
@@ -59,7 +59,7 @@ jobs:
publish_npm:
needs: versioning
name: Publish NPM
runs-on: ubuntu-24.04
runs-on: ubuntu-latest
outputs:
ARTIFACT_VERSION: ${{ steps.artifact_version.outputs.ARTIFACT_VERSION }}
permissions:
@@ -111,7 +111,7 @@ jobs:
needs: [versioning, publish_npm]
if: ${{ needs.versioning.outputs.DRY_RUN == 'false' }}
name: Update release notes
runs-on: ubuntu-24.04
runs-on: ubuntu-latest
# Shared with publish-embedded-packages.yaml, which appends to the same release: two
# appends at once would each write back the body without the other's section
concurrency:
@@ -14,7 +14,7 @@ on:
jobs:
versioning:
name: Versioning
runs-on: ubuntu-24.04
runs-on: ubuntu-latest
outputs:
DRY_RUN: ${{ steps.dry_run.outputs.DRY_RUN }}
PREFIXED_VERSION: ${{ steps.prefixed_version.outputs.PREFIXED_VERSION }}
@@ -76,7 +76,7 @@ jobs:
needs: [build_element_call, versioning]
if: always()
name: Publish tarball
runs-on: ubuntu-24.04
runs-on: ubuntu-latest
permissions:
contents: write # required to upload release asset
steps:
@@ -107,7 +107,7 @@ jobs:
needs: [build_element_call, versioning]
if: always()
name: Publish NPM
runs-on: ubuntu-24.04
runs-on: ubuntu-latest
outputs:
ARTIFACT_VERSION: ${{ steps.artifact_version.outputs.ARTIFACT_VERSION }}
permissions:
@@ -152,7 +152,7 @@ jobs:
needs: [build_element_call, versioning]
if: always()
name: Publish Android AAR
runs-on: ubuntu-24.04
runs-on: ubuntu-latest
outputs:
ARTIFACT_VERSION: ${{ steps.artifact_version.outputs.ARTIFACT_VERSION }}
permissions:
@@ -212,7 +212,7 @@ jobs:
needs: [build_element_call, versioning]
if: always()
name: Publish SwiftPM Library
runs-on: ubuntu-24.04
runs-on: ubuntu-latest
outputs:
ARTIFACT_VERSION: ${{ steps.artifact_version.outputs.ARTIFACT_VERSION }}
permissions:
@@ -282,7 +282,7 @@ jobs:
needs: [versioning, publish_npm, publish_android, publish_ios]
if: always()
name: Update release notes
runs-on: ubuntu-24.04
runs-on: ubuntu-latest
# Shared with publish-component.yaml, which appends to the same release: two appends
# at once would each write back the body without the other's section
concurrency:
+2 -2
View File
@@ -23,7 +23,7 @@ jobs:
needs: build_element_call
if: always()
name: Publish tarball
runs-on: ubuntu-24.04
runs-on: ubuntu-latest
permissions:
contents: write # required to upload release asset
packages: write
@@ -68,7 +68,7 @@ jobs:
add_docker_release_note:
needs: publish_docker
name: Add docker release note
runs-on: ubuntu-24.04
runs-on: ubuntu-latest
steps:
- name: Add release note
uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2
+4 -68
View File
@@ -25,7 +25,7 @@ on:
jobs:
vitest:
name: Run unit tests
runs-on: ubuntu-24.04
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
@@ -71,27 +71,10 @@ jobs:
flags: unittests
fail_ci_if_error: true
playwright:
name: Run end-to-end tests (${{ matrix.part }})
name: Run end-to-end tests
if: ${{ !inputs.disable_e2e_tests }}
timeout-minutes: 60
runs-on: ubuntu-24.04
# We manually shard the tests across runners. Playwright's own shards cut
# the list of tests into equal counts in file order, which would leave the
# widget suite, the slowest tests by far, together at the end.
strategy:
fail-fast: false
matrix:
include:
- part: chromium-1
args: --project=chromium --shard=1/2
- part: chromium-2
args: --project=chromium --shard=2/2
- part: widget-1
args: --project=widget --shard=1/2
- part: widget-2
args: --project=widget --shard=2/2
- part: firefox-mobile
args: --project=firefox --project=mobile
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
with:
@@ -124,64 +107,17 @@ jobs:
run: pnpm playwright install --with-deps
- name: Run backend components
run: |
# Make sure the directory for the embedded build exists as Docker might
# create it with wrong access rights otherwise.
mkdir -p embedded/web/dist
docker compose -f docker-compose-dev.yml -f docker-compose-playwright.yml pull
docker compose -f docker-compose-dev.yml -f docker-compose-playwright.yml up -d
docker ps
- name: Run Playwright tests
env:
USE_DOCKER: 1
ARGS: ${{ matrix.args }}
# Unquoted on purpose: the part's arguments are several words
run: pnpm exec playwright test $ARGS
run: pnpm exec playwright test
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
if: ${{ !cancelled() }}
with:
name: blob-report-${{ matrix.part }}
path: blob-report
if-no-files-found: error
# Only read by the merge below.
retention-days: 1
e2e_report:
# Combines the blobs into a single HTML report and blocks PRs on failure.
name: Run end-to-end tests
needs: playwright
if: ${{ always() && !inputs.disable_e2e_tests }}
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
with:
# Inside a called workflow, github.repository is the caller so we need to make sure to still check out the EC repo here.
repository: ${{ inputs.matrix-js-sdk-sha && 'element-hq/element-call' || github.repository }}
persist-credentials: false
- name: Enable Corepack
run: corepack enable
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
with:
cache: "pnpm"
node-version-file: ".node-version"
- name: Install dependencies
run: scripts/layered.sh --ignore-pnpmfile
env:
PR_HEAD_REPO_OWNER: ${{ github.event.pull_request.head.repo.owner.login }}
JS_SDK_GITHUB_BASE_REF: ${{ inputs.matrix-js-sdk-sha }}
- name: Download the parts' reports
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
with:
pattern: blob-report-*
path: all-blob-reports
merge-multiple: true
- name: Merge into one HTML report
run: pnpm exec playwright merge-reports --reporter html ./all-blob-reports
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: html-report
path: playwright-report
if-no-files-found: error
retention-days: 4
- name: Fail on a failed part
if: ${{ needs.playwright.result != 'success' }}
run: exit 1
+1 -1
View File
@@ -7,7 +7,7 @@ on:
jobs:
download:
runs-on: ubuntu-24.04
runs-on: ubuntu-latest
permissions:
contents: read
+1 -1
View File
@@ -8,7 +8,7 @@ on:
jobs:
upload:
runs-on: ubuntu-24.04
runs-on: ubuntu-latest
permissions:
contents: read
+1 -1
View File
@@ -10,7 +10,7 @@ permissions: {}
jobs:
zizmor:
name: Run zizmor 🌈
runs-on: ubuntu-24.04
runs-on: ubuntu-latest
permissions:
security-events: write # Required for upload-sarif (used by zizmor-action) to upload SARIF files.
steps:
-5
View File
@@ -2,7 +2,6 @@ node_modules
.DS_Store
.env
dist
/dist-widget-rtc-sdk-example
dist-ssr
*.local
*.bkp
@@ -45,7 +44,3 @@ storybook-static
# Nested checkout made by scripts/layered.sh
/matrix-js-sdk/
# Claude
/.claude/*
!/.claude/skills/
+4 -25
View File
@@ -16,35 +16,19 @@ host's page. It is the MatrixRTC reference implementation.
`@media`, global `i18next` or the `widget` global — take it from a provider.
- The change works standalone, as a widget and as a component. Say what you checked.
- Every gate below is green before you push.
- Hand off after the first implementation, before the quality pass. Then commit on
the user's word where the setup allows it, or hand them the message.
## Read before you
| … | … |
| -------------------------------- | ---------------------------------------------------------- |
| start a task of any size | [docs/agents/workflow.md](docs/agents/workflow.md) |
| touch call logic or a view model | [docs/agents/architecture.md](docs/agents/architecture.md) |
| write any code | [docs/agents/code-style.md](docs/agents/code-style.md) |
| write a test, story or e2e spec | [docs/agents/testing.md](docs/agents/testing.md) |
| open a PR | [CONTRIBUTING.md](CONTRIBUTING.md) |
## Scratch files live in `agent-workspace/`
Git-ignored. One kebab-case subfolder per task, matching the branch topic:
`agent-workspace/<slug>/`.
| File | Purpose |
| --------------------------- | -------------------------------------------------------------------------------- |
| `plan.md` | The concrete approach — files, symbols, edit order — once direction is confirmed |
| `commit-msg.txt` | `git commit -F agent-workspace/<slug>/commit-msg.txt` |
| `pr-body.md` | `gh pr create --body-file agent-workspace/<slug>/pr-body.md` |
| `implementation-summary.md` | What was built, decisions, trade-offs |
| `NN-prompt.md` | Raw prompt text, numbered, when worth keeping |
- Nothing here is durable. The folder is ignored, so anything worth keeping is
promoted into the PR body or a doc before the task ends.
- Never write scratch files to the repo root. Stray `load_test_summary.md` and
`config.json_` files are what this folder prevents.
- Start `pr-body.md` from `.github/PULL_REQUEST_TEMPLATE.md` and fill every
section.
| implement from a feature spec | [FEATURES_SPEC/AGENTS.md](FEATURES_SPEC/AGENTS.md) |
## Gates
@@ -58,11 +42,6 @@ pnpm i18n:check
`knip` fails on dead code; a file deliberately inert ahead of its consumer goes in
`knip.ts` `ignoreFiles` with a reason.
Ready means every gate is green and the change is covered at the layers
[testing.md](docs/agents/testing.md) asks for. Read the diff against
[code-style.md](docs/agents/code-style.md) first. Re-run the whole checklist after
any fix.
## A good PR
- One slice, under ~400 changed lines, green on its own, linked to its issue.
+99 -48
View File
@@ -221,68 +221,119 @@ See also:
- [Developing with linked packages](./docs/linking.md)
#### Element Call as a component
#### Element Call as a component (experimental)
Element Call can be rendered inside another React app instead of an iframe.
The host supplies its own Matrix client, and the call stays inside the element
it is mounted in.
Element Call can also be embedded directly into another React application
rather than being loaded in an iframe as a widget. `pnpm build:component`
builds it as a library into `component/dist` (the bundle, its stylesheet and
type declarations), and
##### Install
```sh
pnpm dev:component
```
serves a harness on port 3001 that stands in for such an application: it signs
in twice against the development backend and shows two calls side by side, in
resizable boxes, with page furniture of its own around them. Use it to see how
Element Call behaves when it does not own the page — the size it is given,
whether it stays inside its container, and what it says to its host, which is
logged along the bottom. The harness is served with the same development
certificate as the app, so unless the development CA is trusted, the browser
needs a certificate exception for `https://localhost:3001` as well (see the
note under [Backend](#backend)). It reads the same `public/config.json` as
`pnpm dev` if one exists, and runs with Element Call's defaults otherwise.
The call lays itself out for the size of the element it is mounted in, not the
window: a host that shrinks the container to a corner of its page gets the
picture-in-picture layout, just as a host that shrank the whole iframe used to.
The breakpoints in the stylesheets the component uses are
`@container element-call` queries against its root element for the same reason;
for the standalone app the root is the page, so they mean what the media queries
they replaced did. (The standalone-only views, such as the home and login pages,
still use plain media queries, since the component never shows them.)
The component's stylesheet is confined to the element it is mounted in: the
build rewrites every selector so that it matches only Element Call's root or
what is inside it, with `html`, `body` and `:root` standing for that root (see
`component/build/scopeStylesToRoot.ts`). A host's own page keeps its styles,
and Element Call brings its own fonts and design tokens along.
The component speaks every language the app does. English is bundled in; the
other locales are split into chunks the host's bundler loads the first time
they are needed. It starts in the browser's language, and follows the host's
own language setting through the `language` prop (`supportedLanguages` lists
the tags it accepts, and anything else falls back to its base language or to
English). The `theme` prop works the same way and takes the same values as the
widget's `theme` URL parameter: `light`, `dark`, `light-high-contrast` or
`dark-high-contrast`. Both can change while a call is running without
disturbing it.
A host must call and await `initializeElementCall(config)` once before
rendering the component: it loads the `Intl` polyfills, applies the
deployment-wide `config.json`-style configuration and sets up translations.
The component itself takes the host's `client` and the `roomId` to call in, an
`intent` saying what the user asked for (which decides whether to show the
lobby, ring, and so on), an optional `config` overriding what the intent
implies, and an optional `hostBridge` through which Element Call tells the host
that the user has joined or hung up, that it wants to stay on screen, and so
on. The host makes its own requests (`join`, `hangUp`, `setDeviceMute`) through
the handle exposed on `ref`. The full API is documented in the type declarations
(`component/index.tsx` and `component/host.ts`).
A few things differ from the widget on purpose: the component draws a solid
background rather than a gradient unless told otherwise, never offers to edit
the user's profile (the account is the host's), scopes its keyboard shortcuts to
its own root element so that several instances can share a page, and only shows
its own post-call and error screens when the host has not supplied a `close()`
callback; with one, it asks the host to unmount it instead. The
[global JS controls](./docs/controls.md) on `window` are unchanged and remain
page-wide, so with several instances on one page they apply to all of them.
Each release publishes the package to npm, with the same version as the release:
```sh
npm install @element-hq/element-call-component
```
Releases publish the package with the same version as Element Call. For an
unreleased commit, depend on this repository directly:
To use an unreleased commit, a host can instead install it as a git dependency
on the `component` directory of this repository,
```json
"@element-hq/element-call-component": "github:element-hq/element-call#main&path:/component"
```
This builds on install, so the host needs pnpm (via Corepack) and must allow
the build (`allowBuilds` in its `pnpm-workspace.yaml`).
whose `prepare` script runs the build on install. That build needs pnpm (via
Corepack) on the host's machine, runs a full `pnpm install` of this repository,
since it inherits the `--max-old-space-size` setting of
the app build; the host's pnpm also has to allow it to run at all
(`allowBuilds` in its `pnpm-workspace.yaml`). Note that `component/` is a pnpm
project of its own for this reason, so pnpm commands run from inside that
directory target it rather than the repository; run them from the repository
root. The host imports the component from
`@element-hq/element-call-component` and the stylesheet from
`@element-hq/element-call-component/style.css`, and has to provide `react`,
`react-dom`, `matrix-js-sdk`, `@vector-im/compound-web` and
`@vector-im/compound-design-tokens` itself, since the bundle leaves them
external.
Compound's stylesheet are the host's to load too: the component shares
the host's copy of Compound, (required for example for tooltips)
##### Use
The component is large, and a host will usually load it lazily, only once a
call is shown. Everything a host needs in order to talk about a call before
then is also exported
from `@element-hq/element-call-component/api` (a small entry point that does not
load the component).
1. Provide the externals yourself: `react`, `react-dom`, `matrix-js-sdk`,
`@vector-im/compound-web`, `@vector-im/compound-design-tokens`, and
Compound's stylesheet.
2. Import `@element-hq/element-call-component/style.css`.
3. Await `initializeElementCall(config)` once before rendering.
4. Render the component with `client`, `roomId` and an `intent`; optionally
pass `config`, `hostBridge`, `language` and `theme` (both can change
mid-call). Control it through the `ref` handle (`join`, `hangUp`,
`setDeviceMute`).
Load the component lazily. Types, enums and `configurationForIntent` are
available without it from `@element-hq/element-call-component/api`. The full
API is in `component/index.tsx`, `component/api.ts` and `component/host.ts`.
##### How it differs from the widget
- It lays out for its container, not the window: a small container gets the
picture-in-picture layout.
- Its styles and keyboard shortcuts are scoped to its root, so the host page is
untouched and several instances can share a page. The
[global JS controls](./docs/controls.md) stay page-wide.
- It draws a solid background by default and never offers profile editing.
- If the host bridge implements `close()`, it asks to be unmounted instead of
showing its own post-call and error screens.
- Sounds, images and the MediaPipe runtime are separate files the host's
bundler copies (webpack 5, Vite, Rollup); MediaPipe is fetched only when a
background effect is first used.
##### Develop
- `pnpm build:component` builds the library into `component/dist`.
- `pnpm dev:component` serves a test host on `https://localhost:3001`: two
calls side by side in resizable boxes, with host messages logged at the
bottom. It uses the development backend, `public/config.json` if present,
and needs the same certificate exception as `pnpm dev` (see
[Backend](#backend)).
- Run pnpm commands from the repository root; `component/` is its own pnpm
project.
- props and the types they are made of
- the handle
- the host bridge
- `UserIntent`
- `HeaderStyle`
- `BackgroundStyle` enums
- `configurationForIntent`, the defaults each intent implies
Import from it wherever a value such as
`BackgroundStyle.Solid` is needed on a path that must stay light; the main
entry point re-exports all of it too.
### Backend
+3 -5
View File
@@ -26,7 +26,8 @@
"features": {
"feature_element_call_video_rooms": true,
"feature_video_rooms": true,
"feature_group_calls": true
"feature_group_calls": true,
"feature_release_announcement": false
},
"default_federate": true,
"default_theme": "light",
@@ -39,10 +40,7 @@
},
"setting_defaults": {
"breadcrumbs": true,
"feature_group_calls": true,
"releaseAnnouncementData": {
"room_list_section": true
}
"feature_group_calls": true
},
"jitsi": {
"preferred_domain": "meet.element.io"
+3 -5
View File
@@ -26,7 +26,8 @@
"features": {
"feature_element_call_video_rooms": true,
"feature_video_rooms": true,
"feature_group_calls": true
"feature_group_calls": true,
"feature_release_announcement": false
},
"default_federate": true,
"default_theme": "light",
@@ -39,10 +40,7 @@
},
"setting_defaults": {
"breadcrumbs": true,
"feature_group_calls": true,
"releaseAnnouncementData": {
"room_list_section": true
}
"feature_group_calls": true
},
"jitsi": {
"preferred_domain": "meet.element.io"
+13
View File
@@ -0,0 +1,13 @@
{
"default_server_config": {
"m.homeserver": {
"base_url": "https://call-unstable.ems.host",
"server_name": "call-unstable.ems.host"
}
},
"ssla": "https://static.element.io/legal/element-software-and-services-license-agreement-uk-1.pdf",
"matrix_rtc_session": {
"membership_event_expiry_ms": 180000000,
"network_error_retry_ms": 100
}
}
+2 -2
View File
@@ -3,7 +3,7 @@ networks:
services:
auth-service:
image: ghcr.io/element-hq/lk-jwt-service:0.10.0
image: ghcr.io/element-hq/lk-jwt-service:0.8.0
pull_policy: always
hostname: auth-server
environment:
@@ -33,7 +33,7 @@ services:
- ecbackend
auth-service-1:
image: ghcr.io/element-hq/lk-jwt-service:0.10.0
image: ghcr.io/element-hq/lk-jwt-service:0.8.0
pull_policy: always
hostname: auth-server-1
environment:
+2 -7
View File
@@ -19,12 +19,7 @@ services:
- ./backend/playwright_homeserver-othersite.yaml:/data/cfg/homeserver.yaml:Z
element-web:
# Pin to a SHA so that upstream cannot break our tests. Renovate handles regular updates.
image: ghcr.io/element-hq/element-web:develop@sha256:fd1c291a228374aff127ceb6eea3c20291b233623bbbd96dcae53eca679fd9d7
image: ghcr.io/element-hq/element-web:develop@sha256:a79801206c628a912d57fa6f355d5c5ab9c6c40c2d1b6208d4192678fb6e80c7
element-web-1:
# Pin to a SHA so that upstream cannot break our tests. Renovate handles regular updates.
image: ghcr.io/element-hq/element-web:develop@sha256:fd1c291a228374aff127ceb6eea3c20291b233623bbbd96dcae53eca679fd9d7
volumes:
# The other site opens the embedded package under test in place of the
# copy its image bundles (the first site opens the dev server's app.
# Built by the Playwright run (see build.setup.ts).
- ./embedded/web/dist:/app/widgets/element-call:ro,Z
image: ghcr.io/element-hq/element-web:develop@sha256:a79801206c628a912d57fa6f355d5c5ab9c6c40c2d1b6208d4192678fb6e80c7
+1 -5
View File
@@ -14,11 +14,7 @@ This folder contains documentation for setup, usage, and development of Element
Entry point: [AGENTS.md](../AGENTS.md).
- [Agent workflow](./agents/workflow.md)
- [Architecture](./agents/architecture.md)
- [Code style](./agents/code-style.md)
- [Testing](./agents/testing.md)
Opt-in skills in [`.claude/skills/`](../.claude/skills/):
- `/handoff <task>` — get the first implementation back for a direction check
before lint, the full suite and polish.
+5 -5
View File
@@ -76,11 +76,11 @@ Code that builds in only one is a bug.
Packaged exactly like the component (own pnpm project, run pnpm from the repo
root, types from `sdk/tsconfig.build.json`, externals checked by
`pnpm lint:externals`). Interface only so far; the design is `sdk-plan.md`.
- `build:widget-rtc-sdk` — a single bundle for widget pages that wraps the
`sdk/` package and the widget API, nothing of `src/`; entry
`widget-rtc-sdk/main.ts`. CI builds it as the `widget-rtc-sdk` package
(`build:widget-rtc-sdk:production` / `:development`) and deploys it as the
`pr<N>-widget-rtc-sdk` Netlify preview.
- `build:sdk-target-based-on-call-view-model` — the SDK demo that predates the
package, one bundle of the `CallViewModel`; entry
`sdk-target-based-on-call-view-model/main.ts`. CI's `sdk` package
(`build:sdk:production` / `build:sdk:development`, the Netlify `-sdk` preview)
is still this bundle, not `build:sdk`.
- `build:component` — `@element-hq/element-call-component`, sources in `component/`
(its own pnpm project; run pnpm from the repo root). Host API is in the README;
`pnpm lint:externals` rejects an import of a `react` / `react-dom` /
-7
View File
@@ -34,10 +34,6 @@ translations. The browser is Playwright's, so a fresh clone needs
## End-to-end
The Element Web instance `HOST1` opens the app the dev server serves. The instance
`HOST2` (the other site) opens the embedded package as a host ships it. This package
is built by the `build-embedded` setup project and mounted in `docker-compose-playwright.yml`.
- `playwright/*.spec.ts` — standalone, on Chromium and Firefox.
- `playwright/widget/` — widget mode, where cross-context bugs surface.
- `playwright/component/` — the component in a host page, via the harness on port
@@ -52,13 +48,10 @@ Test what a user observes: the peer sees the change, it survives a reconnect, it
right after a reload. Reuse `playwright/spa-helpers.ts`,
`playwright/widget/test-helpers.ts`, `playwright/fixtures/`.
To mark new release announcements as viewed, add them in `backend/ew.test*.config.json`.
```sh
pnpm test # unit + storybook
pnpm backend # Synapse + LiveKit, required for e2e
pnpm test:playwright # or :open
pnpm dev:component # component harness, port 3001
pnpm dev:sdk # SDK harness, port 3002
pnpm dev:widget-rtc-sdk # widget RTC SDK example widget, port 3003
```
+50
View File
@@ -0,0 +1,50 @@
# Agent workflow
## Scratch files live in `agent-workspace/`
Git-ignored. One kebab-case subfolder per task, matching the branch topic:
`agent-workspace/<slug>/`.
| File | Purpose |
| --------------------------- | -------------------------------------------------------------------------------- |
| `plan.md` | The concrete approach — files, symbols, edit order — once direction is confirmed |
| `commit-msg.txt` | `git commit -F agent-workspace/<slug>/commit-msg.txt` |
| `pr-body.md` | `gh pr create --body-file agent-workspace/<slug>/pr-body.md` |
| `implementation-summary.md` | What was built, decisions, trade-offs |
| `NN-prompt.md` | Raw prompt text, numbered, when worth keeping |
- Nothing here is durable. The folder is ignored, so anything worth keeping is
promoted into the PR body, a doc, or a feature spec before the task ends.
- Never write scratch files to the repo root. Stray `load_test_summary.md` and
`config.json_` files are what this folder prevents.
- Start `pr-body.md` from `.github/PULL_REQUEST_TEMPLATE.md` and fill every
section.
- Commit subjects are plain imperative English. No conventional-commits prefixes.
- A `plan.md` and a feature spec sit at different altitudes; neither replaces the
other. A spec in `FEATURES_SPEC/` is durable and deliberately abstract —
behaviour, decisions, acceptance criteria — so the feature can be rebuilt against
a `main` nobody has seen yet. A plan is one slice of it landing on today's
`main`: the paths, symbols and edit order the spec's decisions and criteria must
not name. `FEATURES_SPEC/AGENTS.md` wins wherever the two genuinely overlap.
## Hand off before the quality pass
- Implement the change, run the narrowest check that rules out an obviously broken
handoff, then stop and ask whether the direction is right.
- Full `pnpm lint`, the whole suite, coverage and benchmarks come after the
direction is confirmed.
- Wider refactors, extra tests and documentation polish are follow-up work, not
part of the first handoff.
- If a check is needed before feedback, keep it to the touched code and say why.
## Commit and PR readiness
- Commit once the user confirms direction, or asks for one. Not before. Where the
setup has no git identity, a sandbox included, write the message to
`commit-msg.txt` and hand it over instead.
- Ready means every gate in [AGENTS.md](../../AGENTS.md#gates) is green and the
change is covered at the layers [testing.md](./testing.md) asks for. Read the diff
against [code-style.md](./code-style.md) first.
- Re-run the whole checklist after any fix. Only on green, either way.
- Once a human has started reviewing, fix forward — never force-push a
regeneration over a review in progress.
+1 -2
View File
@@ -14,8 +14,7 @@ export default {
"vite-embedded.config.ts",
"vite-sdk.config.ts",
"vite-sdk-dev.config.ts",
"vite-widget-rtc-sdk.config.ts",
"vite-widget-rtc-sdk-dev.config.ts",
"vite-sdk-target-based-on-call-view-model.config.ts",
"vite-component.config.ts",
"vite-component-dev.config.ts",
],
-5
View File
@@ -3,7 +3,6 @@
"user_menu": "User menu"
},
"action": {
"add_background_image": "Add image",
"close": "Close",
"copy_link": "Copy link",
"edit": "Edit",
@@ -99,10 +98,6 @@
},
"disconnected_banner": "Connectivity to the server has been lost.",
"error": {
"background_animated": "Animated images cannot be used as a background",
"background_not_an_image": "That file is not a supported image",
"background_not_kept": "That background could not be saved",
"background_undecodable": "That image could not be opened",
"call_is_not_supported": "Call is not supported",
"call_not_found": "Call not found",
"call_not_found_description": "<0>That link doesn't appear to belong to any existing call. Check that you have the right link, or <2>create a new one</2>.</0>",
+4 -6
View File
@@ -8,7 +8,6 @@
"dev:embedded": "vite --config vite-embedded.config.js",
"dev:component": "vite --config vite-component-dev.config.ts",
"dev:sdk": "vite --config vite-sdk-dev.config.ts",
"dev:widget-rtc-sdk": "vite --config vite-widget-rtc-sdk-dev.config.ts",
"build": "pnpm build:full",
"build:full": "node --max-old-space-size=16384 node_modules/vite/bin/vite.js build",
"build:full:production": "pnpm build:full",
@@ -19,9 +18,9 @@
"build:sdk": "pnpm build:sdk:js && pnpm build:sdk:types",
"build:sdk:js": "pnpm build:full --config vite-sdk.config.js",
"build:sdk:types": "tsc -p sdk/tsconfig.build.json",
"build:widget-rtc-sdk": "pnpm build:full --config vite-widget-rtc-sdk.config.js",
"build:widget-rtc-sdk:production": "pnpm build:widget-rtc-sdk",
"build:widget-rtc-sdk:development": "pnpm build:widget-rtc-sdk --mode development",
"build:sdk-target-based-on-call-view-model": "pnpm build:full --config vite-sdk-target-based-on-call-view-model.config.js",
"build:sdk:production": "pnpm build:sdk-target-based-on-call-view-model",
"build:sdk:development": "pnpm build:sdk-target-based-on-call-view-model --mode development",
"build:component": "pnpm build:component:js && pnpm build:component:types",
"build:component:js": "pnpm build:full --config vite-component.config.js",
"build:component:types": "tsc -p component/tsconfig.build.json",
@@ -98,7 +97,6 @@
"copy-to-clipboard": "^3.3.3",
"eslint-plugin-element-call": "link:eslint",
"eslint-plugin-storybook": "^10.3.6",
"fake-indexeddb": "^6.2.5",
"fetch-mock": "11.1.5",
"i18next": "^26.0.0",
"i18next-browser-languagedetector": "^8.0.0",
@@ -108,7 +106,7 @@
"livekit-client": "^2.18.1",
"lodash-es": "^4.17.21",
"loglevel": "^1.9.1",
"matrix-js-sdk": "github:matrix-org/matrix-js-sdk#10e4faabbb49baca655492d523b58faa9eea04b9",
"matrix-js-sdk": "github:matrix-org/matrix-js-sdk#a9ef09fbf2f1644c8663d0aeaf6a921cd16e7b01",
"matrix-widget-api": "^1.18.0",
"node-stdlib-browser": "^1.3.1",
"normalize.css": "^8.0.1",
+21 -49
View File
@@ -13,7 +13,6 @@ import { fileURLToPath } from "node:url";
import { COMPONENT_HARNESS_URL } from "./playwright/component/harness.ts";
import { SDK_HARNESS_URL } from "./playwright/sdk/harness.ts";
import { WIDGET_RTC_SDK_URL } from "./playwright/widget/test-helpers.ts";
const baseURL = process.env.USE_DOCKER
? "http://localhost:8080"
@@ -27,19 +26,6 @@ process.env.NODE_EXTRA_CA_CERTS = join(
"backend/dev_tls_local-ca.crt",
);
const desktopChrome = {
...devices["Desktop Chrome"],
permissions: ["clipboard-write", "clipboard-read", "microphone", "camera"],
ignoreHTTPSErrors: true,
launchOptions: {
args: [
"--use-fake-ui-for-media-stream",
"--use-fake-device-for-media-stream",
"--mute-audio",
],
},
};
/**
* See https://playwright.dev/docs/test-configuration.
*/
@@ -57,11 +43,11 @@ export default defineConfig({
* Reporter to use. See https://playwright.dev/docs/test-reporters
*
* On CI, `list` prints a line per test as it finishes, which keeps the job
* log moving; `github` annotates failures on the PR; the `blob` format is
* uploaded by sharded CI runs and combined into a HTML report by the workflow.
* log moving; `github` annotates failures on the PR; `html` is uploaded as
* an artifact and must never try to open a browser there.
*/
reporter: process.env.CI
? [["list"], ["github"], ["blob"]]
? [["list"], ["github"], ["html", { open: "never" }]]
: [["list"], ["html"]],
/* Stop a broken run early on CI instead of waiting out every retry. */
maxFailures: process.env.CI ? 5 : undefined,
@@ -77,28 +63,29 @@ export default defineConfig({
},
/* Configure projects for major browsers */
projects: [
{
// Builds the embedded package into the directory the other site's
// Element Web container serves it from (see docker-compose-playwright.yml).
name: "build-embedded",
testMatch: "**/widget/embedded/build.setup.ts",
},
{
name: "chromium",
testIgnore: ["**/mobile/**", "**/widget/**"],
use: desktopChrome,
},
{
// Element Call as a widget in Element Web, on Chromium. This uses its
// own project so that it can be sharded on CI.
name: "widget",
dependencies: ["build-embedded"],
testMatch: "**/widget/**/*.@(spec|test).ts",
use: desktopChrome,
testIgnore: "**/mobile/**",
use: {
...devices["Desktop Chrome"],
permissions: [
"clipboard-write",
"clipboard-read",
"microphone",
"camera",
],
ignoreHTTPSErrors: true,
launchOptions: {
args: [
"--use-fake-ui-for-media-stream",
"--use-fake-device-for-media-stream",
"--mute-audio",
],
},
},
},
{
name: "firefox",
dependencies: ["build-embedded"],
testIgnore: "**/mobile/**",
use: {
...devices["Desktop Firefox"],
@@ -184,20 +171,5 @@ export default defineConfig({
timeout: 500,
},
},
{
// The widget RTC SDK's example widget, which Element Web opens as a
// widget. Bundled rather than from the dev server: Firefox now and then
// stalls on the dev server's hundreds of module requests.
command:
"pnpm exec vite build --config vite-widget-rtc-sdk-dev.config.ts && " +
"pnpm exec vite preview --config vite-widget-rtc-sdk-dev.config.ts",
url: WIDGET_RTC_SDK_URL,
reuseExistingServer: !process.env.CI,
ignoreHTTPSErrors: true,
gracefulShutdown: {
signal: "SIGTERM",
timeout: 500,
},
},
],
});
+23 -849
View File
@@ -5,15 +5,9 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { expect, test, type Browser, type Page } from "@playwright/test";
import { expect, test, type Locator, type Page } from "@playwright/test";
import { SpaHelpers } from "./spa-helpers.ts";
import {
averageColour,
cameraColour,
type Colour,
distance,
} from "./utils/colour.ts";
// Background effects need WebGL2, and headless Firefox on a CI runner has none,
// so it rightly offers none of them.
@@ -22,13 +16,15 @@ test.skip(
"Background effects need WebGL2, which headless Firefox on CI does not have",
);
type Colour = [number, number, number];
test.describe("background effects", () => {
test("pre-join preview shows the chosen effect", async ({ page }) => {
test.slow();
await page.goto("/");
await SpaHelpers.createCall(page, "Effect user", "Background effects");
const preview = page.locator("video").first();
const camera = await cameraColour(preview);
const camera = await averageColour(preview);
await page.getByRole("button", { name: "Camera", exact: true }).click();
const tile = page
@@ -106,847 +102,25 @@ async function drawsPerSecond(page: Page): Promise<number> {
});
}
test.describe("the camera menu's preview", () => {
test("shows the camera with the effect in force", async ({ page }) => {
test.setTimeout(120_000);
await page.goto("/");
await SpaHelpers.createCall(page, "Preview user", "Preview in the menu");
const camera = await cameraColour(page.locator("video").first());
await page.getByRole("button", { name: "Camera", exact: true }).click();
const preview = page.getByRole("menu").locator("video");
await expect(preview).toBeVisible();
await expect
.poll(async () => distance(await averageColour(preview), camera), {
timeout: 20_000,
})
.toBeLessThan(30);
const tile = page.getByRole("menuitemradio", { name: "Background 1" });
await tile.click();
const picture = await averageColour(tile.locator("img"));
await expect
.poll(async () => distance(await averageColour(preview), picture), {
timeout: 60_000,
})
.toBeLessThan(distance(camera, picture) / 2);
});
test("shows the call's own camera once joined", async ({ page }) => {
test.setTimeout(120_000);
await page.goto("/");
await SpaHelpers.createCall(
page,
"Call preview user",
"Preview in the call's menu",
true,
);
await expect(page.getByTestId("incall_videomute")).toBeEnabled({
timeout: 10_000,
});
const camera = await cameraColour(page.locator("video").first());
await page.getByRole("button", { name: "Camera", exact: true }).click();
const preview = page.getByRole("menu").locator("video");
await expect(preview).toBeVisible();
await expect
.poll(async () => distance(await averageColour(preview), camera), {
timeout: 20_000,
})
.toBeLessThan(30);
const tile = page.getByRole("menuitemradio", { name: "Background 1" });
await tile.click();
const picture = await averageColour(tile.locator("img"));
await expect
.poll(async () => distance(await averageColour(preview), picture), {
timeout: 60_000,
})
.toBeLessThan(distance(camera, picture) / 2);
});
});
test.describe("background effects section", () => {
test("section stays within the call area at every size", async ({ page }) => {
test.setTimeout(120_000);
await page.goto("/");
await SpaHelpers.createCall(page, "Sizes user", "Background effect sizes");
for (const size of [
{ width: 1280, height: 800 },
{ width: 800, height: 600 },
{ width: 600, height: 420 },
{ width: 480, height: 360 },
{ width: 360, height: 640 },
]) {
await page.setViewportSize(size);
await page.getByRole("button", { name: "Camera", exact: true }).click();
const menu = page.getByRole("menu");
await expect(menu).toBeVisible();
const box = (await menu.boundingBox())!;
const where = `at ${size.width}x${size.height}`;
expect(box.x, where).toBeGreaterThanOrEqual(0);
expect(box.y, where).toBeGreaterThanOrEqual(0);
expect(box.x + box.width, where).toBeLessThanOrEqual(size.width);
expect(box.y + box.height, where).toBeLessThanOrEqual(size.height);
const last = page
.getByRole("group", { name: "Background effects" })
.getByRole("menuitemradio")
.last();
await last.scrollIntoViewIfNeeded();
await expect(last, where).toBeInViewport({ ratio: 1 });
await last.click();
await expect(last, where).toHaveAttribute("aria-checked", "true");
// Back to no effect for the next size, and close.
await page
.getByRole("group", { name: "Background effects" })
.getByRole("menuitemradio", { name: "None" })
.click();
await page.keyboard.press("Escape");
await expect(menu).toBeHidden();
}
});
});
test.describe("a pipeline that fails to build", () => {
test("offers no effects for the session, and keeps the choice", async ({
page,
}) => {
test.setTimeout(120_000);
await page.goto("/");
await SpaHelpers.createCall(page, "Failing user", "Pipeline fails");
const preview = page.locator("video").first();
const camera = await cameraColour(preview);
await page.route("**/*.tflite*", async (route) => route.abort());
await page.getByRole("button", { name: "Camera", exact: true }).click();
const tile = page.getByRole("menuitemradio", { name: "Background 1" });
await tile.click();
await expect(
page.getByText("Background effects are not supported on this platform."),
).toBeVisible({ timeout: 30_000 });
await expect(tile).toBeDisabled();
await page.keyboard.press("Escape");
// The camera and microphone carry on without it.
expect(distance(await cameraColour(preview), camera)).toBeLessThan(30);
await expect(page.getByTestId("incall_mute")).toHaveAttribute(
"aria-checked",
"true",
);
await page.unroute("**/*.tflite*");
await page.reload();
await expect(page.getByTestId("lobby_joinCall")).toBeVisible();
await page.getByRole("button", { name: "Camera", exact: true }).click();
await expect(tile).toHaveAttribute("aria-checked", "true");
const picture = await averageColour(tile.locator("img"));
await expect
.poll(async () => distance(await averageColour(preview), picture), {
timeout: 60_000,
})
.toBeLessThan(distance(camera, picture) / 2);
});
});
test.describe("the chosen effect", () => {
test("chosen effect survives leaving and rejoining", async ({ page }) => {
test.setTimeout(120_000);
await page.goto("/");
await SpaHelpers.createCall(page, "Returning user", "Effect remembered");
const { camera, picture } = await chooseAndWear(page, "Background 1");
await page.getByTestId("lobby_joinCall").click();
await page.getByTestId("incall_leave").click();
await expect(page.getByRole("heading")).toContainText(
"your call has ended",
);
await page.goto("/");
await page.getByTestId("home_callName").fill("Effect remembered again");
await page.getByTestId("home_go").click();
await expect(page.getByTestId("lobby_joinCall")).toBeVisible();
await page.getByRole("button", { name: "Camera", exact: true }).click();
const tile = page
.getByRole("group", { name: "Background effects" })
.getByRole("menuitemradio", { name: "Background 1" });
await expect(tile).toHaveAttribute("aria-checked", "true");
const preview = page.locator("video").first();
await expect
.poll(async () => distance(await averageColour(preview), picture), {
timeout: 60_000,
})
.toBeLessThan(distance(camera, picture) / 2);
});
test("second session still shows the existing error", async ({ page }) => {
test.setTimeout(120_000);
await page.goto("/");
await SpaHelpers.createCall(page, "Two tabs", "Effect in two tabs");
await chooseAndWear(page, "Background 1");
const second = await page.context().newPage();
await second.goto("/");
await expect(
page.getByRole("heading", { name: "Opened in another tab" }),
).toBeVisible();
});
});
/** Chooses an effect in the lobby, and waits until the preview wears it. */
async function chooseAndWear(
page: Page,
name: string,
): Promise<{ camera: Colour; picture: Colour }> {
const preview = page.locator("video").first();
const camera = await cameraColour(preview);
await page.getByRole("button", { name: "Camera", exact: true }).click();
const tile = page.getByRole("menuitemradio", { name });
await tile.click();
const picture = await averageColour(tile.locator("img"));
await expect
.poll(async () => distance(await averageColour(preview), picture), {
timeout: 60_000,
})
.toBeLessThan(distance(camera, picture) / 2);
await page.keyboard.press("Escape");
return { camera, picture };
}
test.describe("a background of one's own", () => {
test("adds an image from the device", async ({ page }) => {
test.setTimeout(120_000);
await page.goto("/");
await SpaHelpers.createCall(page, "Adding user", "Own background");
const preview = page.locator("video").first();
const camera = await cameraColour(preview);
const red = await redImage(page);
await addImage(page, red);
const mine = page
.getByRole("group", { name: "Background effects" })
.getByRole("menuitemradio", { name: "Background 3" });
await expect(mine).toBeVisible();
await expect(mine).toHaveAttribute("aria-checked", "false");
await mine.click();
await expect
.poll(async () => distance(await averageColour(preview), RED), {
timeout: 60_000,
})
.toBeLessThan(distance(camera, RED) / 2);
});
test("adding an image makes no upload", async ({ page }) => {
test.setTimeout(120_000);
await page.goto("/");
await SpaHelpers.createCall(page, "Private user", "Own background kept");
const preview = page.locator("video").first();
const camera = await cameraColour(preview);
const red = await redImage(page);
const sent: { request: string; body: Buffer | null }[] = [];
page.on("request", (request) =>
sent.push({
request: `${request.method()} ${request.url()}`,
body: request.postDataBuffer(),
}),
);
await addImage(page, red);
await page
.getByRole("group", { name: "Background effects" })
.getByRole("menuitemradio", { name: "Background 3" })
.click();
// Kept, chosen and in force: anything that would send it has had its turn.
await expect
.poll(async () => distance(await averageColour(preview), RED), {
timeout: 60_000,
})
.toBeLessThan(distance(camera, RED) / 2);
// The file as given, and the copy kept, which is encoded anew.
const samples = [red, await keptBytes(page)].map((bytes) =>
bytes.subarray(bytes.length - 64),
);
const carriesIt = sent.filter(
({ request, body }) =>
request.includes("/_matrix/media/") ||
samples.some((sample) => body?.includes(sample)),
);
expect(carriesIt.map(({ request }) => request)).toEqual([]);
});
test("added image survives rejoining and is gone after storage is cleared", async ({
page,
}) => {
test.setTimeout(120_000);
await page.goto("/");
await SpaHelpers.createCall(page, "Keeping user", "Own background again");
const preview = page.locator("video").first();
const camera = await cameraColour(preview);
const red = await redImage(page);
await addImage(page, red);
const mine = page
.getByRole("group", { name: "Background effects" })
.getByRole("menuitemradio", { name: "Background 3" });
await mine.click();
await page.keyboard.press("Escape");
await page.reload();
await expect(page.getByTestId("lobby_joinCall")).toBeVisible();
await page.getByRole("button", { name: "Camera", exact: true }).click();
await expect(mine).toHaveAttribute("aria-checked", "true");
await page.keyboard.press("Escape");
await expect
.poll(async () => distance(await averageColour(preview), RED), {
timeout: 60_000,
})
.toBeLessThan(distance(camera, RED) / 2);
await page.evaluate(
async () =>
new Promise<void>((resolve, reject) => {
const request = indexedDB.deleteDatabase(
"element-call-background-images",
);
request.onsuccess = (): void => resolve();
request.onerror = (): void => reject(request.error);
request.onblocked = (): void => resolve();
}),
);
await page.reload();
await expect(page.getByTestId("lobby_joinCall")).toBeVisible();
await page.getByRole("button", { name: "Camera", exact: true }).click();
await expect(
page.getByRole("group", { name: "Background effects" }),
).toBeVisible();
await expect(mine).toHaveCount(0);
await page.keyboard.press("Escape");
await addImage(page, red);
await expect(mine).toBeVisible();
});
test("a removed image stays removed", async ({ page }) => {
test.setTimeout(120_000);
await page.goto("/");
await SpaHelpers.createCall(
page,
"Removing user",
"Own background removed",
);
await addImage(page, await redImage(page));
const mine = page
.getByRole("group", { name: "Background effects" })
.getByRole("menuitemradio", { name: "Background 3" });
await expect(mine).toHaveAttribute("aria-keyshortcuts", "Delete");
await mine.focus();
await page.keyboard.press("Delete");
await expect(mine).toHaveCount(0);
await page.reload();
await expect(page.getByTestId("lobby_joinCall")).toBeVisible();
await page.getByRole("button", { name: "Camera", exact: true }).click();
await expect(
page.getByRole("group", { name: "Background effects" }),
).toBeVisible();
await expect(mine).toHaveCount(0);
expect(await keptImages(page)).toEqual([]);
});
});
test.describe("what can be added", () => {
test("refuses an animated image", async ({ page }) => {
test.setTimeout(120_000);
await page.goto("/");
await SpaHelpers.createCall(page, "Refused user", "Animated refused");
await addImage(page, ANIMATED_GIF, {
name: "moving.gif",
type: "image/gif",
});
await expect(
page.getByText("Animated images cannot be used as a background"),
).toBeVisible();
const section = page.getByRole("group", { name: "Background effects" });
await expect(
section.getByRole("menuitemradio", { name: "Background 3" }),
).toHaveCount(0);
await expect(
section.getByRole("menuitemradio", { name: "None" }),
).toHaveAttribute("aria-checked", "true");
expect(await keptImages(page)).toEqual([]);
});
test("reduces an oversized image", async ({ page }) => {
test.setTimeout(120_000);
await page.goto("/");
await SpaHelpers.createCall(page, "Large user", "Oversized reduced");
await addImage(page, await pngImage(page, { width: 4000, height: 3000 }));
await expect(
page
.getByRole("group", { name: "Background effects" })
.getByRole("menuitemradio", { name: "Background 3" }),
).toBeVisible();
await expect(page.getByRole("alert")).toHaveCount(0);
const [kept] = await keptImages(page);
expect(kept).toMatchObject({ width: 1920, height: 1440 });
});
test("lays every image on an opaque ground", async ({ page }) => {
test.setTimeout(120_000);
await page.goto("/");
await SpaHelpers.createCall(page, "Logo user", "Transparent made opaque");
await addImage(
page,
await pngImage(page, { width: 320, height: 180, transparent: true }),
);
await expect(
page
.getByRole("group", { name: "Background effects" })
.getByRole("menuitemradio", { name: "Background 3" }),
).toBeVisible();
const [kept] = await keptImages(page);
expect(kept.minAlpha).toBe(255);
});
});
/** Two frames of one pixel, red then blue. */
const ANIMATED_GIF = Buffer.from(
"R0lGODlhAQABAPEAAP8AAAAA/wAAAAAAACH/C05FVFNDQVBFMi4wAwEAAAAh+QQACgAAACwAAAAAAQABAAACAkQBACH5BAAKAAAALAAAAAABAAEAAAICTAEAOw==",
"base64",
);
/** A red PNG of the given size; transparent leaves its right half empty. */
async function pngImage(
page: Page,
size: { width: number; height: number; transparent?: boolean },
): Promise<Buffer> {
const dataUrl = await page.evaluate(({ width, height, transparent }) => {
const canvas = document.createElement("canvas");
canvas.width = width;
canvas.height = height;
const context = canvas.getContext("2d")!;
context.fillStyle = "rgb(255, 0, 0)";
context.fillRect(0, 0, transparent ? width / 2 : width, height);
return canvas.toDataURL("image/png");
}, size);
return Buffer.from(dataUrl.split(",")[1], "base64");
}
/** The images the device keeps, as it keeps them. */
async function keptImages(
page: Page,
): Promise<{ width: number; height: number; minAlpha: number }[]> {
return page.evaluate(async () => {
const db = await new Promise<IDBDatabase>((resolve, reject) => {
const request = indexedDB.open("element-call-background-images");
request.onsuccess = (): void => resolve(request.result);
request.onerror = (): void => reject(request.error);
});
if (!db.objectStoreNames.contains("backgrounds")) return [];
const kept = await new Promise<{ image: Blob }[]>((resolve, reject) => {
const request = db
.transaction("backgrounds")
.objectStore("backgrounds")
.getAll();
request.onsuccess = (): void => resolve(request.result);
request.onerror = (): void => reject(request.error);
});
db.close();
return Promise.all(
kept.map(async ({ image }) => {
const bitmap = await createImageBitmap(image);
const canvas = new OffscreenCanvas(bitmap.width, bitmap.height);
const context = canvas.getContext("2d")!;
context.drawImage(bitmap, 0, 0);
const { data } = context.getImageData(
0,
0,
bitmap.width,
bitmap.height,
);
let minAlpha = 255;
for (let i = 3; i < data.length; i += 4)
minAlpha = Math.min(minAlpha, data[i]);
return { width: bitmap.width, height: bitmap.height, minAlpha };
}),
);
});
}
/** The bytes of the one image the device keeps. */
async function keptBytes(page: Page): Promise<Buffer> {
const base64 = await page.evaluate(
async () =>
new Promise<string>((resolve, reject) => {
const open = indexedDB.open("element-call-background-images");
open.onerror = (): void => reject(open.error);
open.onsuccess = (): void => {
const all = open.result
.transaction("backgrounds")
.objectStore("backgrounds")
.getAll();
all.onerror = (): void => reject(all.error);
all.onsuccess = (): void => {
const reader = new FileReader();
reader.onload = (): void =>
resolve((reader.result as string).split(",")[1]);
reader.readAsDataURL((all.result[0] as { image: Blob }).image);
};
};
}),
);
return Buffer.from(base64, "base64");
}
const RED: Colour = [255, 0, 0];
/** A solid red PNG, unlike the camera and the shipped pictures. */
async function redImage(page: Page): Promise<Buffer> {
const dataUrl = await page.evaluate(() => {
const canvas = document.createElement("canvas");
canvas.width = 320;
canvas.height = 180;
const context = canvas.getContext("2d")!;
context.fillStyle = "rgb(255, 0, 0)";
context.fillRect(0, 0, canvas.width, canvas.height);
return canvas.toDataURL("image/png");
});
return Buffer.from(dataUrl.split(",")[1], "base64");
}
/** Adds an image from the camera menu's add tile, leaving the menu open. */
async function addImage(
page: Page,
image: Buffer,
{ name, type } = { name: "mine.png", type: "image/png" },
): Promise<void> {
if (!(await page.getByRole("menu").isVisible()))
await page.getByRole("button", { name: "Camera", exact: true }).click();
const chooser = page.waitForEvent("filechooser");
await page.getByRole("menuitem", { name: "Add image" }).click();
await (await chooser).setFiles({ name, mimeType: type, buffer: image });
}
test.describe("joining with a background chosen", () => {
test("publishes no frame of the room", async ({ browser }) => {
// Two first builds of the pipeline, one of them held back.
test.setTimeout(180_000);
const hostContext = await browser.newContext();
const host = await hostContext.newPage();
await recordIncomingFrames(host);
await host.goto("/");
await SpaHelpers.createCall(host, "Host", "Join with a background", true);
await expect(host.getByTestId("incall_videomute")).toBeEnabled({
timeout: 10_000,
});
const inviteLink = await SpaHelpers.getCallInviteLink(host);
await host.keyboard.press("Escape");
const guestContext = await browser.newContext();
const guest = await guestContext.newPage();
await guest.goto(inviteLink);
await guest.getByTestId("joincall_displayName").fill("Guest");
await guest.getByTestId("joincall_joincall").click();
const preview = guest.locator("video").first();
await expect(preview).toBeVisible();
const camera = await cameraColour(preview);
await guest.getByRole("button", { name: "Camera", exact: true }).click();
const tile = guest.getByRole("menuitemradio", { name: "Background 1" });
await tile.click();
const picture = await averageColour(tile.locator("img"));
await expect
.poll(async () => distance(await averageColour(preview), picture), {
timeout: 60_000,
})
.toBeLessThan(distance(camera, picture) / 2);
// Were the pipeline built again for the call's track, it would take as
// long as on the slower route. Routing bypasses the cache, so it waits.
let heldBack = 0;
await guest.route("**/*.tflite*", async (route) => {
heldBack += 1;
await new Promise((resolve) => setTimeout(resolve, 15_000));
await route.continue();
});
await guest.keyboard.press("Escape");
await guest.getByTestId("lobby_joinCall").click();
const guestTile = host
.getByTestId("videoTile")
.filter({ has: host.getByTestId("name_tag").getByText("Guest") });
await expect(guestTile.locator("video")).toBeVisible({ timeout: 60_000 });
// Visible can come before the stream is attached.
let trackId: string | undefined;
await expect
.poll(async () => {
trackId = await guestTile
.locator("video")
.evaluate(
(video: HTMLVideoElement) =>
(video.srcObject as MediaStream | null)?.getVideoTracks()[0]?.id,
);
return trackId;
})
.toBeDefined();
await expectNoFrameOfTheRoom(host, trackId!, 0, camera, picture);
// The join adopts the lobby's pipeline, so the model is never fetched again.
expect(heldBack).toBe(0);
await hostContext.close();
await guestContext.close();
});
});
test.describe("the first build", () => {
test("survives the camera turned off and on while it runs", async ({
page,
}) => {
test.setTimeout(120_000);
const failures: string[] = [];
page.on("console", (m) => {
if (m.text().includes("Failed to attach video processor"))
failures.push(m.text());
});
await page.goto("/");
await SpaHelpers.createCall(page, "Toggling user", "Toggle while building");
const preview = page.locator("video").first();
const camera = await cameraColour(preview);
const held = await holdBackTheModel(page);
await page.getByRole("button", { name: "Camera", exact: true }).click();
const tile = page.getByRole("menuitemradio", { name: "Background 1" });
await tile.click();
const picture = await averageColour(tile.locator("img"));
await page.keyboard.press("Escape");
// New preview tracks, each attached while the first is still building.
for (let i = 0; i < 2; i++) {
await page.getByTestId("incall_videomute").click();
await page.getByTestId("incall_videomute").click();
}
await expect
.poll(async () => distance(await averageColour(preview), picture), {
timeout: 60_000,
})
.toBeLessThan(distance(camera, picture) / 2);
expect(failures).toEqual([]);
expect(held()).toBeGreaterThan(0);
});
});
test.describe("turning the camera on with a background chosen", () => {
test("publishes no frame of the room when the camera started off", async ({
browser,
}) => {
test.setTimeout(120_000);
const { host, guest, close } = await hostAndGuest(browser);
const camera = await cameraColour(guest.locator("video").first());
await guest.getByTestId("incall_videomute").click();
await guest.getByTestId("lobby_joinCall").click();
const picture = await chooseBackground(guest);
const held = await holdBackTheModel(guest);
await guest.getByTestId("incall_videomute").click();
const trackId = await incomingTrackId(host);
await expectNoFrameOfTheRoom(host, trackId, 0, camera, picture);
expect(held()).toBeGreaterThan(0);
await close();
});
test("publishes no frame of the room when the camera was turned off", async ({
browser,
}) => {
test.setTimeout(120_000);
const { host, guest, close } = await hostAndGuest(browser);
const camera = await cameraColour(guest.locator("video").first());
await guest.getByTestId("lobby_joinCall").click();
const trackId = await incomingTrackId(host);
await expect
.poll(async () => (await framesOf(host, trackId)).length, {
timeout: 20_000,
})
.toBeGreaterThan(0);
await guest.getByTestId("incall_videomute").click();
const picture = await chooseBackground(guest);
const held = await holdBackTheModel(guest);
const before = (await framesOf(host, trackId)).length;
await guest.getByTestId("incall_videomute").click();
await expectNoFrameOfTheRoom(host, trackId, before, camera, picture);
expect(held()).toBeGreaterThan(0);
await close();
});
});
/** A host in a call, recording what it receives, and a guest in its lobby. */
async function hostAndGuest(browser: Browser): Promise<{
host: Page;
guest: Page;
close: () => Promise<void>;
}> {
const hostContext = await browser.newContext();
const host = await hostContext.newPage();
await recordIncomingFrames(host);
await host.goto("/");
await SpaHelpers.createCall(host, "Host", "Camera with a background", true);
await expect(host.getByTestId("incall_videomute")).toBeEnabled({
timeout: 10_000,
});
const inviteLink = await SpaHelpers.getCallInviteLink(host);
await host.keyboard.press("Escape");
const guestContext = await browser.newContext();
const guest = await guestContext.newPage();
await guest.goto(inviteLink);
await guest.getByTestId("joincall_displayName").fill("Guest");
await guest.getByTestId("joincall_joincall").click();
await expect(guest.locator("video").first()).toBeVisible();
return {
host,
guest,
close: async (): Promise<void> => {
await hostContext.close();
await guestContext.close();
},
};
}
/** Chooses the first shipped picture in the call, and answers its colour. */
async function chooseBackground(guest: Page): Promise<Colour> {
// The lobby has a camera button of its own, so wait for the call's.
await expect(guest.getByTestId("incall_leave")).toBeVisible({
timeout: 30_000,
});
await expect(guest.getByTestId("incall_videomute")).toBeEnabled({
timeout: 10_000,
});
await guest.getByRole("button", { name: "Camera", exact: true }).click();
const tile = guest.getByRole("menuitemradio", { name: "Background 1" });
await tile.click();
await expect(tile).toHaveAttribute("aria-checked", "true");
const picture = await averageColour(tile.locator("img"));
await guest.keyboard.press("Escape");
return picture;
}
/** Makes the pipeline's first build as slow as on the slower route. */
/** Answers how often it held the model back, which a check should see. */
async function holdBackTheModel(page: Page): Promise<() => number> {
let held = 0;
// Routing bypasses the cache, so it waits.
await page.route("**/*.tflite*", async (route) => {
held += 1;
await new Promise((resolve) => setTimeout(resolve, 5_000));
await route.continue();
});
return () => held;
}
/** The id of the guest's camera track, as the host receives it. */
async function incomingTrackId(host: Page): Promise<string> {
const guestTile = host
.getByTestId("videoTile")
.filter({ has: host.getByTestId("name_tag").getByText("Guest") });
await expect(guestTile.locator("video")).toBeVisible({ timeout: 60_000 });
// Visible can come before the stream is attached.
let trackId: string | undefined;
await expect
.poll(async () => {
trackId = await guestTile
.locator("video")
.evaluate(
(video: HTMLVideoElement) =>
(video.srcObject as MediaStream | null)?.getVideoTracks()[0]?.id,
);
return trackId;
})
.toBeDefined();
return trackId!;
}
/** Waits for the picture to arrive, then finds no frame of the room since. */
async function expectNoFrameOfTheRoom(
host: Page,
trackId: string,
since: number,
camera: Colour,
picture: Colour,
): Promise<void> {
await expect
.poll(
async () => {
const frames = (await framesOf(host, trackId)).slice(since);
// A black frame, as a track starts, is nearer the picture than the
// camera, but shows neither.
return frames.some(
(f) =>
distance(f, picture) <
Math.min(distance(f, camera), distance(f, [0, 0, 0])),
);
},
{ timeout: 60_000 },
)
.toBe(true);
const frames = (await framesOf(host, trackId)).slice(since);
expect(
frames.filter((f) => distance(f, camera) < distance(f, picture)),
).toEqual([]);
}
/**
* Records the mean colour of every frame of every video track the page
* receives, from the moment it arrives: a tile may attach it later than its
* first frame.
*/
async function recordIncomingFrames(page: Page): Promise<void> {
await page.addInitScript(() => {
const frames: Record<string, [number, number, number][]> = {};
(window as unknown as { receivedFrames: typeof frames }).receivedFrames =
frames;
const record = (track: MediaStreamTrack): void => {
if (track.kind !== "video" || frames[track.id]) return;
const seen: [number, number, number][] = (frames[track.id] = []);
const video = document.createElement("video");
video.muted = true;
video.srcObject = new MediaStream([track]);
/** The mean colour of a video's current frame or an image, drawn small. */
async function averageColour(element: Locator): Promise<Colour> {
return element.evaluate(
async (source: HTMLVideoElement | HTMLImageElement) => {
if (source instanceof HTMLImageElement) await source.decode();
const canvas = document.createElement("canvas");
canvas.width = canvas.height = 16;
const context = canvas.getContext("2d", { willReadFrequently: true })!;
const step = (): void => {
context.drawImage(video, 0, 0, 16, 16);
const { data } = context.getImageData(0, 0, 16, 16);
const sum = [0, 0, 0];
for (let i = 0; i < data.length; i += 4)
for (let c = 0; c < 3; c++) sum[c] += data[i + c];
seen.push(sum.map((v) => v / (data.length / 4)) as Colour);
video.requestVideoFrameCallback(step);
};
video.requestVideoFrameCallback(step);
void video.play();
};
const Connection = window.RTCPeerConnection;
window.RTCPeerConnection = class extends Connection {
public constructor(...args: ConstructorParameters<typeof Connection>) {
super(...args);
this.addEventListener("track", (e) => record(e.track));
}
};
});
}
async function framesOf(page: Page, trackId: string): Promise<Colour[]> {
return page.evaluate(
(id) =>
(
window as unknown as {
receivedFrames: Record<string, [number, number, number][]>;
}
).receivedFrames[id] ?? [],
trackId,
canvas.width = canvas.height = 32;
const context = canvas.getContext("2d")!;
context.drawImage(source, 0, 0, 32, 32);
const { data } = context.getImageData(0, 0, 32, 32);
const sum = [0, 0, 0];
for (let i = 0; i < data.length; i += 4)
for (let c = 0; c < 3; c++) sum[c] += data[i + c];
const pixels = data.length / 4;
return sum.map((s) => s / pixels) as Colour;
},
);
}
function distance(a: Colour, b: Colour): number {
return Math.hypot(a[0] - b[0], a[1] - b[1], a[2] - b[2]);
}
@@ -1,85 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { expect, type Locator, type Page, test } from "@playwright/test";
import { createUserAndRoom, resizeContainer, startHarness } from "./harness.ts";
import { averageColour, cameraColour, expectWearing } from "../utils/colour.ts";
// Background effects need WebGL2, and headless Firefox on a CI runner has none,
// so it rightly offers none of them.
test.skip(
({ browserName }) => browserName === "firefox",
"Background effects need WebGL2, which headless Firefox on CI does not have",
);
/**
* Background effects with Element Call as a component in a host page: the
* pipeline's model and WebAssembly have to load from the component's bundle,
* and the portalled menu has to be sized against the call, not the window.
* Driven from the lobby, which builds the same menu and pipeline.
*/
// Sign-in, crypto setup and sync happen twice before anything shows.
test.describe.configure({ timeout: 180_000 });
test("puts a picture on the call's preview", async ({ page }) => {
const { username, roomId } = await createUserAndRoom("effects");
const panes = await startHarness(page, username, roomId);
const pane = panes.first();
await resizeContainer(pane.getByTestId("call-container"), {
width: 900,
height: 480,
});
await expect(pane.getByTestId("lobby_joinCall")).toBeVisible({
timeout: 60_000,
});
const preview = pane.locator("video").first();
const camera = await cameraColour(preview);
const tile = await openEffect(page, pane, "Background 1");
const picture = await averageColour(tile.locator("img"));
await tile.click();
await expect(tile).toHaveAttribute("aria-checked", "true");
await expectWearing(preview, picture, camera);
});
test("keeps every effect reachable in a small container", async ({ page }) => {
const { username, roomId } = await createUserAndRoom("effectsreach");
const panes = await startHarness(page, username, roomId);
const pane = panes.first();
const container = pane.getByTestId("call-container");
await resizeContainer(container, { width: 400, height: 360 });
await expect(pane.getByTestId("lobby_joinCall")).toBeVisible({
timeout: 60_000,
});
const last = await openEffect(page, pane, "Background 2");
const list = page.locator("[role='menu'] div[role='none']").first();
expect((await list.boundingBox())!.height).toBeLessThanOrEqual(
(await container.boundingBox())!.height,
);
await last.scrollIntoViewIfNeeded();
await expect(last).toBeInViewport({ ratio: 1 });
await last.click();
await expect(last).toHaveAttribute("aria-checked", "true");
});
/** Opens a component's camera menu and returns one of its effects. */
async function openEffect(
page: Page,
pane: Locator,
name: string,
): Promise<Locator> {
await pane
.getByRole("button", { name: "Camera", exact: true })
.click({ timeout: 60_000 });
// Portalled out of the pane, so found on the page.
return page
.getByRole("group", { name: "Background effects" })
.getByRole("menuitemradio", { name });
}
+68 -11
View File
@@ -29,14 +29,49 @@ export interface MyFixtures {
addUser: (username: string, host: string) => Promise<UserBaseFixture>;
}
// Minimal config.json for the local element-web instance
const CONFIG_JSON = {
default_server_config: {
"m.homeserver": {
base_url: "https://synapse.m.localhost",
server_name: "synapse.m.localhost",
},
},
element_call: {
participant_limit: 8,
brand: "Element Call",
},
// The default language is set here for test consistency
setting_defaults: {
language: "en-GB",
feature_group_calls: true,
},
// the location tests want a map style url.
map_style_url:
"https://api.maptiler.com/maps/streets/style.json?key=fU3vlMsMn4Jb6dnEIFsx",
features: {
// We don't want to go through the feature announcement during the e2e test
feature_release_announcement: false,
feature_element_call_video_rooms: true,
feature_video_rooms: true,
feature_group_calls: true,
},
};
export const widgetTest = test.extend<MyFixtures>({
// allow per-test override: `widgetTest.use({ callType: "dm" })`
callType: ["room", { option: true }],
asWidget: async ({ browser, callType }, pUse, testInfo) => {
// Tests in parallel workers can reach this in the same millisecond.
const uniqueSuffix = `${Date.now()}_${testInfo.parallelIndex}`;
const brooksDisplayName = `brooks_${uniqueSuffix}`;
const whistlerDisplayName = `whistler_${uniqueSuffix}`;
asWidget: async ({ browser, context, callType }, pUse) => {
await context.route(`http://localhost:8081/config.json*`, async (route) => {
await route.fulfill({ json: CONFIG_JSON });
});
const brooksDisplayName = `brooks_${Date.now()}`;
const whistlerDisplayName = `whistler_${Date.now()}`;
// Register users
const {
@@ -74,7 +109,15 @@ export const widgetTest = test.extend<MyFixtures>({
await TestHelpers.dismissInviteUnknownUserModal(ewPage1);
// Accept the invite
await TestHelpers.openRoomInvite(ewPage2, "Welcome Room");
await TestHelpers.closeReleaseAnnouncement(
ewPage2,
"Introducing Sections",
);
await TestHelpers.expandAllSections(ewPage2);
await expect(
TestHelpers.roomListItem(ewPage2, "Welcome Room"),
).toBeVisible();
await TestHelpers.roomListItem(ewPage2, "Welcome Room").click();
await ewPage2.getByRole("button", { name: "Accept" }).click();
await expect(
ewPage2
@@ -82,7 +125,17 @@ export const widgetTest = test.extend<MyFixtures>({
.getByRole("heading", { name: "Welcome Room" }),
).toBeVisible();
} else if (callType === "dm") {
await TestHelpers.chooseFromRoomListNewMenu(ewPage1, "Start chat");
await TestHelpers.closeReleaseAnnouncement(
ewPage1,
"Introducing Sections",
);
await ewPage1
.getByRole("navigation", { name: "Room list" })
.getByRole("button", { name: "New", exact: true })
.click();
await ewPage1.getByRole("menuitem", { name: "Start chat" }).click();
await ewPage1.getByRole("textbox", { name: "Search" }).click();
await ewPage1.getByRole("textbox", { name: "Search" }).fill(whistlerMxId);
await ewPage1.getByRole("button", { name: "Go" }).click();
@@ -104,7 +157,12 @@ export const widgetTest = test.extend<MyFixtures>({
).toBeVisible();
// Accept the DM invite from brooks
await TestHelpers.openRoomInvite(ewPage2, brooksDisplayName);
await TestHelpers.closeReleaseAnnouncement(
ewPage2,
"Introducing Sections",
);
await TestHelpers.expandAllSections(ewPage2);
await TestHelpers.roomListItem(ewPage2, brooksDisplayName).click();
await ewPage2.getByRole("button", { name: "Start chatting" }).click();
}
@@ -129,14 +187,13 @@ export const widgetTest = test.extend<MyFixtures>({
* Provide a way to add additional users within a test.
* The returned user will be registered on the default homeserver, the name will be made unique by appending a timestamp.
*/
addUser: async ({ browser }, use, testInfo) => {
addUser: async ({ browser }, use) => {
await use(
async (
username: string,
host: string = HOST1,
): Promise<UserBaseFixture> => {
// Tests in parallel workers can reach this in the same millisecond.
const uniqueSuffix = `${Date.now()}_${testInfo.parallelIndex}`;
const uniqueSuffix = Date.now();
const { page, clientHandle, mxId } = await TestHelpers.registerUser(
browser,
`${username.toLowerCase()}_${uniqueSuffix}`,
+13 -4
View File
@@ -11,18 +11,25 @@ import { startPair, tileOf } from "./harness";
test.describe.configure({ timeout: 300_000 });
test("a message sent beside the media reaches the peer, and only the peer", async ({
test("a message sent on a data track reaches the peer, and only the peer", async ({
browser,
}) => {
const { pages, users } = await startPair(browser, "sdkdata");
const [sender, receiver] = pages;
// Both are in the call before anything is sent, so the receiver knows the
// sender as a member and does not drop the message
// sender as a member
for (const page of pages)
await expect(page.getByTestId("member")).toHaveCount(2, {
timeout: 60_000,
});
await expect(tileOf(receiver, users[0])).toBeVisible();
// The receiver listens on the sender's track, published with the join
await expect(tileOf(receiver, users[0])).toHaveAttribute(
"data-chat",
"true",
{
timeout: 30_000,
},
);
await sender.getByPlaceholder("message").fill("hello from the SDK");
await sender.getByRole("button", { name: "Send" }).click();
@@ -34,7 +41,9 @@ test("a message sent beside the media reaches the peer, and only the peer", asyn
timeout: 30_000,
},
);
await expect(message).toHaveAttribute("data-topic", "chat");
await expect(message).toHaveAttribute("data-name", "chat");
// Decrypted with the sender's key, at the key index the stream carried
await expect(message).toHaveAttribute("data-encrypted", "true");
await expect(message).toHaveAttribute("data-user-id", users[0].userId);
// Our own messages are not echoed back
await expect(sender.getByTestId("message")).toHaveCount(0);
-55
View File
@@ -1,55 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { expect, type Locator } from "@playwright/test";
export type Colour = [number, number, number];
/** The camera's colour, once the preview is showing it rather than nothing. */
export async function cameraColour(preview: Locator): Promise<Colour> {
await expect
.poll(async () => Math.max(...(await averageColour(preview))))
.toBeGreaterThan(40);
return averageColour(preview);
}
/** The mean colour of a video's current frame or an image, drawn small. */
export async function averageColour(element: Locator): Promise<Colour> {
return element.evaluate(
async (source: HTMLVideoElement | HTMLImageElement) => {
if (source instanceof HTMLImageElement) await source.decode();
const canvas = document.createElement("canvas");
canvas.width = canvas.height = 32;
const context = canvas.getContext("2d")!;
context.drawImage(source, 0, 0, 32, 32);
const { data } = context.getImageData(0, 0, 32, 32);
const sum = [0, 0, 0];
for (let i = 0; i < data.length; i += 4)
for (let c = 0; c < 3; c++) sum[c] += data[i + c];
const pixels = data.length / 4;
return sum.map((s) => s / pixels) as Colour;
},
);
}
export function distance(a: Colour, b: Colour): number {
return Math.hypot(a[0] - b[0], a[1] - b[1], a[2] - b[2]);
}
/** Waits until a preview shows a picture rather than the camera. */
export async function expectWearing(
preview: Locator,
picture: Colour,
camera: Colour,
): Promise<void> {
// Long enough for the first effect of a session to be built.
await expect
.poll(async () => distance(await averageColour(preview), picture), {
timeout: 60_000,
})
.toBeLessThan(distance(camera, picture) / 2);
}
@@ -1,43 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { expect, test } from "@playwright/test";
import { widgetTest } from "../fixtures/widget-user.ts";
import { TestHelpers } from "./test-helpers.ts";
import { averageColour, cameraColour, expectWearing } from "../utils/colour.ts";
// Background effects need WebGL2, and headless Firefox on a CI runner has none,
// so it rightly offers none of them.
test.skip(
({ browserName }) => browserName === "firefox",
"Background effects need WebGL2, which headless Firefox on CI does not have",
);
widgetTest(
"puts a picture on the preview inside Element Web",
async ({ asWidget }) => {
test.slow();
const { brooks } = asWidget;
await TestHelpers.startCallInCurrentRoom(brooks.page, false);
const frame = brooks.page
.locator('iframe[title="Element Call"]')
.contentFrame();
await expect(frame.getByTestId("lobby_joinCall")).toBeVisible();
const preview = frame.locator("video").first();
const camera = await cameraColour(preview);
await frame.getByRole("button", { name: "Camera", exact: true }).click();
const tile = frame
.getByRole("group", { name: "Background effects" })
.getByRole("menuitemradio", { name: "Background 1" });
const picture = await averageColour(tile.locator("img"));
await tile.click();
await expect(tile).toHaveAttribute("aria-checked", "true");
await expectWearing(preview, picture, camera);
},
);
-20
View File
@@ -1,20 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { test as setup } from "@playwright/test";
import { execFileSync } from "node:child_process";
import { mkdirSync } from "node:fs";
const EMBEDDED_DIST = "embedded/web/dist";
setup("build the embedded package", () => {
setup.setTimeout(10 * 60 * 1000);
mkdirSync(EMBEDDED_DIST, { recursive: true });
execFileSync("pnpm", ["build:embedded", "--outDir", EMBEDDED_DIST], {
stdio: "inherit",
});
});
@@ -1,56 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { expect, type Route } from "@playwright/test";
import { widgetTest } from "../../fixtures/widget-user.ts";
import { HOST2, TestHelpers } from "../test-helpers.ts";
/** Where the other site's Element Web serves the embedded package from */
const CONFIG_JSON = "**/widgets/element-call/config.json";
const HOST_ANSWERS: Record<string, (route: Route) => Promise<void>> = {
"a 404": async (route) => route.fulfill({ status: 404 }),
"a failed fetch": async (route) => route.abort("failed"),
"an HTML page": async (route) =>
route.fulfill({
status: 200,
contentType: "text/html",
body: "<!doctype html><title>Element</title>",
}),
};
widgetTest.skip(
({ browserName }) => browserName === "firefox",
"The widget suite does not run on Firefox, see the other specs here",
);
for (const [answer, respond] of Object.entries(HOST_ANSWERS)) {
widgetTest(
`reaches the lobby when config.json gets ${answer}`,
async ({ addUser }) => {
const { page } = await addUser("Brooks", HOST2);
await TestHelpers.createRoom("Config Room", page);
let answered = 0;
await page.route(CONFIG_JSON, async (route) => {
answered++;
await respond(route);
});
await TestHelpers.startCallInCurrentRoom(page);
await expect(
page
.locator('iframe[title="Element Call"]')
.contentFrame()
.getByTestId("lobby_joinCall"),
).toBeVisible({ timeout: 15000 });
expect(answered).toBeGreaterThan(0);
},
);
}
+37 -83
View File
@@ -22,12 +22,6 @@ const PASSWORD = "foobarbaz1!";
export const HOST1 = "https://app.m.localhost/#/welcome";
export const HOST2 = "https://app.othersite.m.localhost/#/welcome";
/**
* Where the widget RTC SDK's example widget is served (`pnpm dev:widget-rtc-sdk`):
* `widget-rtc-sdk/index.html`, running the bundle from source.
*/
export const WIDGET_RTC_SDK_URL = "https://localhost:3003";
export type RtcMode = "compat" | "2_0";
export class TestHelpers {
@@ -150,11 +144,7 @@ export class TestHelpers {
await this.dismissStartupToasts(page);
// The other site's Element Web opens the embedded package it serves itself
// (see docker-compose-playwright.yml), not the dev server's app.
if (host !== HOST2) {
await TestHelpers.setDevToolElementCallDevUrl(page);
}
await TestHelpers.setDevToolElementCallDevUrl(page);
const clientHandle = await page.evaluateHandle(() =>
window.mxMatrixClientPeg.get(),
@@ -163,40 +153,29 @@ export class TestHelpers {
return { page, clientHandle, mxId };
}
// Dismisses the toasts a fresh device gets after sign-in.
//
// Toasts stack and only the top one can be dismissed, so look
// at what is on top each time.
// Dismisses any toasts that appear on startup, such as "Failed to load service worker" or "Back up your chats".
// Toast can be stacked, and only the top one can be dismiss, so just look at what is on top and
// dismiss (if part of expected toats)
public static async dismissStartupToasts(page: Page): Promise<void> {
const expectedToasts: {
title: string;
button: string;
confirm?: string;
}[] = [
const expectedToasts = [
{ title: "Failed to load service worker", button: "OK" },
{ title: "Verify this device", button: "Later" },
{ title: "Back up your chats", button: "Not now" },
{
title: "Turn on key storage",
button: "Dismiss",
confirm: "Yes, dismiss",
},
{ title: "Notifications", button: "Dismiss" },
{ title: "Turn on key storage", button: "Dismiss" },
{ title: "Element does not support this browser", button: "Dismiss" },
];
const toast = page.locator(".mx_Toast_toast");
const toastTitle = toast.locator(".mx_Toast_title h2");
// eslint-disable-next-line no-constant-condition
while (true) {
try {
await toast.waitFor({ state: "visible", timeout: 700 });
const title = await toastTitle.textContent();
if (!title) break;
const title = await toast.locator(".mx_Toast_title h2").textContent();
// Find the matching toast config
const toastConfig = expectedToasts.find((t) => title.includes(t.title));
const toastConfig = expectedToasts.find((t) =>
title?.includes(t.title),
);
if (toastConfig) {
// Bounded so that a renamed button leaves the toast in place and the
@@ -204,16 +183,6 @@ export class TestHelpers {
await toast
.getByRole("button", { name: toastConfig.button })
.click({ timeout: 5000 });
if (toastConfig.confirm) {
const dialog = page.getByRole("dialog");
await dialog
.getByRole("button", { name: toastConfig.confirm })
.click({ timeout: 5000 });
await expect(dialog).toBeHidden({ timeout: 5000 });
}
// The same element shows the next toast a moment after the click, so
// reading it straight away acts on the dismissed one a second time
await expect(toastTitle).not.toHaveText(title, { timeout: 5000 });
} else {
// Unknown toast. We don't want to act on unknown toasts
break;
@@ -225,6 +194,20 @@ export class TestHelpers {
}
}
public static async closeReleaseAnnouncement(
page: Page,
name: string,
): Promise<void> {
try {
await page
.getByRole("dialog", { name })
.getByRole("button", { name: "OK" })
.click({ timeout: 2000 });
} catch {
// Announcement not shown; nothing to do
}
}
public static async expandAllSections(page: Page): Promise<void> {
try {
await page
@@ -240,7 +223,16 @@ export class TestHelpers {
page: Page,
andInvite: string[] = [],
): Promise<void> {
await TestHelpers.chooseFromRoomListNewMenu(page, "New Room");
await TestHelpers.closeReleaseAnnouncement(page, "Introducing Sections");
await page
.getByRole("navigation", { name: "Room list" })
.getByRole("button", { name: "New", exact: true })
.click();
await page.getByRole("menuitem", { name: "New Room" }).click({
timeout: 5000,
});
await page.getByRole("textbox", { name: "Name" }).fill(name);
await page.getByRole("button", { name: "Create room" }).click();
await expect(page.getByText("You created this room.")).toBeVisible({
@@ -267,24 +259,6 @@ export class TestHelpers {
}
}
/**
* Opens the room list's "New" menu and picks the given item.
*
* Driven by keyboard, as a toast arriving late sits over the button and
* swallows clicks.
*/
public static async chooseFromRoomListNewMenu(
page: Page,
itemName: string,
): Promise<void> {
await TestHelpers.dismissStartupToasts(page);
await page
.getByRole("navigation", { name: "Room list" })
.getByRole("button", { name: "New", exact: true })
.press("Enter");
await page.getByRole("menuitem", { name: itemName }).press("Enter");
}
/**
* Accepts a room invite using the room name.
* Locatest the invite in the room list.
@@ -294,7 +268,9 @@ export class TestHelpers {
roomName: string,
page: Page,
): Promise<void> {
await TestHelpers.openRoomInvite(page, roomName);
await TestHelpers.closeReleaseAnnouncement(page, "Introducing Sections");
await TestHelpers.expandAllSections(page);
await TestHelpers.roomListItem(page, roomName).click({ timeout: 10000 });
await page.getByRole("button", { name: "Accept" }).click({
timeout: 5000,
});
@@ -400,28 +376,6 @@ export class TestHelpers {
return page.locator(`[aria-label^="Open room ${roomName}"]`);
}
/**
* Opens a pending invite from the room list by its room name.
*
* Element Web collapses the Invites section every time it appears, so it is
* expanded here once the invite has arrived.
*/
public static async openRoomInvite(
page: Page,
roomName: string,
): Promise<void> {
const invitesSection = page.getByRole("button", {
name: "Toggle Invites section",
});
await expect(invitesSection).toBeVisible({ timeout: 10000 });
// Toasts sit over the top of the room list, where this header is.
await TestHelpers.dismissStartupToasts(page);
if ((await invitesSection.getAttribute("aria-expanded")) !== "true") {
await invitesSection.click();
}
await TestHelpers.roomListItem(page, roomName).click({ timeout: 10000 });
}
/**
* Switches to a room in the room list by its name.
* @param page - The EW page
-129
View File
@@ -1,129 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { expect, type FrameLocator, type Page } from "@playwright/test";
import { widgetTest } from "../fixtures/widget-user.ts";
import { WIDGET_RTC_SDK_URL } from "./test-helpers.ts";
/**
* The widget RTC SDK's example widget, `widget-rtc-sdk/index.html`, added to a
* room as a custom widget and opened in two Element Webs: the bundle's whole
* public API, from the widget API to the data channel.
*/
widgetTest.describe.configure({ timeout: 300_000 });
const WIDGET_ID = "widget-rtc-sdk-example";
const WIDGET_NAME = "Widget RTC SDK";
/** Accepts the prompts Element Web shows before it runs a widget, and finds it. */
async function openWidget(page: Page): Promise<FrameLocator> {
// Only for a widget someone else added
const loadPrompt = page.getByRole("button", { name: "Continue" });
const capabilitiesPrompt = page.getByRole("button", { name: "Approve" });
await expect(loadPrompt.or(capabilitiesPrompt)).toBeVisible({
timeout: 30_000,
});
if (await loadPrompt.isVisible()) await loadPrompt.click();
await capabilitiesPrompt.click({ timeout: 30_000 });
// The SDK authenticates with the media transport by an OpenID token, and
// asks for more than one, so the answer has to be remembered
const openIdPrompt = page.getByRole("dialog", {
name: "Allow this widget to verify your identity",
});
await openIdPrompt.getByText("Remember this").click({ timeout: 30_000 });
await openIdPrompt.getByRole("button", { name: "Continue" }).click();
return page.frameLocator(`iframe[title="${WIDGET_NAME}"]`);
}
widgetTest(
"data sent from one widget arrives in the other",
async ({ asWidget }) => {
const { brooks, whistler } = asWidget;
// Brooks adds the example widget to the room and pins it to the top, so
// that both Element Webs open it without anyone looking for it
await brooks.clientHandle.evaluate(
async (client, { widgetId, name, url }) => {
const roomId = client
.getRooms()
.find((room) => room.name === "Welcome Room")!.roomId;
await client.sendStateEvent(
roomId,
// @ts-expect-error not in the js-sdk's state event map
"im.vector.modular.widgets",
{
id: widgetId,
type: "m.custom",
name,
url,
data: {},
creatorUserId: client.getUserId(),
},
widgetId,
);
await client.sendStateEvent(
roomId,
// @ts-expect-error not in the js-sdk's state event map
"io.element.widgets.layout",
{
widgets: {
[widgetId]: {
container: "top",
index: 0,
width: 100,
height: 50,
},
},
},
"",
);
},
{
widgetId: WIDGET_ID,
name: WIDGET_NAME,
// Element Web fills the $ variables in and adds widgetId and parentUrl
url:
`${WIDGET_RTC_SDK_URL}/?widgetId=$matrix_widget_id&roomId=$matrix_room_id` +
"&userId=$matrix_user_id&deviceId=$org.matrix.msc3819.matrix_device_id" +
"&baseUrl=$org.matrix.msc4039.matrix_base_url",
},
);
const brooksWidget = await openWidget(brooks.page);
const whistlerWidget = await openWidget(whistler.page);
// The example joins on load; each sees the other as a member
for (const [widget, other] of [
[brooksWidget, whistler],
[whistlerWidget, brooks],
] as const) {
await expect(widget.locator("#connect_status")).toHaveText("Connected", {
timeout: 60_000,
});
await expect(widget.locator("#members")).toContainText(other.mxId, {
timeout: 60_000,
});
}
// Data is not queued: the receiver drops what arrives before it has matched
// the sender on the media transport, which the example does not show. So
// send until it arrives.
await expect(async () => {
await brooksWidget.getByRole("button", { name: "Send Text" }).click();
await expect(whistlerWidget.locator("#data")).toContainText(
"Hello, world!",
{ timeout: 2_000 },
);
}).toPass({ timeout: 60_000 });
// Our own data is not echoed back
await expect(brooksWidget.locator("#data")).not.toContainText(
"Hello, world!",
);
},
);
+31 -53
View File
@@ -8,7 +8,7 @@ overrides:
'@livekit/components-core>rxjs': ^7.8.1
'@livekit/track-processors>@mediapipe/tasks-vision': ^0.10.18
tar: ^7.5.11
minimatch: ^11.0.0
minimatch: ^10.2.3
glob: ^10.5.0
qs: ^6.15.2
js-yaml: ^4.2.0
@@ -160,9 +160,6 @@ importers:
eslint-plugin-storybook:
specifier: ^10.3.6
version: 10.6.2(eslint@8.57.1(supports-color@7.2.0))(supports-color@7.2.0)(typescript@5.9.3)
fake-indexeddb:
specifier: ^6.2.5
version: 6.2.5
fetch-mock:
specifier: 11.1.5
version: 11.1.5(node-fetch@2.7.0)
@@ -191,8 +188,8 @@ importers:
specifier: ^1.9.1
version: 1.9.2
matrix-js-sdk:
specifier: github:matrix-org/matrix-js-sdk#10e4faabbb49baca655492d523b58faa9eea04b9
version: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/10e4faabbb49baca655492d523b58faa9eea04b9
specifier: github:matrix-org/matrix-js-sdk#a9ef09fbf2f1644c8663d0aeaf6a921cd16e7b01
version: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/a9ef09fbf2f1644c8663d0aeaf6a921cd16e7b01
matrix-widget-api:
specifier: ^1.18.0
version: 1.20.0
@@ -1145,9 +1142,6 @@ packages:
resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==}
engines: {node: '>=12'}
'@isaacs/nonrecursive-glob-match@1.0.3':
resolution: {integrity: sha512-SI7kyKHq4XXDRozw+r9GlRZkia1ZkFrpLW98r2IC2Bpr8YkFGE3fvjzmxCG0x78A++69HmDNrDvSMYBZmHGpKA==}
'@joshwooding/vite-plugin-react-docgen-typescript@0.7.0':
resolution: {integrity: sha512-qvsTEwEFefhdirGOPnu9Wp6ChfIwy2dBCRuETU3uE+4cC+PFoxMSiiEhxk4lOluA34eARHA0OxqsEUYDqRMgeQ==}
peerDependencies:
@@ -1195,8 +1189,8 @@ packages:
'@types/dom-mediacapture-transform': ^0.1.9
livekit-client: ^1.12.0 || ^2.1.0
'@matrix-org/matrix-sdk-crypto-wasm@18.9.2':
resolution: {integrity: sha512-GHeuSeb5Fj6Wg2+NE/WoLJJs9U3nOzttO+2sj7lOaS5gNBpGIksc48N/Q/kK/Z+Lxh7YMhopB7M8b6HyASXnGQ==}
'@matrix-org/matrix-sdk-crypto-wasm@18.9.0':
resolution: {integrity: sha512-NtLzNDTEfNASBjNCTGZHuk7N0jrQhS6Zx2CM857rCHZYYh/rpbKGQNrJ7azktm4S1P5Jy3WU3nwJTGohGw26qQ==}
engines: {node: '>= 18'}
'@mdx-js/react@3.1.1':
@@ -3533,8 +3527,8 @@ packages:
boolbase@1.0.0:
resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==}
brace-expansion@5.0.12:
resolution: {integrity: sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ==}
brace-expansion@5.0.9:
resolution: {integrity: sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==}
engines: {node: 20 || >=22}
braces@3.0.3:
@@ -4054,10 +4048,6 @@ packages:
resolution: {integrity: sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==}
engines: {node: '>=12.0.0'}
fake-indexeddb@6.2.5:
resolution: {integrity: sha512-CGnyrvbhPlWYMngksqrSSUT1BAVP49dZocrHuK0SvtR0D5TMs5wP0o3j7jexDJW01KSadjBp1M/71o/KR3nD1w==}
engines: {node: '>=18'}
fast-deep-equal@3.1.3:
resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==}
@@ -4734,9 +4724,9 @@ packages:
matrix-events-sdk@0.0.1:
resolution: {integrity: sha512-1QEOsXO+bhyCroIe2/A5OwaxHvBm7EsSQ46DEDn8RBIfQwN5HWBpFvyWWR4QY0KHPPnnJdI99wgRiAl7Ad5qaA==}
matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/10e4faabbb49baca655492d523b58faa9eea04b9:
resolution: {gitHosted: true, integrity: sha512-7cbPN96wGt93B73jZieW2HBtTnasgKJ7wzTgiV0SjwkNyb50EK3vDa17LdSJpyOwEjFP5edyPRprk865HQJqww==, tarball: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/10e4faabbb49baca655492d523b58faa9eea04b9}
version: 45.1.0
matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/a9ef09fbf2f1644c8663d0aeaf6a921cd16e7b01:
resolution: {gitHosted: true, integrity: sha512-ZAhn/zB44MgrXqUzi9ffSFcmZY3+2nx7bEgfEJR7chYDFyT/XmlmfCJx7Ar4rOTAYZJRVnMxrEwiEHGDapUi0Q==, tarball: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/a9ef09fbf2f1644c8663d0aeaf6a921cd16e7b01}
version: 45.0.0
engines: {node: '>=22.0.0'}
matrix-widget-api@1.20.0:
@@ -4771,9 +4761,9 @@ packages:
minimalistic-crypto-utils@1.0.1:
resolution: {integrity: sha512-JIYlbt6g8i5jKfJ3xz7rF0LXmv2TkDxBLUkiBeZ7bAx4GnnNMr8xFpGnOxn6GhTEHx3SjRrZEoU+j04prX1ktg==}
minimatch@11.0.4:
resolution: {integrity: sha512-/UJ1dEoVQTjPNDh6SGlUo+ueLMmoHTas4q+8yaAR/5t6nLAIiBDJtoYUh20IYleix7ehiQkgczb84PE3uzHkxg==}
engines: {node: 20 || >=22}
minimatch@10.2.6:
resolution: {integrity: sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==}
engines: {node: 18 || 20 || >=22}
minimist@1.2.8:
resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==}
@@ -4946,8 +4936,8 @@ packages:
resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==}
engines: {node: '>=10'}
p-retry@9.0.0:
resolution: {integrity: sha512-HuEats8gYn/UUYa5uyveyQGv75mQmwdFtuMgnZlS12dj/7bdnJSO6keaTjgcZUhvkANTj5EO3lACUmV3xmgj0Q==}
p-retry@8.0.1:
resolution: {integrity: sha512-NAigyu8hfvJe7MsS18mnc4is0MZtau3QMdBklaJKK23oBQb6occO3UPM3vHmTckW8KhfyjMZaUOqdTFEYliHgg==}
engines: {node: '>=22'}
p-try@2.2.0:
@@ -5318,10 +5308,6 @@ packages:
randomfill@1.0.4:
resolution: {integrity: sha512-87lcbR8+MhcWcUiQ+9e+Rwx8MyR2P7qnt15ynUlbm3TU/fjbgz4GsvfSUDTemtCCtVCqb4ZcEFlyPNTh9bBTLw==}
re2js@2.9.1:
resolution: {integrity: sha512-JIT3bEEUZTOR2YS1walNEIfH4cF/cQeZcBUFq7ldqO6lR6g0X5CBq/S8rsa9omzZcsNf8qc36jCkWpKKtncrZg==}
engines: {node: '>=20.0.0'}
react-docgen-typescript@2.4.0:
resolution: {integrity: sha512-ZtAp5XTO5HRzQctjPU0ybY0RRCQO19X/8fxn3w7y2VVTUbGHDKULPTL4ky3vB05euSgG5NpALhEhDPvQ56wvXg==}
peerDependencies:
@@ -6909,7 +6895,7 @@ snapshots:
ignore: 5.3.2
import-fresh: 3.3.1
js-yaml: 4.3.1
minimatch: 11.0.4
minimatch: 10.2.6
strip-json-comments: 3.1.1
transitivePeerDependencies:
- supports-color
@@ -6988,7 +6974,7 @@ snapshots:
dependencies:
'@humanwhocodes/object-schema': 2.0.3
debug: 4.4.3(supports-color@7.2.0)
minimatch: 11.0.4
minimatch: 10.2.6
transitivePeerDependencies:
- supports-color
@@ -7124,8 +7110,6 @@ snapshots:
wrap-ansi: 8.1.0
wrap-ansi-cjs: wrap-ansi@7.0.0
'@isaacs/nonrecursive-glob-match@1.0.3': {}
'@joshwooding/vite-plugin-react-docgen-typescript@0.7.0(typescript@5.9.3)(vite@8.2.4(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.103.1)(terser@5.46.1)(yaml@2.9.0))':
dependencies:
glob: 10.5.0
@@ -7178,7 +7162,7 @@ snapshots:
'@types/dom-mediacapture-transform': 0.1.11
livekit-client: 2.23.0(@types/dom-mediacapture-record@1.0.22)
'@matrix-org/matrix-sdk-crypto-wasm@18.9.2': {}
'@matrix-org/matrix-sdk-crypto-wasm@18.9.0': {}
'@mdx-js/react@3.1.1(@types/react@19.2.18)(react@19.3.1)':
dependencies:
@@ -8785,7 +8769,7 @@ snapshots:
'@typescript-eslint/types': 8.69.0
'@typescript-eslint/visitor-keys': 8.69.0
debug: 4.4.3(supports-color@7.2.0)
minimatch: 11.0.4
minimatch: 10.2.6
semver: 7.8.5
tinyglobby: 0.2.17
ts-api-utils: 2.5.0(typescript@5.9.3)
@@ -8800,7 +8784,7 @@ snapshots:
'@typescript-eslint/types': 8.72.0
'@typescript-eslint/visitor-keys': 8.72.0
debug: 4.4.3(supports-color@7.2.0)
minimatch: 11.0.4
minimatch: 10.2.6
semver: 7.8.5
tinyglobby: 0.2.17
ts-api-utils: 2.5.0(typescript@5.9.3)
@@ -9125,7 +9109,7 @@ snapshots:
boolbase@1.0.0: {}
brace-expansion@5.0.12:
brace-expansion@5.0.9:
dependencies:
balanced-match: 4.0.4
@@ -9670,7 +9654,7 @@ snapshots:
json-stable-stringify-without-jsonify: 1.0.1
levn: 0.4.1
lodash.merge: 4.6.2
minimatch: 11.0.4
minimatch: 10.2.6
natural-compare: 1.4.0
optionator: 0.9.4
strip-ansi: 6.0.1
@@ -9728,8 +9712,6 @@ snapshots:
expect-type@1.4.0: {}
fake-indexeddb@6.2.5: {}
fast-deep-equal@3.1.3: {}
fast-glob@3.3.3:
@@ -9790,7 +9772,7 @@ snapshots:
filelist@1.0.6:
dependencies:
minimatch: 11.0.4
minimatch: 10.2.6
fill-range@7.1.1:
dependencies:
@@ -9890,7 +9872,7 @@ snapshots:
dependencies:
foreground-child: 3.3.1
jackspeak: 3.4.3
minimatch: 11.0.4
minimatch: 10.2.6
minipass: 7.1.3
package-json-from-dist: 1.0.1
path-scurry: 1.11.1
@@ -10015,7 +9997,7 @@ snapshots:
jiti: 2.7.0
jsonc-parser: 3.3.1
magic-string: 1.2.3
minimatch: 11.0.4
minimatch: 10.2.6
ora: 9.4.1
react: 19.2.8
react-i18next: 17.0.13(i18next@26.4.1(typescript@5.9.3))(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(typescript@5.9.3)
@@ -10413,18 +10395,17 @@ snapshots:
matrix-events-sdk@0.0.1: {}
matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/10e4faabbb49baca655492d523b58faa9eea04b9:
matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/a9ef09fbf2f1644c8663d0aeaf6a921cd16e7b01:
dependencies:
'@babel/runtime': 8.0.8
'@matrix-org/matrix-sdk-crypto-wasm': 18.9.2
'@matrix-org/matrix-sdk-crypto-wasm': 18.9.0
another-json: 0.2.0
bs58: 6.0.0
content-type: 3.1.1
loglevel: 1.9.2
matrix-events-sdk: 0.0.1
matrix-widget-api: 1.20.0
p-retry: 9.0.0
re2js: 2.9.1
p-retry: 8.0.1
sdp-transform: 3.0.0
unhomoglyph: 1.0.6
@@ -10459,10 +10440,9 @@ snapshots:
minimalistic-crypto-utils@1.0.1: {}
minimatch@11.0.4:
minimatch@10.2.6:
dependencies:
'@isaacs/nonrecursive-glob-match': 1.0.3
brace-expansion: 5.0.12
brace-expansion: 5.0.9
minimist@1.2.8: {}
@@ -10746,7 +10726,7 @@ snapshots:
dependencies:
p-limit: 3.1.0
p-retry@9.0.0:
p-retry@8.0.1:
dependencies:
is-network-error: 1.3.2
@@ -11170,8 +11150,6 @@ snapshots:
randombytes: 2.1.0
safe-buffer: 5.2.1
re2js@2.9.1: {}
react-docgen-typescript@2.4.0(typescript@5.9.3):
dependencies:
typescript: 5.9.3
+1 -1
View File
@@ -27,7 +27,7 @@ overrides:
# Security fixes:
# - https://github.com/advisories/GHSA-7r86-cg39-jmmj
# - https://github.com/advisories/GHSA-23c5-xmqv-rm74
"minimatch": "^11.0.0"
"minimatch": "^10.2.3"
# Security fix: https://github.com/element-hq/element-call/security/dependabot/109
"glob": "^10.5.0"
# Security fixes:
@@ -0,0 +1,68 @@
# SDK mode (LEGACY)
> Superseded by the `sdk/` package (`@element-hq/matrixrtc-sdk`) and its harness in
> `sdk/dev`; see `sdk/SdkArchitecture.md`. This bundle is in production use, so it
> keeps building as a thin layer over that package until its consumers have moved
> to the package directly. It will then be removed. Its members are the package's
> `RTCMember`s now, in place of its own wrapper around the raw membership.
EC can be build in sdk mode. This will result in a compiled js file that can be imported in very simple webapps.
It allows to use matrixRTC in combination with livekit without relying on element call.
This is done by instantiating the call view model and exposing some useful behaviors (observables) and methods.
This folder contains an example index.html file that showcases the sdk in use (hosted on localhost:8123 with a webserver allowing cors (for example `npx serve -l 81234 --cors`)) as a godot engine HTML export template.
## Getting started
To get started run
```
pnpm install
pnpm build:sdk-target-based-on-call-view-model
```
in the repository root. (`pnpm build:sdk` builds the `sdk/` package instead.)
CI's `sdk` package (`build:sdk:production`, `build:sdk:development`, the
`build-output-sdk` artifact and the `pr<N>-sdk` Netlify preview) is still this
bundle, not the `sdk/` package, until its consumers have moved over.
It will create `dist` containing the compiled js file, `matrixrtc-sdk.js`.
This file needs to be hosted. Locally (via `npx serve -l 81234 --cors`) or on a remote server.
Now you just need to add the widget to element web via:
```
/addwidget http://localhost:3000?widgetId=$matrix_widget_id&userId=$matrix_user_id&deviceId=$org.matrix.msc3819.matrix_device_id&baseUrl=$org.matrix.msc4039.matrix_base_url&roomId=$matrix_room_id
```
## Widgets
The sdk mode is particularly interesting to be used in widgets. In widgets you do not need to pay attention to matrix login/cs api ...
To create a widget see the example `index.html` file in this folder. And add it to EW via:
`/addwidget <widgetUrl>` (see **url parameters** for more details on `<widgetUrl>`)
### url parameters
The url parameters are needed to pass initial data to the widget. They will automatically be used
by the matrixRTCSdk to start the postmessage widget api (communication between the client (e.g. Element Web) and the widget)
```
widgetId = $matrix_widget_id
userId = $matrix_user_id
deviceId = $org.matrix.msc3819.matrix_device_id
baseUrl = $org.matrix.msc4039.matrix_base_url
```
`parentUrl = // will be inserted automatically`
Full template use as `<widgetUrl>`:
```
http://localhost:3000?widgetId=$matrix_widget_id&userId=$matrix_user_id&deviceId=$org.matrix.msc3819.matrix_device_id&baseUrl=$org.matrix.msc4039.matrix_base_url&roomId=$matrix_room_id
```
the `$` prefixed variables will be replaced by EW on widget instantiation. (e.g. `$matrix_user_id` -> `@user:example.com` (url encoding will also be applied automatically by EW) -> `%40user%3Aexample.com`)
@@ -1,13 +1,11 @@
<!doctype html>
<html>
<head>
<title>Widget RTC SDK example</title>
<title>MatrixRTC Widget</title>
<meta charset="utf-8" />
<script type="module">
// The bundle a pull request's CI deploys: replace 1234 with the PR number.
// For a local build use the url `dist/` is served from, for example
// "http://localhost:8123/widget-rtc-sdk.js".
import { createMatrixRTCSdk } from "https://pr1234-widget-rtc-sdk--element-call.netlify.app/widget-rtc-sdk.js";
// TODO use the url where the matrixrtc-sdk.js file from dist is hosted
import { createMatrixRTCSdk } from "http://localhost:8123/matrixrtc-sdk.js";
try {
window.matrixRTCSdk = await createMatrixRTCSdk(
+231
View File
@@ -0,0 +1,231 @@
/*
Copyright 2025-2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
/**
* LEGACY, in production use
*
* The entrypoint of `pnpm build:sdk-target-based-on-call-view-model`, a
* bundle for widgets that exposes `createMatrixRTCSdk`: join a slot and
* exchange realtime data, with the `@element-hq/matrixrtc-sdk` package and
* Element Call's call view model doing the work underneath. New consumers use
* the package directly; this bundle goes away once the existing ones have.
*/
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import {
distinct,
distinctUntilChanged,
EMPTY,
filter,
map,
mergeMap,
type Observable,
of,
switchMap,
take,
takeUntil,
} from "rxjs";
import {
type Behavior,
createRTCSlot,
type DataTrack,
E2eeType,
type LocalDataTrack,
type LocalRTCMember,
ObservableScope,
type RemoteRTCMember,
type RTCParticipation,
} from "@element-hq/matrixrtc-sdk";
import {
callViewModelOptionsFromParams,
type CallViewModel,
createCallViewModel$,
initialPublishRequests,
} from "../src/state/CallViewModel/CallViewModel";
import {
captureSettings,
rtcParticipationOptions,
rtcSlotOptions,
} from "../src/room/InCallView";
import { createWidgetHostBridge } from "../src/HostBridge";
import { MediaDevices } from "../src/state/MediaDevices";
import { MuteStates } from "../src/state/MuteStates";
import { getUrlParams } from "../src/UrlParams";
import { observeElementSize$ } from "../src/utils/elementSize";
import { initializeWidget } from "../src/widget";
/** The name of our data track: the topic this bundle has always used, so that old and new builds talk. */
const TOPIC = "matrixRTC";
export interface MatrixRTCSdk {
/** Joins the slot; `connected$` tracks the progress. A second call does nothing. */
join: () => void;
leave: () => void;
/** Ends everything. The sdk cannot be restarted; create a new one. */
stop: () => void;
connected$: Behavior<boolean>;
/** Our own member, null until our membership has been seen in the room. */
localMember$: Behavior<LocalRTCMember | null>;
/** The other members of the slot, with the media the participation carries for them. */
remoteMembers$: Behavior<RemoteRTCMember[]>;
/** What other members sent with `sendData`, on their data track. */
data$: Observable<{ rtcBackendIdentity: string; data: string }>;
/** Sends on our data track, published with the first message. Rejects while not joined. */
sendData: (data: unknown) => Promise<void>;
sendRoomMessage: (message: string) => Promise<void>;
}
export async function createMatrixRTCSdk(
application = "m.call",
id = "",
sticky = false,
): Promise<MatrixRTCSdk> {
const logger = rootLogger.getChild("[MatrixRTCSdk]");
const scope = new ObservableScope();
const widget = initializeWidget(application, true);
if (!widget) throw new Error("No widget: this bundle only runs as a widget");
const client = await widget.client;
const hostBridge = createWidgetHostBridge(widget);
const urlParams = getUrlParams();
const { roomId, controlledAudioDevices, callIntent } = urlParams;
const room = roomId === null ? null : client.getRoom(roomId);
if (room === null) throw new Error("No room: the url needs a joined roomId");
const mediaDevices = new MediaDevices(scope, {
controlledAudioDevices,
callIntent,
});
const muteStates = new MuteStates(
scope,
mediaDevices,
{ audioEnabled: false, videoEnabled: false },
hostBridge,
);
const encryptionSystem = { kind: E2eeType.PER_PARTICIPANT } as const;
const capture = captureSettings();
const slot = createRTCSlot(scope, client, room, {
...rtcSlotOptions(encryptionSystem),
application,
id,
});
let vm: CallViewModel | undefined;
let dataTrack: Promise<LocalDataTrack> | undefined;
const encoder = new TextEncoder();
const decoder = new TextDecoder();
const failed =
(what: string) =>
(e: unknown): void =>
logger.error(what, e);
const over = <T>(
pick: (participation: RTCParticipation) => Observable<T>,
fallback: T,
): Behavior<T> =>
scope.behavior(
slot.participation$.pipe(
switchMap((p) => (p === null ? of(fallback) : pick(p))),
),
);
const remoteMembers$ = over((p) => p.remoteMembers$, []);
return {
join: (): void => {
if (slot.participation$.value !== null) return;
if (sticky)
widget.api
.setAlwaysOnScreen(true)
.catch(failed("Failed to make the widget sticky"));
dataTrack = undefined;
const participation = slot.join({
...rtcParticipationOptions(urlParams),
publish: initialPublishRequests(muteStates, mediaDevices, capture),
});
vm = createCallViewModel$(
scope,
participation,
room,
mediaDevices,
muteStates,
{
...callViewModelOptionsFromParams(urlParams),
encryptionSystem,
capture,
hostBridge,
// The bundle owns its page, so the body is the space it has
windowSize$: scope.behavior(observeElementSize$(document.body)),
},
of({}),
of({}),
);
// Once the hang-up has gone through, unstick and ask the host to close us
vm.leave$
.pipe(
switchMap(() =>
participation.state$.pipe(
filter((state) => state.kind === "left"),
take(1),
),
),
scope.bind(),
)
.subscribe(() => {
hostBridge
.setAlwaysOnScreen(false)
.catch(failed("Failed to unstick the widget"))
.then(() => hostBridge.close?.())
.catch(failed("Failed to ask the host to close"));
});
},
leave: (): void => vm?.leave(),
stop: (): void => scope.end(),
connected$: over(
(p) => p.state$.pipe(map((state) => state.kind === "connected")),
false,
),
localMember$: over((p) => p.localMember$, null),
remoteMembers$,
data$: remoteMembers$.pipe(
// Each member once, for as long as it stays: a data track is only
// subscribed while observed, so it is not dropped on every list change
mergeMap((members) => members),
distinct(),
mergeMap((member) =>
member.tracks$.pipe(
map((tracks) =>
tracks?.find(
(t): t is DataTrack => t.kind === "data" && t.name === TOPIC,
),
),
distinctUntilChanged(),
switchMap((track) => track?.data$ ?? EMPTY),
map((payload) => ({
rtcBackendIdentity: member.rtcBackendIdentity,
data: decoder.decode(payload),
})),
takeUntil(
remoteMembers$.pipe(filter((members) => !members.includes(member))),
),
),
),
),
sendData: async (data): Promise<void> => {
const participation = slot.participation$.value;
if (participation === null) throw new Error("Not joined");
dataTrack ??= participation.publish({ source: "data", name: TOPIC });
const track = await dataTrack.catch((e: unknown) => {
dataTrack = undefined;
throw e;
});
await track.send(encoder.encode(JSON.stringify(data)));
},
sendRoomMessage: async (message): Promise<void> => {
await client.sendTextMessage(room.roomId, message);
},
};
}
+2 -1
View File
@@ -5,7 +5,8 @@ it joins a MatrixRTC slot in a Matrix room, writes and watches the memberships,
connects to the media transport (LiveKit today), publishes the local media and
exposes every member and their tracks as observables. It has no UI. Element
Call's own `CallViewModel` is one consumer of it; any host can build a different
view on top. The design is in [`SdkArchitecture.md`](./SdkArchitecture.md).
view on top. The design is in [`SdkArchitecture.md`](./SdkArchitecture.md), what
moved out of Element Call's `src/` in [`SdkMigration.md`](./SdkMigration.md).
**Status:** first implementation. `createRTCSlot` and `slot.join()` join the
session, connect to the transport, publish the local media and expose every
+96 -42
View File
@@ -10,7 +10,8 @@ interface.
Element Call becomes one consumer of it: `CallViewModel` keeps what makes a session
a _call_ (ringing, auto-leave, layout, sounds, reactions, settings, the host
bridge) and takes everything else from the client. The dependency direction is
therefore `Element Call → sdk`, never the reverse.
therefore `Element Call → sdk`, never the reverse. What moved out of Element
Call's `src/` to get there is listed in [`SdkMigration.md`](./SdkMigration.md).
## The architecture
@@ -24,7 +25,7 @@ therefore `Element Call → sdk`, never the reverse.
│ │
│ public API, all Behaviors: │
│ join() leave() state$ │
│ localMember$ remoteMembers$ memberCount$ sendData() data$ │
│ localMember$ remoteMembers$ memberCount$ publish(data) │
│ keyRotationSuppressed$ connectedTransports$ │
│ │
│ per member RTCMember { id userId deviceId membership$ displayName$ │
@@ -47,7 +48,7 @@ therefore `Element Call → sdk`, never the reverse.
│ │ (LocalMembership) │ local.publish, │ Publisher: local tracks, │ │
│ │ transport registry │ setPublishing▶│ upstream paused until │ │
│ │ (TransportMetadata) │ ◀─ connections$ │ joined │ │
│ │ data packet → member │ ◀─ data$ │ LivekitMemberMedia: │ │
│ │ │ │ LivekitMemberMedia: │ │
│ │ │ │ participant → tracks │ │
│ └───────────────────────────┘ └───────────────────────────┘ │
│ │
@@ -145,12 +146,12 @@ the sections below still describe `MatrixRTCClient` until that lands.
| Word | Meaning here | Elsewhere |
| --------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------- |
| slot | an `RTCSlot`: the namespace in a room that memberships join into, `application` plus `id`. It exists while the room does, with nobody in it too. Holds `members$`, its open or closed `status$`, and `join()` | the js-sdk `MatrixRTCSession` is one per slot; the MSC4143 slot event |
| participation | an `RTCParticipation`: the span between our `join()` of a slot and our `leave()`. Holds the connection and its `state$`, `localMember$`, `remoteMembers$`, the connected transports and the data channel. Made by `slot.join()`, never reused | "session" is not used here, because in the js-sdk it means the slot |
| participation | an `RTCParticipation`: the span between our `join()` of a slot and our `leave()`. Holds the connection and its `state$`, `localMember$`, `remoteMembers$`, and the connected transports. Made by `slot.join()`, never reused | "session" is not used here, because in the js-sdk it means the slot |
| member | an `RTCMember`: one membership in a slot with what the SDK derives from it, `rtcBackendIdentity`, `userId`, `deviceId`, `memberId`, `displayName$`, `avatarUrl$`, `transport$` and `applicationData$`. No media. What the slot hands out in `members$` | the js-sdk `CallMembership` behind it, which never leaves the SDK; Element Call's old "matrixLivekitMember" |
| local member, remote member | `LocalRTCMember` and `RemoteRTCMember`: an `RTCMember` plus its `MemberMedia`, which only a participation can carry. The local one's tracks carry the controls. What the participation hands out in `localMember$` and `remoteMembers$` | LiveKit `LocalParticipant` and `RemoteParticipant`, which never leave the SDK |
| media | a `MemberMedia`: what a member sends once it has arrived on its transport, `tracks$`, null while waiting, and `encryptionError$` beside it. The media backend supplies exactly these two fields per member | Element Call's `MediaViewModel` is a view of it |
| track | a `MediaTrack`: one publication of a member, audio or video, with `source`, `kind`, `id` and the behaviors on it. `LocalAudioMediaTrack` and `LocalVideoMediaTrack` add the controls over our own | LiveKit `TrackPublication` and `Track`, which never leave the SDK |
| publish request | a `PublishRequest`: a source, where to capture it from and how to encode it. Given at the join in `publish`, or later to `publish` on the participation | |
| track | a `Track`: one publication of a member. A `MediaTrack` is audio or video, with `source`, `kind`, `id` and the behaviors on it; a `DataTrack` is a named channel of bytes. The `Local…` variants add the controls over our own, and `send` for data | LiveKit `TrackPublication` and `Track`, which never leave the SDK |
| publish request | a `PublishRequest`: a source, where to capture it from and how to encode it. Given at the join in `publish`, or later to `publish` on the participation. A `DataPublishRequest` names a data track, and only goes to `publish` | |
| attach, detach | handing a `<video>` or `<audio>` element to a track and taking it back. The SDK owns the stream and the observers on the element | LiveKit `Track.attach` |
| transport | an `RTCTransport`: where media is exchanged, a LiveKit service url today. Carries the raw js-sdk `Transport` from the membership and, in `resolved$`, what the backend fetched to connect to it | MSC4143 "focus" |
| media backend | a `MediaBackend`: what carries the media, behind `sdk/src/media-backend/api.ts`. One per participation, chosen with `backend` in the join options; LiveKit under `sdk/src/media-backend/livekit/` is the one that exists | "SFU", a LiveKit room, a peer connection |
@@ -339,28 +340,6 @@ export interface MatrixRTCClient {
* the browser's choice, never calls it.
*/
setAudioOutputDeviceId(deviceId: string): Promise<void>;
/**
* Sends a short text to every member on the local transport, over a
* reliable data channel beside the media. A packet holds about 15 KiB;
* anything larger belongs in a room event. Rejects while the local
* transport is not connected. Encrypted on the wire like the media, but
* not end to end with the media key.
*/
sendData(topic: string, text: string): Promise<void>;
/**
* What remote members sent with `sendData`, on every transport the client
* is connected to. A message from an identity that is not a member is
* dropped, so a host only ever hears from attested members.
*/
data$: Observable<DataMessage>;
}
/** One `sendData` call as it arrives at a remote member. */
export interface DataMessage {
member: RemoteRTCMember;
topic: string;
text: string;
}
```
@@ -451,11 +430,15 @@ export interface LocalRTCMember extends RTCMember {
* user closing the picker. One publication per source: publishing a source
* again unmutes it.
*/
publish(request: DataPublishRequest): Promise<LocalDataTrack>;
publish(
request: PublishRequest,
): Promise<LocalAudioMediaTrack | LocalVideoMediaTrack>;
/** Removes one of our tracks; a screen share takes its audio with it. */
unpublish(id: string): Promise<void>;
/**
* Removes one of our tracks, as `publish` handed it out; a screen share
* takes its audio with it. The track's controls and `send` reject from then on.
*/
unpublish(track: LocalTrack): Promise<void>;
}
```
@@ -556,8 +539,6 @@ export interface MediaBackend {
membership$: Behavior<RTCMembership>,
): MemberMediaFields;
readonly connections$: Behavior<BackendConnection[]>; // transport, state, resolved; diagnostics
sendData(topic: string, text: string): Promise<void>;
readonly data$: Observable<DataPacket>;
}
```
@@ -567,8 +548,7 @@ LiveKit backend only, a last resort of another member's SFU that lets us
publish (`findRemotePublishLivekitTransport`, wired in by `RTCParticipation` rather
than by the backend interface, since it is an MSC4195 matter), the membership
state machine in `LocalMembership.ts` and its combination with
`local.connectionState$` into `state$`, the choice of delayed leave timings,
mapping a data packet's sender to a member, and the transport registry, which keys on the raw
`local.connectionState$` into `state$`, the choice of delayed leave timings, and the transport registry, which keys on the raw
transport serialised with sorted keys and derives `connectedTransports$` and each
`TransportMetadata.resolved$` from `connections$`.
@@ -612,6 +592,43 @@ the local transport, and keys its connections on the transport plus the server
name where the transport has a `url`. A JWT service is reached directly, so
members of several homeservers advertising the same one share a connection.
### Data tracks on LiveKit
A data track is a track like the media, in `tracks$`, so the backend carries it
the way it carries the media: `local.publish` takes a `DataPublishRequest`, and
`mediaFor` lists a remote member's data tracks after its media. Its sender is
attested by construction: a track only exists on a member. The session never
sees a key, a frame or a stream.
Every data track is one LiveKit data track, whose publication announces it: a
receiver sees the track before its first message, and sees it go. LiveKit's
data tracks are lossy only, so the LiveKit name says how the payloads travel
(`dataTrackName.ts`):
| Request | LiveKit name | Payloads |
| ----------------- | ----------------- | --------------------------------------------------------------- |
| `reliable: false` | `lossy:<name>` | frames on the data track: unordered, no retransmits |
| `reliable: true` | `reliable:<name>` | one byte stream per `send` on topic `reliable:<name>`, in order |
A track without a prefix, from a plain LiveKit publisher, is read as lossy. A
room takes one byte stream handler per topic, so `ByteStreams.ts` shares it
between every reliable track on that topic and filters by sender identity.
Encryption is LiveKit's. The rooms are built with `encryption` rather than
`e2ee`, which encrypts the data channel as well as the media with the same
E2EE manager and our key provider. The sender encrypts with its current key
and puts the key index and IV on the wire: in each frame's E2EE extension for a
lossy track, in each packet's `EncryptedPacket` for a stream. The receiver
decrypts with the sender's key at that index. A frame that does not decrypt is
dropped. A stream cannot be checked: LiveKit 2.23 sends an encrypted packet
without its encryption type, so every received stream reads as unencrypted,
and a cleartext stream injected on the way would be accepted as the sender's.
A local track is published on LiveKit once publishing is allowed, like the
media, and again on every new connection to the local transport; `publish`
resolves with that first publication. The object a host holds outlives the
connections.
## Media: `tracks$` and `MediaTrack`
Every member has media. Today, in Element Call, that media _is_ a `livekit-client`
@@ -706,14 +723,52 @@ export interface LocalVideoMediaTrack extends VideoMediaTrack, LocalMediaTrack {
setProcessor(processor: VideoProcessor | undefined): Promise<void>;
}
/** A named channel of bytes a member publishes beside its media. */
export interface DataPublishRequest {
source: "data";
/** Unique per member; what receivers find the track by. */
name: string;
/** Whether every message arrives, in order. Default true. */
reliable?: boolean;
}
/** One data track of a member; it exists only on a member, so its sender is attested. */
export interface DataTrack {
source: "data";
kind: "data";
id: string;
name: string;
reliable: boolean;
/** Whether payloads are end-to-end encrypted with the media key. */
encrypted: boolean;
/** What the member sends from now on, subscribed while observed. */
data$: Observable<Uint8Array>;
}
/** Our own data track; its `data$` replays what we sent. */
export interface LocalDataTrack extends DataTrack {
send(payload: Uint8Array): Promise<void>;
}
export type Track = AudioMediaTrack | VideoMediaTrack | DataTrack;
export type LocalTrack =
LocalAudioMediaTrack | LocalVideoMediaTrack | LocalDataTrack;
export type EncryptionError = "MissingKey" | "InvalidKey";
/** The member's first track of a source; undefined while there is none, or no media yet. */
export function trackBySource$<S extends MediaSource>(
scope: ObservableScope,
tracks$: Behavior<MediaTrack[] | null>,
tracks$: Behavior<readonly (MediaTrack | DataTrack)[] | null>,
source: S,
): Behavior<TrackOfSource<S> | undefined>;
/** The member's data track of this name; undefined while there is none. */
export function dataTrackByName$(
scope: ObservableScope,
tracks$: Behavior<readonly (MediaTrack | DataTrack)[] | null>,
name: string,
): Behavior<DataTrack | undefined>;
```
### Rendering: `attach` and `detach`
@@ -776,6 +831,7 @@ sdk/
src/utils/ mapScoped, network retry, display names, test helpers
dev/ the harness (below)
SdkArchitecture.md this document
SdkMigration.md what moved from Element Call's src/ into the SDK
```
The lint rule `element-call/sdk-import-boundary` keeps it that way: nothing under
@@ -820,13 +876,11 @@ public API only.
- **Developer panel.** `resolved$` on each connected transport covers what the
panel shows today. Anything beyond that (LiveKit room state, connection
quality) needs an opaque `debug$` per transport.
- **Data beside the media.** `sendData`/`data$` ride on LiveKit's reliable data
packets: topic-agnostic, matched to members by identity, received on every
connection but sent only on the local transport, so a member on another
transport does not hear us. They are transport-encrypted, not end to end:
LiveKit can encrypt data packets with the media key, but that is not wired
through the key provider yet. A host that needs either reach or end-to-end
encryption uses room events.
- **Data before a receiver listens.** Nothing is replayed: a message sent
before a receiver has subscribed to the track, which it does once it has
seen the track published, does not reach it. Old builds of the legacy
widget bundle, which sent bare packets, do not hear new builds or the other
way round.
- **Unencrypted rooms with per-participant keys.** Keys only reach devices the
crypto tracks, which are the members of encrypted rooms. If the SDK is to work
in an unencrypted room, tracking the members' devices becomes its job.
+53
View File
@@ -0,0 +1,53 @@
# What moved from `src/` into the SDK
Element Call's `CallViewModel` used to do everything; now it consumes the SDK and
keeps only what makes a session a call. This is the inventory of what moved out
of `src/` on the way, so that a reader of either side knows where to look. It
names source files, so it ages with the code; the shape they make is in
[`SdkArchitecture.md`](./SdkArchitecture.md).
Everything in the table is gone from `src/`. Each module came with its tests;
`sdk/src/utils/test.ts` and `test-fixtures.ts` are the subset of Element Call's
test helpers they need. `Behavior`, `ObservableScope`, the observable operators,
`E2eeType`, `EncryptionSystem` and `MatrixRTCMode` are re-exported from the SDK
by their old `src/` paths, so the rest of Element Call did not have to change.
| Was in `src/` | Is in `sdk/src/` | What changed on the way |
| ----------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- |
| `state/Behavior.ts`, `state/ObservableScope.ts`, `utils/observable.ts` | `reactive/` | as is |
| `state/SessionBehaviors.ts` | `matrixrtc/JsRtcSessionBehaviors.ts` | the memberships and key rotation behaviors only |
| `state/CallViewModel/localMember/HomeserverConnected.ts` | `matrixrtc/HomeserverConnected.ts` | the grace period defaults to `defaultSessionTimings` instead of `Config.get()` |
| `state/CallViewModel/localMember/LocalMember.ts` | `matrixrtc/LocalMembership.ts`, `media-backend/livekit/LocalMedia.ts`, `participationState.ts` | split into the Matrix side, the media side and the state machine that combines them |
| `state/CallViewModel/localMember/LocalTransport.ts`, `RtcTransportAutoDiscovery.ts` | `matrixrtc/LocalTransport.ts`, `TransportAutoDiscovery.ts` | the override and the fallback are transports from the options, not a setting and a resolved config |
| `state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts` | `media-backend/livekit/LivekitBackend.ts` (`mediaFor`) | matches participants to members inside the backend; the SDK hands out `RTCMember`s |
| `state/CallViewModel/remoteMembers/MatrixMemberMetadata.ts`, `utils/displayname.ts` | `matrixrtc/MatrixMemberMetadata.ts`, `utils/displayname.ts` | as is |
| `state/CallViewModel/remoteMembers/Connection.ts` | `media-backend/livekit/Connection.ts` | throws `MatrixRTCError`s; `ConnectionState` is a discriminated union; fetches its own token for its role |
| `state/CallViewModel/remoteMembers/ConnectionFactory.ts` | `media-backend/livekit/ConnectionFactory.ts` | `LivekitConnectionFactory` builds the room options from the publish requests and `mediaQuality`, not from settings |
| `state/CallViewModel/remoteMembers/ConnectionManager.ts` | `media-backend/livekit/ConnectionMap.ts` | keyed by `TransportLocator`; the publisher role is the local transport's |
| `state/CallViewModel/localMember/Publisher.ts` | `media-backend/livekit/Publisher.ts` | rewritten on publish requests instead of `MuteStates` |
| `livekit/openIDSFU.ts`, later `livekit/auth/` | `media-backend/livekit/auth/` | the flow is chosen by the transport's shape and our role; the delegated leave timeout is a parameter |
| `livekit/options.ts` | `media-backend/livekit/livekitOptions.ts` | `buildLiveKitOptions(mediaQuality?)` only; the `Config` singleton is gone |
| `e2ee/matrixKeyProvider.ts`, the key provider selection | `media-backend/livekit/MatrixKeyProvider.ts`, `KeyProvider.ts` | as is; selection keyed off `encryptionSystem` |
| `e2ee/e2eeType.ts`, `EncryptionSystem` from `e2ee/sharedKeyManagement.ts` | `encryption.ts` | the React hooks stayed behind |
| `MatrixRTCMode`, the timings and media quality types from `config/ConfigOptions.ts` | `config.ts` | `defaultSessionTimings` and `defaultMediaQuality` replace `DEFAULT_CONFIG` |
| `utils/errors.ts` | `errors.ts` | `MatrixRTCError` with `code` and `category` replaces `ElementCallError`; messages are plain English, the host translates by code |
| `doNetworkOperationWithRetry` from `utils/matrix.ts` | `utils/network.ts` | takes a `shouldRetry` predicate |
| the join in `CallViewModel.ts` (`enterRTCSession`) | `matrixrtc/joinJsSdkSession.ts` | the timings and the delayed leave variant are parameters |
| `livekit/MatrixAudioRenderer.tsx`, the participant reads in `state/media/` | `media-backend/livekit/LivekitMediaTrack.ts`, `LivekitMemberMedia.ts` | the view models get `MediaTrack`s with `attach`; nothing in `src/` imports a LiveKit participant any more |
## What still binds Element Call to LiveKit
Each is a gap in the SDK's API rather than something to rename; the import goes
away when the SDK grows the piece. The names Element Call owes to the outside
stay as they are: the `livekit.livekit_service_url` key in `config.json`, the
persisted setting keys `extended-livekit-logs` and `custom-transport`, the
Posthog fields `callBackend` and `callReconnectingCountLivekit`, the rageshake
field `call_backend`, and the translated error strings.
| Gap | Where it shows |
| ----------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------ |
| Preview before `join()`: local tracks for the lobby, handed out as `MediaTrack`s | `LobbyView` (`usePreviewTracks`), `VideoPreview` (`LocalVideoTrack`, `facingModeFromLocalTrack`) |
| Device enumeration without LiveKit's helpers | `MediaDevices` (`createMediaDeviceObserver`), `CallView` (`Room.getLocalDevices`) |
| A `logging` option: log level and a sink for the transport's logs | `initializer.tsx`, the `extended-livekit-logs` setting |
| A `capabilities` field: output selection and background processors, asked of the SDK rather than of LiveKit | `CallFooterViewModel` (`supportsAudioOutputSelection`, `supportsBackgroundProcessors`) |
| Element Call's error and analytics types on the SDK's | `PosthogEvents` (`DisconnectReason`), `errors.ts` (`ConnectionError`, `LivekitConnectionError`) |
+62 -21
View File
@@ -12,12 +12,22 @@ Please see LICENSE in the repository root for full details.
*/
import { logger } from "matrix-js-sdk/lib/logger";
import { combineLatest, type Observable, of, switchMap } from "rxjs";
import {
combineLatest,
EMPTY,
map,
type Observable,
of,
switchMap,
} from "rxjs";
import {
type AudioMediaTrack,
createRTCSlot,
dataTrackByName$,
E2eeType,
type LocalAudioMediaTrack,
type LocalRTCMember,
type LocalVideoMediaTrack,
MatrixRTCMode,
ObservableScope,
type RemoteRTCMember,
@@ -82,7 +92,7 @@ async function start(
state.kind === "failed" ? `Error: ${state.error.message}` : state.kind;
});
showMembers(scope, rtcParticipation);
showMessages(scope, rtcParticipation);
showMessages(rtcParticipation);
toggle(buttons.microphone, async (enabled) =>
setPublished(rtcParticipation, "microphone", enabled),
@@ -120,10 +130,11 @@ async function setPublished(
enabled: boolean,
): Promise<boolean> {
const track = rtcParticipation.localMember$.value?.tracks$.value?.find(
(t) => t.source === source,
(t): t is LocalAudioMediaTrack | LocalVideoMediaTrack =>
t.source === source,
);
if (track && source === "screenShare") {
if (!enabled) await rtcParticipation.unpublish(source);
if (!enabled) await rtcParticipation.unpublish(track);
return enabled;
}
if (track) return track.setEnabled(enabled);
@@ -145,30 +156,58 @@ function toggle(
};
}
/** Sends what the form holds on the "chat" topic, and lists what arrives. */
function showMessages(
scope: ObservableScope,
rtcParticipation: RTCParticipation,
): void {
/**
* Publishes our "chat" data track with the join, so that the others hear the
* first message, and sends what the form holds on it. What arrives is listed
* per member, in `memberTile`.
*/
function showMessages(rtcParticipation: RTCParticipation): void {
const encoder = new TextEncoder();
const chat = rtcParticipation.publish({ source: "data", name: "chat" });
chat.catch((e: unknown) => {
status.textContent = `Error: ${e}`;
});
dataForm.hidden = false;
dataForm.onsubmit = (event): void => {
event.preventDefault();
const text = new FormData(dataForm).get("text") as string;
rtcParticipation.sendData("chat", text).then(
() => dataForm.reset(),
(e: unknown) => {
status.textContent = `Error: ${e}`;
},
);
chat
.then(async (track) => track.send(encoder.encode(text)))
.then(
() => dataForm.reset(),
(e: unknown) => {
status.textContent = `Error: ${e}`;
},
);
};
rtcParticipation.data$
.pipe(scope.bind())
.subscribe(({ member, topic, text }) => {
}
/**
* Lists what a remote member sends on its "chat" data track, and marks its
* tile while it has one.
*/
function listMessages(
scope: ObservableScope,
member: RemoteRTCMember,
tile: HTMLElement,
): void {
const decoder = new TextDecoder();
dataTrackByName$(scope, member.tracks$, "chat")
.pipe(
switchMap((track) => {
tile.dataset.chat = String(track !== undefined);
return (
track?.data$.pipe(map((payload) => ({ track, payload }))) ?? EMPTY
);
}),
scope.bind(),
)
.subscribe(({ track, payload }) => {
const line = messages.appendChild(document.createElement("li"));
line.dataset.testid = "message";
line.dataset.topic = topic;
line.dataset.encrypted = String(track.encrypted);
line.dataset.userId = member.userId;
line.textContent = `${member.displayName$.value}: ${text}`;
line.textContent = `${member.displayName$.value}: ${decoder.decode(payload)}`;
});
}
@@ -218,12 +257,14 @@ function memberTile(
tile.appendChild(document.createElement("video")),
);
// Our own microphone would only echo
if (!member.local)
if (!member.local) {
render(
scope,
trackBySource$(scope, member.tracks$, "microphone"),
tile.appendChild(document.createElement("audio")),
);
listMessages(scope, member, tile);
}
return tile;
}
+5 -1
View File
@@ -55,7 +55,11 @@ export {
export * from "./src/api";
export * from "./src/media-api";
export * from "./src/media-backend/api";
export { trackBySource$, type TrackOfSource } from "./src/utils/tracks";
export {
dataTrackByName$,
trackBySource$,
type TrackOfSource,
} from "./src/utils/tracks";
export { createRTCSlot } from "./src/RTCSlot";
export {
createLivekitBackend,
-1
View File
@@ -162,7 +162,6 @@ describe("createLocalMedia", () => {
media.release("camera");
expect(trackBehind(camera).stop).toHaveBeenCalled();
expect(media.tracks$.value).toEqual([microphone]);
expect(media.requests.map((r) => r.source)).toEqual(["microphone"]);
media.stop();
expect(trackBehind(microphone).stop).toHaveBeenCalled();
-1
View File
@@ -107,7 +107,6 @@ export function createLocalMedia(
if (source !== "microphone" && source !== "camera") return;
capturedTracks.get(source)?.stop();
capturedTracks.delete(source);
requests.delete(source);
publishCapturedTracks();
};
+29 -28
View File
@@ -14,11 +14,10 @@ import {
type Transport,
type UnstableLivekitTransport,
} from "matrix-js-sdk/lib/matrixrtc";
import { BehaviorSubject, EMPTY, Subject } from "rxjs";
import { BehaviorSubject, EMPTY } from "rxjs";
import { describe, expect, it, type Mock, vi } from "vitest";
import {
type DataMessage,
type ParticipationState,
type RTCMember,
type RTCParticipationOptions,
@@ -29,13 +28,16 @@ import { defaultSessionTimings, MatrixRTCMode } from "./config";
import { E2eeType } from "./encryption";
import { MatrixRTCError } from "./errors";
import {
type DataPacket,
type MediaBackend,
type MediaBackendContext,
MediaConnectionState,
type MediaKey,
} from "./media-backend/api";
import { type MemberMedia } from "./media-api";
import {
type LocalDataTrack,
type LocalVideoMediaTrack,
type MemberMedia,
} from "./media-api";
import { constant } from "./reactive/Behavior";
import { ObservableScope } from "./reactive/ObservableScope";
import { createRTCParticipation } from "./RTCParticipation";
@@ -103,7 +105,6 @@ interface FakeBackend {
context: MediaBackendContext;
prepared: PromiseWithResolvers<{ canDelegateDelayedLeave: boolean }>;
connectionState$: BehaviorSubject<MediaConnectionState | Error>;
data$: Subject<DataPacket>;
setPublishing: Mock<(publish: boolean) => void>;
delegateDelayedLeave: Mock<(delayId: string) => Promise<void>>;
mediaFor: Mock<(scope: ObservableScope, member: RTCMember) => MemberMedia>;
@@ -140,7 +141,7 @@ function setup(
getDeviceId: () => deviceId,
getDomain: () => "example.org",
getSyncState: () => SyncState.Syncing,
getRTCTransports: vi.fn().mockResolvedValue(homeserverTransports),
_unstable_getRTCTransports: vi.fn().mockResolvedValue(homeserverTransports),
_unstable_getLivekitToken: vi.fn().mockResolvedValue({ jwt: publisherJWT }),
} as unknown as MatrixClient;
const members$ = new BehaviorSubject<RTCMember[]>([]);
@@ -151,7 +152,6 @@ function setup(
connectionState$: new BehaviorSubject<MediaConnectionState | Error>(
MediaConnectionState.Initialized,
),
data$: new Subject<DataPacket>(),
setPublishing: vi.fn(),
delegateDelayedLeave: vi.fn(async () => Promise.resolve()),
mediaFor: vi.fn(() => ({
@@ -179,8 +179,6 @@ function setup(
mediaFor: fake.mediaFor,
connections$: constant([]),
setAudioOutputDeviceId: vi.fn(),
sendData: vi.fn(),
data$: fake.data$,
};
fake.context = context;
fake.backend = backend;
@@ -414,27 +412,30 @@ describe("createRTCParticipation", () => {
scope.end();
});
it("delivers data from a member with its member and drops the rest", () => {
const { participation, members$, fake, scope } = setup();
members$.next([
member({
userId: "@bob:example.org",
memberId: "bob-member",
backendIdentities: ["bob-hashed", "bob-member"],
}),
]);
const messages: DataMessage[] = [];
participation.data$.subscribe((message) => messages.push(message));
it("hands publications to the backend, data tracks too", async () => {
const { participation, fake, scope } = setup();
const chat = { kind: "data", name: "chat" } as LocalDataTrack;
const screenShare = {
kind: "video",
source: "screenShare",
} as LocalVideoMediaTrack;
fake.data$.next({ senderId: "bob-hashed", topic: "chat", text: "hi" });
fake.data$.next({ senderId: "stranger", topic: "chat", text: "boo" });
expect(messages).toHaveLength(1);
expect(messages[0]).toMatchObject({
topic: "chat",
text: "hi",
member: { memberId: "bob-member" },
await participation.publish({ source: "data", name: "chat" });
await participation.publish({ source: "camera" });
expect(fake.backend.local.publish).toHaveBeenNthCalledWith(1, {
source: "data",
name: "chat",
});
expect(fake.backend.local.publish).toHaveBeenNthCalledWith(2, {
source: "camera",
});
await participation.unpublish(chat);
await participation.unpublish(screenShare);
expect(fake.backend.local.unpublish).toHaveBeenNthCalledWith(1, chat);
expect(fake.backend.local.unpublish).toHaveBeenNthCalledWith(
2,
screenShare,
);
scope.end();
});
+23 -23
View File
@@ -20,7 +20,6 @@ import { v4 as uuidv4 } from "uuid";
import {
combineLatest,
distinctUntilChanged,
filter,
from,
fromEvent,
map,
@@ -35,7 +34,6 @@ import { type ObservableScope } from "./reactive/ObservableScope";
import { generateItems } from "./reactive/observable";
import { mapScoped } from "./utils/mapScoped";
import {
type DataMessage,
type LocalRTCMember,
type ParticipationState,
type RemoteRTCMember,
@@ -45,6 +43,14 @@ import {
type RTCSlot,
type RTCSlotOptions,
} from "./api";
import {
type DataPublishRequest,
type LocalAudioMediaTrack,
type LocalDataTrack,
type LocalTrack,
type LocalVideoMediaTrack,
type PublishRequest,
} from "./media-api";
import {
type MediaBackend,
type MediaBackendContext,
@@ -184,7 +190,6 @@ export function createRTCParticipation(
encryptMedia: encryptionSystem.kind !== E2eeType.NONE,
matrixRTCMode,
sendNotificationType: options.sendNotificationType,
callIntent: options.callIntent,
applicationData: options.applicationData,
timings,
delayedLeave,
@@ -246,22 +251,6 @@ export function createRTCParticipation(
).pipe(map((member) => member ?? null)),
);
// A packet from an identity that is not a member is dropped, so a host only
// ever hears from attested members
const data$ = backend.data$.pipe(
map(({ senderId, topic, text }): DataMessage | null => {
const member = remoteMembers$.value.find((m) =>
m.backendIdentities.includes(senderId),
);
if (member === undefined) {
logger.warn(`Dropping data from ${senderId}: not a member`);
return null;
}
return { member, topic, text };
}),
filter((message) => message !== null),
);
// Nothing leaves this device while it may already have been dropped from
// the session: the member would show as away while still being heard
localMembership.state$
@@ -286,6 +275,19 @@ export function createRTCParticipation(
),
);
function publish(request: DataPublishRequest): Promise<LocalDataTrack>;
function publish(
request: PublishRequest,
): Promise<LocalAudioMediaTrack | LocalVideoMediaTrack>;
async function publish(
request: DataPublishRequest | PublishRequest,
): Promise<LocalTrack> {
// Narrowed for the overloads; both go to the backend alike
return request.source === "data"
? backend.local.publish(request)
: backend.local.publish(request);
}
const leave = (): void => {
if (state$.value.kind === "left") return;
localMembership.leave();
@@ -300,14 +302,12 @@ export function createRTCParticipation(
state$,
localMember$,
remoteMembers$,
publish: async (request) => backend.local.publish(request),
unpublish: async (source) => backend.local.unpublish(source),
publish,
unpublish: async (track) => backend.local.unpublish(track),
keyRotationSuppressed$: createKeyRotationSuppressed$(scope, jsSdkSession),
connectedTransports$: transports.connected$,
setAudioOutputDeviceId: async (deviceId) =>
backend.setAudioOutputDeviceId(deviceId),
sendData: async (topic, text) => backend.sendData(topic, text),
data$,
},
};
}
+16 -38
View File
@@ -12,19 +12,18 @@ Please see LICENSE in the repository root for full details.
*/
import {
type RTCCallIntent,
type RTCNotificationType,
type Transport,
} from "matrix-js-sdk/lib/matrixrtc";
import { type Observable } from "rxjs";
import { type Behavior } from "./reactive/Behavior";
import {
type DataPublishRequest,
type LocalAudioMediaTrack,
type LocalDataTrack,
type LocalMedia,
type LocalMemberMedia,
type LocalTrack,
type LocalVideoMediaTrack,
type MediaSource,
type MemberMedia,
type PublishRequest,
} from "./media-api";
@@ -104,14 +103,11 @@ export interface RTCParticipationOptions {
* to a notification (ringing, timeouts, declines) is the application's job.
*/
sendNotificationType?: RTCNotificationType;
/**
* The kind of call the local member intends to take part in: `"m.audio"`
* or `"m.video"`. Carried by the membership as a field of its own.
*/
callIntent?: RTCCallIntent;
/**
* Whatever the application wants to say about itself in the membership,
* under namespaced keys. Opaque to the SDK.
* under namespaced keys. Opaque to the SDK, apart from `m.call.intent`
* (`"audio"` or `"video"`), which the membership format carries as a
* field of its own.
*/
applicationData?: Record<string, unknown>;
/** Timings the host has configured; the defaults otherwise. */
@@ -180,15 +176,19 @@ export interface RTCParticipation {
remoteMembers$: Behavior<RemoteRTCMember[]>;
/**
* Publishes a source and resolves with its track, which also shows up in
* `localMember$.tracks$`. Works from the moment `join()` returns: before the
* transport is connected.
* Publishes a source, or a data track, and resolves with its track, which
* also shows up in `localMember$.tracks$`. Works from the moment `join()`
* returns: before the transport is connected.
*/
publish(request: DataPublishRequest): Promise<LocalDataTrack>;
publish(
request: PublishRequest,
): Promise<LocalAudioMediaTrack | LocalVideoMediaTrack>;
/** Removes one of our publications; a screen share takes its audio with it. */
unpublish(source: MediaSource): Promise<void>;
/**
* Removes one of our publications; a screen share takes its audio with it.
* The track's object is finished: its controls and `send` reject from then on.
*/
unpublish(track: LocalTrack): Promise<void>;
/**
* Whether the slot has grown large enough that MatrixRTC has stopped
@@ -205,28 +205,6 @@ export interface RTCParticipation {
* the browser's choice, never calls it.
*/
setAudioOutputDeviceId(deviceId: string): Promise<void>;
/**
* Sends a short text to every member on the local transport, over a
* reliable data channel beside the media. A packet holds about 15 KiB;
* anything larger belongs in a room event. Rejects while the local
* transport is not connected. Encrypted on the wire like the media, but
* not end to end with the media key.
*/
sendData(topic: string, text: string): Promise<void>;
/**
* What remote members sent with `sendData`, on every transport the
* participation is connected to. A message from an identity that is not a
* member is dropped, so a host only ever hears from attested members.
*/
data$: Observable<DataMessage>;
}
/** One `sendData` call as it arrives at a remote member. */
export interface DataMessage {
member: RemoteRTCMember;
topic: string;
text: string;
}
// ---------------------------------------------------------------------------
@@ -291,7 +269,7 @@ export interface RTCMember {
memberId: string;
/** Which transport this member is on; undefined when the membership has none. */
transport$: Behavior<TransportMetadata | undefined>;
/** What the member's application says about itself, e.g. `intent`. */
/** What the member's application says about itself, e.g. `m.call.intent`. */
applicationData$: Behavior<Record<string, unknown>>;
displayName$: Behavior<string>;
+2 -2
View File
@@ -31,7 +31,7 @@ function client(
): Parameters<typeof discoverLocalTransport>[0] {
return {
getDomain: () => "example.org",
getRTCTransports: vi.fn().mockResolvedValue(advertised),
_unstable_getRTCTransports: vi.fn().mockResolvedValue(advertised),
};
}
@@ -54,7 +54,7 @@ describe("discoverLocalTransport", () => {
logger,
),
).resolves.toBe(hostTransport);
expect(c.getRTCTransports).not.toHaveBeenCalled();
expect(c._unstable_getRTCTransports).not.toHaveBeenCalled();
});
it("prefers the homeserver's transport over the host's fallback", async () => {
+1 -1
View File
@@ -22,7 +22,7 @@ import { TransportAutoDiscovery } from "./TransportAutoDiscovery";
* @throws {MatrixRTCTransportMissingError} where none of these exists.
*/
export async function discoverLocalTransport(
client: Pick<MatrixClient, "getDomain" | "getRTCTransports">,
client: Pick<MatrixClient, "getDomain" | "_unstable_getRTCTransports">,
transportTypes: readonly string[],
{
transport: override,
@@ -36,7 +36,11 @@ const configTransport: Transport = {
function makeClient(): MockedObject<DiscoveryClient> {
return {
getRTCTransports: vi.fn().mockResolvedValue([]),
getDomain: vi.fn().mockReturnValue("example.org"),
baseUrl: "https://matrix.example.org",
_unstable_getRTCTransports: vi.fn().mockResolvedValue([]),
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
} as unknown as MockedObject<DiscoveryClient>;
}
@@ -53,7 +57,7 @@ describe("TransportAutoDiscovery", () => {
"prefers backend transport other app config $transports",
async ({ transports }) => {
const client = makeClient();
client.getRTCTransports.mockResolvedValue(transports);
client._unstable_getRTCTransports.mockResolvedValue(transports);
const discovery = new TransportAutoDiscovery({
client,
@@ -67,7 +71,7 @@ describe("TransportAutoDiscovery", () => {
expect(discoveredTransport).toStrictEqual(backendTransport);
expect(discoveredTransport).not.toStrictEqual(configTransport);
expect(client.getRTCTransports).toHaveBeenCalledTimes(1);
expect(client._unstable_getRTCTransports).toHaveBeenCalledTimes(1);
},
);
@@ -77,7 +81,10 @@ describe("TransportAutoDiscovery", () => {
url: "wss://sfu.example.org",
};
const client = makeClient();
client.getRTCTransports.mockResolvedValue([backendTransport, mLivekit]);
client._unstable_getRTCTransports.mockResolvedValue([
backendTransport,
mLivekit,
]);
const discovery = new TransportAutoDiscovery({
client,
@@ -93,7 +100,7 @@ describe("TransportAutoDiscovery", () => {
it("Retries limit_exceeded backend transport", async () => {
const client = makeClient();
client.getRTCTransports
client._unstable_getRTCTransports
.mockRejectedValueOnce(
new MatrixError(
{
@@ -117,7 +124,7 @@ describe("TransportAutoDiscovery", () => {
backendTransport,
);
expect(client.getRTCTransports).toHaveBeenCalledTimes(2);
expect(client._unstable_getRTCTransports).toHaveBeenCalledTimes(2);
});
const INVALID_TEST_CASES: Array<{ transports: Transport[] }> = [
@@ -128,7 +135,7 @@ describe("TransportAutoDiscovery", () => {
"falls back to config when backend has no (valid) livekit transports $transports",
async ({ transports }) => {
const client = makeClient();
client.getRTCTransports.mockResolvedValue(transports);
client._unstable_getRTCTransports.mockResolvedValue(transports);
const discovery = new TransportAutoDiscovery({
client,
@@ -145,7 +152,7 @@ describe("TransportAutoDiscovery", () => {
it("returns null when backend and config are all unavailable", async () => {
const client = makeClient();
client.getRTCTransports.mockResolvedValue([]);
client._unstable_getRTCTransports.mockResolvedValue([]);
const discovery = new TransportAutoDiscovery({
client,
+5 -2
View File
@@ -10,7 +10,10 @@ import { type Logger } from "matrix-js-sdk/lib/logger";
import { doNetworkOperationWithRetry } from "../utils/network";
type TransportDiscoveryClient = Pick<MatrixClient, "getRTCTransports">;
type TransportDiscoveryClient = Pick<
MatrixClient,
"getDomain" | "_unstable_getRTCTransports"
>;
export interface TransportAutoDiscoveryProps {
client: TransportDiscoveryClient;
@@ -71,7 +74,7 @@ export class TransportAutoDiscovery {
this.logger.info("First try to use getRTCTransports end point ...");
try {
const transportList = await doNetworkOperationWithRetry(async () =>
client.getRTCTransports(),
client._unstable_getRTCTransports(),
);
for (const type of this.transportTypes) {
const first = transportList.find(
+6 -8
View File
@@ -60,7 +60,7 @@ describe("joinJsSdkSession", () => {
expect(s.join).toHaveBeenCalledWith(ownMemberMock, [transportWithAlias], {
notificationType: "ring",
callIntent: undefined,
applicationData: undefined,
applicationData: {},
manageMediaKeys: true,
delayedLeaveEventRestartMs: 300_000,
delayedLeaveEventDelayMs: 3_600_000,
@@ -95,14 +95,13 @@ describe("joinJsSdkSession", () => {
);
});
it("passes the call intent and application data through and sends sticky events in Matrix 2.0 mode", () => {
it("lifts the call intent out of the application data and sends sticky events in Matrix 2.0 mode", () => {
const s = session();
joinJsSdkSession("!room:example.org", s, ownMemberMock, transport, {
encryptMedia: true,
matrixRTCMode: MatrixRTCMode.Matrix_2_0,
callIntent: "m.audio",
applicationData: { custom: 1 },
applicationData: { "m.call.intent": "audio", custom: 1 },
timings: defaultSessionTimings,
delayedLeave: defaultSessionTimings.delayedLeave,
});
@@ -111,7 +110,7 @@ describe("joinJsSdkSession", () => {
ownMemberMock,
[transportWithAlias],
expect.objectContaining({
callIntent: "m.audio",
callIntent: "audio",
applicationData: { custom: 1 },
unstableSendStickyEvents: true,
}),
@@ -125,8 +124,7 @@ describe("joinJsSdkSession", () => {
joinJsSdkSession("!room:example.org", s, ownMemberMock, transport, {
encryptMedia: true,
matrixRTCMode: MatrixRTCMode.Matrix_2_0,
callIntent: "m.audio",
applicationData: { custom: 1 },
applicationData: { "m.call.intent": "audio", custom: 1 },
timings: defaultSessionTimings,
delayedLeave: defaultSessionTimings.delayedLeave,
});
@@ -135,7 +133,7 @@ describe("joinJsSdkSession", () => {
ownMemberMock,
[transport],
expect.objectContaining({
callIntent: "m.audio",
callIntent: "audio",
applicationData: { custom: 1 },
unstableSendStickyEvents: true,
}),
+5 -3
View File
@@ -23,7 +23,6 @@ interface Options {
encryptMedia: boolean;
matrixRTCMode: MatrixRTCMode;
sendNotificationType?: RTCNotificationType;
callIntent?: RTCCallIntent;
applicationData?: Record<string, unknown>;
timings: SessionTimings;
/** The delayed leave timings to use: delegated to the SFU or not. */
@@ -44,12 +43,15 @@ export function joinJsSdkSession(
encryptMedia,
matrixRTCMode,
sendNotificationType,
callIntent,
applicationData,
timings,
delayedLeave,
}: Options,
): void {
// The membership format carries the intent as a field of its own
const { "m.call.intent": intent, ...rest } = applicationData ?? {};
const callIntent =
typeof intent === "string" ? (intent as RTCCallIntent) : undefined;
// Give up on the network as soon as either the sync has been down for the
// grace period or the delayed leave has probably been sent, whichever is
// sooner
@@ -76,7 +78,7 @@ export function joinJsSdkSession(
{
notificationType: sendNotificationType,
callIntent,
applicationData,
applicationData: rest,
manageMediaKeys: encryptMedia,
delayedLeaveEventRestartMs: delayedLeave.restart_ms,
delayedLeaveEventDelayMs: delayedLeave.delay_ms,
+65 -7
View File
@@ -69,6 +69,21 @@ export type PublishRequest =
capture?: VideoCaptureSettings;
};
/** A named channel of bytes a member publishes beside its media. */
export interface DataPublishRequest {
source: "data";
/**
* What receivers find the track by, with `dataTrackByName$`. Unique per
* member: publishing a name that is published already rejects.
*/
name: string;
/**
* Whether every message arrives, in order. Default true. Lossy delivery is
* for high-rate state where only the newest value matters, such as cursors.
*/
reliable?: boolean;
}
export type MediaSource =
| "microphone"
| "camera"
@@ -82,7 +97,7 @@ export type MediaStreamStats =
| RTCOutboundRtpStreamStats
| undefined;
/** One published track of a member. */
/** One published audio or video track of a member. */
export interface MediaTrack {
source: MediaSource;
kind: "audio" | "video";
@@ -163,6 +178,48 @@ export interface LocalVideoMediaTrack extends VideoMediaTrack, LocalMediaTrack {
setProcessor(processor: VideoProcessor | undefined): Promise<void>;
}
/**
* One data track of a member. It exists only on a member of the slot, so
* whatever arrives on it comes from an attested member.
*/
export interface DataTrack {
source: "data";
kind: "data";
/** Stable for the life of the track. */
id: string;
/** As the publisher named it in its `DataPublishRequest`. */
name: string;
reliable: boolean;
/** Whether payloads are end-to-end encrypted with the media key. */
encrypted: boolean;
/**
* What the member sends from now on. The track is subscribed while this is
* observed and not at all otherwise; nothing sent earlier is replayed.
*/
data$: Observable<Uint8Array>;
}
/**
* Our own data track. Its `data$` replays what we sent: each payload once
* `send` has handed it to the transport, so a host renders its own messages
* from the same stream as a remote member's.
*/
export interface LocalDataTrack extends DataTrack {
/**
* Sends to every member subscribed to this track. Rejects while the local
* transport is not connected, and where the payload is larger than the
* backend carries.
*/
send(payload: Uint8Array): Promise<void>;
}
/** Any track of a member: its media and its data. */
export type Track = AudioMediaTrack | VideoMediaTrack | DataTrack;
export type LocalTrack =
| LocalAudioMediaTrack
| LocalVideoMediaTrack
| LocalDataTrack;
export type EncryptionError = "MissingKey" | "InvalidKey";
/**
@@ -172,18 +229,19 @@ export type EncryptionError = "MissingKey" | "InvalidKey";
*/
export interface MemberMedia {
/**
* The member's tracks, in publication order, once it has shown up on its
* transport; null until then ("waiting for media"). An entry stays the same
* object for as long as the same publication is behind it. Which track is
* which is in its `source`; `trackBySource$` picks one out.
* The member's tracks, media and data, in publication order, once it has
* shown up on its transport; null until then ("waiting for media"). An
* entry stays the same object for as long as the same publication is
* behind it. Which track is which is in its `source`: `trackBySource$`
* picks a media track out, `dataTrackByName$` a data track.
*/
tracks$: Behavior<(AudioMediaTrack | VideoMediaTrack)[] | null>;
tracks$: Behavior<Track[] | null>;
/** Emits when the SFU reports a key problem for this member. */
encryptionError$: Observable<EncryptionError>;
}
export interface LocalMemberMedia extends MemberMedia {
tracks$: Behavior<(LocalAudioMediaTrack | LocalVideoMediaTrack)[] | null>;
tracks$: Behavior<LocalTrack[] | null>;
}
/**
+10 -20
View File
@@ -19,11 +19,13 @@ import { type Observable } from "rxjs";
import { type ResolvedTransport, type RTCMember } from "../api";
import {
type DataPublishRequest,
type LocalAudioMediaTrack,
type LocalDataTrack,
type LocalMedia,
type LocalMemberMedia,
type LocalTrack,
type LocalVideoMediaTrack,
type MediaSource,
type MemberMedia,
type PublishRequest,
} from "../media-api";
@@ -106,13 +108,6 @@ export interface BackendConnection {
resolved: ResolvedTransport | undefined;
}
export interface DataPacket {
/** The backend identity of the sender. */
senderId: string;
topic: string;
text: string;
}
export interface MediaBackend {
/**
* The membership transport types this backend serves, ordered decreasingly
@@ -158,14 +153,6 @@ export interface MediaBackend {
* exists keeps the device it was created with.
*/
setAudioOutputDeviceId(deviceId: string): Promise<void>;
/**
* A text channel beside the media: sent on the local transport, received on every one.
* This could well be just another track kind. Not needed for element call. just for the sdk target.
* Keeping it on the top level for visibility. Should be tackled in a future refactor.
*/
sendData(topic: string, text: string): Promise<void>;
readonly data$: Observable<DataPacket>;
}
/** `tracks$` is null until the connection carries a local participant. */
@@ -187,14 +174,17 @@ export interface LocalMediaBackend extends LocalMemberMedia {
readonly publishError$: Behavior<Error | null>;
/**
* Publishes a source and resolves with its track once it is in `tracks$`.
* Remembered across connections, so a request before the
* Publishes a source, or a data track, and resolves with its track once it
* is in `tracks$`. Remembered across connections, so a request before the
* transport is up is applied once it is, and republished after a
* reconnection. Rejects where the device could not be used.
* reconnection. Rejects where the device could not be used. A data track
* is end-to-end encrypted by the backend wherever the media is; the session
* never sees how.
*/
publish(request: DataPublishRequest): Promise<LocalDataTrack>;
publish(
request: PublishRequest,
): Promise<LocalAudioMediaTrack | LocalVideoMediaTrack>;
/** Removes one of our publications; a screen share takes its audio with it. */
unpublish(source: MediaSource): Promise<void>;
unpublish(track: LocalTrack): Promise<void>;
}
@@ -0,0 +1,85 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type ByteStreamHandler,
type ByteStreamReader,
type Room as LivekitRoom,
} from "livekit-client";
import { describe, expect, it, vi } from "vitest";
import { byteStreams$, type ReceivedBytes } from "./ByteStreams";
import { flushPromises } from "../../utils/test";
function fakeRoom(): {
room: LivekitRoom;
handlers: Map<string, ByteStreamHandler>;
} {
const handlers = new Map<string, ByteStreamHandler>();
const room = {
registerByteStreamHandler: vi.fn(
(topic: string, handler: ByteStreamHandler) => {
if (handlers.has(topic)) throw new Error("already registered");
handlers.set(topic, handler);
},
),
unregisterByteStreamHandler: vi.fn((topic: string) =>
handlers.delete(topic),
),
} as unknown as LivekitRoom;
return { room, handlers };
}
function reader(chunks: number[][]): {
reader: ByteStreamReader;
finish: () => void;
} {
const done = Promise.withResolvers<Uint8Array[]>();
return {
reader: {
readAll: async () => done.promise,
} as unknown as ByteStreamReader,
finish: (): void => done.resolve(chunks.map((c) => new Uint8Array(c))),
};
}
describe("byteStreams$", () => {
it("shares one handler per topic, released with the last observer", () => {
const { room, handlers } = fakeRoom();
const first = byteStreams$(room, "reliable:chat").subscribe();
const second = byteStreams$(room, "reliable:chat").subscribe();
expect(room.registerByteStreamHandler).toHaveBeenCalledTimes(1);
first.unsubscribe();
expect(handlers.has("reliable:chat")).toBe(true);
second.unsubscribe();
expect(handlers.has("reliable:chat")).toBe(false);
});
it("joins the chunks and keeps the order the streams started in", async () => {
const { room, handlers } = fakeRoom();
const received: ReceivedBytes[] = [];
const subscription = byteStreams$(room, "reliable:chat").subscribe((r) =>
received.push(r),
);
const handler = handlers.get("reliable:chat")!;
const one = reader([[1, 2], [3]]);
const two = reader([[4]]);
handler(one.reader, { identity: "alice" });
handler(two.reader, { identity: "bob" });
two.finish();
await flushPromises();
expect(received).toEqual([]);
one.finish();
await flushPromises();
expect(received).toEqual([
{ identity: "alice", payload: new Uint8Array([1, 2, 3]) },
{ identity: "bob", payload: new Uint8Array([4]) },
]);
subscription.unsubscribe();
});
});
@@ -0,0 +1,82 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import {
type ByteStreamReader,
type Room as LivekitRoom,
} from "livekit-client";
import { catchError, concatMap, EMPTY, from, Observable, share } from "rxjs";
/** One byte stream, read to its end. */
export interface ReceivedBytes {
/** The participant identity of the sender. */
identity: string;
payload: Uint8Array;
}
const byTopicByRoom = new WeakMap<
LivekitRoom,
Map<string, Observable<ReceivedBytes>>
>();
/**
* The byte streams a room receives on a topic, in the order they started. A
* room takes one handler per topic, so every observer of a topic shares it:
* it is registered with the first subscription and released with the last.
*/
export function byteStreams$(
room: LivekitRoom,
topic: string,
): Observable<ReceivedBytes> {
let byTopic = byTopicByRoom.get(room);
if (byTopic === undefined) {
byTopic = new Map();
byTopicByRoom.set(room, byTopic);
}
let streams$ = byTopic.get(topic);
if (streams$ === undefined) {
streams$ = new Observable<{ reader: ByteStreamReader; identity: string }>(
(subscriber) => {
room.registerByteStreamHandler(topic, (reader, { identity }) =>
subscriber.next({ reader, identity }),
);
return (): void => room.unregisterByteStreamHandler(topic);
},
).pipe(
concatMap(({ reader, identity }) =>
from(readStream(reader, identity)).pipe(
catchError((e: unknown) => {
rootLogger
.getChild("[ByteStreams]")
.warn(`Dropping a stream on ${topic} from ${identity}`, e);
return EMPTY;
}),
),
),
share(),
);
byTopic.set(topic, streams$);
}
return streams$;
}
async function readStream(
reader: ByteStreamReader,
identity: string,
): Promise<ReceivedBytes> {
const chunks = await reader.readAll();
const payload = new Uint8Array(
chunks.reduce((length, chunk) => length + chunk.length, 0),
);
let offset = 0;
for (const chunk of chunks) {
payload.set(chunk, offset);
offset += chunk.length;
}
return { identity, payload };
}
@@ -77,6 +77,7 @@ describe("LivekitConnectionFactory", () => {
resolution: expect.objectContaining({ height: 720 }),
});
expect(options.audioOutput).toEqual({ deviceId: undefined });
expect(options.encryption).toBeUndefined();
expect(options.e2ee).toBeUndefined();
});
@@ -19,6 +19,7 @@ import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/En
import { BehaviorSubject, combineLatest, map } from "rxjs";
import { type ResolvedTransport } from "../../api";
import { watchRemoteDataTracks } from "./RemoteDataTracks";
import { type MediaQuality } from "../../config";
import { type PublishRequest } from "../../media-api";
import { type Behavior } from "../../reactive/Behavior";
@@ -128,12 +129,14 @@ export class LivekitConnectionFactory implements ConnectionFactory {
client: this.client,
scope,
livekitRoomFactory: () =>
new LivekitRoom(
roomOptions(
this.publish,
this.audioOutputDeviceId$.value,
this.keyProvider,
this.mediaQuality,
watchRemoteDataTracks(
new LivekitRoom(
roomOptions(
this.publish,
this.audioOutputDeviceId$.value,
this.keyProvider,
this.mediaQuality,
),
),
),
ownMembershipIdentity,
@@ -169,8 +172,10 @@ function roomOptions(
},
audioOutput: { deviceId: audioOutputDeviceId },
// Every room needs a worker of its own: one gets confused by streams from
// several rooms
e2ee: keyProvider && { keyProvider, worker: new E2EEWorker() },
// several rooms. `encryption` rather than `e2ee`, so that LiveKit encrypts
// the data channel too, which carries the reliable data tracks; media is
// encrypted the same either way
encryption: keyProvider && { keyProvider, worker: new E2EEWorker() },
};
}
@@ -1,143 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { EventEmitter } from "events";
import {
ConnectionState as LivekitConnectionState,
type RemoteParticipant,
RoomEvent,
} from "livekit-client";
import { BehaviorSubject } from "rxjs";
import { describe, expect, it, vi } from "vitest";
import { Epoch } from "../../reactive/ObservableScope";
import { constant } from "../../reactive/Behavior";
import { type DataPacket } from "../api";
import { type Connection, type ConnectionState } from "./Connection";
import { ConnectionMap } from "./ConnectionMap";
import { createDataChannel$ } from "./DataChannel";
import { testScope } from "../../utils/test";
import { exampleTransport } from "./test";
const encode = (text: string): Uint8Array => new TextEncoder().encode(text);
const connected: ConnectionState = {
state: "authenticated",
livekitState: LivekitConnectionState.Connected,
};
function fakeConnection(
url: string,
state: ConnectionState = connected,
): {
connection: Connection;
room: EventEmitter;
publishData: ReturnType<typeof vi.fn>;
} {
const room = new EventEmitter();
const publishData = vi.fn(async () => Promise.resolve());
const connection = {
transport: { ...exampleTransport, livekit_service_url: url },
serverName: "example.org",
state$: constant(state),
livekitRoom: Object.assign(room, { localParticipant: { publishData } }),
} as unknown as Connection;
return { connection, room, publishData };
}
function setup(connections: Connection[]): {
received: DataPacket[];
connections$: BehaviorSubject<Epoch<ConnectionMap>>;
sendData: (topic: string, text: string) => Promise<void>;
} {
const data = new ConnectionMap();
for (const connection of connections) data.add(connection, []);
const connections$ = new BehaviorSubject(new Epoch(data));
const { data$, sendData } = createDataChannel$({
scope: testScope(),
connectionMap$: connections$,
connection$: constant(connections[0] ?? null),
});
const received: DataPacket[] = [];
data$.subscribe((message) => received.push(message));
return { received, connections$, sendData };
}
describe("data$", () => {
it("delivers a packet with its sender, topic and text", () => {
const { connection, room } = fakeConnection("https://sfu");
const { received } = setup([connection]);
room.emit(
RoomEvent.DataReceived,
encode("hello"),
{ identity: "alice" } as RemoteParticipant,
undefined,
"chat",
);
expect(received).toEqual([
{ senderId: "alice", topic: "chat", text: "hello" },
]);
});
it("drops packets from the server", () => {
const { connection, room } = fakeConnection("https://sfu");
const { received } = setup([connection]);
room.emit(RoomEvent.DataReceived, encode("server"), undefined);
expect(received).toEqual([]);
});
it("listens on every connection, including ones added later", () => {
const first = fakeConnection("https://sfu-1");
const second = fakeConnection("https://sfu-2");
const { received, connections$ } = setup([first.connection]);
const data = new ConnectionMap();
data.add(first.connection, []);
data.add(second.connection, []);
connections$.next(new Epoch(data, 1));
first.room.emit(RoomEvent.DataReceived, encode("one"), {
identity: "alice",
} as RemoteParticipant);
second.room.emit(RoomEvent.DataReceived, encode("two"), {
identity: "bob",
} as RemoteParticipant);
expect(received.map((m) => [m.senderId, m.text])).toEqual([
["alice", "one"],
["bob", "two"],
]);
});
});
describe("sendData", () => {
it("publishes the text as a reliable packet on the local connection", async () => {
const { connection, publishData } = fakeConnection("https://sfu");
const { sendData } = setup([connection]);
await sendData("chat", "hello");
expect(publishData).toHaveBeenCalledWith(encode("hello"), {
reliable: true,
topic: "chat",
});
});
it("rejects while the local transport is not connected", async () => {
const { connection, publishData } = fakeConnection("https://sfu", {
state: "initialized",
});
const { sendData } = setup([connection]);
await expect(sendData("chat", "hello")).rejects.toThrow("not connected");
expect(publishData).not.toHaveBeenCalled();
});
});
@@ -1,119 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
ConnectionState as LivekitConnectionState,
type RemoteParticipant,
type Room as LivekitRoom,
RoomEvent,
} from "livekit-client";
import {
distinctUntilChanged,
filter,
fromEvent,
map,
merge,
type Observable,
share,
switchMap,
} from "rxjs";
import { type Behavior } from "../../reactive/Behavior";
import {
type Epoch,
type ObservableScope,
} from "../../reactive/ObservableScope";
import { type DataPacket } from "../api";
import { type Connection } from "./Connection";
import { type ConnectionMap } from "./ConnectionMap";
interface Props {
scope: ObservableScope;
connectionMap$: Behavior<Epoch<ConnectionMap>>;
/** The local member's connection, which carries what we send. */
connection$: Behavior<Connection | null>;
}
export interface DataChannel {
data$: Observable<DataPacket>;
sendData: (topic: string, text: string) => Promise<void>;
}
/**
* A text channel beside the media: LiveKit's reliable data packets, received
* on every connection the client holds and sent on the local member's. The
* sender is reported by its identity; the packet's own claim of who sent it
* never reaches the host.
*/
export function createDataChannel$({
scope,
connectionMap$,
connection$,
}: Props): DataChannel {
const decoder = new TextDecoder();
const encoder = new TextEncoder();
const rooms$ = connectionMap$.pipe(
map(({ value }) => value.getConnections().map((c) => c.livekitRoom)),
distinctUntilChanged(
(a, b) => a.length === b.length && a.every((room, i) => room === b[i]),
),
);
const data$ = rooms$.pipe(
switchMap((rooms) => merge(...rooms.map(receivedOn))),
map(({ payload, participant, topic }): DataPacket | null =>
// Only the server sends without a participant; a host has no use for it
participant === undefined
? null
: {
senderId: participant.identity,
topic: topic ?? "",
text: decoder.decode(payload),
},
),
filter((packet) => packet !== null),
scope.bind(),
share(),
);
const sendData = async (topic: string, text: string): Promise<void> => {
const connection = connection$.value;
const state = connection?.state$.value;
if (
connection === null ||
state?.state !== "authenticated" ||
state.livekitState !== LivekitConnectionState.Connected
)
throw new Error("Cannot send data: the local transport is not connected");
await connection.livekitRoom.localParticipant.publishData(
encoder.encode(text),
{ reliable: true, topic },
);
};
return { data$, sendData };
}
interface Received {
payload: Uint8Array;
participant: RemoteParticipant | undefined;
topic: string | undefined;
}
function receivedOn(room: LivekitRoom): Observable<Received> {
return fromEvent(
room,
RoomEvent.DataReceived,
(
payload: Uint8Array,
participant?: RemoteParticipant,
_kind?: unknown,
topic?: string,
): Received => ({ payload, participant, topic }),
);
}
@@ -58,7 +58,6 @@ import {
ResolvedConnection,
} from "./ConnectionFactory";
import { toMediaConnectionState } from "./connectionState";
import { createDataChannel$ } from "./DataChannel";
import { createKeyProvider } from "./KeyProvider";
import {
createLivekitMemberMedia,
@@ -263,7 +262,12 @@ export function createLivekitBackend(
`${member.rtcBackendIdentity}: LiveKit participant missing on ${transportString}`,
);
});
return createLivekitMemberMedia(memberScope, source$, encryptionSystem);
return createLivekitMemberMedia(
memberScope,
source$,
encryptionSystem,
logger,
);
};
const connections$ = scope.behavior<BackendConnection[]>(
@@ -276,12 +280,6 @@ export function createLivekitBackend(
),
);
const { data$, sendData } = createDataChannel$({
scope,
connectionMap$,
connection$: localMedia_.connection$,
});
return {
transportTypes: ["m.livekit", "livekit"],
prepareLocalTransport,
@@ -290,8 +288,6 @@ export function createLivekitBackend(
mediaFor,
connections$,
setAudioOutputDeviceId: localMedia_.setAudioOutputDeviceId,
sendData,
data$,
};
}
@@ -0,0 +1,124 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { logger } from "matrix-js-sdk/lib/logger";
import {
type ByteStreamHandler,
type ByteStreamReader,
type DataTrackFrame,
type RemoteDataTrack,
type Room as LivekitRoom,
} from "livekit-client";
import { describe, expect, it, vi } from "vitest";
import { createLivekitDataTrack } from "./LivekitDataTrack";
import { flushPromises } from "../../utils/test";
function remoteTrack(
name: string,
usesE2ee = true,
): {
track: RemoteDataTrack;
frames: ReadableStreamDefaultController<DataTrackFrame>[];
cancel: ReturnType<typeof vi.fn>;
} {
const frames: ReadableStreamDefaultController<DataTrackFrame>[] = [];
const cancel = vi.fn();
const track = {
info: { sid: "DT_1", name, usesE2ee },
publisherIdentity: "alice",
subscribe: vi.fn(
() =>
new ReadableStream<DataTrackFrame>({
start: (controller): void => {
frames.push(controller);
},
cancel,
}),
),
} as unknown as RemoteDataTrack;
return { track, frames, cancel };
}
function roomWithStreams(): {
room: LivekitRoom;
send: (identity: string, bytes: number[]) => void;
} {
const handlers = new Map<string, ByteStreamHandler>();
return {
room: {
registerByteStreamHandler: (topic: string, h: ByteStreamHandler) =>
handlers.set(topic, h),
unregisterByteStreamHandler: (topic: string) => handlers.delete(topic),
} as unknown as LivekitRoom,
send: (identity, bytes): void =>
handlers.get("reliable:chat")!(
{
readAll: async () => Promise.resolve([new Uint8Array(bytes)]),
} as unknown as ByteStreamReader,
{ identity },
),
};
}
describe("a remote data track", () => {
it("names itself without the LiveKit prefix", () => {
const { track } = remoteTrack("reliable:chat", false);
expect(
createLivekitDataTrack(roomWithStreams().room, track, logger),
).toMatchObject({
source: "data",
kind: "data",
id: "DT_1",
name: "chat",
reliable: true,
encrypted: false,
});
});
it("subscribes a lossy track while observed, once for all observers", async () => {
const { track, frames, cancel } = remoteTrack("lossy:cursors");
const dataTrack = createLivekitDataTrack(
roomWithStreams().room,
track,
logger,
);
expect(track.subscribe).not.toHaveBeenCalled();
const received: Uint8Array[] = [];
const first = dataTrack.data$.subscribe((p) => received.push(p));
const second = dataTrack.data$.subscribe();
expect(track.subscribe).toHaveBeenCalledTimes(1);
frames[0].enqueue({ payload: new Uint8Array([7]) });
await flushPromises();
expect(received).toEqual([new Uint8Array([7])]);
first.unsubscribe();
expect(cancel).not.toHaveBeenCalled();
second.unsubscribe();
expect(cancel).toHaveBeenCalled();
});
it("takes a reliable track's streams from its publisher only", async () => {
const { room, send } = roomWithStreams();
const dataTrack = createLivekitDataTrack(
room,
remoteTrack("reliable:chat").track,
logger,
);
const received: Uint8Array[] = [];
const subscription = dataTrack.data$.subscribe((p) => received.push(p));
send("bob", [1]);
send("alice", [2]);
await flushPromises();
// Bob's is not this track's
expect(received).toEqual([new Uint8Array([2])]);
subscription.unsubscribe();
});
});
@@ -0,0 +1,74 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type Logger } from "matrix-js-sdk/lib/logger";
import { type RemoteDataTrack, type Room as LivekitRoom } from "livekit-client";
import { filter, map, Observable, share } from "rxjs";
import { type DataTrack } from "../../media-api";
import { byteStreams$ } from "./ByteStreams";
import { fromLivekitDataTrackName } from "./dataTrackName";
/**
* A remote member's data track. Decryption is LiveKit's: its E2EE manager
* reads the key index and IV off each frame or stream and decrypts with the
* sender's key, so nothing here sees a key. A frame of an encrypted track that
* does not decrypt is dropped by LiveKit. A stream is not checked: LiveKit
* 2.23 sends an encrypted packet without its encryption type, so a received
* stream reads as unencrypted either way, and a cleartext one would pass.
*/
export function createLivekitDataTrack(
room: LivekitRoom,
track: RemoteDataTrack,
logger: Logger,
): DataTrack {
const { name, reliable } = fromLivekitDataTrackName(track.info.name);
const encrypted = track.info.usesE2ee;
const data$ = reliable
? byteStreams$(room, track.info.name).pipe(
filter(({ identity }) => identity === track.publisherIdentity),
map(({ payload }) => payload),
)
: framesOf(track, logger);
return {
source: "data",
kind: "data",
id: track.info.sid,
name,
reliable,
encrypted,
data$,
};
}
/** The track's frames, subscribed at the SFU while observed. */
function framesOf(
track: RemoteDataTrack,
logger: Logger,
): Observable<Uint8Array> {
return new Observable<Uint8Array>((subscriber) => {
const reader = track.subscribe().getReader();
let cancelled = false;
void (async (): Promise<void> => {
try {
for (;;) {
const { done, value } = await reader.read();
if (done) break;
subscriber.next(value.payload);
}
} catch (e) {
// A failed subscription ends the stream; a host's pipeline survives it
if (!cancelled) logger.warn(`Data track ${track.info.name} ended`, e);
}
subscriber.complete();
})();
return (): void => {
cancelled = true;
reader.cancel().catch(() => {});
};
}).pipe(share());
}
@@ -225,51 +225,6 @@ describe("createLivekitMediaTrack", () => {
);
});
// LiveKit's media stream track is the processed one while a processor runs,
// and that one names no device; the camera behind it does
it.each([
{ deviceId: "cam-1", switches: false },
{ deviceId: "cam-2", switches: true },
])(
"leaves a processed camera alone unless its device changes: $deviceId",
async ({ deviceId, switches }) => {
vi.stubGlobal("MediaStreamTrack", class {});
onTestFinished(() => {
vi.unstubAllGlobals();
});
const camera = Object.assign(Object.create(LocalVideoTrack.prototype), {
source: Track.Source.Camera,
_mediaStreamTrack: {
readyState: "live",
label: "",
getSettings: () => ({ deviceId: "cam-1" }),
},
// What LiveKit shows as the track, with no device of its own
processor: {
processedTrack: { label: "", getSettings: () => ({}) },
},
}) as LocalVideoTrack;
const switchActiveDevice = vi.fn().mockResolvedValue(true);
const { participant, publication, room } = fakes({
track: camera as unknown as FakeTrack,
});
const track = createLocalLivekitMediaTrack(
scope,
Object.assign(participant, {
isLocal: true,
}) as unknown as LocalParticipant,
publication as unknown as TrackPublication,
Object.assign(room, { switchActiveDevice }),
vi.fn(),
);
await track.setDevice(deviceId);
expect(switchActiveDevice.mock.calls).toEqual(
switches ? [["videoinput", deviceId, false]] : [],
);
},
);
// LiveKit stops a camera that is turned off; the processor is built when it
// is turned on again
it.each([
@@ -199,13 +199,11 @@ export function createLocalLivekitMediaTrack(
throw new MatrixRTCError(`A ${base.source} track has no device`);
// LiveKit decides whether to restart by the constraints it keeps, which
// a browser that does not echo the device id back (Firefox) leaves
// without one; a track already on the device is left alone here. Asked
// of the camera track itself: with a processor on, LiveKit's media
// stream track is the processed one, which names no device
// without one; a track already on the device is left alone here
const current = publication.track;
if (
current instanceof LocalTrack &&
(await current.getDeviceId(false)) === deviceId
current.mediaStreamTrack.getSettings().deviceId === deviceId
)
return;
await room.switchActiveDevice(
@@ -6,9 +6,12 @@ Please see LICENSE in the repository root for full details.
*/
import { EventEmitter } from "events";
import { logger } from "matrix-js-sdk/lib/logger";
import {
ParticipantEvent,
type RemoteDataTrack,
type RemoteParticipant,
RoomEvent,
type Room as LivekitRoom,
Track,
type TrackPublication,
@@ -27,6 +30,7 @@ describe("createLivekitMemberMedia", () => {
testScope(),
constant({ participants: [participant], room }),
{ kind: E2eeType.NONE },
logger,
);
expect(media.tracks$.value).toEqual([]);
@@ -54,6 +58,7 @@ describe("createLivekitMemberMedia", () => {
testScope(),
constant({ participants: [participant], room }),
{ kind: E2eeType.NONE },
logger,
);
emitter.emit(
ParticipantEvent.TrackPublished,
@@ -85,6 +90,44 @@ function publish(
return publication;
}
describe("a member's data tracks", () => {
it("follow the room's data track events, after the media", () => {
const { participant, publications, room, emitter } = fakes();
const media = createLivekitMemberMedia(
testScope(),
constant({ participants: [participant], room }),
{ kind: E2eeType.NONE },
logger,
);
const camera = publish(publications, "TR_1", Track.Source.Camera);
emitter.emit(ParticipantEvent.TrackPublished, camera);
const chat = {
info: { sid: "DT_1", name: "reliable:chat", usesE2ee: true },
publisherIdentity: participant.identity,
} as unknown as RemoteDataTrack;
(room as unknown as EventEmitter).emit(RoomEvent.DataTrackPublished, chat);
const [video, data] = media.tracks$.value!;
expect(video.source).toBe("camera");
expect(data).toMatchObject({
kind: "data",
name: "chat",
reliable: true,
encrypted: true,
});
// An unrelated media event keeps the same object
emitter.emit(ParticipantEvent.TrackMuted, camera);
expect(media.tracks$.value![1]).toBe(data);
(room as unknown as EventEmitter).emit(
RoomEvent.DataTrackUnpublished,
"DT_1",
);
expect(media.tracks$.value).toEqual([video]);
});
});
function fakes(): {
participant: RemoteParticipant;
publications: Map<string, TrackPublication>;
@@ -9,6 +9,7 @@ import {
observeParticipantMedia,
roomEventSelector,
} from "@livekit/components-core";
import { type Logger } from "matrix-js-sdk/lib/logger";
import {
type LocalParticipant,
type Participant,
@@ -31,11 +32,14 @@ import { generateItems } from "../../reactive/observable";
import { E2eeType, type EncryptionSystem } from "../../encryption";
import { type Behavior } from "../../reactive/Behavior";
import {
type DataTrack,
type EncryptionError,
type LocalMemberMedia,
type MediaSource,
type MemberMedia,
} from "../../media-api";
import { createLivekitDataTrack } from "./LivekitDataTrack";
import { remoteDataTracks$ } from "./RemoteDataTracks";
import {
createLivekitMediaTrack,
createLocalLivekitMediaTrack,
@@ -70,13 +74,21 @@ export function createLivekitMemberMedia(
scope: ObservableScope,
source$: Behavior<ParticipantSource>,
encryptionSystem: EncryptionSystem,
logger: Logger,
): MemberMedia {
const media$ = memberTracks$(
scope,
source$,
(trackScope, participant, publication, room) =>
createLivekitMediaTrack(trackScope, participant, publication, room),
);
const data$ = memberDataTracks$(scope, source$, logger);
return {
tracks$: memberTracks$(
scope,
source$,
(trackScope, participant, publication, room) =>
createLivekitMediaTrack(trackScope, participant, publication, room),
// The data tracks after the media, both null while there is no participant
tracks$: scope.behavior(
combineLatest([media$, data$], (media, data) =>
media === null ? null : [...media, ...data],
),
),
encryptionError$: encryptionErrors$(source$, encryptionSystem),
};
@@ -145,6 +157,40 @@ function memberTracks$<P extends Participant, T>(
);
}
/**
* One data track per LiveKit data track of the member's participants, each
* living as long as it is published.
*/
function memberDataTracks$(
scope: ObservableScope,
source$: Behavior<ParticipantSource>,
logger: Logger,
): Behavior<DataTrack[]> {
return scope.behavior(
source$.pipe(
switchMap((source) => {
if (source === null) return of([]);
const { participants, room } = source;
const identities = new Set(participants.map((p) => p.identity));
return remoteDataTracks$(room).pipe(
map((tracks) =>
tracks.filter((t) => identities.has(t.publisherIdentity)),
),
generateItems(
`${[...identities].join(",")} dataTracks$`,
function* (tracks) {
for (const track of tracks)
yield { keys: [track.info.sid], data: track };
},
(_trackScope, track$) =>
createLivekitDataTrack(room, track$.value, logger),
),
);
}),
),
);
}
function encryptionErrors$(
source$: Behavior<ParticipantSource>,
encryptionSystem: EncryptionSystem,
@@ -0,0 +1,195 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
ConnectionState as LivekitConnectionState,
type LocalDataTrack as LivekitLocalDataTrack,
type LocalParticipant,
} from "livekit-client";
import { type Logger } from "matrix-js-sdk/lib/logger";
import {
BehaviorSubject,
combineLatest,
distinctUntilChanged,
first,
map,
type Observable,
of,
startWith,
Subject,
switchMap,
} from "rxjs";
import { type DataPublishRequest, type LocalDataTrack } from "../../media-api";
import { type Behavior } from "../../reactive/Behavior";
import { ObservableScope } from "../../reactive/ObservableScope";
import { type Connection } from "./Connection";
import { toLivekitDataTrackName } from "./dataTrackName";
interface Props {
scope: ObservableScope;
/** The connection the local member publishes on. */
connection$: Behavior<Connection | null>;
/** Whether anything may leave this device: joined and the homeserver reachable. */
publishing$: Observable<boolean>;
/** Whether the rooms carry a key provider, so LiveKit encrypts what we send. */
encrypted: boolean;
logger: Logger;
}
export interface LocalDataTracks {
tracks$: Behavior<LocalDataTrack[]>;
/**
* Resolves once the track is published on LiveKit, which waits for the
* join. Rejects where a track of that name is published already, or where
* LiveKit refuses it.
*/
publish(request: DataPublishRequest): Promise<LocalDataTrack>;
unpublish(track: LocalDataTrack): void;
}
interface Published {
track: LivekitLocalDataTrack;
participant: LocalParticipant;
}
interface Entry {
track: LocalDataTrack;
finish: () => void;
}
/**
* The local member's data tracks. Each is one LiveKit data track, published
* on every connection to the local transport while publishing is allowed, so
* a track requested before the join goes out once we are joined. The object a
* host holds outlives the connections. Encryption is LiveKit's: its E2EE
* manager encrypts with our current key and writes the key index and IV into
* each frame, or each stream for a reliable track.
*/
export function createLocalDataTracks({
scope,
connection$,
publishing$,
encrypted,
logger,
}: Props): LocalDataTracks {
const entries = new Map<string, Entry>();
const tracks$ = new BehaviorSubject<LocalDataTrack[]>([]);
const update = (): void =>
tracks$.next([...entries.values()].map(({ track }) => track));
scope.onEnd(() => {
for (const entry of entries.values()) entry.finish();
});
// Where to publish: the local connection once it has connected, while
// publishing is allowed
const target$ = scope.behavior(
combineLatest([connection$, publishing$]).pipe(
switchMap(([connection, publishing]) =>
connection === null || !publishing
? of(null)
: connection.state$.pipe(
first(
(s) =>
s.state === "authenticated" &&
s.livekitState === LivekitConnectionState.Connected,
),
map(() => connection),
startWith(null),
),
),
distinctUntilChanged(),
),
);
const publish = async ({
name,
reliable = true,
}: DataPublishRequest): Promise<LocalDataTrack> => {
if (entries.has(name))
throw new Error(`A data track named ${name} is published already`);
const firstPublished = Promise.withResolvers<void>();
const livekitName = toLivekitDataTrackName(name, reliable);
const trackScope = new ObservableScope();
const published$ = new BehaviorSubject<Published | null>(null);
const sent$ = new Subject<Uint8Array>();
let finished = false;
trackScope.reconcile(target$, async (connection) => {
if (connection === null) return;
const participant = connection.livekitRoom.localParticipant;
let track: LivekitLocalDataTrack;
try {
track = await participant.publishDataTrack({ name: livekitName });
} catch (e) {
logger.error(`Failed to publish the data track ${name}`, e);
firstPublished.reject(e);
return;
}
published$.next({ track, participant });
firstPublished.resolve();
return async (): Promise<void> => {
published$.next(null);
await track.unpublish().catch((e: unknown) => {
logger.warn(`Failed to unpublish the data track ${name}`, e);
});
};
});
const track: LocalDataTrack = {
source: "data",
kind: "data",
id: `data:${name}`,
name,
reliable,
encrypted,
data$: sent$.asObservable(),
send: async (payload): Promise<void> => {
if (finished) throw new Error(`The data track ${name} is unpublished`);
const published = published$.value;
if (published === null)
throw new Error(
`Cannot send on ${name}: not published on the transport yet`,
);
if (reliable)
await published.participant.sendBytes(payload, {
topic: livekitName,
});
else
await published.track.tryPush({ payload: new Uint8Array(payload) });
sent$.next(payload);
},
};
entries.set(name, {
track,
finish: (): void => {
finished = true;
trackScope.end();
sent$.complete();
},
});
update();
try {
await firstPublished.promise;
} catch (e) {
unpublish(track);
throw e;
}
return track;
};
const unpublish = (track: LocalDataTrack): void => {
const entry = entries.get(track.name);
if (entry?.track !== track) return;
entries.delete(track.name);
entry.finish();
update();
};
return { tracks$: scope.behavior(tracks$), publish, unpublish };
}
@@ -7,6 +7,7 @@ Please see LICENSE in the repository root for full details.
import {
ConnectionState as LivekitConnectionState,
type LocalDataTrack as LivekitLocalDataTrack,
type LocalParticipant,
type LocalTrack,
type LocalTrackPublication,
@@ -28,6 +29,10 @@ import { type Connection, type ConnectionState } from "./Connection";
import { ConnectionMap } from "./ConnectionMap";
import { createLivekitLocalMedia } from "./LocalMedia";
import { exampleTransport } from "./test";
import {
type LocalAudioMediaTrack,
type LocalDataTrack,
} from "../../media-api";
const localTransport: TransportLocator = {
transport: exampleTransport,
@@ -270,9 +275,8 @@ describe("createLivekitLocalMedia", () => {
media.setPublishing(true);
await flushPromises();
await expect(media.tracks$.value![0].setEnabled(false)).resolves.toBe(
false,
);
const microphone = media.tracks$.value![0] as LocalAudioMediaTrack;
await expect(microphone.setEnabled(false)).resolves.toBe(false);
expect(first.participant.setMicrophoneEnabled).toHaveBeenLastCalledWith(
false,
);
@@ -293,10 +297,16 @@ describe("createLivekitLocalMedia", () => {
await flushPromises();
await media.publish({ source: "screenShare" });
await media.unpublish("screenShareAudio");
await media.unpublish({
kind: "audio",
source: "screenShareAudio",
} as LocalAudioMediaTrack);
expect(participant.setScreenShareEnabled).toHaveBeenLastCalledWith(false);
await media.unpublish("microphone");
await media.unpublish({
kind: "audio",
source: "microphone",
} as LocalAudioMediaTrack);
expect(participant.unpublishTrack).toHaveBeenCalledOnce();
expect(media.tracks$.value).toEqual([]);
});
@@ -385,6 +395,116 @@ function fakeLocalTrack(source: Track.Source): LocalTrack {
return t as unknown as LocalTrack;
}
describe("local data tracks", () => {
function joined(): ReturnType<typeof setup> & FakeConnection {
const fixture = setup();
const fake = fakeConnection();
fixture.connectionMap$.next(mapOf(fake.connection));
fixture.localTransport$.next(localTransport);
return { ...fixture, ...fake };
}
it("publishes on LiveKit only once publishing is allowed, and again on a new connection", async () => {
const { media, participant, connectionMap$ } = joined();
let published = false;
const publishing = media
.publish({ source: "data", name: "chat" })
.then((track) => {
published = true;
return track;
});
await flushPromises();
expect(participant.publishDataTrack).not.toHaveBeenCalled();
expect(published).toBe(false);
// Listed from the request on, like a media publication
const listed = media.tracks$.value!.find((t) => t.kind === "data");
expect(listed).toMatchObject({ name: "chat", reliable: true });
media.setPublishing(true);
const chat = await publishing;
expect(chat).toBe(listed);
expect(participant.publishDataTrack).toHaveBeenCalledWith({
name: "reliable:chat",
});
const next = fakeConnection();
connectionMap$.next(mapOf(next.connection));
await flushPromises();
expect(next.participant.publishDataTrack).toHaveBeenCalledWith({
name: "reliable:chat",
});
// The host's object outlives the connection
expect(media.tracks$.value).toContain(chat);
});
it("sends reliable data as a byte stream and lossy data as frames, replaying what was sent", async () => {
const { media, participant } = joined();
media.setPublishing(true);
const chat = await media.publish({ source: "data", name: "chat" });
const cursors = await media.publish({
source: "data",
name: "cursors",
reliable: false,
});
await flushPromises();
const sent: Uint8Array[] = [];
chat.data$.subscribe((p) => sent.push(p));
await chat.send(new Uint8Array([1]));
expect(participant.sendBytes).toHaveBeenCalledWith(new Uint8Array([1]), {
topic: "reliable:chat",
});
expect(sent).toEqual([new Uint8Array([1])]);
await cursors.send(new Uint8Array([2]));
const livekitCursors = await vi.mocked(participant.publishDataTrack).mock
.results[1].value;
expect(livekitCursors.tryPush).toHaveBeenCalledWith({
payload: new Uint8Array([2]),
});
});
it("rejects a second track of the same name, or one LiveKit refuses", async () => {
const { media, participant } = joined();
media.setPublishing(true);
await media.publish({ source: "data", name: "chat" });
await expect(
media.publish({ source: "data", name: "chat", reliable: false }),
).rejects.toThrow("published already");
vi.mocked(participant.publishDataTrack).mockRejectedValueOnce(
new Error("DuplicateName"),
);
await expect(
media.publish({ source: "data", name: "cursors" }),
).rejects.toThrow("DuplicateName");
expect(
media.tracks$.value!.filter((t) => t.kind === "data").map((t) => t.name),
).toEqual(["chat"]);
});
it("finishes a track on unpublish", async () => {
const { media, participant } = joined();
media.setPublishing(true);
const chat: LocalDataTrack = await media.publish({
source: "data",
name: "chat",
});
await flushPromises();
let completed = false;
chat.data$.subscribe({ complete: () => (completed = true) });
await media.unpublish(chat);
await flushPromises();
const livekitChat = await vi.mocked(participant.publishDataTrack).mock
.results[0].value;
expect(livekitChat.unpublish).toHaveBeenCalled();
expect(completed).toBe(true);
expect(media.tracks$.value).not.toContain(chat);
await expect(chat.send(new Uint8Array([1]))).rejects.toThrow("unpublished");
});
});
/**
* The slice of a local participant the publisher and the local tracks use:
* enabling a source publishes a fake track at once, muting an existing one
@@ -442,6 +562,14 @@ function fakeLocalParticipant(): LocalParticipant {
const participant = {
...emitter,
publishDataTrack: vi.fn(async ({ name }: { name: string }) =>
Promise.resolve({
name,
tryPush: vi.fn(async () => Promise.resolve()),
unpublish: vi.fn(async () => Promise.resolve()),
} as unknown as LivekitLocalDataTrack),
),
sendBytes: vi.fn(async () => Promise.resolve({})),
identity: "@alice:example.org:DEVICE",
isLocal: true,
isSpeaking: false,
+49 -11
View File
@@ -26,13 +26,16 @@ import {
} from "rxjs";
import {
type DataPublishRequest,
type LocalAudioMediaTrack,
type LocalDataTrack,
type LocalMedia,
type LocalTrack,
type LocalVideoMediaTrack,
type MediaSource,
type PublishRequest,
} from "../../media-api";
import { type EncryptionSystem } from "../../encryption";
import { E2eeType, type EncryptionSystem } from "../../encryption";
import { FailToStartLivekitConnection, toMatrixRTCError } from "../../errors";
import { type Behavior } from "../../reactive/Behavior";
import {
@@ -50,6 +53,7 @@ import {
} from "./LivekitMemberMedia";
import { livekitSources } from "./LivekitMediaTrack";
import { type DesiredMedia, Publisher } from "./Publisher";
import { createLocalDataTracks } from "./LocalDataTracks";
import { toMediaConnectionState } from "./connectionState";
interface Props {
@@ -67,7 +71,7 @@ interface Props {
}
export interface LivekitLocalMedia extends LocalMediaBackend {
/** The connection the local member publishes on; the data channel sends on it. */
/** The connection the local member publishes on. */
connection$: Behavior<Connection | null>;
setAudioOutputDeviceId(deviceId: string): Promise<void>;
}
@@ -171,14 +175,42 @@ export function createLivekitLocalMedia({
return result;
};
const { tracks$, encryptionError$ } = createLocalLivekitMemberMedia(
const { tracks$: mediaTracks$, encryptionError$ } =
createLocalLivekitMemberMedia(
scope,
participantAndRoom$(scope, participant$, connection$),
encryptionSystem,
setEnabled,
);
const dataTracks = createLocalDataTracks({
scope,
participantAndRoom$(scope, participant$, connection$),
encryptionSystem,
setEnabled,
connection$,
publishing$,
encrypted: encryptionSystem.kind !== E2eeType.NONE,
logger,
});
// The data tracks after the media, both null while there is no participant
const tracks$ = scope.behavior<LocalTrack[] | null>(
combineLatest([mediaTracks$, dataTracks.tracks$], (media, data) =>
media === null ? null : [...media, ...data],
),
);
const publish = async (
function publish(request: DataPublishRequest): Promise<LocalDataTrack>;
function publish(
request: PublishRequest,
): Promise<LocalAudioMediaTrack | LocalVideoMediaTrack>;
async function publish(
request: DataPublishRequest | PublishRequest,
): Promise<LocalTrack> {
if (request.source === "data")
return dataTracks.publish(request).catch((e: unknown) => {
throw toMatrixRTCError(e);
});
return publishMedia(request);
}
const publishMedia = async (
request: PublishRequest,
): Promise<LocalAudioMediaTrack | LocalVideoMediaTrack> => {
desired.set(request.source, { request, enabled: true });
@@ -195,8 +227,9 @@ export function createLivekitLocalMedia({
}
};
const unpublish = async (source: MediaSource): Promise<void> => {
const published = publishedSource(source);
const unpublish = async (track: LocalTrack): Promise<void> => {
if (track.kind === "data") return dataTracks.unpublish(track);
const published = publishedSource(track.source);
if (published === undefined) return;
desired.delete(published);
await publisher$.value?.unpublish(published);
@@ -259,11 +292,16 @@ function publication$(
}
function trackWithId$(
tracks$: Behavior<(LocalAudioMediaTrack | LocalVideoMediaTrack)[] | null>,
tracks$: Behavior<LocalTrack[] | null>,
id: string,
): Observable<LocalAudioMediaTrack | LocalVideoMediaTrack> {
return tracks$.pipe(
map((tracks) => tracks?.find((track) => track.id === id)),
map((tracks) =>
tracks?.find(
(track): track is LocalAudioMediaTrack | LocalVideoMediaTrack =>
track.kind !== "data" && track.id === id,
),
),
filter((track) => track !== undefined),
);
}
+5 -3
View File
@@ -72,9 +72,11 @@ export class Publisher {
private readonly media?: LocalMedia,
) {
this.room = room;
room.setE2EEEnabled(room.options.e2ee !== undefined)?.catch((e: Error) => {
this.logger.error("Failed to enable E2EE on the room", e);
});
room
.setE2EEEnabled(room.options.encryption !== undefined)
?.catch((e: Error) => {
this.logger.error("Failed to enable E2EE on the room", e);
});
this.onLocalTrackPublished = this.onLocalTrackPublished.bind(this);
room.localParticipant.on(
ParticipantEvent.LocalTrackPublished,
@@ -0,0 +1,39 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { EventEmitter } from "events";
import {
type RemoteDataTrack,
type Room as LivekitRoom,
RoomEvent,
} from "livekit-client";
import { describe, expect, it } from "vitest";
import { remoteDataTracks$, watchRemoteDataTracks } from "./RemoteDataTracks";
const track = (sid: string): RemoteDataTrack =>
({ info: { sid }, publisherIdentity: "alice" }) as RemoteDataTrack;
describe("remoteDataTracks$", () => {
it("keeps what the room announced before anyone observed it", () => {
const emitter = new EventEmitter();
const room = watchRemoteDataTracks(emitter as unknown as LivekitRoom);
// As in a join response: announced before any member asks
emitter.emit(RoomEvent.DataTrackPublished, track("DT_1"));
emitter.emit(RoomEvent.DataTrackPublished, track("DT_2"));
emitter.emit(RoomEvent.DataTrackUnpublished, "DT_1");
const seen: string[][] = [];
remoteDataTracks$(room).subscribe((tracks) =>
seen.push(tracks.map((t) => t.info.sid)),
);
expect(seen).toEqual([["DT_2"]]);
emitter.emit(RoomEvent.Disconnected);
expect(seen.at(-1)).toEqual([]);
});
});
@@ -0,0 +1,60 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type RemoteDataTrack,
type Room as LivekitRoom,
RoomEvent,
} from "livekit-client";
import { BehaviorSubject, type Observable } from "rxjs";
const tracksByRoom = new WeakMap<
LivekitRoom,
BehaviorSubject<readonly RemoteDataTrack[]>
>();
/**
* Starts keeping the room's remote data tracks, from its own events. Called
* as the room is created: LiveKit announces the tracks in the join response
* before the participants that publish them exist, so a participant's
* `dataTracks` misses them, and it adds a track to that map only after
* announcing it.
*/
export function watchRemoteDataTracks(room: LivekitRoom): LivekitRoom {
remoteDataTracksOf(room);
return room;
}
/** Every remote data track the room knows of, as LiveKit announced them. */
export function remoteDataTracks$(
room: LivekitRoom,
): Observable<readonly RemoteDataTrack[]> {
return remoteDataTracksOf(room);
}
function remoteDataTracksOf(
room: LivekitRoom,
): BehaviorSubject<readonly RemoteDataTrack[]> {
let tracks$ = tracksByRoom.get(room);
if (tracks$ === undefined) {
const subject$ = new BehaviorSubject<readonly RemoteDataTrack[]>([]);
room.on(RoomEvent.DataTrackPublished, (track: RemoteDataTrack) =>
subject$.next([
...subject$.value.filter((t) => t.info.sid !== track.info.sid),
track,
]),
);
room.on(RoomEvent.DataTrackUnpublished, (sid: string) =>
subject$.next(subject$.value.filter((t) => t.info.sid !== sid)),
);
// A new session announces its tracks again
room.on(RoomEvent.Disconnected, () => subject$.next([]));
tracksByRoom.set(room, subject$);
tracks$ = subject$;
}
return tracks$;
}
@@ -0,0 +1,28 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { describe, expect, it } from "vitest";
import {
fromLivekitDataTrackName,
toLivekitDataTrackName,
} from "./dataTrackName";
describe("data track names on LiveKit", () => {
it.each([true, false])("round-trips reliable=%s", (reliable) => {
expect(
fromLivekitDataTrackName(toLivekitDataTrackName("chat:room", reliable)),
).toEqual({ name: "chat:room", reliable });
});
it("reads a name without a prefix as lossy", () => {
expect(fromLivekitDataTrackName("telemetry")).toEqual({
name: "telemetry",
reliable: false,
});
});
});
@@ -0,0 +1,36 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
/**
* How a data track is named on LiveKit. Every data track of a member is one
* LiveKit data track, which announces it; the prefix says how its payloads
* travel. A lossy track pushes them as frames on the data track itself. A
* reliable one sends each as a byte stream on a topic of the same name,
* because LiveKit's data tracks are lossy only.
*/
const lossyPrefix = "lossy:";
const reliablePrefix = "reliable:";
export function toLivekitDataTrackName(
name: string,
reliable: boolean,
): string {
return (reliable ? reliablePrefix : lossyPrefix) + name;
}
/** A track without a prefix, from a plain LiveKit publisher, is lossy under its own name. */
export function fromLivekitDataTrackName(livekitName: string): {
name: string;
reliable: boolean;
} {
if (livekitName.startsWith(reliablePrefix))
return { name: livekitName.slice(reliablePrefix.length), reliable: true };
if (livekitName.startsWith(lossyPrefix))
return { name: livekitName.slice(lossyPrefix.length), reliable: false };
return { name: livekitName, reliable: false };
}
+39 -2
View File
@@ -8,9 +8,9 @@ Please see LICENSE in the repository root for full details.
import { BehaviorSubject } from "rxjs";
import { describe, expect, it } from "vitest";
import { type MediaTrack } from "../media-api";
import { type DataTrack, type MediaTrack } from "../media-api";
import { testScope } from "./test";
import { trackBySource$ } from "./tracks";
import { dataTrackByName$, trackBySource$ } from "./tracks";
const camera = { source: "camera", kind: "video", id: "1" } as MediaTrack;
const microphone = {
@@ -39,3 +39,40 @@ describe("trackBySource$", () => {
expect(emissions).toEqual([camera]);
});
});
const chat = {
source: "data",
kind: "data",
id: "3",
name: "chat",
} as DataTrack;
const cursors = {
source: "data",
kind: "data",
id: "4",
name: "cursors",
} as DataTrack;
describe("dataTrackByName$", () => {
it("picks the data track of a name and follows the list", () => {
const tracks$ = new BehaviorSubject<(MediaTrack | DataTrack)[] | null>(
null,
);
const chat$ = dataTrackByName$(testScope(), tracks$, "chat");
expect(chat$.value).toBeUndefined();
tracks$.next([camera, cursors, chat]);
expect(chat$.value).toBe(chat);
tracks$.next([camera, cursors]);
expect(chat$.value).toBeUndefined();
});
it("does not emit when the list changes elsewhere", () => {
const tracks$ = new BehaviorSubject<(MediaTrack | DataTrack)[]>([chat]);
const chat$ = dataTrackByName$(testScope(), tracks$, "chat");
const emissions: unknown[] = [];
chat$.subscribe((t) => emissions.push(t));
tracks$.next([chat, camera]);
tracks$.next([microphone, chat, cursors]);
expect(emissions).toEqual([chat]);
});
});
+20 -1
View File
@@ -11,6 +11,7 @@ import { type Behavior } from "../reactive/Behavior";
import { type ObservableScope } from "../reactive/ObservableScope";
import {
type AudioMediaTrack,
type DataTrack,
type MediaSource,
type MediaTrack,
type VideoMediaTrack,
@@ -27,7 +28,7 @@ export type TrackOfSource<S extends MediaSource> = S extends
/** The member's first track of a source; undefined while there is none, or no media yet. */
export function trackBySource$<S extends MediaSource>(
scope: ObservableScope,
tracks$: Behavior<MediaTrack[] | null>,
tracks$: Behavior<readonly (MediaTrack | DataTrack)[] | null>,
source: S,
): Behavior<TrackOfSource<S> | undefined> {
return scope.behavior(
@@ -42,3 +43,21 @@ export function trackBySource$<S extends MediaSource>(
),
);
}
/** The member's data track of this name; undefined while there is none, or no media yet. */
export function dataTrackByName$(
scope: ObservableScope,
tracks$: Behavior<readonly (MediaTrack | DataTrack)[] | null>,
name: string,
): Behavior<DataTrack | undefined> {
return scope.behavior(
tracks$.pipe(
map((tracks) =>
tracks?.find(
(t): t is DataTrack => t.kind === "data" && t.name === name,
),
),
distinctUntilChanged(),
),
);
}
+2 -12
View File
@@ -30,7 +30,7 @@ import { HomePage } from "./home/HomePage";
import { LoginPage } from "./auth/LoginPage";
import { RegisterPage } from "./auth/RegisterPage";
import { RoomPage } from "./room/RoomPage";
import { type ClientLoader, ClientProvider } from "./ClientContext";
import { ClientProvider } from "./ClientContext";
import { ErrorPage, LoadingPage } from "./FullScreenView";
import { Initializer } from "./initializer";
import { type WidgetHelpers } from "./widget";
@@ -57,16 +57,6 @@ import { LeaveToHomeProvider } from "./LeaveToHomeContext";
const SentryRoute = Sentry.withSentryReactRouterV7Routing(Route);
// Element Call's own session, for when no host supplies a client.
// - Passed in here, not imported by ClientProvider: this module is only in the
// app builds, so the client setup and the store worker behind it stay out of
// the component package (see the `loadClient` prop).
// - Imported on demand, as it was before, so that a widget, which gets its
// client from the host, never downloads it.
// - Module-level, so that the provider's effect sees one stable loader.
const loadStandaloneClient: ClientLoader = async (...args) =>
(await import("./utils/spa")).initSPA(...args);
interface SimpleProviderProps {
children: JSX.Element;
}
@@ -156,7 +146,7 @@ export const App: FC<Props> = ({ vm, widget }) => {
const content =
loaded && clientReady ? (
<ClientProvider client={widgetClient} loadClient={loadStandaloneClient}>
<ClientProvider client={widgetClient}>
<MediaDevicesContext value={vm.mediaDevices}>
<BackgroundEffectsProvider effects={vm.backgroundEffects}>
<Sentry.ErrorBoundary
+9 -28
View File
@@ -11,11 +11,7 @@ import { BrowserRouter } from "react-router-dom";
import { type MatrixClient } from "matrix-js-sdk";
import { type FC } from "react";
import {
type ClientLoader,
ClientProvider,
useClientState,
} from "./ClientContext";
import { ClientProvider, useClientState } from "./ClientContext";
const mockClient = (userId = "@alice:example.org"): MatrixClient =>
({
@@ -99,29 +95,10 @@ test("follows the client when the host swaps it", () => {
test("finds a client of its own when the host supplies none", async () => {
const client = mockClient();
const loadClient = vi
.fn<ClientLoader>()
.mockResolvedValue({ client, passwordlessUser: true });
vi.doMock("./utils/spa", () => ({
initSPA: vi.fn().mockResolvedValue({ client, passwordlessUser: true }),
}));
const { container } = render(
<BrowserRouter>
<ClientProvider loadClient={loadClient}>
<ShowClientState />
</ClientProvider>
</BrowserRouter>,
);
// Nothing to show until a session has been restored or created
expect(container.textContent).toBe("loading");
await waitFor(() => expect(container.textContent).toBe("@alice:example.org"));
// Given the session store to restore from and to clear on a stale session
expect(loadClient).toHaveBeenCalledWith(
expect.any(Function),
expect.any(Function),
);
});
test("has no one signed in when given neither a client nor a way to load one", async () => {
const { container } = render(
<BrowserRouter>
<ClientProvider>
@@ -130,5 +107,9 @@ test("has no one signed in when given neither a client nor a way to load one", a
</BrowserRouter>,
);
await waitFor(() => expect(container.textContent).toBe("unauthenticated"));
// Nothing to show until a session has been restored or created
expect(container.textContent).toBe("loading");
await waitFor(() => expect(container.textContent).toBe("@alice:example.org"));
vi.doUnmock("./utils/spa");
});
+12 -32
View File
@@ -143,34 +143,9 @@ interface Props {
* own.
*/
client?: MatrixClient;
/**
* How Element Call finds a client when none is supplied: the standalone app
* restores or creates a session of its own. Stable, as a hook dependency.
* Left out along with the client, no one is signed in.
*
* A prop, rather than this module importing `utils/spa` itself, because:
* - This module is in the component build, and so is everything it imports,
* dynamically or not: a dynamic import becomes a chunk in the package,
* not a module left out of it.
* - Behind `utils/spa` sits `utils/initClient` and the client store worker,
* which the bundler emits as soon as it loads the import (see there).
* - `App.tsx`, which the component never imports, is where the standalone
* app passes the loader in; the component passes nothing, having the
* host's client.
*/
loadClient?: ClientLoader;
}
/**
* Restores or creates a session of Element Call's own, kept where the given
* functions keep it. Resolves to null when no one is signed in.
*/
export type ClientLoader = (
loadSession: () => Session | undefined,
clearSession: () => void,
) => Promise<InitResult | null>;
export const ClientProvider: FC<Props> = ({ children, client, loadClient }) => {
export const ClientProvider: FC<Props> = ({ children, client }) => {
const leaveToHome = useLeaveToHome();
const hostBridge = useHostBridge();
@@ -204,13 +179,9 @@ export const ClientProvider: FC<Props> = ({ children, client, loadClient }) => {
// React does in strict mode), we need to make sure not to doubly initialize
// the client.
if (initializing.current) return;
if (loadClient === undefined) {
setInitClientState(null);
return;
}
initializing.current = true;
loadClient(loadSession, clearSession)
loadClient()
.then((initResult) => {
setInitClientState(initResult);
if (PosthogAnalytics.instance.isEnabled())
@@ -218,7 +189,7 @@ export const ClientProvider: FC<Props> = ({ children, client, loadClient }) => {
})
.catch((err) => logger.error(err))
.finally(() => (initializing.current = false));
}, [client, loadClient]);
}, [client]);
const changePassword = useCallback(
async (password: string) => {
@@ -389,6 +360,15 @@ export type InitResult = {
passwordlessUser: boolean;
};
/**
* Restores or creates a session of Element Call's own. Only reached when no
* client was supplied for it to use.
*/
async function loadClient(): Promise<InitResult | null> {
const { initSPA } = await import("./utils/spa");
return initSPA(loadSession, clearSession);
}
export interface Session {
user_id: string;
device_id: string;
+8 -10
View File
@@ -18,8 +18,6 @@ Please see LICENSE in the repository root for full details.
*/
import {
RTC_CALL_INTENT_AUDIO,
RTC_CALL_INTENT_VIDEO,
type RTCCallIntent,
type RTCNotificationType,
} from "matrix-js-sdk/lib/matrixrtc";
@@ -181,40 +179,40 @@ export function configurationForIntent(intent: UserIntent): UrlConfiguration {
switch (intent) {
case UserIntent.StartNewCall:
preset.skipLobby = false;
preset.callIntent = RTC_CALL_INTENT_VIDEO;
preset.callIntent = "video";
break;
case UserIntent.JoinExistingCall:
// On desktop this will be overridden based on which button was used to join the call
preset.skipLobby = false;
preset.callIntent = RTC_CALL_INTENT_VIDEO;
preset.callIntent = "video";
break;
case UserIntent.StartNewCallVoice:
preset.skipLobby = false;
preset.callIntent = RTC_CALL_INTENT_AUDIO;
preset.callIntent = "audio";
break;
case UserIntent.JoinExistingCallVoice:
// On desktop this will be overridden based on which button was used to join the call
preset.skipLobby = false;
preset.callIntent = RTC_CALL_INTENT_AUDIO;
preset.callIntent = "audio";
break;
case UserIntent.StartNewCallDMVoice:
preset.callIntent = RTC_CALL_INTENT_AUDIO;
preset.callIntent = "audio";
// Fall through
case UserIntent.StartNewCallDM:
preset.skipLobby = true;
preset.sendNotificationType = "ring";
preset.autoLeaveWhenOthersLeft = true;
preset.waitForCallPickup = true;
preset.callIntent = preset.callIntent ?? RTC_CALL_INTENT_VIDEO;
preset.callIntent = preset.callIntent ?? "video";
break;
case UserIntent.JoinExistingCallDMVoice:
preset.callIntent = RTC_CALL_INTENT_AUDIO;
preset.callIntent = "audio";
// Fall through
case UserIntent.JoinExistingCallDM:
// On desktop this will be overridden based on which button was used to join the call
preset.skipLobby = true;
preset.autoLeaveWhenOthersLeft = true;
preset.callIntent = preset.callIntent ?? RTC_CALL_INTENT_VIDEO;
preset.callIntent = preset.callIntent ?? "video";
break;
// Non widget usecase defaults
default:
+6 -10
View File
@@ -21,10 +21,6 @@ import {
UserIntent,
} from "../src/UrlParams";
import { mockConfig } from "./utils/test";
import {
RTC_CALL_INTENT_AUDIO,
RTC_CALL_INTENT_VIDEO,
} from "matrix-js-sdk/lib/matrixrtc";
const ROOM_NAME = "roomNameHere";
const ROOM_ID = "!d45f138fsd";
@@ -279,7 +275,7 @@ describe("UrlParams", () => {
).toMatchObject({
...startNewCallDefaults("desktop"),
skipLobby: false,
callIntent: RTC_CALL_INTENT_VIDEO,
callIntent: "video",
});
});
@@ -326,7 +322,7 @@ describe("UrlParams", () => {
).toMatchObject({
...startNewCallDefaults("desktop"),
skipLobby: false,
callIntent: RTC_CALL_INTENT_AUDIO,
callIntent: "audio",
});
});
@@ -337,7 +333,7 @@ describe("UrlParams", () => {
),
).toMatchObject({
...joinExistingCallDefaults("desktop"),
callIntent: RTC_CALL_INTENT_AUDIO,
callIntent: "audio",
});
});
@@ -352,7 +348,7 @@ describe("UrlParams", () => {
sendNotificationType: "ring",
autoLeaveWhenOthersLeft: true,
waitForCallPickup: true,
callIntent: RTC_CALL_INTENT_AUDIO,
callIntent: "audio",
});
});
@@ -367,7 +363,7 @@ describe("UrlParams", () => {
skipLobby: true,
autoLeaveWhenOthersLeft: true,
waitForCallPickup: false,
callIntent: RTC_CALL_INTENT_VIDEO,
callIntent: "video",
});
});
@@ -381,7 +377,7 @@ describe("UrlParams", () => {
skipLobby: true,
autoLeaveWhenOthersLeft: true,
waitForCallPickup: false,
callIntent: RTC_CALL_INTENT_AUDIO,
callIntent: "audio",
});
});
});
+1 -1
View File
@@ -13,7 +13,7 @@ import {
type MatrixClient,
} from "matrix-js-sdk";
import { initClient } from "../utils/initClient";
import { initClient } from "../utils/matrix";
import { type Session } from "../ClientContext";
/**
* This provides the login method to login using user credentials.
+1 -1
View File
@@ -14,7 +14,7 @@ import {
} from "matrix-js-sdk";
import { logger } from "matrix-js-sdk/lib/logger";
import { initClient } from "../utils/initClient";
import { initClient } from "../utils/matrix";
import { type Session } from "../ClientContext";
import { Config } from "../config/Config";
import { useUrlParams } from "../UrlParams";
@@ -90,42 +90,3 @@ Please see LICENSE in the repository root for full details.
text-align: center;
overflow-wrap: anywhere;
}
.removable {
position: relative;
display: flex;
min-inline-size: 0;
}
.removable > .tile {
flex: 1;
}
/* Clear of the target for choosing the tile. */
.remove {
position: absolute;
inset-block-start: 0;
inset-inline-end: 0;
translate: 20% -20%;
display: none;
align-items: center;
justify-content: center;
inline-size: 28px;
block-size: 28px;
border-radius: 50%;
background: var(--cpd-color-bg-canvas-default);
color: var(--cpd-color-icon-critical-primary);
box-shadow: 0 1.2px 2.4px 0 rgb(0 0 0 / 15%);
cursor: pointer;
}
.remove:hover {
background: var(--cpd-color-bg-critical-subtle-hovered);
box-shadow: 0 0 0 var(--cpd-border-width-1)
var(--cpd-color-border-critical-subtle);
}
.removable:hover .remove,
.removable:focus-within .remove {
display: flex;
}
+9 -82
View File
@@ -5,16 +5,14 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type FC, type KeyboardEvent, type ReactElement } from "react";
import { type FC, type ReactElement } from "react";
import { useTranslation } from "react-i18next";
import { VisuallyHidden } from "@radix-ui/react-visually-hidden";
import { InlineSpinner, MenuItem, Tooltip } from "@vector-im/compound-web";
import { InlineSpinner, MenuItem } from "@vector-im/compound-web";
import {
BlockIcon,
BlurIcon,
CheckCircleSolidIcon,
CloseIcon,
PlusIcon,
} from "@vector-im/compound-design-tokens/assets/web/icons";
import styles from "./BackgroundEffectGrid.module.css";
@@ -29,8 +27,6 @@ export interface BackgroundEffectGridItem {
kind: "none" | "blur" | "image";
/** For kind "image". */
imageUrl?: string;
/** Whether it is the user's to remove. */
removable?: boolean;
}
interface Props {
@@ -42,12 +38,6 @@ interface Props {
settling?: boolean;
/** Id of the text saying what the user should know before choosing. */
describedBy?: string;
/** Offers to add a background of one's own. Omit to leave that tile out. */
onAdd?: () => void;
addLabel?: string;
/** Removes a removable one, never the one in force. Omit to offer none. */
onRemove?: (id: string) => void;
removeLabel?: string;
}
/**
@@ -61,10 +51,6 @@ export const BackgroundEffectGrid: FC<Props> = ({
onSelect,
settling = false,
describedBy,
onAdd,
addLabel,
onRemove,
removeLabel,
}) => {
const { t } = useTranslation();
const choose = (id: string): void => {
@@ -80,30 +66,7 @@ export const BackgroundEffectGrid: FC<Props> = ({
: t("background_effects.numbered", { n: ++pictures });
const checked = effect.id === selected;
const disabled = onSelect === undefined && effect.kind !== "none";
const remove =
effect.removable && !checked && onRemove !== undefined
? (): void => onRemove(effect.id)
: undefined;
const removeProps =
remove === undefined
? {}
: {
"aria-keyshortcuts": "Delete",
onKeyDown: (e: KeyboardEvent): void => {
if (e.key !== "Delete" && e.key !== "Backspace") return;
e.preventDefault();
remove();
},
};
const content = (
<TileContent
effect={effect}
label={label}
checked={checked}
settling={checked && settling}
/>
);
const tile = (
return (
<MenuItem
key={effect.id}
role="menuitemradio"
@@ -125,57 +88,21 @@ export const BackgroundEffectGrid: FC<Props> = ({
choose(effect.id);
}
}
{...removeProps}
>
{content}
<TileContent
effect={effect}
label={label}
checked={checked}
settling={checked && settling}
/>
</MenuItem>
);
if (remove === undefined) return tile;
// Beside the item, not in it: inside, the item takes the press first and
// the tile is chosen instead.
return (
<div key={effect.id} className={styles.removable}>
{tile}
<Tooltip label={removeLabel ?? ""}>
{/* Not a control of its own, so the menu keeps one keyboard model:
the keyboard reaches the same action through Delete. */}
<span aria-hidden className={styles.remove} onClick={remove}>
<CloseIcon width={20} height={20} />
</span>
</Tooltip>
</div>
);
});
// A command rather than a choice, so an item beside the radios.
const addContent = (
<>
<span aria-hidden className={styles.swatch}>
<PlusIcon width={24} height={24} />
</span>
<VisuallyHidden>{addLabel}</VisuallyHidden>
</>
);
const addTile =
onAdd === undefined ? null : (
<MenuItem
label={null}
hideChevron
className={styles.tile}
onSelect={(e) => {
e.preventDefault();
onAdd();
}}
>
{addContent}
</MenuItem>
);
return (
// A group of its own, as the section's can't carry the description.
<div role="group" aria-describedby={describedBy} className={styles.grid}>
{tiles}
{addTile}
</div>
);
};
+2 -7
View File
@@ -127,21 +127,16 @@ export const Default: Story = {
selectBackgroundEffect: fn(),
backgroundEffect: "blur",
backgroundEffects: [
{ id: "none", kind: "none", removable: false },
{ id: "blur", kind: "blur", removable: false },
{ id: "none", kind: "none" },
{ id: "blur", kind: "blur" },
...shippedBackgrounds.map((background) => ({
id: `image:${background.id}`,
kind: "image" as const,
imageUrl: background.imagePath,
removable: false,
})),
],
backgroundEffectNotice: undefined,
backgroundEffectSettling: false,
backgroundImageRefusal: undefined,
addBackgroundImage: fn(),
removeBackgroundEffect: fn(),
cameraTrack: null,
hangup: fn(),
buttonSize: "lg",
showFooter: true,
-37
View File
@@ -33,13 +33,6 @@ import { useBehavior } from "../useBehavior";
import { type LayoutSwitchViewModel } from "../state/LayoutSwitchViewModel";
import { LayoutSwitch } from "../room/LayoutSwitch";
import { type BackgroundEffectGridItem } from "./BackgroundEffectGrid";
import { SelfPreview } from "./SelfPreview";
import { type UnusableReason } from "../tracks/backgroundImages";
import { type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk";
export interface BackgroundImageRefusal {
reason: UnusableReason | "not-kept";
}
export interface AudioOutputSwitcher {
targetOutput: string;
@@ -67,10 +60,6 @@ export interface FooterActions {
toggleVideo: (() => void) | undefined;
/** Undefined where background effects can't be chosen. */
selectBackgroundEffect: ((id: string) => void) | undefined;
/** Undefined where no background can be added. */
addBackgroundImage: ((file: File) => void) | undefined;
/** Removes an added background by its option id; never the one in force. */
removeBackgroundEffect: ((id: string) => void) | undefined;
toggleScreenSharing: (() => void) | undefined;
/** Also controls if the settings button is visible */
openSettings: (() => void) | undefined;
@@ -92,10 +81,6 @@ export interface FooterState {
backgroundEffectNotice: "unavailable" | "slow" | undefined;
/** Whether the first effect chosen is still being prepared. */
backgroundEffectSettling: boolean;
/** Why the last file offered couldn't be kept, if it couldn't. */
backgroundImageRefusal: BackgroundImageRefusal | undefined;
/** The camera to preview, with the effect in force; null while it is off. */
cameraTrack: LocalVideoMediaTrack | null;
showFooter: boolean;
/* This is needed for WindowMode = "flat" */
@@ -183,24 +168,6 @@ export const CallFooter: FC<FooterProps> = ({
const selectBackgroundEffect = useBehavior(vm.selectBackgroundEffect$);
const backgroundEffectNotice = useBehavior(vm.backgroundEffectNotice$);
const backgroundEffectSettling = useBehavior(vm.backgroundEffectSettling$);
const addBackgroundImage = useBehavior(vm.addBackgroundImage$);
const removeBackgroundEffect = useBehavior(vm.removeBackgroundEffect$);
const cameraTrack = useBehavior(vm.cameraTrack$);
const refusal = useBehavior(vm.backgroundImageRefusal$);
const refusalMessage = useMemo(() => {
switch (refusal?.reason) {
case undefined:
return undefined;
case "not-an-image":
return { text: t("error.background_not_an_image") };
case "animated":
return { text: t("error.background_animated") };
case "undecodable":
return { text: t("error.background_undecodable") };
case "not-kept":
return { text: t("error.background_not_kept") };
}
}, [refusal, t]);
const backgroundEffects = useBehavior(vm.backgroundEffects$);
const buttonSize = useBehavior(vm.buttonSize$);
const showLogo = useBehavior(vm.showLogo$);
@@ -267,10 +234,6 @@ export const CallFooter: FC<FooterProps> = ({
selectedBackgroundEffect={backgroundEffect}
onSelectBackgroundEffect={selectBackgroundEffect}
backgroundEffectSettling={backgroundEffectSettling}
onAddBackgroundImage={addBackgroundImage}
onRemoveBackgroundEffect={removeBackgroundEffect}
selfPreview={<SelfPreview track={cameraTrack} />}
backgroundImageRefusal={refusalMessage}
backgroundEffectNotice={
backgroundEffectNotice === "unavailable"
? t("background_effects.unavailable")

Some files were not shown because too many files have changed in this diff Show More